commit 3a2c66702caee9178b1715749206fb718290b0ec Author: wehub-resource-sync Date: Mon Jul 13 12:49:27 2026 +0800 chore: import upstream snapshot with attribution diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 0000000..c0ca2dd --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,51 @@ +# see https://docs.codecov.io/docs/codecov-yaml +# Validation check: +# $ curl --data-binary @.codecov.yml https://codecov.io/validate + + +# https://docs.codecov.io/docs/codecovyml-reference +codecov: + bot: "codecov-io" + strict_yaml_branch: "yaml-config" + require_ci_to_pass: yes + notify: + # after_n_builds: 2 + wait_for_ci: yes + +coverage: + precision: 2 # 2 = xx.xx%, 0 = xx% + round: nearest # how coverage is rounded: down/up/nearest + range: 70...100 # custom range of coverage colors from red -> yellow -> green + status: + # https://codecov.readme.io/v1.0/docs/commit-status + project: + default: + against: auto + target: 87% # The target should be the same of the pyproject `fail_under` + # threshold: 30% # allow this little decrease on project + # https://github.com/codecov/support/wiki/Filtering-Branches + # branches: main + if_ci_failed: error + # https://github.com/codecov/support/wiki/Patch-Status + patch: # is the coverage percentage of the number of lines touched in your patch (the git diff) + default: + against: auto + target: 87% # The target should be the same of the pyproject `fail_under` + # threshold: 50% # allow this much decrease on patch + changes: false + +parsers: + gcov: + branch_detection: + conditional: true + loop: true + macro: false + method: false + javascript: + enable_partials: false + +comment: + layout: header, diff + require_changes: false + behavior: default # update if exists else create new + # branches: * diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..7595f66 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,5 @@ +# These are supported funding model platforms + +github: [kornia, edgarriba, shijianjian] +open_collective: kornia +custom: # Replace with a single custom sponsorship URL diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 0000000..d8efa8a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,124 @@ +name: 🐛 Bug Report +description: Create a report to help us improve +title: "[Bug]: " +labels: ["bug", "help wanted"] + +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + + **Before submitting:** + - [ ] I have searched existing issues to ensure this bug hasn't been reported + - [ ] I have checked the [help wanted issues](https://github.com/kornia/kornia/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22help%20wanted%22) list + - [ ] I have read the [Contributing Guide](https://github.com/kornia/kornia/blob/main/CONTRIBUTING.md) + + --- + + **⚠️ Important: Before submitting a PR** + + If you plan to fix this bug yourself: + + 1. **Wait for maintainer approval**: A maintainer must review and approve this issue first + 2. **Wait for assignment**: You must be assigned to this issue by a maintainer before submitting a PR + 3. **Do not start work until assigned**: PRs submitted without prior issue approval and assignment may be closed + + This helps us ensure the issue aligns with project goals, avoid duplicate work, and coordinate contributions effectively. See the [Contributing Guide](https://github.com/kornia/kornia/blob/main/CONTRIBUTING.md#pull-request) for more details. + + - type: textarea + id: describe-bug + attributes: + label: 🐛 Describe the bug + description: A clear and concise description of what the bug is. + placeholder: | + Example: When I call function X with parameters Y, I get error Z instead of the expected result. + validations: + required: true + + - type: textarea + id: reproduction-steps + attributes: + label: 🔄 Steps to Reproduce + description: Steps to reproduce the behavior. Include minimal code example if possible. + value: | + 1. + 2. + 3. + ... + render: bash + validations: + required: true + + - type: textarea + id: code-example + attributes: + label: 💻 Minimal Code Example + description: If applicable, provide a minimal code example that reproduces the bug + placeholder: | + ```python + import kornia + # Your minimal code here + ``` + render: python + validations: + required: false + + - type: textarea + id: expected-behavior + attributes: + label: ✅ Expected behavior + description: A clear and concise description of what you expected to happen. + validations: + required: true + + - type: textarea + id: actual-behavior + attributes: + label: ❌ Actual behavior + description: What actually happened? Include error messages if any. + validations: + required: true + + - type: textarea + id: environment + attributes: + label: 🔧 Environment + description: Please provide your environment details + value: | + - Kornia version: + - PyTorch version: + - Python version: + - OS (e.g., Linux, macOS, Windows): + - Installation method (pip, conda, from source): + - CUDA/cuDNN version (if applicable): + - GPU model (if applicable): + + You can also run: + ```bash + wget https://raw.githubusercontent.com/pytorch/pytorch/main/torch/utils/collect_env.py + # For security purposes, please check the contents of collect_env.py before running it. + python collect_env.py + ``` + render: shell + validations: + required: true + + - type: textarea + id: additional-context + attributes: + label: 📝 Additional context + description: Add any other context, screenshots, or relevant information about the problem here. + validations: + required: false + + - type: checkboxes + id: contribution-intent + attributes: + label: 🤝 Contribution Intent + description: Are you planning to contribute a fix for this bug? + options: + - label: I plan to submit a PR to fix this bug + required: false + - label: I'm reporting this bug but not planning to fix it + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..dd931a9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,14 @@ +blank_issues_enabled: false +contact_links: + - name: ❓ Ask a question + url: https://github.com/kornia/kornia/discussions + about: Please post questions related to the library in discussions. + - name: 🆘 Help Wanted Issues + url: https://github.com/kornia/kornia/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22help%20wanted%22 + about: Browse issues looking for contributors + - name: 💬 Discord + url: https://discord.gg/HfnywwpBnD + about: Chat with our community + - name: 📖 Contributing Guide + url: https://github.com/kornia/kornia/blob/main/CONTRIBUTING.md + about: Read our contribution guidelines and policies diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 0000000..2184ae3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,124 @@ +name: 🚀 Feature Request +description: Suggest an idea or new feature for this project +title: "[Feature]: " +labels: ["enhancement", "help wanted"] + +body: + - type: markdown + attributes: + value: | + Thanks for suggesting a new feature! + + **Before submitting:** + - [ ] I have searched existing issues to ensure this feature hasn't been requested + - [ ] I have checked the [help wanted issues](https://github.com/kornia/kornia/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22help%20wanted%22) list + - [ ] I have read the [Contributing Guide](https://github.com/kornia/kornia/blob/main/CONTRIBUTING.md) + - [ ] This feature aligns with Kornia's focus on end-to-end vision models and SOTA Vision/Robotics models + + --- + + **⚠️ Important: Before submitting a PR** + + If you plan to implement this feature yourself: + + 1. **Wait for maintainer approval**: A maintainer must review and approve this issue first + 2. **Wait for assignment**: You must be assigned to this issue by a maintainer before submitting a PR + 3. **Do not start work until assigned**: PRs submitted without prior issue approval and assignment may be closed + + This helps us ensure the issue aligns with project goals, avoid duplicate work, and coordinate contributions effectively. See the [Contributing Guide](https://github.com/kornia/kornia/blob/main/CONTRIBUTING.md#pull-request) for more details. + + - type: textarea + id: feature-description + attributes: + label: 🚀 Feature Description + description: A clear and concise description of the feature you'd like to see + placeholder: | + Example: Add support for X model/functionality that enables Y use case. + validations: + required: true + + - type: dropdown + id: feature-category + attributes: + label: 📂 Feature Category + description: What category does this feature belong to? + options: + - VLM/VLA Models (Vision Language Models/Agents) - Priority + - Robotics Models + - Image Processing + - Augmentation + - Feature Detection/Matching + - Geometry + - Loss Functions + - Models (General) + - Documentation + - Testing Infrastructure + - Other + validations: + required: true + + - type: textarea + id: motivation + attributes: + label: 💡 Motivation + description: | + What problem does this feature solve? Why is this feature needed? + If this is related to another GitHub issue, please link it here. + placeholder: | + Example: I'm always frustrated when [problem]. This feature would help by [benefit]. + validations: + required: true + + - type: textarea + id: proposed-solution + attributes: + label: 💭 Proposed Solution + description: A clear and concise description of what you want to happen. Include API design if applicable. + placeholder: | + Example: Add a new function `kornia.feature.new_function()` that accepts X and returns Y. + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: 🔄 Alternatives Considered + description: A clear and concise description of any alternative solutions or features you've considered + validations: + required: false + + - type: textarea + id: use-cases + attributes: + label: 🎯 Use Cases + description: Describe specific use cases or examples where this feature would be valuable + validations: + required: false + + - type: textarea + id: additional-context + attributes: + label: 📝 Additional Context + description: Add any other context, mockups, diagrams, or references about the feature request here. + validations: + required: false + + - type: checkboxes + id: contribution-intent + attributes: + label: 🤝 Contribution Intent + description: Are you planning to contribute this feature? + options: + - label: I plan to submit a PR to implement this feature + required: false + - label: I'm requesting this feature but not planning to implement it + required: false + + - type: markdown + attributes: + value: | + --- + + #### 📚 Consider also contributing to Kornia universe projects: + + - [**Tutorials**](https://github.com/kornia/tutorials): Our repository containing the tutorials. diff --git a/.github/actions/coverage/action.yml b/.github/actions/coverage/action.yml new file mode 100644 index 0000000..026974e --- /dev/null +++ b/.github/actions/coverage/action.yml @@ -0,0 +1,62 @@ +name: tests-coverage +description: "Run CPU tests with coverage collection and artifact upload" + +inputs: + python-version: + description: "Python version to use" + required: false + default: "3.11" + pytorch-dtype: + description: "PyTorch data type for testing" + required: false + default: float32 + pytorch-device: + description: "PyTorch device for testing" + required: false + default: cpu + pytest-extra: + description: "Extra pytest arguments" + required: false + default: --timeout=30 -k "not test_dynamo" + coverage-artifact: + description: "Name of coverage artifact" + required: false + default: coverage + continue-on-error: + description: "Whether to continue on error" + required: false + default: "false" + +runs: + using: "composite" + steps: + - name: Set pixi environment name + id: pixi-env + shell: bash + run: echo "name=py$(echo '${{ inputs.python-version }}' | tr -d '.')" >> $GITHUB_OUTPUT + + - name: Setup pixi + uses: prefix-dev/setup-pixi@v0.8.1 + with: + pixi-version: v0.40.0 + cache: true + environments: ${{ steps.pixi-env.outputs.name }} + + - name: Install dependencies + shell: bash + run: pixi run -e ${{ steps.pixi-env.outputs.name }} install + + - name: Run CPU tests coverage ${{ inputs.pytorch-dtype }} + continue-on-error: ${{ inputs.continue-on-error == 'true' }} + shell: bash + run: | + pixi run -e ${{ steps.pixi-env.outputs.name }} uv run coverage erase + pixi run -e ${{ steps.pixi-env.outputs.name }} uv run coverage run --data-file=${{ inputs.coverage-artifact }} -m pytest -v ./tests/ --device=${{ inputs.pytorch-device }} --dtype=${{ inputs.pytorch-dtype }} ${{ inputs.pytest-extra }} + + - name: Upload coverage results + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.coverage-artifact }} + path: ${{ inputs.coverage-artifact }} + if-no-files-found: error + retention-days: 1 diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..9713a46 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,82 @@ +# Kornia Repository Instructions + +This file provides instructions for GitHub Copilot when working with code in this repository. + +## Coding Standards + +Follow the coding standards and best practices defined in [CONTRIBUTING.md](../CONTRIBUTING.md#coding-standards). + +## AI Policy + +**CRITICAL**: All contributions must comply with the [AI_POLICY.md](AI_POLICY.md). Review that document for complete requirements. + +### Core Principles: +- Code and comments must not be direct, unreviewed outputs of AI agents +- All AI-assisted contributions require human oversight and validation +- Ensure code logic reflects genuine understanding, not copied AI output +- The submitter is the Sole Responsible Author for every line of code + +## Instructions for AI Reviewers (Copilot / CodeRabbit) + +AI-based reviewers (e.g. GitHub Copilot, CodeRabbit) must follow the repository's AI usage policy and review rules. + +For the complete and authoritative AI reviewer instructions, see [AI_POLICY.md](AI_POLICY.md), section 3. + +When generating or reviewing suggestions, prefer: +- Enforcing the coding standards in [CONTRIBUTING.md](../CONTRIBUTING.md#coding-standards) +- Enforcing the AI usage rules and review heuristics defined in [AI_POLICY.md](AI_POLICY.md) +- Highlighting missing tests, missing proof of local execution, and misuse of `kornia` vs. raw PyTorch utilities +## Key Guidelines + +- **Code style**: Follow PEP8, use 120 character line length, Ruff linting, and f-strings +- **Type hints**: Required for all function inputs and outputs +- **Documentation**: Follow documentation and docstring guidelines in [CONTRIBUTING.md](../CONTRIBUTING.md#coding-standards) and match the existing codebase style +- **Testing**: Use `BaseTester` pattern with smoke, exception, cardinality, feature, gradcheck, and dynamo tests +- **Dependencies**: Only PyTorch is allowed as a dependency +- **Use kornia**: Always prefer `kornia` utilities over raw PyTorch functions + +## Running Checks + +```bash +pixi run lint # Linting +pixi run typecheck # Type checking +pixi run test # Testing +pixi run doctest # Documentation tests +``` + +## Review Checklist + +When reviewing code changes, verify: + +- Code and comments are not direct, unreviewed AI agent outputs +- Code follows guidelines in [CONTRIBUTING.md](../CONTRIBUTING.md) +- Code complies with [AI_POLICY.md](AI_POLICY.md) +- Tests are included for new functionality +- Code passes `pixi run lint` and `pixi run typecheck` +- PR includes proof of local test execution (test logs) +- Code uses `kornia` utilities instead of reinventing existing functionality +- Comments are written in English and verified by a human with a good understanding of the code + +## PR-Issue Alignment Review + +When reviewing pull requests, ensure strict alignment with the linked issue: + +1. **Issue Link Verification**: + - Verify the PR description contains a valid issue reference (e.g., "Fixes #123" or "Closes #123") + - Confirm the linked issue exists and is open (or was open when the PR was created) + +2. **Assignment Verification**: + - Check that the PR author is assigned to the linked issue + - If not assigned, request that a maintainer assign the issue before proceeding with review + +3. **Scope Matching**: + - **Critical**: Verify that the PR implementation strictly matches what the issue describes + - The PR should not include changes beyond the scope of the linked issue + - If the PR includes additional features or changes not mentioned in the issue, request that those be split into separate issues/PRs + - Compare the PR description, code changes, and tests against the issue description to ensure alignment + +4. **Issue Approval Status**: + - Verify the linked issue has been reviewed and approved by a maintainer + - Issues with the `triage` label may not have been fully reviewed yet + +**Reviewer Action**: If the PR does not match the issue scope or requirements, clearly explain the mismatch and request that the PR be updated to strictly align with the issue, or that additional changes be moved to separate issues/PRs. diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..52a2a61 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,28 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + # Python packages + - package-ecosystem: "pip" + directory: "/" + open-pull-requests-limit: 10 + schedule: + interval: weekly + day: sunday + + # GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + commit-message: + prefix: ⬆️ + open-pull-requests-limit: 5 + # this will group all dependencies together into single PRs for all github-actions + groups: + github-actions: + patterns: + - "*" diff --git a/.github/download-models-weights.py b/.github/download-models-weights.py new file mode 100644 index 0000000..a48a4b8 --- /dev/null +++ b/.github/download-models-weights.py @@ -0,0 +1,80 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +import logging +import os + +import torch + +logging.basicConfig(level=logging.INFO, format="%(message)s") +logger = logging.getLogger(__name__) + +# Models used in doctests and tests +# Format: "name": url +# All models use torch.hub.load_state_dict_from_url +MODELS = { + # SOLD2 - line detection + "sold2_wireframe": "http://cmp.felk.cvut.cz/~mishkdmy/models/sold2_wireframe.pth", + # DexiNed - edge detection (used by EdgeDetector doctest) + "dexined": "http://cmp.felk.cvut.cz/~mishkdmy/models/DexiNed_BIPED_10.pth", + # DISK - feature extraction (used by DISK.from_pretrained doctest) + "disk_depth": "https://raw.githubusercontent.com/cvlab-epfl/disk/master/depth-save.pth", + # DeDoDe - feature detection/description (used by DeDoDe.from_pretrained doctest) + "dedode_detector_L_v2": "https://github.com/Parskatt/DeDoDe/releases/download/v2/dedode_detector_L_v2.pth", + "dedode_descriptor_B_SO2": "https://github.com/georg-bn/rotation-steerers/releases/download/release-2/B_SO2_Spread_descriptor_setting_C.pth", + "dedode_descriptor_B_upright": "https://github.com/Parskatt/DeDoDe/releases/download/dedode_pretrained_models/dedode_descriptor_B.pth", + "dedode_descriptor_G_upright": "https://github.com/Parskatt/DeDoDe/releases/download/dedode_pretrained_models/dedode_descriptor_G.pth", + # DINOv2 - used by DeDoDe encoder + "dinov2_vitl14": "https://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_pretrain.pth", + # YuNet - face detection + "yunet": "https://github.com/kornia/data/raw/main/yunet_final.pth", + # RT-DETR - object detection (used by RTDETR.from_pretrained doctest) + "rtdetr_r18vd": "https://github.com/lyuwenyu/storage/releases/download/v0.1/rtdetr_r18vd_dec3_6x_coco_from_paddle.pth", + # VisionTransformer - used by VisionTransformer.from_config doctest + "vit_b_16": "https://huggingface.co/kornia/vit_b16_augreg_i21k_r224/resolve/main/vit_b-16.pth", + # LocalFeatureMatcher - AffNet and HardNet (used by LocalFeatureMatcher doctest) + "affnet": "https://github.com/ducha-aiki/affnet/raw/master/pretrained/AffNet.pth", + "hardnet_liberty": "https://github.com/DagnyT/hardnet/raw/master/pretrained/train_liberty_with_aug/checkpoint_liberty_with_aug.pth", + # LoFTR - feature matching (used by LoFTR doctest) + "loftr_outdoor": "http://cmp.felk.cvut.cz/~mishkdmy/models/loftr_outdoor.ckpt", + # MKD - descriptor (used by MKDDescriptor doctest) + "mkd_concat": "https://github.com/manyids2/mkd_pytorch/raw/master/mkd_pytorch/mkd-concat-64.pth", +} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("WeightsDownloader") + parser.add_argument("--target_directory", "-t", required=False, default="weights") + + args = parser.parse_args() + + # Set torch.hub directory - files will go to {target_directory}/checkpoints/ + torch.hub.set_dir(args.target_directory) + # For HuggingFace model caching + os.environ["HF_HOME"] = args.target_directory + + logger.info(f"Downloading models to: {torch.hub.get_dir()}/checkpoints/") + + for name, url in MODELS.items(): + logger.info(f"Downloading `{name}` from `{url}`...") + # Don't pass model_dir - use the default from torch.hub.set_dir() + # This ensures files go to {hub_dir}/checkpoints/ matching test behavior + torch.hub.load_state_dict_from_url(url, map_location=torch.device("cpu")) + + logger.info("All models downloaded successfully!") + raise SystemExit(0) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..0d9e77d --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,47 @@ +## 📝 Description + +**⚠️ Issue Link Required**: This PR must be linked to an approved and assigned issue. See [Contributing Guide](https://github.com/kornia/kornia/blob/main/CONTRIBUTING.md#pull-request) for details. + +**Fixes/Relates to:** # (issue number) + +**Important**: +- Ensure you are assigned to the linked issue before submitting this PR +- This PR should strictly implement what the linked issue describes +- Do not include changes beyond the scope of the linked issue + +--- + +## 🛠️ Changes Made +- [ ] Item 1 +- [ ] Item 2 + +--- + +## 🧪 How Was This Tested? +- [ ] **Unit Tests:** (List new/updated tests) +- [ ] **Manual Verification:** (Describe the steps you took) +- [ ] **Performance/Edge Cases:** (How does this handle nulls, large data, etc.?) + +--- + +## 🕵️ AI Usage Disclosure +*Check one of the following:* +- [ ] 🟢 **No AI used.** +- [ ] 🟡 **AI-assisted:** I used AI for boilerplate/refactoring but have manually reviewed and tested every line. +- [ ] 🔴 **AI-generated:** (Note: These PRs may be subject to stricter scrutiny or immediate closure if the logic is not explained). + +--- + +## 🚦 Checklist +- [ ] I am assigned to the linked issue (required before PR submission) +- [ ] The linked issue has been approved by a maintainer +- [ ] I have performed a **self-review** of my code (no "ghost" variables or hallucinations). +- [ ] My code follows the existing style guidelines of this project. +- [ ] I have commented my code, particularly in hard-to-understand areas. +- [ ] I have added tests that prove my fix is effective or that my feature works. +- [ ] (Optional) I have attached screenshots/recordings for UI changes. + +--- + +## 💭 Additional Context +Add any other context or screenshots about the pull request here. diff --git a/.github/workflows/codeflash.yml b/.github/workflows/codeflash.yml new file mode 100644 index 0000000..b78d4c0 --- /dev/null +++ b/.github/workflows/codeflash.yml @@ -0,0 +1,39 @@ +name: Codeflash Optimization + +on: + pull_request: + paths: + - 'kornia/**' + workflow_dispatch: + +concurrency: + # Any new push to the PR will cancel the previous run, so that only the latest code is optimized + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + + +jobs: + optimize: + name: Optimize new Python code + # Don't run codeflash on codeflash-ai[bot] commits, prevent duplicate optimizations + if: ${{ github.actor != 'codeflash-ai[bot]' }} + runs-on: ubuntu-latest + env: + CODEFLASH_API_KEY: ${{ secrets.CODEFLASH_API_KEY }} + + steps: + - name: 🛎️ Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: 🐍 Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + - name: 📦 Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev,x]" + pip install codeflash + - name: ⚡️Codeflash Optimization + run: codeflash diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000..178508a --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,173 @@ +on: + workflow_call: + inputs: + python-version: + required: false + type: string + default: "3.11" + pytorch-device: + required: false + type: string + default: "cpu" + os: + required: false + type: string + default: ubuntu-latest + ref: + required: false + type: string + default: ${{ github.sha }} + cache-path: + required: false + type: string + cache-key: + required: false + type: string + cache-restore-keys: + required: false + type: string + +permissions: + contents: read + +jobs: + slow: + # Only run tests with slow marker which aren't dynamo tests + runs-on: ${{ inputs.os }} + strategy: + fail-fast: false + matrix: + pytorch-dtype: ["float32", "float64"] + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + + - name: Restore cache + if: inputs.cache-path != '' + uses: actions/cache/restore@v5 + with: + path: ${{ inputs.cache-path }} + key: ${{ inputs.cache-key }} + restore-keys: ${{ inputs.cache-restore-keys }} + + - uses: ./.github/actions/coverage + with: + python-version: ${{ inputs.python-version }} + pytorch-dtype: ${{ matrix.pytorch-dtype }} + pytorch-device: ${{ inputs.pytorch-device }} + pytest-extra: '--runslow -k "slow and not test_dynamo"' + coverage-artifact: coverage-slow-${{ matrix.pytorch-dtype }} + + dynamo: + # Run all dynamo (torch.compile) related tests + runs-on: ${{ inputs.os }} + strategy: + fail-fast: false + matrix: + pytorch-dtype: ["float32"] + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + + - name: Restore cache + if: inputs.cache-path != '' + uses: actions/cache/restore@v5 + with: + path: ${{ inputs.cache-path }} + key: ${{ inputs.cache-key }} + restore-keys: ${{ inputs.cache-restore-keys }} + + - uses: ./.github/actions/coverage + env: + KORNIA_TEST_OPTIMIZER: inductor + with: + python-version: ${{ inputs.python-version }} + pytorch-dtype: ${{ matrix.pytorch-dtype }} + pytorch-device: ${{ inputs.pytorch-device }} + pytest-extra: '--runslow -k "test_dynamo"' + coverage-artifact: coverage-dynamo-${{ matrix.pytorch-dtype }} + + overall: + # Run all tests which didn't have a slow marker or are dynamo tests + runs-on: ${{ inputs.os }} + strategy: + fail-fast: false + matrix: + pytorch-dtype: ["float32", "float64"] + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + + - name: Restore cache + if: inputs.cache-path != '' + uses: actions/cache/restore@v5 + with: + path: ${{ inputs.cache-path }} + key: ${{ inputs.cache-key }} + restore-keys: ${{ inputs.cache-restore-keys }} + + - uses: ./.github/actions/coverage + with: + python-version: ${{ inputs.python-version }} + pytorch-dtype: ${{ matrix.pytorch-dtype }} + pytorch-device: ${{ inputs.pytorch-device }} + coverage-artifact: coverage-overall-${{ matrix.pytorch-dtype }} + + report: + runs-on: ${{ inputs.os }} + needs: [slow, dynamo, overall] + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + repository: kornia/kornia + ref: ${{ inputs.ref }} + + - name: Download coverage reports + uses: actions/download-artifact@v8 + with: + merge-multiple: true + + - name: Set pixi environment name + id: pixi-env + shell: bash + run: echo "name=py$(echo '${{ inputs.python-version }}' | tr -d '.')" >> $GITHUB_OUTPUT + + - name: Setup pixi + uses: prefix-dev/setup-pixi@v0.9.6 + with: + pixi-version: v0.40.0 + cache: true + environments: ${{ steps.pixi-env.outputs.name }} + + - name: Install dependencies + run: pixi run -e ${{ steps.pixi-env.outputs.name }} install + + - name: Run coverage + run: | + pixi run -e ${{ steps.pixi-env.outputs.name }} uv run coverage combine --keep \ + $GITHUB_WORKSPACE/coverage-dynamo-float32 \ + $GITHUB_WORKSPACE/coverage-slow-float32 \ + $GITHUB_WORKSPACE/coverage-slow-float64 \ + $GITHUB_WORKSPACE/coverage-overall-float32 \ + $GITHUB_WORKSPACE/coverage-overall-float64 + pixi run -e ${{ steps.pixi-env.outputs.name }} uv run coverage xml -o coverage.xml + pixi run -e ${{ steps.pixi-env.outputs.name }} uv run coverage report --show-missing --skip-covered + + - if: always() + name: Upload coverage + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v3 + with: + file: coverage.xml + token: ${{ secrets.CODECOV_TOKEN }} + flags: cpu,${{ inputs.os }}_py-${{ inputs.python-version }}_float32,float64 + name: cpu-coverage diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..9621f75 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,79 @@ +on: + pull_request: + branches: [main] + paths: + - 'docs/**' + - '*.rst' + workflow_call: + inputs: + python-version: + required: false + type: string + default: '["3.11"]' + fail-fast: + required: false + type: boolean + default: false + ref: + required: false + type: string + default: ${{ github.sha }} + cache-path: + required: false + type: string + cache-key: + required: false + type: string + cache-restore-keys: + required: false + type: string + +permissions: + contents: read + +jobs: + docs: + name: docs + runs-on: ubuntu-latest + strategy: + fail-fast: ${{ inputs.fail-fast }} + matrix: + python-version: ${{ fromJSON(inputs.python-version) }} + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha || inputs.ref }} + + - name: Restore cache + if: inputs.cache-path != '' + uses: actions/cache/restore@v5 + with: + path: ${{ inputs.cache-path }} + key: ${{ inputs.cache-key }} + restore-keys: ${{ inputs.cache-restore-keys }} + + - name: Set pixi environment name + id: pixi-env + shell: bash + run: echo "name=py$(echo '${{ matrix.python-version }}' | tr -d '.')" >> $GITHUB_OUTPUT + + - name: Setup pixi + uses: prefix-dev/setup-pixi@v0.9.6 + with: + pixi-version: v0.40.0 + cache: true + environments: ${{ steps.pixi-env.outputs.name }} + + - name: Install environment + run: pixi install -e ${{ steps.pixi-env.outputs.name }} --locked + + - name: Install docs dependencies + run: pixi run -e ${{ steps.pixi-env.outputs.name }} install-docs + + - name: Run doctest + run: pixi run -e ${{ steps.pixi-env.outputs.name }} doctest + + - name: Build Documentation + run: pixi run -e ${{ steps.pixi-env.outputs.name }} build-docs diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml new file mode 100644 index 0000000..696b156 --- /dev/null +++ b/.github/workflows/issue-triage.yml @@ -0,0 +1,41 @@ +name: Issue Triage + +on: + issues: + types: [opened] + +permissions: + issues: write + contents: read + +jobs: + triage: + name: Triage New Issue + runs-on: ubuntu-latest + steps: + - name: Add triage label + uses: actions/github-script@v9 + with: + script: | + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + labels: ['triage'] + }); + + - name: Post welcome message + uses: actions/github-script@v9 + with: + script: | + const welcomeMessage = '## ⚠️ Before Submitting a PR\n\n' + + 'Please wait for a maintainer to approve and assign this issue to you or someone else before submitting a PR. ' + + 'PRs submitted without prior approval and assignment may be closed. ' + + 'See our [Contributing Guide](https://github.com/kornia/kornia/blob/main/CONTRIBUTING.md) for details.'; + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: welcomeMessage + }); diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..48c8c61 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,35 @@ +name: Lint + +on: + push: + branches: + - main + paths: + - '*.toml' + - 'pixi.toml' + - 'pyproject.toml' + pull_request: + branches: + - main + paths: + - '*.toml' + - 'pixi.toml' + - 'pyproject.toml' + +permissions: + contents: read + +jobs: + toml-fmt: + name: TOML Format + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: prefix-dev/setup-pixi@v0.9.6 + with: + pixi-version: v0.40.0 + cache: true + + - name: Check TOML formatting + run: pixi run -e default toml-fmt-check diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml new file mode 100644 index 0000000..f853f4e --- /dev/null +++ b/.github/workflows/pr-validation.yml @@ -0,0 +1,129 @@ +name: PR Validation + +on: + pull_request_target: + types: [opened, synchronize, reopened, edited] + +permissions: + pull-requests: write + issues: read + contents: read + +jobs: + validate: + name: Validate PR Requirements + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Validate PR requirements + uses: actions/github-script@v9 + with: + script: | + const pr = context.payload.pull_request; + const prBody = pr.body || ''; + const prAuthor = pr.user.login; + + // Extract issue numbers from PR description + // Matches patterns like: Fixes #123, Closes #456, Relates to #789, etc. + const issuePattern = /(?:fixes?|closes?|resolves?|relates?\s+to|see|refs?)\s+#(\d+)/gi; + const matches = [...prBody.matchAll(issuePattern)]; + const issueNumbers = [...new Set(matches.map(m => parseInt(m[1])))]; + + const warnings = []; + + if (issueNumbers.length === 0) { + warnings.push('❌ **No linked issue found**: This PR does not reference any issue. Please link to an issue using "Fixes #123" or "Closes #123" in the PR description.'); + } else { + // Check each linked issue + for (const issueNumber of issueNumbers) { + try { + const issue = await github.rest.issues.get({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber + }); + + if (issue.data.state === 'closed') { + warnings.push(`⚠️ **Issue #${issueNumber} is closed**: The linked issue #${issueNumber} is already closed. Please ensure you're linking to the correct issue.`); + } + + // Check if PR author is assigned to the issue + const assignees = issue.data.assignees.map(a => a.login); + if (assignees.length === 0) { + warnings.push(`⚠️ **Issue #${issueNumber} has no assignee**: This issue has not been assigned to anyone. Please wait for a maintainer to assign it to you before submitting a PR.`); + } else if (!assignees.includes(prAuthor)) { + warnings.push(`⚠️ **Assignment mismatch**: You (@${prAuthor}) are not assigned to issue #${issueNumber}. The issue is assigned to: ${assignees.map(a => `@${a}`).join(', ')}. Please wait for a maintainer to assign the issue to you.`); + } + + // Check if issue has triage label (indicating it might not be approved yet) + const labels = issue.data.labels.map(l => l.name); + if (labels.includes('triage')) { + warnings.push(`ℹ️ **Issue #${issueNumber} is still in triage**: This issue may not have been reviewed and approved by a maintainer yet. Please ensure the issue has been approved before submitting a PR.`); + } + } catch (error) { + if (error.status === 404) { + warnings.push(`❌ **Issue #${issueNumber} not found**: The linked issue #${issueNumber} does not exist. Please check the issue number.`); + } else { + warnings.push(`⚠️ **Error checking issue #${issueNumber}**: ${error.message}`); + } + } + } + } + + // Post comment if there are warnings + if (warnings.length > 0) { + const commentBody = '## ⚠️ PR Validation Warnings\n\n' + + warnings.join('\n\n') + '\n\n' + + '---\n\n' + + '**Note**: This PR can remain open, but please address these issues to ensure a smooth review process. For more information, see our [Contributing Guide](https://github.com/kornia/kornia/blob/main/CONTRIBUTING.md).'; + + // Check if we already posted a comment + const comments = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number + }); + + const botComment = comments.data.find( + c => c.user.type === 'Bot' && c.body.includes('PR Validation Warnings') + ); + + if (botComment) { + // Update existing comment + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: commentBody + }); + } else { + // Create new comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + body: commentBody + }); + } + } else { + // Remove any existing warning comments if validation passes + const comments = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number + }); + + const botComment = comments.data.find( + c => c.user.type === 'Bot' && c.body.includes('PR Validation Warnings') + ); + + if (botComment) { + await github.rest.issues.deleteComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id + }); + } + } diff --git a/.github/workflows/pr_test_cpu.yml b/.github/workflows/pr_test_cpu.yml new file mode 100644 index 0000000..0ac7db6 --- /dev/null +++ b/.github/workflows/pr_test_cpu.yml @@ -0,0 +1,100 @@ +name: Tests on CPU (PR) + +on: + pull_request: + branches: [main] + types: [opened, reopened, synchronize, ready_for_review] + paths-ignore: + - 'docs/**' + - '*.md' + - 'LICENSE' + - 'COPYRIGHT' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + pre-tests: + runs-on: ubuntu-latest + outputs: + hash: ${{ steps.hashid.outputs.weights-hash }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup pixi + uses: prefix-dev/setup-pixi@v0.9.6 + with: + pixi-version: v0.40.0 + cache: true + + - uses: actions/cache@v5 + id: cache-weights + with: + path: weights/ + key: model-weights-${{ hashFiles('.github/download-models-weights.py') }} + enableCrossOsArchive: true + + - name: Download models weights... + if: steps.cache-weights.outputs.cache-hit != 'true' + run: pixi run uv run python .github/download-models-weights.py -t weights/ + + - name: write hashid + id: hashid + run: echo "weights-hash=${{ hashFiles('.github/download-models-weights.py') }}" >> "$GITHUB_OUTPUT" + + tests: + needs: [pre-tests] + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest", "windows-latest", "macos-latest"] + pytorch-dtype: ["float32", "float64"] + python-version: ["3.11", "3.12", "3.13"] + pytorch-version: ["2.5.1", "2.9.1"] + exclude: + # PyTorch 2.5.1 doesn't have wheels for Python 3.13 on Windows + - os: windows-latest + python-version: "3.13" + pytorch-version: "2.5.1" + # PyTorch 2.5.1 doesn't have wheels for Python 3.13 on macOS + - os: macos-latest + python-version: "3.13" + pytorch-version: "2.5.1" + # macOS only runs float32 tests + - os: macos-latest + pytorch-dtype: "float64" + + uses: ./.github/workflows/tests.yml + with: + os: ${{ matrix.os }} + python-version: '["${{ matrix.python-version }}"]' + pytorch-version: '["${{ matrix.pytorch-version }}"]' + pytorch-dtype: ${{ matrix.pytorch-dtype }} + cache-path: weights/ + cache-key: model-weights-${{ needs.pre-tests.outputs.hash }} + cache-restore-keys: | + model-weights-${{ needs.pre-tests.outputs.hash }} + model-weights- + + typecheck: + needs: [pre-tests] + uses: ./.github/workflows/typecheck.yml + + tutorials: + needs: [pre-tests] + uses: ./.github/workflows/tutorials.yml + + collector: + needs: [tests, typecheck] + if: always() + runs-on: ubuntu-latest + steps: + - name: check for failures + if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: echo job failed && exit 1 diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml new file mode 100644 index 0000000..89c1907 --- /dev/null +++ b/.github/workflows/pypi-release.yml @@ -0,0 +1,70 @@ +name: PyPI Release + +on: + release: + types: [published] + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Setup pixi + uses: prefix-dev/setup-pixi@v0.9.6 + with: + pixi-version: v0.40.0 + cache: true + + - name: Build package + run: pixi run uv build + + - name: List built packages + run: ls -lh dist/ + + - uses: actions/upload-artifact@v7 + with: + name: pypi-packages-${{ github.sha }} + path: dist + + upload-release: + runs-on: ubuntu-latest + needs: build + if: github.event_name == 'release' + permissions: + contents: write # Required to attach assets to the GitHub Release + steps: + - uses: actions/download-artifact@v8 + with: + name: pypi-packages-${{ github.sha }} + path: dist + + - name: Upload to release + uses: AButler/upload-release-assets@v4.0 + with: + files: 'dist/*' + repo-token: ${{ secrets.GITHUB_TOKEN }} + + publish-pypi: + runs-on: ubuntu-latest + needs: build + if: github.event_name == 'release' + permissions: + id-token: write # Required for trusted publishing + steps: + - uses: actions/download-artifact@v8 + with: + name: pypi-packages-${{ github.sha }} + path: dist + + - name: List packages + run: ls -lh dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/scheduled_test_cpu.yml b/.github/workflows/scheduled_test_cpu.yml new file mode 100644 index 0000000..b40f5c9 --- /dev/null +++ b/.github/workflows/scheduled_test_cpu.yml @@ -0,0 +1,205 @@ +name: Tests on CPU (scheduled) + +on: + push: + branches: [main] + paths-ignore: + - 'docs/**' + - '*.md' + - 'LICENSE' + - 'COPYRIGHT' + schedule: + - cron: '0 4 * * *' + workflow_call: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + check-skip: + # For scheduled runs, check if only documentation files changed + # If so, skip the expensive test runs + runs-on: ubuntu-latest + outputs: + should-skip: ${{ steps.check.outputs.should-skip }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 2 # Need at least 2 commits to compare + + - name: Check if only docs changed + id: check + run: | + # For non-schedule events, always run tests + if [[ "${{ github.event_name }}" != "schedule" ]]; then + echo "should-skip=false" >> "$GITHUB_OUTPUT" + echo "Not a scheduled run, will proceed with tests" + exit 0 + fi + + # For scheduled runs, check if only documentation files changed + # Get the last commit on main + LAST_COMMIT=$(git rev-parse HEAD) + + # Get list of changed files in the last commit + CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r $LAST_COMMIT) + + # Check if all changed files are in paths-ignore patterns + SHOULD_SKIP="true" + for file in $CHANGED_FILES; do + # Check if file matches any ignore pattern + if [[ ! "$file" =~ ^docs/ ]] && \ + [[ ! "$file" =~ \.md$ ]] && \ + [[ "$file" != "LICENSE" ]] && \ + [[ "$file" != "COPYRIGHT" ]]; then + SHOULD_SKIP="false" + break + fi + done + + echo "should-skip=$SHOULD_SKIP" >> "$GITHUB_OUTPUT" + echo "Changed files: $CHANGED_FILES" + echo "Should skip: $SHOULD_SKIP" + + pre-tests: + runs-on: ubuntu-latest + needs: [check-skip] + if: needs.check-skip.outputs.should-skip == 'false' + outputs: + hash: ${{ steps.hashid.outputs.weights-hash }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup pixi + uses: prefix-dev/setup-pixi@v0.9.6 + with: + pixi-version: v0.40.0 + cache: true + + - uses: actions/cache@v5 + id: cache-weights + with: + path: weights/ + key: model-weights-${{ hashFiles('.github/download-models-weights.py') }} + enableCrossOsArchive: true + + - name: Download models weights... + if: steps.cache-weights.outputs.cache-hit != 'true' + run: pixi run uv run python .github/download-models-weights.py -t weights/ + + - name: write hashid + id: hashid + run: echo "weights-hash=${{ hashFiles('.github/download-models-weights.py') }}" >> "$GITHUB_OUTPUT" + + tests-cpu-ubuntu: + needs: [check-skip, pre-tests] + if: needs.check-skip.outputs.should-skip == 'false' + strategy: + fail-fast: false + matrix: + pytorch-dtype: ["float32", "float64"] + + uses: ./.github/workflows/tests.yml + with: + os: "ubuntu-latest" + python-version: '["3.11", "3.12", "3.13"]' + pytorch-version: '["2.1.2", "2.2.2", "2.3.1", "2.4.0", "2.5.1", "2.9.1"]' + pytorch-dtype: ${{ matrix.pytorch-dtype }} + cache-path: weights/ + cache-key: model-weights-${{ needs.pre-tests.outputs.hash }} + cache-restore-keys: | + model-weights-${{ needs.pre-tests.outputs.hash }} + model-weights- + + tests-cpu-windows: + needs: [check-skip, pre-tests] + if: needs.check-skip.outputs.should-skip == 'false' + strategy: + fail-fast: false + matrix: + pytorch-dtype: ["float32", "float64"] + python-version: ["3.11", "3.12", "3.13"] + pytorch-version: ["2.5.1", "2.9.1"] + exclude: + # PyTorch 2.5.1 doesn't have wheels for Python 3.13 on Windows + - python-version: "3.13" + pytorch-version: "2.5.1" + + uses: ./.github/workflows/tests.yml + with: + os: "windows-latest" + python-version: '["${{ matrix.python-version }}"]' + pytorch-version: '["${{ matrix.pytorch-version }}"]' + pytorch-dtype: ${{ matrix.pytorch-dtype }} + cache-path: weights/ + cache-key: model-weights-${{ needs.pre-tests.outputs.hash }} + cache-restore-keys: | + model-weights-${{ needs.pre-tests.outputs.hash }} + model-weights- + + tests-cpu-mac: + needs: [check-skip, pre-tests] + if: needs.check-skip.outputs.should-skip == 'false' + strategy: + fail-fast: false + matrix: + pytorch-dtype: ["float32", "float64"] + python-version: ["3.11", "3.12", "3.13"] + pytorch-version: ["2.5.1", "2.9.1"] + exclude: + # PyTorch 2.5.1 doesn't have wheels for Python 3.13 on macOS + - python-version: "3.13" + pytorch-version: "2.5.1" + # macOS only runs float32 tests + - pytorch-dtype: "float64" + + uses: ./.github/workflows/tests.yml + with: + os: "macos-latest" + python-version: '["${{ matrix.python-version }}"]' + pytorch-version: '["${{ matrix.pytorch-version }}"]' + pytorch-dtype: ${{ matrix.pytorch-dtype }} + cache-path: weights/ + cache-key: model-weights-${{ needs.pre-tests.outputs.hash }} + cache-restore-keys: | + model-weights-${{ needs.pre-tests.outputs.hash }} + model-weights- + + coverage: + needs: [check-skip, pre-tests] + if: needs.check-skip.outputs.should-skip == 'false' + uses: ./.github/workflows/coverage.yml + with: + cache-path: weights/ + cache-key: model-weights-${{ needs.pre-tests.outputs.hash }} + cache-restore-keys: | + model-weights-${{ needs.pre-tests.outputs.hash }} + model-weights- + + typing: + needs: [check-skip] + if: needs.check-skip.outputs.should-skip == 'false' + uses: ./.github/workflows/typecheck.yml + + tutorials: + needs: [check-skip] + if: needs.check-skip.outputs.should-skip == 'false' + uses: ./.github/workflows/tutorials.yml + + docs: + needs: [check-skip, pre-tests] + if: needs.check-skip.outputs.should-skip == 'false' + uses: ./.github/workflows/docs.yml + with: + python-version: '["3.11"]' + cache-path: weights/ + cache-key: model-weights-${{ needs.pre-tests.outputs.hash }} + cache-restore-keys: | + model-weights-${{ needs.pre-tests.outputs.hash }} + model-weights- diff --git a/.github/workflows/scheduled_test_pypi_package.yml b/.github/workflows/scheduled_test_pypi_package.yml new file mode 100644 index 0000000..09550d4 --- /dev/null +++ b/.github/workflows/scheduled_test_pypi_package.yml @@ -0,0 +1,33 @@ +name: Test PyPI package + +on: + schedule: + - cron: '0 4 * * *' + +permissions: + contents: read + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ['3.11', '3.12', '3.13'] + + steps: + - name: Checkout kornia + uses: actions/checkout@v6 + + - name: Config Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + + - name: Pip install Kornia + run: pip install kornia + + - name: Check dependencies and kornia version + run: | + python -c "import torch;print('Pytorch version: ', torch.__version__)" + python -c "import kornia;print('Kornia version: ', kornia.__version__)" diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..e84f322 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,26 @@ +name: Stale + +on: + schedule: + - cron: "0 0 * * *" # Run daily at midnight UTC + workflow_dispatch: + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + name: Stale Pull Requests + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v10 + with: + stale-pr-message: "This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs within 7 days. Thank you for your contributions!" + close-pr-message: "This pull request has been automatically closed due to inactivity. Feel free to reopen it if you would like to continue working on it." + days-before-pr-stale: 15 + days-before-pr-close: 7 + stale-pr-label: "stale" + exempt-pr-labels: "pinned,work-in-progress,on-hold" + days-before-issue-stale: -1 + days-before-issue-close: -1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..924aa40 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,93 @@ +on: + workflow_call: + inputs: + python-version: + required: false + type: string + default: '["3.11", "3.12", "3.13"]' + pytorch-version: + required: false + type: string + default: '["2.9.1"]' + os: + required: false + type: string + default: ubuntu-latest + pytorch-dtype: + required: false + type: string + default: float32 + pytest-extra: + required: false + type: string + default: "" + fail-fast: + required: false + type: boolean + default: false + runslow: + required: false + type: boolean + default: false + ref: + required: false + type: string + default: ${{ github.sha }} + cache-path: + required: false + type: string + cache-key: + required: false + type: string + cache-restore-keys: + required: false + type: string + +permissions: + contents: read + +jobs: + tests: + runs-on: ${{ inputs.os }} + strategy: + fail-fast: ${{ inputs.fail-fast }} + matrix: + python-version: ${{ fromJSON(inputs.python-version) }} + pytorch-version: ${{ fromJSON(inputs.pytorch-version) }} + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + + - name: Restore cache + if: inputs.cache-path != '' + uses: actions/cache/restore@v5 + with: + path: ${{ inputs.cache-path }} + key: ${{ inputs.cache-key }} + restore-keys: ${{ inputs.cache-restore-keys }} + + - name: Set pixi environment name + id: pixi-env + shell: bash + run: echo "name=py$(echo '${{ matrix.python-version }}' | tr -d '.')" >> $GITHUB_OUTPUT + + - name: Setup pixi + uses: prefix-dev/setup-pixi@v0.9.6 + with: + pixi-version: v0.40.0 + cache: true + environments: ${{ steps.pixi-env.outputs.name }} + + - name: Install dependencies + run: pixi run -e ${{ steps.pixi-env.outputs.name }} install + + - name: Run tests + env: + KORNIA_TEST_DEVICE: cpu + KORNIA_TEST_DTYPE: ${{ inputs.pytorch-dtype }} + KORNIA_TEST_RUNSLOW: ${{ inputs.runslow }} + KORNIA_TEST_OPTIMIZER: "" + run: pixi run -e ${{ steps.pixi-env.outputs.name }} test ${{ inputs.pytest-extra }} diff --git a/.github/workflows/tutorials.yml b/.github/workflows/tutorials.yml new file mode 100644 index 0000000..98d171a --- /dev/null +++ b/.github/workflows/tutorials.yml @@ -0,0 +1,80 @@ +on: + workflow_call: + inputs: + python-version: + required: false + type: string + default: '["3.12"]' + os: + required: false + type: string + default: ubuntu-latest + fail-fast: + required: false + type: boolean + default: false + ref: + required: false + type: string + default: ${{ github.sha }} + +permissions: + contents: read + +jobs: + tutorials: + runs-on: ${{ inputs.os }} + continue-on-error: true + strategy: + fail-fast: ${{ inputs.fail-fast }} + matrix: + python-version: ${{ fromJSON(inputs.python-version) }} + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + + - name: Set pixi environment name + id: pixi-env + shell: bash + run: echo "name=py$(echo '${{ matrix.python-version }}' | tr -d '.')" >> $GITHUB_OUTPUT + + - name: Setup pixi + uses: prefix-dev/setup-pixi@v0.9.6 + with: + pixi-version: v0.40.0 + cache: true + environments: ${{ steps.pixi-env.outputs.name }} + + - name: Install kornia + run: pixi run -e ${{ steps.pixi-env.outputs.name }} install + + - uses: actions/checkout@v6 + with: + repository: "kornia/tutorials" + path: "tutorials-repo/" + + - name: Install dependencies + working-directory: ./tutorials-repo/ + env: + PIP_NO_CACHE_DIR: "1" + PIP_INDEX_URL: "https://download.pytorch.org/whl/cpu" + PIP_EXTRA_INDEX_URL: "https://pypi.org/simple" + run: make setup + + - name: Install local kornia + run: pip install -e . --no-deps + + - name: Check deps + working-directory: ./tutorials-repo/ + run: make check-deps + + - name: Generate tutorials + working-directory: ./tutorials-repo/ + run: make generate + + - name: Run tutorials + working-directory: ./tutorials-repo/ + run: make execute diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml new file mode 100644 index 0000000..771d05c --- /dev/null +++ b/.github/workflows/typecheck.yml @@ -0,0 +1,34 @@ +name: Type Check + +on: + workflow_call: + +permissions: + contents: read + +jobs: + typecheck: + name: typecheck + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup pixi + uses: prefix-dev/setup-pixi@v0.9.6 + with: + pixi-version: v0.40.0 + cache: true + + - name: Install dependencies + run: pixi run -e default install + + - name: Debug - Show ty version and config + run: | + pixi run -e default ty --version + echo "=== pyproject.toml ty config ===" + grep -A 30 '\[tool.ty' pyproject.toml || echo "No ty config found" + + - name: Run type checking with ty + run: pixi run -e default typecheck diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..936a7d3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,41 @@ +build/ +dist/ +kornia.egg-info/ +.idea +*/**/__pycache__ +*/**/*.pyc +*/**/*~ +*~ +*.swp +*.swo +*.swn +*.swm +.eggs/* +.cache/* +.pytest_cache/* +.dev_env +.mypy_cache/* +.coverage* +*bak +*ipynb_checkpoints* +docs/source/_static/*jpg +docs/source/_static/img/*jpg +docs/source/_static/scripts/* +*DS_Store +*jpg + +# onnx tests create temp.onnx file +*.onnx +venv/ +weights/ + +# uv +.uv_cache/ + +.python-version + +# Benchmark profiling outputs +benchmarks/profile/ + +# Editor / agent metadata +.omc/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..fb3c4a3 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,55 @@ +ci: + autofix_prs: true + autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions' + # submodules: true + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-yaml + - id: check-toml + - id: check-merge-conflict + - id: check-added-large-files + args: ['--maxkb=1000'] + - id: mixed-line-ending + args: ['--fix=lf'] + - id: requirements-txt-fixer + - id: pretty-format-json + exclude: ^$|.devcontainer + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.1 + hooks: + - id: ruff-check + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format + + - repo: https://github.com/codespell-project/codespell + rev: v2.4.1 + hooks: + - id: codespell + + - repo: https://github.com/arkinmodi/add-license-header + rev: v2.4.1 + hooks: + - id: add-license-header + types: [python] + args: + - --license + - | + Copyright 2018 Kornia Team + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..a32e2b4 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "python-envs.defaultEnvManager": "ms-python.python:conda", + "python-envs.defaultPackageManager": "ms-python.python:conda", + "python-envs.pythonProjects": [] +} diff --git a/AI_POLICY.md b/AI_POLICY.md new file mode 100644 index 0000000..d35d073 --- /dev/null +++ b/AI_POLICY.md @@ -0,0 +1,76 @@ +# 🤖 Kornia AI & Authorship Policy + +**Version:** 1.0 +**Enforcement:** Strict +**Applicability:** All Pull Requests (Human & Bot) + +## 1. Core Philosophy + +Kornia accepts AI-assisted code (e.g., using Copilot, Cursor, etc.), but strictly rejects AI-generated contributions where the submitter acts merely as a proxy. The submitter is the **Sole Responsible Author** for every line of code, comment, and design decision. + +## 2. The 3 Laws of Contribution + +### Law 1: Proof of Verification + +AI tools frequently write code that looks correct but fails execution. Therefore, "vibe checks" are insufficient. + +**Requirement:** Every PR introducing functional changes must include a pasted snippet of the local test logs (e.g., `pixi run test ...`). This is mandatory for all contributors and is particularly important for first-time contributors. + +**Failure Condition:** If a PR lacks execution proof and contains complex logic, it will be flagged as **Unverified**. + +**Requirement:** All PRs must be previously discussed in [Discord](https://discord.gg/HfnywwpBnD) or via a [GitHub issue](https://github.com/kornia/kornia/issues) before implementation. The PR must reference the discussion or issue. + +**Requirement:** Implementations must be based on an existing library reference (e.g., PyTorch, OpenCV, scikit-image, etc.) that must be provided in the PR description for verification. This reference serves as proof that the implementation follows established algorithms and is not hallucinated. + +### Law 2: The Hallucination & Redundancy Ban + +AI models often hallucinate comments or reinvent existing utilities. + +**Requirement:** You must use existing `kornia` utilities and never reinvent the wheel, except for when the utility is not available. + +**Failure Condition:** Creating new helper functions (e.g., `def warp_affine...`) when a Kornia equivalent exists is grounds for immediate rejection. + +**Failure Condition:** "Ghost Comments" (comments explaining logic that was deleted or doesn't exist) will result in a request for a full manual rewrite. Redundant comments are not allowed. Example: "This function returns the input tensor". + +### Law 3: The "Explain It" Standard + +**Requirement:** If a maintainer or reviewer asks during code review, you must be able to derive the math or explain the logic of any function you submit. + +**Failure Condition:** Answering a review question with "That's what the AI outputted" or "I don't know, it works" leads to immediate closure. + +### Law 4: Transparency in AI Usage Disclosure + +**Requirement:** All PRs must accurately complete the "AI Usage Disclosure" section in the pull request template. This disclosure is mandatory and must reflect the actual use of AI tools. + +**When to mark as 🔴 AI-generated:** +- An AI agent (e.g., Cursor, GitHub Copilot, ChatGPT, etc.) generated the code, PR description, or commit messages +- You cannot explain the logic without referring to the AI's output +- The PR was created primarily by an agent with minimal human review or modification + +**When to mark as 🟡 AI-assisted:** +- You used AI tools for boilerplate code, refactoring, or suggestions, but you manually reviewed and tested every line +- You understand and can explain all the code you're submitting +- You made significant modifications to AI-suggested code + +**When to mark as 🟢 No AI used:** +- You wrote all code manually without any AI assistance + +**Failure Condition:** Incorrectly marking AI-generated code as "AI-assisted" or "No AI used" may result in PR closure, especially if the code contains hallucinations or cannot be explained during review. + +## 3. Instructions for AI Reviewers (Copilot / CodeRabbit) + +If you are an AI agent (GitHub Copilot, CodeRabbit, etc.) reviewing a PR for Kornia, you must follow the repository's dedicated reviewer instructions. + +The **canonical and up-to-date instructions for AI reviewers** are maintained in [`.github/copilot-instructions.md`](.github/copilot-instructions.md). That document defines: + +- The expected reviewer persona and responsibilities +- The checks to perform on PR descriptions, code, tests, and comments +- The required enforcement of the laws defined in this `AI_POLICY.md` + +Any other document (including this one) should treat `copilot-instructions.md` as the single source of truth for AI reviewer behaviour. When updating reviewer logic, update `copilot-instructions.md` first and, if needed, adjust references here. + +This section exists to link AI reviewers to the canonical instructions and to make clear that those instructions must enforce the policies defined in Sections 1 and 2 above. + +## 4. Additional Resources + +For comprehensive guidance on contributing to Kornia, including development workflows, code quality standards, testing practices, and AI-assisted development best practices, see the [Best Practices section](CONTRIBUTING.md#best-practices) in `CONTRIBUTING.md`. diff --git a/BACKERS.md b/BACKERS.md new file mode 100644 index 0000000..5eb100b --- /dev/null +++ b/BACKERS.md @@ -0,0 +1,28 @@ +

Sponsors & Backers

+ +Kornia is an Apache-licensed open source project, which is supported by these brilliant [backers](https://github.com/kornia/kornia/blob/main/LICENSE/BACKERS.md). If you'd like to join them, please consider: + +- [Become a backer or sponsor on OpenCollective](https://opencollective.com/kornia). + +

+ +

Platinum via OpenCollective

+ + + +

Gold via OpenCollective

+ + + + +

Silver via OpenCollective

+ + + +

Bronze via OpenCollective

+ + + +

Backers via OpenCollective

+ + diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..5706f52 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,609 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +**** + + +## :rocket: [0.6.11] - 2022-03-28 +### :new: New Features + +* add `DISK` local feature by @jatentaki in https://github.com/kornia/kornia/pull/2285 +* Add Joint Bilateral Filter by @gau-nernst https://github.com/kornia/kornia/pull/2244 +* Add Bilateral Filter by @gau-nernst https://github.com/kornia/kornia/pull/2242 +* Add random snow by @just1ce415 https://github.com/kornia/kornia/pull/2229 + + +## :rocket: [0.6.10] - 2022-02-17 +### :new: New Features + +* add `depth_from_disparity` function by @pri1311 in https://github.com/kornia/kornia/pull/2096 +* Add Vector2 by @cjpurackal in https://github.com/kornia/kornia/pull/2134 +* Add 3D-SSIM loss by @pri1311 in https://github.com/kornia/kornia/pull/2130 +* [Feat] Initiate AutoAugment modules by @shijianjian in https://github.com/kornia/kornia/pull/2181 +* Add Common Regression Losses by @ChristophReich1996 in https://github.com/kornia/kornia/pull/2109 +* Add `integral_image` and `integral_tensor` by @AnimeshMaheshwari22 in https://github.com/kornia/kornia/pull/1779 + + +### :lady_beetle: Bug fixes + +* Fix AugmentationSequential to return list of boxes by @johnnv1 in https://github.com/kornia/kornia/pull/2114 +* Fix support for (*, 3, H, W) tensors in yuv by @ChristophReich1996 in https://github.com/kornia/kornia/pull/2108 +* fix TensorWrapper serialization by @edgarriba in https://github.com/kornia/kornia/pull/2132 +* Split the half precision tests workflow by @johnnv1 in https://github.com/kornia/kornia/pull/2118 +* Fixed DoG accuracy, add `upscale_double` by @vicsyl in https://github.com/kornia/kornia/pull/2105 +* Added Face detection Interactive demo by @jeffin07 in https://github.com/kornia/kornia/pull/2142 +* Bump pytest from 7.2.0 to 7.2.1 by @dependabot in https://github.com/kornia/kornia/pull/2148 +* add SSIM3D and `depth_from_disparity` to docs by @pri1311 in https://github.com/kornia/kornia/pull/2150 +* Explicitly cast output to input type to avoid type mismatch errors by @JanSellner in https://github.com/kornia/kornia/pull/1842 +* Fix params computation for `LongestMaxSize` and `SmallestMaxSize` by @johnnv1 in https://github.com/kornia/kornia/pull/2131 +* torch_version_geq -> torch_version_ge according to todo by @ducha-aiki in https://github.com/kornia/kornia/pull/2157 +* fix doc build - `sphinx-autodoc-typehints==1.21.3` by @johnnv1 in https://github.com/kornia/kornia/pull/2159 +* ScaleSpaceDetector -> Fast ScaleSpaceDetector by @ducha-aiki in https://github.com/kornia/kornia/pull/2154 +* Improve losses tests, add `TestSSIM3d`, and `BaseTester.gradcheck` by @johnnv1 in https://github.com/kornia/kornia/pull/2152 +* modify comments of rgb and lab conversion by @gravitychen in https://github.com/kornia/kornia/pull/2153 +* add __repr__ and __getitem__ to vector by @cjpurackal in https://github.com/kornia/kornia/pull/2163 +* Fix adalam-config by @ducha-aiki in https://github.com/kornia/kornia/pull/2170 +* Fix docs of `boxes`, `MultiResolutionDetector`. `apply colormap`, `AugmentationSequential` by @johnnv1 in https://github.com/kornia/kornia/pull/2167 +* add exception test for se2 + small bug fix by @cjpurackal in https://github.com/kornia/kornia/pull/2160 +* Fix MobileViT by @chinhsuanwu in https://github.com/kornia/kornia/pull/2172 +* Fix output types of augmentations on autocast regions by @johnnv1 in https://github.com/kornia/kornia/pull/2168 +* Fix planckian jitter for cuda by @johnnv1 in https://github.com/kornia/kornia/pull/2177 +* Fix: resample method None default missing for inverse masks by @miquelmarti in https://github.com/kornia/kornia/pull/2185 +* Move padding_size to device in pad for boxes by @miquelmarti in https://github.com/kornia/kornia/pull/2197 +* Return boxes tensor directly if no boxes by @miquelmarti in https://github.com/kornia/kornia/pull/2196 +* Make value an attribute of RandomErasing instances again by @miquelmarti in https://github.com/kornia/kornia/pull/2195 +* TensorWrapper bug fix + add __radd__, __rmul__, __rsub__ by @cjpurackal in https://github.com/kornia/kornia/pull/2190 +* Fix/repr bug by @neyazbasheer in https://github.com/kornia/kornia/pull/2207 +* Replace `assert_allclose` by `assert_close` by @johnnv1 in https://github.com/kornia/kornia/pull/2210 +* Fix random crop for keypoints on CUDA device by @johnnv1 in https://github.com/kornia/kornia/pull/2209 +* Remove outdated augmentation example by @johnnv1 in https://github.com/kornia/kornia/pull/2206 +* Fix CUDA failing tests of same device on `Augmentations` by @johnnv1 in https://github.com/kornia/kornia/pull/2215 + + + +## :zap: Improvements + +* add `PadTo` to docs by @johnnv1 in https://github.com/kornia/kornia/pull/2122 +* add colormap and `apply_ColorMap` for integer tensor by @johnnv1 in https://github.com/kornia/kornia/pull/1996 +* Fix numerical stability for binary focal loss by @zimka in https://github.com/kornia/kornia/pull/2125 +* Add RandomGaussianBlur with instance-level gaussian kernel generation by @juliendenize in https://github.com/kornia/kornia/pull/1663 +* add transparent pad to `CenterCrop` docs example by @johnnv1 in https://github.com/kornia/kornia/pull/2124 +* Ensure support to Python 3.9 and 3.10 by @johnnv1 in https://github.com/kornia/kornia/pull/2025 +* improve `TestUpscaleDouble` by @johnnv1 in https://github.com/kornia/kornia/pull/2147 +* DataKey: add 'image' as alias of 'input' by @adamjstewart in https://github.com/kornia/kornia/pull/2193 +* add `fail-fast:false` as default on tests workflow by @johnnv1 in https://github.com/kornia/kornia/pull/2146 + [enhance] improve flipping and cropping speed by @shijianjian in https://github.com/kornia/kornia/pull/2179 +* Replace jit test method in favor of dynamo in `BaseTester` by @johnnv1 in https://github.com/kornia/kornia/pull/2120 +* Small refactor on `filters` module: Dropping JIT support by @johnnv1 in https://github.com/kornia/kornia/pull/2187 +* Augmentation Base Refactor by @shijianjian in https://github.com/kornia/kornia/pull/2117 + + +### Deprecation + +* move kornia check api to kornia.core.check by @edgarriba in https://github.com/kornia/kornia/pull/2143 +* Remove py 3.7 for nightly CI by @johnnv1 in https://github.com/kornia/kornia/pull/2204 + + +## :rocket: [0.6.9] - 2022-12-21 +### :new: New Features + +* Feat/randombrightness contrast saturation hue by @duc12111 in https://github.com/kornia/kornia/pull/1955 +* Liegroups by @edgarriba in https://github.com/kornia/kornia/pull/1887 +* Add sepia by @johnnv1 in https://github.com/kornia/kornia/pull/1947 +* Normalize with intrinsics by @ducha-aiki in https://github.com/kornia/kornia/pull/1727 +* [feat] liegroup so2 by @cjpurackal in https://github.com/kornia/kornia/pull/1973 +* [feat] adjoint for se2, so2 by @cjpurackal in https://github.com/kornia/kornia/pull/2101 +* add trans, trans_x, trans_y + minor changes se2 by @cjpurackal in https://github.com/kornia/kornia/pull/2103 +* Motion blur by @nitaifingerhut in https://github.com/kornia/kornia/pull/2075 +* Add `Hyperplane` and `Ray` API by @edgarriba in https://github.com/kornia/kornia/pull/1963 + + +### :lady_beetle: Bug fixes + +* Quaternion pow bug fix (div by zero) by @cjpurackal in https://github.com/kornia/kornia/pull/1946 +* fix cuda init by @ducha-aiki in https://github.com/kornia/kornia/pull/1953 +* Documentation: proper Sørensen–Dice coefficient by @sergiev in https://github.com/kornia/kornia/pull/1961 +* quaternion, so3 and se3 as non batched by @edgarriba in https://github.com/kornia/kornia/pull/1997 +* Bump pytest-mypy from 0.10.0 to 0.10.1 by @dependabot in https://github.com/kornia/kornia/pull/2005 +* Join the gh-actions for docs by @johnnv1 in https://github.com/kornia/kornia/pull/2003 +* [pre-commit.ci] pre-commit suggestions by @pre-commit-ci in https://github.com/kornia/kornia/pull/2010 +* So2 bug fix by @cjpurackal in https://github.com/kornia/kornia/pull/2015 +* Fix type annotation for torch 1.13.0 by @johnnv1 in https://github.com/kornia/kornia/pull/2023 +* Fix an error in `match_smnn` by @anstadnik in https://github.com/kornia/kornia/pull/2020 +* Set equal_nan to False in assert_close by @edgarriba in https://github.com/kornia/kornia/pull/1986 + +## :zap: Improvements + +* minor improvements to So3 by @cjpurackal in https://github.com/kornia/kornia/pull/1966 +* Add `TensorWrapper`, `Vector3`, `Scalar` and improvements in `fit_plane` by @edgarriba in https://github.com/kornia/kornia/pull/ +* [feat] add vee to so2, se2 by @cjpurackal in https://github.com/kornia/kornia/pull/2091 +* Remove deprecated code in `kornia.augmentation` by @johnnv1 in https://github.com/kornia/kornia/pull/2052 +* [feat] Implement se2 by @cjpurackal in https://github.com/kornia/kornia/pull/2019 +* add quaternion to euler conversion by @edgarriba in https://github.com/kornia/kornia/pull/1994 +* use resample instead of mode argument in RandomElasticTransform per default by @JanSellner in https://github.com/kornia/kornia/pull/2017 +* replacing .repeat(...) with .expand(...) by @nitaifingerhut in https://github.com/kornia/kornia/pull/2059 +* making `RandomGaussianNoise` play nicely on GPU by @nitaifingerhut in https://github.com/kornia/kornia/pull/2050 +* None for align_corners arg of resize op with nearest mode by @miquelmarti in https://github.com/kornia/kornia/pull/2049 +* facedetector now returns a list of tensors containing the boxes x image by @lferraz in https://github.com/kornia/kornia/pull/2034 +* add random for liegroups by @cjpurackal in https://github.com/kornia/kornia/pull/2041 +* add rotation and translation classmethods in se3 and so3 by @edgarriba in https://github.com/kornia/kornia/pull/2001 +* implement `kornia.geometry.linalg.euclidean_distance` by @edgarriba in https://github.com/kornia/kornia/pull/2000 + + +### Deprecation + +* Drop pytorch 1.8 (LTS) support by @johnnv1 in https://github.com/kornia/kornia/pull/2024 + + +## :rocket: [0.6.8] - 2022-10-13 +### :new: New Features + +* NeRF Implementation by @YanivHollander in https://github.com/kornia/kornia/pull/1911 +* [Feat] Added AugmentationDispatcher by @shijianjian in https://github.com/kornia/kornia/pull/1914 +* Add `EdgeDetection` api by @edgarriba in https://github.com/kornia/kornia/pull/1483 +* [feat] slerp implementation for Quaternion by @cjpurackal in https://github.com/kornia/kornia/pull/1931 +* add laplacian pyramid by @lafith in https://github.com/kornia/kornia/pull/1816 +* Added homography from line segment correspondences by @ducha-aiki in https://github.com/kornia/kornia/pull/1851 +* [feat] Added Jigsaw Augmentation by @shijianjian in https://github.com/kornia/kornia/pull/1852 + +### :lady_beetle: Bug fixes + +* Fix svdvals usage by @ducha-aiki in https://github.com/kornia/kornia/pull/1926 +* fix shift_rgb stack dimension by @nmichlo in https://github.com/kornia/kornia/pull/1930 +* Update kernels.py by @farhankhot in https://github.com/kornia/kornia/pull/1940 +* Quaternion.norm bug fix by @cjpurackal in https://github.com/kornia/kornia/pull/1935 +* Fix quaternion doctests by @edgarriba in https://github.com/kornia/kornia/pull/1943 +* Remove unnecessary CI jobs by @johnnv1 in https://github.com/kornia/kornia/pull/1933 +* fix cuda tests failing by @ducha-aiki in https://github.com/kornia/kornia/pull/1941 +* No crash in local feature matching if empty tensor output by @ducha-aiki in https://github.com/kornia/kornia/pull/1890 + + +### :zap: Improvements + +* RANSAC improvements by @ducha-aiki in https://github.com/kornia/kornia/pull/1435 +* Make AdaLAM output match confidence by @ducha-aiki in https://github.com/kornia/kornia/pull/1862 +* Enlargen LoFTR positional encoding map if large images are input by @georg-bn in https://github.com/kornia/kornia/pull/1853 + + +## :rocket: [0.6.7] - 2022-08-30 +### :new: New Features + +* Added FGINN matching by @ducha-aiki in https://github.com/kornia/kornia/pull/1813 +* Added SOLD2 by @rpautrat https://github.com/kornia/kornia/pull/1507 https://github.com/kornia/kornia/pull/1844 +* edge aware blur2d by @nitaifingerhut in https://github.com/kornia/kornia/pull/1822 +* Adds conversions between graphics and vision coordinate frames by @ducha-aiki in https://github.com/kornia/kornia/pull/1823 +* Add Quaternion API by @edgarriba in https://github.com/kornia/kornia/pull/1801 +* AdaLAM match filtering by @ducha-aiki in https://github.com/kornia/kornia/pull/1831 +* Init Mosaic Augmentation by @shijianjian in https://github.com/kornia/kornia/pull/1713 + + +### :lady_beetle: Bug fixes + +* fix tests float16 module losses by @MrShevan in https://github.com/kornia/kornia/pull/1809 + +### :zap: Improvements + +* Allowing more than 3/4 dims for `total_variation` + adding `reduction` by @nitaifingerhut in https://github.com/kornia/kornia/pull/1815 + + +## :rocket: [0.6.6] - - 2022-07-16 + +### :new: New Features + +* Add `ParametrizedLine` and `fit_line` by @edgarriba in https://github.com/kornia/kornia/pull/1794 +* Implement `project` and `unproject` in `PinholeCamera` by @YanivHollander in https://github.com/kornia/kornia/pull/1729 +* adding `rgb_to_y` by @nitaifingerhut in https://github.com/kornia/kornia/pull/1734 +* add `KORNIA_CHECK_SAME_DEVICES` by @MrShevan in https://github.com/kornia/kornia/pull/1788 + + +### Deprecation + +* deprecate `filter2D` `filter3D` api by @edgarriba in https://github.com/kornia/kornia/pull/1725 + + +### :lady_beetle: Bug fixes + +* fixes for half precision in imgwarp by @edgarriba in https://github.com/kornia/kornia/pull/1723 +* Fix transforms for empty boxes and keypoints inputs by @hal-314 in https://github.com/kornia/kornia/pull/1741 +* fixing doctest in pinhole by @edgarriba in https://github.com/kornia/kornia/pull/1743 +* Fix/crop transforms by @hal-314 in https://github.com/kornia/kornia/pull/1739 +* Fix Boxes.from_tensor(boxes, mode="vertices") by @hal-314 in https://github.com/kornia/kornia/pull/1740 +* fix typing callable in load storage by @edgarriba in https://github.com/kornia/kornia/pull/1768 +* Fix bug preventing sample wise augmentations by @ashnair1 in https://github.com/kornia/kornia/pull/1761 +* Refactor and add tests in `get_perspective_transform` by @edgarriba in https://github.com/kornia/kornia/pull/1767 + + +## :rocket: [0.6.5] - 2022-05-16 +### :new: New Features +- Create `kornia.io` and implement `load_image` with rust (#1701) +- Implement `diamond_square` and plasma augmentations: `RandomPlasmaBrightness`, `RandomPlasmaContrast`, `RandomPlasmaShadow` (#1700) +- Added `RandomRGBShift` augmentations (#1694) +- Added STE gradient estimator (#1666) +- More epipolar geometry metrics (+linalg utility) (#1674) +- Add Lovasz-Hinge/Softmax losses (#1682) +- Add `adjust_sigmoid` and `adjust_log` initial implementation (#1685) +- Added distribution mapper (#1667) +- `pos_weight` param to focal loss (#1744) + +### :lady_beetle: Bug fixes +- Fixes filter2d's output shape shrink when padding='same' (#1661) +- fix: added eps in geometry/rotmat_to_quaternion (#1665) +- [fix] receive num_features as an arg to KeyNetDetector constructor (#1686 + +### :zap: Improvements +- Add reduction option to `MS_SSIMLoss` (#1655) +- Making epipolar metrics work with volumetric tensors (#1656) +- Add get_safe_device util (#1662) +- Added antialiasing option to Resize augmentation (#1687) +- Use nearest neighbour interpolation for masks (#1630) +- grayscale to rgb for `torch.uint8` (#1705) +- Add `KORNIA_CHECK_SAME_DEVICES` (#1775) + +## :rocket: [0.6.4] - 2022-03-19 +### :new: New Features +- Adds MS-SSIMLoss reconstruction loss function (#1551) +- Added HyNet descriptor (#1573) +- Add KeyNet detector (#1574) +- Add RandomPlanckianJitter in color augmentations (#1607) +- Add Jina AI QAbot to Kornia documentation (#1628) +- Add `draw_convex_polygon` (#1636) + +### :lady_beetle: Bug fixes +- RandomCrop fix and improvement (#1571) +- Fix draw_line produce wrong output for coordinates larger than uint8 +- Fix mask bug for loftr (#1580) +- Fix gradient bug for distance_transform (#1584) +- Fix translation sampling in AffineGenerator3D (#1581) +- Fix AugmentationSequential bbox keypoints transformation fix (#1570) +- Fix CombineTensorPatches (#1558) +- Fix overblur in AA (#1612) + +### :exclamation: Changes +- Deprecated `return_transform`, enabled 3D augmentations in AugmentionSequential (#1590) + +### :zap: Improvements +- Making compute_correspond_epilines work with fundamental and point of volumetric tensor (#1585) +- Update batch shape when augmentations change size of image (#1609) +- Remap accepts arbitrary grid size (#1617) +- Rename variables named 'input' to 'sample' (in tests). (#1614) +- Remove half log2 in extract_patches (#1616) +- Add orientation-preserving option for AffNet and make it default (#1620) +- Add option for sampling_method in 2d perspective transform generation (#1591) (#1592) +- Fix adjust brightness (#1586) +- Added default params for laf construction from xy and new tensor shape check (#1633) +- Make nms2d jittable (#1637) +- Add fn to automatically compute padding (#1634) +- Add pillow_like option for ColorJitter to match torchvision. (#1611) + +## :rocket: [0.6.3] - 2022-01-30 +### :new: New Features +- Update CI to pytorch 1.10.1 (#1518) +- Added Hanning kernel, prepare for KCF tracking (#1519) +- Add distance transform implementation (#1490) +- Add Resize augmentation module (#1545) + +### :lady_beetle: Bug fixes +- Precompute padding parameters when RandomCrop aug in container (#1494) +- Padding error with RandomCrop #1520 +- Fix correct shape after cropping when forwarding parameters (#1533) +- Fixed #1534 nested augmentation sequential bug (#1536) +- Fixes to device in augmentations (#1546) +- Bugfix for larger MotionBlur kernel size ranges (#1543) +- Fix RandomErasing applied to mask keys (#1541) + +### :exclamation: Changes +- Restructure augmentation package (#1515) + +### :zap: Improvements +- Add missing keepdims with fixed type (#1488) +- Allow to pass a second K to distort and undistort points (#1506) +- Augmentation Sequential with a list of bboxes as a batch (#1497) +- Adde Devcontainer for development (#1515) +- Improve the histogram_matching function (#1532) + +## :rocket: [0.6.2] - 2021-12-03 +### :new: New Features +- Add face detection API (#1469) +- Add `ObjectDetectorTrainer` (#1414) +- Add container operation weights and `OneOf` documentation (#1443) +- Add oriented constraint check to Homography RANSAC (#1453) +- Add background color selection in `warp_perspective` (#1452) +- Add `draw_line` image utility (#1456) +- Add Bounding Boxes API (#1304) +- Add histogram_matching functionality (#1395) + +### :lady_beetle: Bug fixes +- fix catch type for torch.svd error (#1431) +- Fix for nested AugmentationSequential containers (#1467) +- Use common bbox format xywh (#1472) +- Fix motion blur kernel size bug for larger random generator ranges (#1540) + +### :exclamation: Changes +- Add padding_mode for RandomElasticTransform augmentation (#1439) +- Expose inliers sum to HomographyTracker (#1463) + +### :zap: Improvements +- Switch to one-way error RANSAC for speed-up (#1454) +- Few improvements on homography tracking (#1434) +- Enable all bandit tests, add separate hook for tests (#1437) +- Merge homography_warp to warp_perspective (#1438) +- Random generator refactor (#1459) + + +## :rocket: [0.6.1] - 2021-10-22 +### :lady_beetle: Bug fixes +- Fixes PyPI tarball missing required files #1421 +- hotfix: remove mutable object in constructor #1423 + + +## :rocket: [0.6.0] - 2021-10-22 + +### :new: New Features +- Add Training API (#1307) +- Added combine patches (#1309) +- Add semantic segmentation trainer (#1323) +- Add vanilla LO-RANSAC (#1335) +- Add Lambda function module (#1346) +- Add support for YUV420 and YUV422 to complement current YUV444 (#1360) +- Add raw to rgb color conversion (#1380) +- Implement separable_filter2d (#1385) +- Add MobileViT to contrib (#1388) +- Add solve_pnp_dlt (#1349) +- Add function image_list_to_tensor to utils (#1393) +- Add undistort_image function (#1303) +- Create kormia.metrics submodule (#1325) +- Add Image Stitching API (#1358) +- Add Homography Tracker API (#1389) + +### :exclamation: Changes +- Refactor library namespaces [pre-release][0.6-rc1] (#1412) +- deprecate 1.6/1.7 and add 1.9.1 (#1399) + +### :zap: Improvements +- Improve bbox_to_mask (#1351) +- Refactor unfold->conv for morphology backbone (#1107) +- Improve focal loss for numerical stability (#1362) +- Add more border_type options for filter2D (#1375) +- Replace deprecated torch.qr (#1376) +- Add special case hardcoded implementtion for local features speed up (#1387) +- Enable non/batched connected components (#1193) +- Remove warnings during testing (#1401) + +### :lady_beetle: Bug fixes +- Fix binary focal loss (#1313) +- Fix kornia.geometry.subpix.spatial_soft_argmax imports (#1318) +- Fixed a simple typo in __init__.py (#1319) +- Fix path to dev requirements file in a setup_dev_env.sh (#1324) +- Fix bug in create_meshgrid3d along depth (#1330) +- Fix anisotropic scale error (#1340) +- Fix rgb_to_hsv for onnx (#1329) +- Fixed useless return in ransac.py (#1352) +- Fixed classificationhead typo and leave out some of the guesswork (#1354) +- Fix clahe differentiability and tests (#1356) +- Fixes singular matrix inverse/solve for RANSAC and ConvQuad3d (#1408) +- Change intermediate datatype to fix imgwarp (#1413) + +## :rocket: [0.5.11] - 2021-08-30 +### :new: New Features +- Add Vision Transformer (ViT) ([#1296](https://github.com/kornia/kornia/pull/1296)) +- Add ImageRegistrator API ([#1253](https://github.com/kornia/kornia/pull/1253)) +- Add LoFTR inference ([#1218](https://github.com/kornia/kornia/pull/1218)) +- Added differentiable Hausdorff Distance (HD) loss ([#1254](https://github.com/kornia/kornia/pull/1254)) +- Add PadTo to kornia.augmentation ([#1286](https://github.com/kornia/kornia/pull/1286)) + +### :zap: Code refactor +- Return all learned modules by default in eval() mode ([#1266](https://github.com/kornia/kornia/pull/1266)) +- Enable ImageSequential and VideoSequential to AugmentationSequential (#1231) +- Specify that angles are in radians ([#1287](https://github.com/kornia/kornia/pull/1287)) +- Removed deprecated codes for v6.0 ([#1281](https://github.com/kornia/kornia/pull/1281)) + +### :lady_beetle: Bug fixes +- Fix save_pointcloud_ply fn counting point with inf coordinates ([#1263](https://github.com/kornia/kornia/pull/1263)) +- Fixes torch version parse and add temporal packaging dependency ([#1284](https://github.com/kornia/kornia/pull/1284)) +- Fix issue of image_histogram2d ([#1295](https://github.com/kornia/kornia/pull/1295)) + + +## [0.5.10] - 2021-08-30 + +### Added +- Added Basic pool request for DeFMO. ([#1135](https://github.com/kornia/kornia/pull/1135)) +- Added homography error metrics, and improved find_homography_iter ([#1222](https://github.com/kornia/kornia/pull/1222)) + +### Fixed +- Fixed wrong param name ([#1197](https://github.com/kornia/kornia/pull/1197)) +- Fixed NotImplementedError for the rtvec ([#1215)](https://github.com/kornia/kornia/pull/1215)) +- Fixes warnings and add compatibility stub in torch solve ([#1235](https://github.com/kornia/kornia/pull/1235)) + +### Changed +- Ensure CenterCrop indices are integers ([#1208](https://github.com/kornia/kornia/pull/1208)) +- Added tests, fixed docstrings and made some other changes ([#1211](https://github.com/kornia/kornia/pull/1211)) +- Upgrade to modern Python syntax ([#1213](https://github.com/kornia/kornia/pull/1213)) +- Code health improvements [#1199, #1200, #1198, #1202, #1203, #1205, #1208, #1210, #1214, #1220] +- Enable pyupgrade as pre-commit ([#1221](https://github.com/kornia/kornia/pull/1221)) +- Add bandit tool in the pre-commit ([#1228](https://github.com/kornia/kornia/pull/1228)) + + +## [0.5.8] - 2021-08-06 + +### Added +- Add the connected components labeling algorithm ([#1184](https://github.com/kornia/kornia/pull/1184)) + +### Fixed +- Partial fix for horizontal and vertical flips ([#1166](https://github.com/kornia/kornia/pull/1166)) +- Fix even kernel and add test ([#1183](https://github.com/kornia/kornia/pull/1183)) +- Fix wrong source points for RandomThinPlateSpline ([#1187](https://github.com/kornia/kornia/pull/1187)) +- Fix RandomElasticTransform ignores same_on_batch ([#1189](https://github.com/kornia/kornia/pull/1189)) +- Fixed bugs in patchsequential. Remove fill_diagonal operation for better ONNX support ([#1178](https://github.com/kornia/kornia/pull/1178)) + +### Changed +- Differentiable image histogram using kernel density estimation ([#1172](https://github.com/kornia/kornia/pull/1172)) + + +## [0.5.7] - 2021-07-27 + +### Added +- Grayscale to RGB image conversion. ([#1162](https://github.com/kornia/kornia/pull/1162)) +- Add keepdim param to tensor_to_image function. ([#1168](https://github.com/kornia/kornia/pull/1168)) + +### Fixed +- Fix checks on wrong tensor shape condition in depth.py ([#1164](https://github.com/kornia/kornia/pull/1164)) + + +## [0.5.6] - 2021-07-12 + +### Added +- Added mix augmentations in containers ([#1139](https://github.com/kornia/kornia/pull/1139)) + +### Fixed +- Fixed non-4-dim input error for sequential ([#1146](https://github.com/kornia/kornia/pull/1146)) + +### Changed +- Moving bbox-related functionality to bbox module ([#1103](https://github.com/kornia/kornia/pull/1103)) +- Optimized version of hls_to_rgb and rgb_to_hls ([#1154](https://github.com/kornia/kornia/pull/1154)) + +### Removed +- Remove numpy dependency ([#1136](https://github.com/kornia/kornia/pull/1136)) + + +## [0.5.5] - 2021-06-26 + +### Added +- Added Stereo camera class ([#1102](https://github.com/kornia/kornia/pull/1102)) +- Added auto-generated images in docs ([#1105](https://github.com/kornia/kornia/pull/1105)) ([#1108](https://github.com/kornia/kornia/pull/1108)) ([#1127](https://github.com/kornia/kornia/pull/1127)) ([#1128](https://github.com/kornia/kornia/pull/1128)) ([#1129](https://github.com/kornia/kornia/pull/1129)) ([#1131](https://github.com/kornia/kornia/pull/1131)) +- Added chinese version README ([#1112](https://github.com/kornia/kornia/pull/1112)) +- Added random_apply to augmentaton containers ([#1125](https://github.com/kornia/kornia/pull/1125)) + +### Changed +- Change GaussianBlur to RandomGaussianBlur ([#1118](https://github.com/kornia/kornia/pull/1118)) +- Update ci with pytorch 1.9.0 ([#1120](https://github.com/kornia/kornia/pull/1120)) +- Changed option for mean and std to be tuples in normalization ([#987](https://github.com/kornia/kornia/pull/987)) +- Adopt torch.testing.assert_close ([#1031](https://github.com/kornia/kornia/pull/1031)) + +### Removed +- Remove numpy import ([#1116](https://github.com/kornia/kornia/pull/1116)) + + +## [0.5.4] - 2021-06-11 + +### Added +- Add Canny edge detection ([#1020](https://github.com/kornia/kornia/pull/1020)) +- Added Batched forward function ([#1058](https://github.com/kornia/kornia/pull/1058)) +- Added denormalize homography function [(#1061](https://github.com/kornia/kornia/pull/1061)) +- Added more augmentations containers ([#1014](https://github.com/kornia/kornia/pull/1014)) +- Added calibration module and Undistort 2D points function ([#1026](https://github.com/kornia/kornia/pull/1026)) +- Added patch augmentation container ([#1095](https://github.com/kornia/kornia/pull/1095)) + +### Fixed +- Remove lena ([#1059](https://github.com/kornia/kornia/pull/1059)) :) + +### Changed +- Resize regardless of number of dims, considering the last two dims as image ([#1047](https://github.com/kornia/kornia/pull/1047)) +- Raise error if converting to unit8 image to gray with float weights ([#1057](https://github.com/kornia/kornia/pull/1057)) +- Filter 2D->2d, 3D->3d ([#1069](https://github.com/kornia/kornia/pull/1069)) +- Removed augmentation functional module. ([#1067](https://github.com/kornia/kornia/pull/1067)) +- Make Morphology compatible with both OpenCV and Scipy ([#1084](https://github.com/kornia/kornia/pull/1084)) + + +## [0.5.3] - 2021-05-29 + +### Added +- Added inverse for augmentations ([#1013](https://github.com/kornia/kornia/pull/1013)) +- Add advanced augmentations: RandomFisheye, RandomElasticTransform, RandomThinPlateSpline, RandomBloxBlur ([#1015](https://github.com/kornia/kornia/pull/1015) + +### Fixed +- Correct Sobel test_noncontiguous. Nothing was tested before. ([#1018](https://github.com/kornia/kornia/pull/1018)) +- Fixing #795: find_homography_dlt_iterated sometimes fails ([#1022](https://github.com/kornia/kornia/pull/1022)) + +### Changed +- Refactorization of the morphology package ([#1034](https://github.com/kornia/kornia/pull/1034)) +- Optimised clipping in clahe and some other minor optimisation ([#1035](https://github.com/kornia/kornia/pull/1035)) + + +## [0.5.2] - 2021-05-14 + +## Added +- Added unsharp mask filtering ([#1004](https://github.com/kornia/kornia/pull/1004)) + +### Fixed +- Fixed angle axis to quaternion order bug ([#926](https://github.com/kornia/kornia/pull/926)) +- Fixed type error for lab_to_rgb conversion when using coremltools. ([#1002](https://github.com/kornia/kornia/pull/1002)) + +### Changed +- Mask with unbatched motion from essential choose solution ([#998](https://github.com/kornia/kornia/pull/998)) + + +## [0.5.1] - 2021-04-30 + +### Added +- Added dtype for create_mesh ([#919](https://github.com/kornia/kornia/pull/919)) +- Added Hardnet8 ([#955](https://github.com/kornia/kornia/pull/955)) +- Added normalize boolean for remap ([#921](https://github.com/kornia/kornia/pull/921)) +- Added custom weights option for rgb2gray ([#944](https://github.com/kornia/kornia/pull/944)) +- Added fp16 support ([#963](https://github.com/kornia/kornia/pull/963)) +- Added ImageToTensor module and resize for non-batched images ([#978](https://github.com/kornia/kornia/pull/978)) +- Add more augmentations ([#960](https://github.com/kornia/kornia/pull/960)) +- Anti alias resize ([#989](https://github.com/kornia/kornia/pull/989)) + +## Changed +- Improve kornia porphology ([#965](https://github.com/kornia/kornia/pull/965)) +- Improve cuda ci workflow speed ([#975](https://github.com/kornia/kornia/pull/975)) +- Refactor augmentation module ([#948](https://github.com/kornia/kornia/pull/948)) +- Implement fast version of crop function in augmentations ([#967](https://github.com/kornia/kornia/pull/967)) +- Implement missing jit ops in kornia.geometry.transform ([#981](https://github.com/kornia/kornia/pull/981)) + +### Fixed +- Fixed RandomAffine translation range check ([#917](https://github.com/kornia/kornia/pull/917) +- Fixed the issue of NaN gradients by adding epsilon in focal loss ([#924](https://github.com/kornia/kornia/pull/924)) +- Allow crop size greater than input size. ([#957](https://github.com/kornia/kornia/pull/957)) +- Fixed RandomCrop bug ([#951](https://github.com/kornia/kornia/pull/951)) + +### Removed +- Deprecate some augmentation functionals ([#943](https://github.com/kornia/kornia/pull/943)) + + +## [0.4.1] - 2020-10-20 +### Added +- Update docs for `get_affine_matrix2d` and `get_affine_matrix3d` ([#618](https://github.com/kornia/kornia/pull/618)) +- Added docs for `solarize`, `posterize`, `sharpness`, `equalize` ([#623](https://github.com/kornia/kornia/pull/623)) +- Added tensor device conversion for solarize params ([#624](https://github.com/kornia/kornia/pull/624)) +- Added rescale functional and transformation ([#631](https://github.com/kornia/kornia/pull/631)) +- Added Mixup data augmentation ([#609](https://github.com/kornia/kornia/pull/609)) +- Added `equalize3d` ([#639](https://github.com/kornia/kornia/pull/639)) +- Added `decompose 3x4projection matrix` ([#650](https://github.com/kornia/kornia/pull/650)) +- Added `normalize_min_max` functionality ([#684](https://github.com/kornia/kornia/pull/684)) +- Added `random equalize3d` ([#653](https://github.com/kornia/kornia/pull/653)) +- Added 3D motion blur ([#713](https://github.com/kornia/kornia/pull/713)) +- Added 3D volumetric crop implementation ([#689](https://github.com/kornia/kornia/pull/689)) + - `warp_affine3d` + - `warp_perspective3d` + - `get_perspective_transform3d` + - `crop_by_boxes3d` + - `warp_grid3d` + + +### Changed +- Replace convolution with `unfold` in `contrib.extract_tensor_patches` ([#626](https://github.com/kornia/kornia/pull/626)) +- Updates Affine scale with non-isotropic values ([#646](https://github.com/kornia/kornia/pull/646)) +- Enabled param p for each augmentation ([#664](https://github.com/kornia/kornia/pull/664)) +- Enabled RandomResizedCrop batch mode when same_on_batch=False ([#683](https://github.com/kornia/kornia/pull/683)) +- Increase speed of transform_points ([#687](https://github.com/kornia/kornia/pull/687)) +- Improves `find_homography_dlt` performance improvement and weights params made optional ([#690](https://github.com/kornia/kornia/pull/690)) +- Enable variable side resizing in `kornia.resize` ([#628](https://github.com/kornia/kornia/pull/628)) +- Added `Affine` transformation as `nn.Module` ([#630](https://github.com/kornia/kornia/pull/630)) +- Accelerate augmentations ([#708](https://github.com/kornia/kornia/pull/708)) + +### Fixed +- Fixed error in normal_transform_pixel3d ([#621](https://github.com/kornia/kornia/pull/621)) +- Fixed pipelining multiple augmentations return wrong transformation matrix (#645)([645](https://github.com/kornia/kornia/pull/645)) +- Fixed flipping returns wrong transformation matrices ([#648](https://github.com/kornia/kornia/pull/648)) +- Fixed 3d augmentations return wrong transformation matrix ([#665](https://github.com/kornia/kornia/pull/665)) +- Fix the SOSNet loading bug ([#668](https://github.com/kornia/kornia/pull/668)) +- Fix/random perspective returns wrong transformation matrix ([#667](https://github.com/kornia/kornia/pull/667)) +- Fixes Zca inverse transform ([#695](https://github.com/kornia/kornia/pull/695)) +- Fixes Affine scale bug ([#714](https://github.com/kornia/kornia/pull/714)) + +## Removed +- Removed `warp_projective` ([#689](https://github.com/kornia/kornia/pull/689)) diff --git a/CITATION.md b/CITATION.md new file mode 100644 index 0000000..66a72d5 --- /dev/null +++ b/CITATION.md @@ -0,0 +1,44 @@ + + ## Cite Kornia papers + 1. Kornia: an Open Source Differentiable Computer Vision Library for PyTorch + 2. A survey on Kornia: an Open Source Differentiable Computer Vision Library for PyTorch + 3. Differentiable Data Augmentation with Kornia + 4. torchgeometry: when PyTorch meets geometry + ```bash + @inproceedings{eriba2019kornia, + author = {E. Riba, D. Mishkin, D. Ponsa, E. Rublee and G. Bradski}, + title = {Kornia: an Open Source Differentiable Computer Vision Library for PyTorch}, + booktitle = {Winter Conference on Applications of Computer Vision}, + year = {2020}, + url = {https://arxiv.org/pdf/1910.02190.pdf} + } + ``` + ```bash + @misc{riba2020survey, + title={A survey on Kornia: an Open Source Differentiable Computer Vision Library for PyTorch}, + author={E. Riba and D. Mishkin and J. Shi and D. Ponsa and F. Moreno-Noguer and G. Bradski}, + year={2020}, + eprint={2009.10521}, + archivePrefix={arXiv}, + primaryClass={cs.CV} + } + ``` + ```bash + @misc{shi2020differentiable, + title={Differentiable Data Augmentation with Kornia}, + author={Jian Shi and Edgar Riba and Dmytro Mishkin and Francesc Moreno and Anguelos Nicolaou}, + year={2020}, + eprint={2011.09832}, + archivePrefix={arXiv}, + primaryClass={cs.CV} + } + ``` + ```bash + @misc{Arraiy2018, + author = {E. Riba, M. Fathollahi, W. Chaney, E. Rublee and G. Bradski}, + title = {torchgeometry: when PyTorch meets geometry}, + booktitle = {PyTorch Developer Conference}, + year = {2018}, + url = {https://drive.google.com/file/d/1xiao1Xj9WzjJ08YY_nYwsthE-wxfyfhG/view?usp=sharing} + } + ``` diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..30b7d4c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,173 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## About Kornia + +Kornia is a differentiable computer vision library built on PyTorch. It provides differentiable image processing, geometric vision algorithms, augmentation pipelines, and pre-trained AI models (feature detection/matching, segmentation, etc.). + +## Development Environment + +The project uses [pixi](https://pixi.sh) for environment management and `uv` for Python package management. + +```bash +# Set up development environment (Python 3.11 default) +pixi install + +# For specific Python versions +pixi install -e py312 +pixi install -e py313 + +# For CUDA development +pixi run -e cuda install +``` + +## Common Commands + +```bash +# Run all tests +pixi run test + +# Run a specific test file +pixi run test tests/test_geometry_transform.py + +# Run tests with pytest options (dtype: bfloat16, float16, float32, float64, all; device: cpu, cuda, mps, tpu, all) +pixi run test tests/test_geometry_transform.py --dtype=float32,float64 --device=all + +# Run quick tests (excludes jit, grad, nn) +pixi run test-quick + +# Linting (ruff via pre-commit) +pixi run lint + +# Type checking (uses `ty`) +pixi run typecheck + +# Doctests +pixi run doctest + +# Build documentation +pixi run build-docs + +# Run tests with specific device/dtype via env vars +KORNIA_TEST_DEVICE=cuda KORNIA_TEST_DTYPE=float32 pixi run test +KORNIA_TEST_RUNSLOW=true pixi run test-slow +``` + +## Code Architecture + +The library is structured as submodules under `kornia/`: + +- **`kornia/filters/`** — Image filtering (Gaussian, Sobel, Median, Canny, etc.). Imported first in `__init__.py` as it's core. +- **`kornia/geometry/`** — Geometric transformations (affine, homography, camera models, stereo, 3D). Also core, imported first. +- **`kornia/augmentation/`** — Augmentation pipeline (`AugmentationSequential`, `RandomAffine`, etc.) +- **`kornia/color/`** — Color space conversions (RGB, HSV, grayscale, etc.) +- **`kornia/feature/`** — Feature detection and description (SIFT, HardNet, DISK, DeDoDe, LoFTR, LightGlue, etc.) +- **`kornia/enhance/`** — Image enhancement (histogram equalization, CLAHE, gamma correction) +- **`kornia/losses/`** — Loss functions (SSIM, PSNR, Dice, Hausdorff, etc.) +- **`kornia/models/`** — Pre-trained AI models (YuNet face detection, SAM segmentation, etc.) +- **`kornia/morphology/`** — Morphological operations (dilation, erosion, etc.) +- **`kornia/onnx/`** — ONNX export and inference (`ONNXSequential`) +- **`kornia/contrib/`** — Experimental/contributed modules +- **`kornia/core/`** — Base classes (`ImageModule`, `ImageSequential`, `TensorWrapper`, ONNX mixins) +- **`kornia/transpiler/`** — Multi-framework support via ivy (JAX, TensorFlow, NumPy backends) +- **`testing/`** — Test utilities (not tests). `testing/base.py` contains `BaseTester` and `assert_close`. + +**Import order matters**: `filters` and `geometry` must be imported before other modules to avoid circular dependencies (see `kornia/__init__.py`). + +## Testing Patterns + +All tests should inherit from `BaseTester` (from `testing.base`): + +```python +from testing.base import BaseTester + +class TestMyFunction(BaseTester): + def test_smoke(self, device, dtype): ... # Basic run with all arg combinations + def test_exception(self, device, dtype): ... # Exception cases + def test_cardinality(self, device, dtype): ... # Output shapes + def test_feature(self, device, dtype): ... # Correctness / numerical accuracy + def test_gradcheck(self, device): ... # Gradient checking via self.gradcheck() + def test_dynamo(self, device, dtype, torch_optimizer): ... # torch.compile compat +``` + +The `device` and `dtype` fixtures are injected automatically. Use `self.assert_close()` for tensor comparisons. Test configurations are driven by env vars `KORNIA_TEST_DEVICE` and `KORNIA_TEST_DTYPE`, or by `--device`/`--dtype` pytest args. + +## Coding Standards + +- **Python >= 3.11** with `from __future__ import annotations` for non-JIT modules +- **Line length**: 120 characters (ruff enforced) +- **Type hints required** on all function inputs and outputs; use `torch.Tensor` directly (not string annotations for tensor types in JIT-compatible code) +- **Only PyTorch** as a third-party dependency — no other libraries +- **Use existing `kornia` utilities** rather than reimplementing with raw PyTorch +- **Docstrings**: Follow existing codebase style; all public APIs need docstrings +- Every source file must start with the Apache 2.0 license header (managed by `add-license-header`) + +## Benchmarks + +Scripts under `benchmarks/` measure the speed and/or quality of existing kornia functions, modules, or models. Each benchmark must: + +- Report **CPU and CUDA timings** in a table. +- Include **quality metrics** where applicable (see `benchmarks/feature/` for an example with local-feature matching scores). +- Record the **date, hardware description, and git commit hash** being evaluated at the top of the output or in a results file. +- Benchmark **only the public kornia API** — no custom reimplementations or alternative snippets inside the script. + +### Workflow for performance PRs + +1. Check out `main` (or the relevant release tag) and run the benchmark to establish a baseline. +2. Apply your changes on a new branch and run the same benchmark again. +3. Include both result tables in the PR description so reviewers can compare before and after. + +## Pre-commit Hooks + +Install hooks with `pre-commit install`. CI enforces ruff formatting, linting, and docformatter. + +**Always run `pixi run lint` before pushing** to catch ruff errors (formatting, style, ambiguous Unicode characters, etc.) that will cause CI to fail. Fix any reported issues before committing. + +## Documentation and Visualizations + +**Every public class or function added to a `kornia/` submodule must also be listed in the corresponding `docs/source/*.rst` file** — otherwise it will not appear in the rendered docs. Check the relevant `.rst` after adding any public API symbol and add an `.. autoclass::` or `.. autofunction::` directive if it is missing. + +When adding a new feature detector or descriptor to `kornia/feature/`: +- Add an `.. autoclass::` entry to `docs/source/feature.rst` in the appropriate section (Detectors, Descriptors, or Local Features). +- Add an entry to the `responses` list in `docs/generate_examples.py` with a corresponding `elif` block that produces a heatmap/score visualization (`(B, 3, H, W)` BGR image in `img_in`, `(B, 3, H, W)` response map in `out`). +- See existing entries (`DISK`, `ALIKED`, `XFeat`) for the expected pattern. + +## PR Requirements + +All PRs must: +- Be linked to a previously discussed GitHub issue or Discord discussion (`Fixes #123`) +- Include pasted local test log output as proof of execution (`pixi run test ...`) +- Reference an algorithm source (PyTorch, OpenCV, scikit-image, paper, etc.) for any new implementation + +**Comments**: No redundant or ghost comments (e.g., "this returns the input tensor", or comments explaining deleted code). Violation triggers mandatory manual rewrite request. + +## Test-First Debugging + +**Always run the relevant tests before answering a question about a bug or test failure.** +Do not assume a failure is pre-existing or unrelated to recent changes without running it: + +```bash +# Run the failing test(s) first, then investigate +pixi run -e default uv run pytest -q --dtype=float32,float64 +``` + +If the test passes on the current branch, verify on `main` with `git stash` before concluding it is pre-existing. + +## Hardcoded Reference Values in Tests + +**Never use `pytest.importorskip` to gate correctness tests on optional third-party libraries** (OpenCV, scipy, etc.). Instead, compute the expected output once offline and embed it as a hardcoded tensor literal, following the pattern used in `tests/geometry/epipolar/test_fundamental.py`: + +```python +# Snippet used to generate expected (requires numpy only): +# import numpy as np +# ... run the reference algorithm ... +# expected = result # <-- print and paste below + +expected = torch.tensor([[...]], dtype=dtype, device=device) +self.assert_close(actual, expected, atol=1e-4, rtol=1e-4) +``` + +Leave a comment above the literal showing the generation snippet so it can be reproduced. This makes tests self-contained, deterministic, and runnable without optional dependencies. + +See `AI_POLICY.md` for the full contribution policy, including AI usage disclosure requirements. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..ca5899f --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socioeconomic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies within all project spaces, and it also applies when +an individual is representing the project or its community in public spaces. +Examples of representing a project or community include using an official +project e-mail address, posting via an official social media account, or acting +as an appointed representative at an online or offline event. Representation of +a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at edgar.riba@arraiy.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..ee3a591 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,424 @@ +# Contributing to Kornia + +Welcome! This guide will help you contribute to Kornia. + +## Policies and Guidelines + +- **AI Policy & Authorship**: See [AI_POLICY.md](AI_POLICY.md) for the complete policy. Summary: + - Kornia accepts AI-assisted code but strictly rejects AI-generated contributions where the submitter acts as a proxy. + - **Proof of Verification**: PRs must include local test logs proving execution. + - **Hallucination & Redundancy Ban**: Use existing `kornia` utilities and never reinvent the wheel, except for when the utility is not available. + - **The "Explain It" Standard**: You must be able to explain any code you submit. + - Violations result in immediate closure or rejection. + +- **15-Day Rule**: PRs with no activity for 15+ days will be automatically closed. + +- **Transparency**: All discussions must be public. + +We're all volunteers. These policies help us focus on high-impact work. + +## Ways to Contribute + +1. **Ask/Answer questions:** + - [GitHub Discussions](https://github.com/kornia/kornia/discussions) + - `#kornia` tag in [PyTorch Discuss](https://discuss.pytorch.org) + - [Discord](https://discord.gg/HfnywwpBnD) + - Don't use GitHub issues for Q&A. + +2. **Report bugs** via [GitHub issues](https://github.com/kornia/kornia/issues): + - Search for existing issues first. + - Use the bug report template. + - Include: clear description, reproduction steps, package versions, and code sample. + +3. **Fix bugs or add features:** + - Check [help wanted issues](https://github.com/kornia/kornia/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22help%20wanted%22) for starting points. + - Follow the [development setup](#developing-kornia) below. + - See [Pull Request](#pull-request) section for PR requirements. + +4. **Donate resources:** + - [Open Collective](https://opencollective.com/kornia) + - [GitHub Sponsors](https://github.com/sponsors/kornia) + - We're looking for CUDA server donations for testing. + +# Developing Kornia + +## Setup + +1. **Fork** the [repository](https://github.com/kornia/kornia/fork) + +2. **Clone your fork** and add upstream: + ```bash + $ git clone git@github.com:/kornia.git + $ cd kornia + $ git remote add upstream https://github.com/kornia/kornia.git + ``` + +3. **Create a branch** (don't work on `main`): + ```bash + git checkout upstream/main -b feat/foo_feature + # or + git checkout upstream/main -b fix/bar_bug + ``` + +4. **Development environment** + + We use [pixi](https://pixi.sh) for package and environment management. + + **Install Pixi:** + + ```bash + # On Linux/macOS + curl -fsSL https://pixi.sh/install.sh | bash + + # On Windows (PowerShell) + irm https://pixi.sh/install.ps1 | iex + + # Or using conda/mamba + conda install -c conda-forge pixi + ``` + + **Set up the development environment:** + + ```bash + # Install all dependencies (defaults to Python 3.11) + pixi install + + # For specific Python versions + pixi install -e py312 # Python 3.12 + pixi install -e py313 # Python 3.13 + + # For CUDA development (requires reinstall of PyTorch) + pixi run -e cuda install + ``` + + **Available tasks:** + + Kornia provides several tasks via pixi for common development workflows: + + ```bash + # Installation + pixi run install # Install dev dependencies + pixi run install-docs # Install dev + docs dependencies + + # Testing + pixi run test # Run tests (configure via KORNIA_TEST_* env vars) + pixi run test-f32 # Run tests with float32 + pixi run test-f64 # Run tests with float64 + pixi run test-slow # Run slow tests + pixi run test-quick # Run quick tests (excludes jit, grad, nn) + + # CUDA testing (requires cuda environment) + pixi run -e cuda test-cuda # Run tests on CUDA + pixi run -e cuda test-cuda-f32 # Run CUDA tests with float32 + pixi run -e cuda test-cuda-f64 # Run CUDA tests with float64 + + # Code quality + pixi run lint # Run ruff linting + pixi run typecheck # Run type checking with ty + pixi run doctest # Run doctests + + # Documentation + pixi run build-docs # Build documentation + + # Utilities + pixi run clean # Clean Python cache files + ``` + + **Environment variables for tests:** + + Tests can be configured using environment variables: + + ```bash + # Set device (cpu, cuda, mps, tpu) + export KORNIA_TEST_DEVICE=cuda + + # Set dtype (float32, float64, float16, bfloat16) + export KORNIA_TEST_DTYPE=float32 + + # Run slow tests + export KORNIA_TEST_RUNSLOW=true + + # Then run tests + pixi run test + ``` + + **Dependencies:** Defined in `pyproject.toml`. Update it and run `pixi install`. + + **CUDA:** The CUDA environment uses PyTorch with CUDA 12.1. Run `pixi run -e cuda install` to set it up. + +5. **Develop and test:** + + Create test cases for your code. Run tests with: + ```bash + # Run all tests + pixi run test + + # Run specific test file + pixi run test tests/.py + + # For specific test with pytest options + pixi run test tests/.py --dtype=float32,float64 --device=all + ``` + + **dtype options:** `bfloat16`, `float16`, `float32`, `float64`, `all` + **device options:** `cpu`, `cuda`, `tpu`, `mps`, `all` + + We use [pre-commit](https://pre-commit.com) for code quality. Install it with `pre-commit install`. See [coding standards](#coding-standards) below. + +# Contributing to Documentation + +1. Set up your development environment (see [above](#developing-kornia)) +2. Edit files in `docs/` +3. Build docs: `make build-docs` +4. Preview: `open docs/build/html/index.html` +5. Submit a PR following the [Pull Request](#pull-request) guidelines + +# Coding Standards + +- **Write small incremental changes:** + - Commit small, logical changes + - Write clear commit messages + - Avoid large files + +- **Add tests:** + - Write unit tests for each functionality + - Use helpers from [testing/](./testing/) + - Put test utilities (not tests or fixtures) in `testing/` + + ```python + from testing.base import BaseTester + + class TestMyFunction(BaseTester): + # To compare the actual and expected tensors use `self.assert_close(...)` + + + def test_smoke(self, device, dtype): + # test the function with different parameters arguments, to check if the function at least runs with all the + # arguments allowed. + pass + + def test_exception(self, device, dtype): + # tests the exceptions which can occur on your function + + # example of how to properly test your exceptions + # with pytest.raises() as errinfo: + # your_function() + # assert '' in str(errinfo) + + pass + + def test_cardinality(self, device, dtype): + # test if with different parameters the shape of the output is the expected + pass + + def test_feature_foo(self, device, dtype): + # test basic functionality + pass + + def test_feature_bar(self, device, dtype): + # test another functionality + pass + + def test_gradcheck(self, device): + # test the functionality gradients + # Uses `self.gradcheck(...)` + pass + + def test_dynamo(self, device, dtype, torch_optimizer): + # test the functionality using dynamo optimizer + + # Example of how to properly test your function for dynamo + # inputs = (...) + # op = your_function + # op_optimized = torch_optimizer(op) + # self.assert_close(op(inputs), op_optimized(inputs)) + + pass + ``` + +- **Test coverage:** Cover different devices, dtypes, and batch sizes. Use `--dtype` and `--device` pytest arguments to generate test combinations: + + ```python + import pytest + + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_smoke(batch_size, device, dtype): + x = torch.rand(batch_size, 2, 3, device=device, dtype=dtype) + assert x.shape == (batch_size, 2, 3) + ``` + +- **Type hints** (Python >= 3.11): + - Use typing when it improves readability + - **Use `torch.Tensor` directly** for type hints (preferred) or import from `kornia.core` for backward compatibility + - Use `torch.nn.Module` directly for module classes (preferred) or import from `kornia.core` for backward compatibility + - For non-JIT modules, use `from __future__ import annotations` + - **Always** type function inputs and outputs: + - Run type checking with `pixi run typecheck` (uses `ty`) + ```python + from __future__ import annotations + import torch + + def homography_warp( + patch_src: torch.Tensor, + dst_homo_src: torch.Tensor, + dsize: tuple[int, int], + mode: str = 'bilinear', + padding_mode: str = 'zeros' + ) -> torch.Tensor: + ``` + + For module classes: + ```python + from __future__ import annotations + import torch.nn as nn + + class MyModule(nn.Module): + def forward(self, x: torch.Tensor) -> torch.Tensor: + return x + ``` + +- **Code style:** + - Follow [PEP8](https://www.python.org/dev/peps/pep-0008/) + - Use f-strings: [PEP 498](https://peps.python.org/pep-0498/) + - Line length: 120 characters + - Comments must be written in English and verified by a human with a good understanding of the code + - Obvious or redundant comments are not allowed (see [Best Practices](#best-practices) for comment guidelines) + - W504 (line break after binary operator) is sometimes acceptable. Example: + + ```python + determinant = A[:, :, 0:1, 0:1] * A[:, :, 1:2, 1:2] - + A[:, :, 0:1, 1:2] * A[:, :, 1:2, 0:1]) + ``` + +- **Third-party libraries:** Not allowed. Only PyTorch. + +# Best Practices + +This section provides guidance for contributing to Kornia, with a focus on Python and PyTorch best practices, performance, and maintainability. + +## Before You Start + +1. **Discuss First**: Always discuss your proposed changes in Discord or via a GitHub issue before starting implementation. This ensures your work aligns with project goals and avoids duplicate effort. + +2. **Start Small**: If you're new to the project, start with small bug fixes or documentation improvements to familiarize yourself with the codebase and contribution process. + +3. **Understand the Codebase**: Take time to explore existing code patterns, architecture, and conventions before implementing new features. + +4. **Review Existing Utilities**: Before implementing new functionality, search the codebase for existing utilities in `kornia`. This aligns with the AI Policy's Hallucination & Redundancy Ban (see [Policies and Guidelines](#policies-and-guidelines)). + +## Development Workflow + +1. **Keep PRs Focused**: Each PR should address a single concern. If you're working on multiple features, create separate PRs for each. + +2. **Test Locally First**: Always run all relevant tests locally before submitting (see [Pull Request](#pull-request) for requirements): + ```bash + pixi run lint # Check formatting and linting + pixi run test # Run all tests + pixi run typecheck # Verify type checking + ``` + +3. **Update Documentation**: When adding new features or changing behavior, update docstrings for public APIs. For documentation contributions, see [Contributing to Documentation](#contributing-to-documentation). + +## Code Quality + +1. **Performance Considerations**: + - Prefer in-place operations when possible (e.g., `tensor.add_(other)` vs `tensor = tensor.add(other)`) + - Use tensor views and slicing instead of copying when possible + - Leverage PyTorch's vectorized operations over Python loops + - Profile before optimizing (use `torch.profiler` or `cProfile`) + - Consider memory efficiency for large tensors (use appropriate dtypes, avoid unnecessary copies) + - Use `torch.jit.script` or `torch.compile` for performance-critical paths when appropriate + +2. **Code Clarity**: + - Use descriptive variable and function names that convey intent + - Keep functions focused and single-purpose + - Prefer clear code over comments; when comments are needed, explain "why" not "what" + - Avoid over-engineering; start simple and refactor when needed + +3. **Tensor Operations**: + - Use `kornia` utilities instead of reimplementing common operations (see [AI Policy](#policies-and-guidelines)) + - Ensure operations are device-agnostic (work on CPU, CUDA, MPS, etc.) + - Support multiple dtypes (float32, float64, float16, bfloat16) when applicable + - Handle batched and non-batched inputs consistently + +## Testing Best Practices + +- Write tests for happy paths, error cases, edge conditions, boundary conditions, and integration scenarios +- Use `BaseTester` from `testing.base` for consistent test structure (see [Coding Standards](#coding-standards) for examples) +- Test across different devices and dtypes using pytest parametrization (see [Coding Standards](#coding-standards) for examples) +- Make tests deterministic, fast, and independent +- Use descriptive test names; test both forward pass and gradients when applicable + +## Review Process + +- Review your own PR first: check for typos/formatting, verify tests pass, ensure documentation is updated, and confirm AI policy compliance +- Respond promptly to review feedback +- Be open to feedback and explain your decisions when questioned +- See [Pull Request](#pull-request) section for review requirements + +## AI-Assisted Development + +- Understand every line of code you submit; you must be able to explain it during review (see [AI Policy](#policies-and-guidelines)) +- Review AI output thoroughly: check for unnecessary complexity, verify it follows project conventions, ensure it uses existing utilities, and test it +- Be transparent in PR descriptions about what was AI-assisted and what you manually reviewed (see [Pull Request](#pull-request) for AI Usage Disclosure requirements) +- **AI Usage Disclosure in PR Template**: When completing the PR template's "AI Usage Disclosure" section: + - Mark as **🟢 No AI used** only if you wrote all code manually without any AI assistance + - Mark as **🟡 AI-assisted** if you used AI tools (Copilot, Cursor, etc.) for boilerplate/refactoring but manually reviewed and tested every line + - Mark as **🔴 AI-generated** if an AI agent generated the code, PR description, or commit messages, or if you cannot explain the logic without referring to the AI's output. **Important**: PRs marked as AI-generated are subject to stricter scrutiny and may be immediately closed if the logic cannot be explained + +## Communication + +- Write clear, concise PR descriptions (see [Pull Request](#pull-request) for requirements) +- Always link to related issues or discussions in your PR description +- Ask questions in Discord or PR comments if unsure; it's better to clarify early than to rework later + +# Pull Request + +## Issue Approval and Assignment Workflow + +**Before submitting a PR, you must:** + +1. **Open an issue first**: All PRs must be linked to an existing issue. If no issue exists for your work, create one using the appropriate template (bug report or feature request). + +2. **Wait for maintainer approval**: A maintainer must review and approve the issue before you start working on it. New issues are automatically labeled with `triage` and will receive a welcome message explaining this process. + +3. **Wait for assignment**: You must be assigned to the issue by a maintainer before submitting a PR. This ensures: + - The issue aligns with project goals + - No duplicate work is being done + - Proper coordination of contributions + +4. **Do not start work until assigned**: PRs submitted without prior issue approval and assignment may be closed or receive warnings during automated validation. + +This workflow helps maintain quality, avoid conflicts, and ensure contributions align with the project's direction. The automated PR validation workflow will check these requirements and post warnings if they're not met. + +**Requirements:** +- **Issue approval and assignment**: The linked issue must be approved by a maintainer and you must be assigned to it (see workflow above) +- Link PR to an issue (use "Closes #123" or "Fixes #123") +- Pass all local tests before submission +- Provide proof of local test execution in the PR description (this is especially important for first-time contributors) +- Fill the [pull request template](.github/pull_request_template.md) +- **AI Policy Compliance**: Must comply with [AI_POLICY.md](AI_POLICY.md). This includes: + - Using existing `kornia` utilities instead of reinventing + - Being able to explain all submitted code + - Completing the AI Usage Disclosure in the PR template accurately (see [AI-Assisted Development](#ai-assisted-development) for guidance on when to mark as AI-generated) +- 15-Day Rule: Inactive PRs (>15 days) will be closed +- Transparency: Keep discussions public + +**Code review:** +- By default, GitHub Copilot will check the PR against the AI Policy and the coding standards. +- Code must be reviewed by the repository owner or a senior contributor, who have the final say on the quality and acceptance of the PR. + +**Note:** Tickets may be closed during cleanup. Feel free to reopen if you plan to finish the work. + +**CI checks:** +- All tests pass +- Test coverage maintained +- Type checking (ty) +- Documentation builds successfully +- Code formatting (ruff, docformatter via pre-commit) + +Fix any failing checks before your PR will be considered. + +# License + +By contributing, you agree to license your contributions under the Apache License. See [LICENSE](./LICENSE). diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 0000000..ba676bc --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,14 @@ + Copyright (C) 2017-2019, Arraiy, Inc., all rights reserved. + Copyright (C) 2019- , Kornia authors, all rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d9a10c0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/README.md b/README.md new file mode 100644 index 0000000..5809237 --- /dev/null +++ b/README.md @@ -0,0 +1,320 @@ +
+

+ +

+ +--- + +English | [简体中文](README_zh-CN.md) + + +Docs • +Try it Now • +Tutorials • +Examples • +Blog • +Community + +[![PyPI version](https://badge.fury.io/py/kornia.svg)](https://pypi.org/project/kornia) +[![Downloads](https://static.pepy.tech/badge/kornia)](https://pepy.tech/project/kornia) +[![star](https://gitcode.com/kornia/kornia/star/badge.svg)](https://gitcode.com/kornia/kornia) +[![Discord](https://img.shields.io/badge/Discord-5865F2?logo=discord&logoColor=white)](https://discord.gg/HfnywwpBnD) +[![Twitter](https://img.shields.io/twitter/follow/kornia_foss?style=social)](https://twitter.com/kornia_foss) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) + +

+
+ +**Kornia** is a differentiable computer vision library that provides a rich set of differentiable image processing and geometric vision algorithms. Built on top of [PyTorch](https://pytorch.org), Kornia integrates seamlessly into existing AI workflows, allowing you to leverage powerful [batch transformations](), [auto-differentiation]() and [GPU acceleration](). Whether you're working on image transformations, augmentations, or AI-driven image processing, Kornia equips you with the tools you need to bring your ideas to life. + +> **📢 Announcement**: Kornia is shifting towards end-to-end vision models. We are focusing on integrating state-of-the-art Vision Language Models (VLM) and Vision Language Agents (VLA) to provide comprehensive end-to-end vision solutions. + +## Key Components +1. **Differentiable Image Processing**
+ Kornia provides a comprehensive suite of image processing operators, all differentiable and ready to integrate into deep learning pipelines. + - **Filters**: Gaussian, Sobel, Median, Box Blur, etc. + - **Transformations**: Affine, Homography, Perspective, etc. + - **Enhancements**: Histogram Equalization, CLAHE, Gamma Correction, etc. + - **Edge Detection**: Canny, Laplacian, Sobel, etc. + - ... check our [docs](https://kornia.readthedocs.io) for more. +2. **Advanced Augmentations**
+Perform powerful data augmentation with Kornia’s built-in functions, ideal for training AI models with complex augmentation pipelines. + - **Augmentation Pipeline**: AugmentationSequential, PatchSequential, VideoSequential, etc. + - **Automatic Augmentation**: AutoAugment, RandAugment, TrivialAugment. +3. **AI Models**
+Leverage pre-trained AI models optimized for a variety of vision tasks, all within the Kornia ecosystem. + - **Face Detection**: YuNet + - **Feature Matching**: LoFTR, LightGlue + - **Feature Descriptor**: DISK, DeDoDe, SOLD2 + - **Segmentation**: SAM + - **Classification**: MobileViT, VisionTransformer. + +
+See here for some of the methods that we support! (>500 ops in total !) + +| **Category** | **Methods/Models** | +|----------------------------|---------------------------------------------------------------------------------------------------------------------| +| **Image Processing** | - Color conversions (RGB, Grayscale, HSV, etc.)
- Geometric transformations (Affine, Homography, Resizing, etc.)
- Filtering (Gaussian blur, Median blur, etc.)
- Edge detection (Sobel, Canny, etc.)
- Morphological operations (Erosion, Dilation, etc.) | +| **Augmentation** | - Random cropping, Erasing
- Random geometric transformations (Affine, flipping, Fish Eye, Perspecive, Thin plate spline, Elastic)
- Random noises (Gaussian, Median, Motion, Box, Rain, Snow, Salt and Pepper)
- Random color jittering (Contrast, Brightness, CLAHE, Equalize, Gamma, Hue, Invert, JPEG, Plasma, Posterize, Saturation, Sharpness, Solarize)
- Random MixUp, CutMix, Mosaic, Transplantation, etc. | +| **Feature Detection** | - Detector (Harris, GFTT, Hessian, DoG, KeyNet, DISK and DeDoDe)
- Descriptor (SIFT, HardNet, TFeat, HyNet, SOSNet, and LAFDescriptor)
- Matching (nearest neighbor, mutual nearest neighbor, geometrically aware matching, AdaLAM LightGlue, and LoFTR) | +| **Geometry** | - Camera models and calibration
- Stereo vision (epipolar geometry, disparity, etc.)
- Homography estimation
- Depth estimation from disparity
- 3D transformations | +| **Deep Learning Layers** | - Custom convolution layers
- Recurrent layers for vision tasks
- Loss functions (e.g., SSIM, PSNR, etc.)
- Vision-specific optimizers | +| **Photometric Functions** | - Photometric loss functions
- Photometric augmentations | +| **Filtering** | - Bilateral filtering
- DexiNed
- Dissolving
- Guided Blur
- Laplacian
- Gaussian
- Non-local means
- Sobel
- Unsharp masking | +| **Color** | - Color space conversions
- Brightness/contrast adjustment
- Gamma correction | +| **Stereo Vision** | - Disparity estimation
- Depth estimation
- Rectification | +| **Image Registration** | - Affine and homography-based registration
- Image alignment using feature matching | +| **Pose Estimation** | - Essential and Fundamental matrix estimation
- PnP problem solvers
- Pose refinement | +| **Optical Flow** | - Farneback optical flow
- Dense optical flow
- Sparse optical flow | +| **3D Vision** | - Depth estimation
- Point cloud operations
| +| **Image Denoising** | - Gaussian noise removal
- Poisson noise removal | +| **Edge Detection** | - Sobel operator
- Canny edge detection | | +| **Transformations** | - Rotation
- Translation
- Scaling
- Shearing | +| **Loss Functions** | - SSIM (Structural Similarity Index Measure)
- PSNR (Peak Signal-to-Noise Ratio)
- Cauchy
- Charbonnier
- Depth Smooth
- Dice
- Hausdorff
- Tversky
- Welsch
| | +| **Morphological Operations**| - Dilation
- Erosion
- Opening
- Closing | + +
+ +## Half-Precision Support + +| Module | float16 | bfloat16 | Notes | +|--------|:-------:|:--------:|-------| +| `kornia.color` | ⚠️ | ⚠️ | Most conversions work for both; FFT-based ops may fail | +| `kornia.filters` | ⚠️ | ⚠️ | Basic filters work; FFT-based ops may fail on CUDA | +| `kornia.enhance` | ⚠️ | ⚠️ | Histogram eq / gamma / ZCA work (linalg ops use cast helpers) | +| `kornia.morphology` | ✅ | ✅ | Pure conv/pool ops; no dtype restrictions | +| `kornia.augmentation` | ⚠️ | ⚠️ | Most ops work; precision-sensitive transforms may be inaccurate | +| `kornia.geometry.transform` | ⚠️ | ⚠️ | Affine/warp/resize work via cast helpers; thin-plate spline may fail | +| `kornia.geometry.camera` | ⚠️ | ⚠️ | Pinhole model and most camera ops work; `StereoCamera` accepts both | +| `kornia.geometry.calibration` | ❌ | ❌ | Explicitly accepts float32/float64 only (PnP solver) | +| `kornia.geometry.epipolar` | ⚠️ | ⚠️ | SVD/inverse use cast helpers; both dtypes work | +| `kornia.geometry.homography` | ⚠️ | ⚠️ | Uses `_torch_svd_cast` — both dtypes work via casting | +| `kornia.geometry.liegroup` | ⚠️ | ⚠️ | Most ops work via cast helpers; some linalg paths may fail | +| `kornia.geometry.solvers` | ⚠️ | ⚠️ | Uses `_torch_solve_cast` — both dtypes work via casting | +| `kornia.geometry.subpix` | ⚠️ | ⚠️ | Soft-argmax works; precision-sensitive ops may be inaccurate | +| `kornia.losses` | ⚠️ | ⚠️ | Photometric losses work; linalg-based losses may not | +| `kornia.feature` | ⚠️ | ⚠️ | Detectors/descriptors work; matching uses manual cdist fallback | +| `kornia.metrics` | ⚠️ | ⚠️ | Pixel-level metrics work; linalg-based metrics may not | +| `kornia.models` | ⚠️ | ⚠️ | Conv-based models work; attention-based models may have dtype mismatches | + +✅ Supported   ⚠️ Partial   ❌ Not supported + +**Test results** (commit `6131e98`, 2026-03-21): + +| Run | Passed | Failed | Skipped | Pass% | +|-----|-------:|-------:|--------:|------:| +| CPU float32 *(baseline)* | 7647 | 3 | 3269 | **99.9%** | +| CUDA float32 *(baseline)* | 7634 | 3 | 3280 | **99.9%** | +| CPU float16 | 6866 | 747 | 3306 | **90.1%** | +| CPU bfloat16 | 6838 | 812 | 3269 | **89.3%** | +| CUDA float16 *(KORNIA_TEST_IN_SUBPROCESS=1)* | 6727 | 643 | 3556 | **91.3%** | +| CUDA bfloat16 *(KORNIA_TEST_IN_SUBPROCESS=1)* | 6695 | 713 | 3518 | **90.4%** | + +See the [full precision guide](https://kornia.readthedocs.io/en/stable/get-started/precision.html) for details. + +## Sponsorship + +Kornia is an open-source project that is developed and maintained by volunteers. Whether you're using it for research or commercial purposes, consider sponsoring or collaborating with us. Your support will help ensure Kornia's growth and ongoing innovation. Reach out to us today and be a part of shaping the future of this exciting initiative! + + + + + +## Installation + +[![PyPI python](https://img.shields.io/pypi/pyversions/kornia)](https://pypi.org/project/kornia) +[![pytorch](https://img.shields.io/badge/PyTorch_2.0.0+-ee4c2c?logo=pytorch&logoColor=white)](https://pytorch.org/get-started/locally/) + +### From pip + + ```bash + pip install kornia + ``` + +
+ Other installation options + +#### From source with editable mode + + ```bash + pip install -e . + ``` + +#### For development with Pixi (Recommended) + +For development, Kornia uses [pixi](https://pixi.sh) for fast Python package management and environment management. The project includes a `pixi.toml` configuration file for reproducible dependency management. + + ```bash + # Install pixi (if not already installed) + curl -fsSL https://pixi.sh/install.sh | bash + + # Install dependencies and set up the development environment + pixi install + + # Run tests + pixi run test + + # For CUDA development + pixi run -e cuda install + pixi run -e cuda test-cuda + ``` + +This will set up a complete development environment with all dependencies. For more details on dependency management and available tasks, see [CONTRIBUTING.md](CONTRIBUTING.md). + +#### From Github url (latest version) + + ```bash + pip install git+https://github.com/kornia/kornia + ``` + +
+ +## Quick Start + +Kornia is not just another computer vision library — it's your gateway to effortless Computer Vision and AI. + +
+Get started with Kornia image transformation and augmentation! + +```python +import numpy as np +import kornia_rs as kr + +from kornia.augmentation import AugmentationSequential, RandomAffine, RandomBrightness +from kornia.filters import StableDiffusionDissolving + +# Load and prepare your image +img: np.ndarray = kr.read_image_any("img.jpeg") +img = kr.resize(img, (256, 256), interpolation="bilinear") + +# alternatively, load image with PIL +# img = Image.open("img.jpeg").resize((256, 256)) +# img = np.array(img) + +img = np.stack([img] * 2) # batch images + +# Define an augmentation pipeline +augmentation_pipeline = AugmentationSequential( + RandomAffine((-45., 45.), p=1.), + RandomBrightness((0.,1.), p=1.) +) + +# Leveraging StableDiffusion models +dslv_op = StableDiffusionDissolving() + +img = augmentation_pipeline(img) +dslv_op(img, step_number=500) + +dslv_op.save("Kornia-enhanced.jpg") +``` + +
+ +
+Find out Kornia ONNX models with ONNXSequential! + +```python +import numpy as np +from kornia.onnx import ONNXSequential +# Chain ONNX models from HuggingFace repo and your own local model together +onnx_seq = ONNXSequential( + "hf://operators/kornia.geometry.transform.flips.Hflip", + "hf://models/kornia.models.detection.rtdetr_r18vd_640x640", # Or you may use "YOUR_OWN_MODEL.onnx" +) +# Prepare some input data +input_data = np.random.randn(1, 3, 384, 512).astype(np.float32) +# Perform inference +outputs = onnx_seq(input_data) +# Print the model outputs +print(outputs) + +# Export a new ONNX model that chains up all three models together! +onnx_seq.export("chained_model.onnx") +``` +
+ +## Multi-framework support + +You can now use Kornia with [TensorFlow](https://www.tensorflow.org/), [JAX](https://jax.readthedocs.io/en/latest/index.html), and [NumPy](https://numpy.org/). See [Multi-Framework Support](docs/source/get-started/multi-framework-support.rst) for more details. + +```python +import kornia +tf_kornia = kornia.to_tensorflow() +``` + +

+ Powered by + +

+ +

+ +## Call For Contributors + +Are you passionate about computer vision, AI, and open-source development? Join us in shaping the future of Kornia! We are actively seeking contributors to help expand and enhance our library, making it even more powerful, accessible, and versatile. Whether you're an experienced developer or just starting, there's a place for you in our community. + +### Accessible AI Models + +We are excited to announce our latest advancement: a new initiative designed to seamlessly integrate lightweight AI models into Kornia. +We aim to run any models as smooth as big models such as StableDiffusion, to support them well in many perspectives. + +**Priority Focus: VLM/VLA Models** + +Our primary focus is on integrating **Vision Language Models (VLM)** and **Vision Language Agents (VLA)** to enable end-to-end vision solutions. We're actively seeking contributors to help us: + +- **VLM/VLA Integration (Priority)**: Implement and integrate state-of-the-art Vision Language Models and Vision Language Agents. This includes models like Qwen2.5-VL, SAM-3, and other cutting-edge VLM/VLA architectures. If you are a researcher working on VLM/VLA models, Kornia is an excellent place for you to promote your model! +- Expand the Model Selection: Import decent models into our library. If you are a researcher, Kornia is an excellent place for you to promote your model! +- Model Optimization: Work on optimizing models to reduce their computational footprint while maintaining accuracy and performance. You may start from offering ONNX support! +- Model Documentation: Create detailed guides and examples to help users get the most out of these models in their projects. + +### Documentation And Tutorial Optimization + +Kornia's foundation lies in its extensive collection of classic computer vision operators, providing robust tools for image processing, feature extraction, and geometric transformations. We continuously seek for contributors to help us improve our documentation and present nice tutorials to our users. + + +## Cite + +If you are using kornia in your research-related documents, it is recommended that you cite the paper. See more in [CITATION](./CITATION.md). + + ```bibtex + @inproceedings{eriba2019kornia, + author = {E. Riba, D. Mishkin, D. Ponsa, E. Rublee and G. Bradski}, + title = {Kornia: an Open Source Differentiable Computer Vision Library for PyTorch}, + booktitle = {Winter Conference on Applications of Computer Vision}, + year = {2020}, + url = {https://arxiv.org/pdf/1910.02190.pdf} + } + ``` + +## Contributing + +We appreciate all contributions. If you are planning to contribute back bug-fixes, please do so without any further discussion. If you plan to contribute new features, utility functions or extensions, please first open an issue and discuss the feature with us. Please, consider reading the [CONTRIBUTING](./CONTRIBUTING.md) notes. The participation in this open source project is subject to [Code of Conduct](./CODE_OF_CONDUCT.md). + +### AI Policy + +Kornia accepts AI-assisted code but strictly rejects AI-generated contributions where the submitter acts as a proxy. All contributors must be the **Sole Responsible Author** for every line of code. Please review our [AI Policy](AI_POLICY.md) before submitting pull requests. Key requirements include: + +- **Proof of Verification**: PRs must include local test logs proving execution +- **Pre-Discussion**: All PRs must be discussed in Discord or via a GitHub issue before implementation +- **Library References**: Implementations must be based on existing library references (PyTorch, OpenCV, etc.) +- **Use Existing Utilities**: Use existing `kornia` utilities instead of reinventing the wheel +- **Explain It**: You must be able to explain any code you submit + +Automated AI reviewers (e.g., GitHub Copilot) will check PRs against these policies. See [AI_POLICY.md](AI_POLICY.md) for complete details. + +## Community +- **Discord:** Join our workspace to keep in touch with our core contributors, get latest updates on the industry and be part of our community. [JOIN HERE](https://discord.gg/HfnywwpBnD) +- **GitHub Issues:** bug reports, feature requests, install issues, RFCs, thoughts, etc. [OPEN](https://github.com/kornia/kornia/issues/new/choose) +- **Forums:** discuss implementations, research, etc. [GitHub Forums](https://github.com/kornia/kornia/discussions) + + + + + +Made with [contrib.rocks](https://contrib.rocks). + +## License + +Kornia is released under the Apache 2.0 license. See the [LICENSE](./LICENSE) file for more information. diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..5178dda --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目:`kornia/kornia` +- 原始仓库:https://github.com/kornia/kornia +- 导入方式:上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源,不代表 WeHub 是原项目作者 diff --git a/README_zh-CN.md b/README_zh-CN.md new file mode 100644 index 0000000..dae7798 --- /dev/null +++ b/README_zh-CN.md @@ -0,0 +1,174 @@ +
+

+ +

+ +**The open-source and Computer Vision 2.0 library** + +--- + +[English](README.md) | 简体中文 + + +Docs • +Try it Now • +Tutorials • +Examples • +Blog • +Community + +[![PyPI version](https://badge.fury.io/py/kornia.svg)](https://pypi.org/project/kornia) +[![Downloads](https://static.pepy.tech/badge/kornia)](https://pepy.tech/project/kornia) +[![star](https://gitcode.com/kornia/kornia/star/badge.svg)](https://gitcode.com/kornia/kornia) +[![Discord](https://img.shields.io/badge/Discord-5865F2?logo=discord&logoColor=white)](https://discord.gg/HfnywwpBnD) +[![Twitter](https://img.shields.io/twitter/follow/kornia_foss?style=social)](https://twitter.com/kornia_foss) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) + +

+
+ +*Kornia* 是一款基于 [PyTorch](https://pytorch.org) 的可微分的计算机视觉库。 + +它由一组用于解决通用计算机视觉问题的操作模块和可微分模块组成。其核心使用 *PyTorch* 作为主要后端,以提高效率并利用反向模式自动微分来定义和计算复杂函数的梯度。 + +
+ +
+ + + +## 概览 + +受现有开源库的启发,Kornia可以由包含各种可以嵌入神经网络的操作符组成,并可以训练模型来执行图像变换、对极几何、深度估计和低级图像处理,例如过滤和边缘检测。此外,整个库都可以直接对张量进行操作。 + +详细来说,Kornia 是一个包含以下组件的库: + +| **Component** | **Description** | +|----------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------| +| [kornia](https://kornia.readthedocs.io/en/latest/index.html) | 具有强大 GPU 支持的可微计算机视觉库 | +| [kornia.augmentation](https://kornia.readthedocs.io/en/latest/augmentation.html) | 在 GPU 中执行数据增强的模块 | +| [kornia.color](https://kornia.readthedocs.io/en/latest/color.html) | 执行色彩空间转换的模块 | +| [kornia.contrib](https://kornia.readthedocs.io/en/latest/contrib.html) | 未进入稳定版本的实验性模块 | +| [kornia.enhance](https://kornia.readthedocs.io/en/latest/enhance.html) | 执行归一化和像素强度变换的模块 | +| [kornia.feature](https://kornia.readthedocs.io/en/latest/feature.html) | 执行特征检测的模块 | +| [kornia.filters](https://kornia.readthedocs.io/en/latest/filters.html) | 执行图像滤波和边缘检测的模块 | +| [kornia.geometry](https://kornia.readthedocs.io/en/latest/geometry.html) | 执行几何计算的模块,用于使用不同的相机模型执行图像变换、3D线性代数和转换 | +| [kornia.losses](https://kornia.readthedocs.io/en/latest/losses.html) | 损失函数模块 | +| [kornia.morphology](https://kornia.readthedocs.io/en/latest/morphology.html) | 执行形态学操作的模块 | +| [kornia.utils](https://kornia.readthedocs.io/en/latest/utils.html) | 图像/张量常用工具以及metrics | + +## 支持我们 + + + + + +## 安装说明 + +### 通过 pip 安装: + + ```bash + pip install kornia + ``` + +
+ 其他安装方法 + + #### 通过源码安装(软链接至当前路径): + + ```bash + pip install -e . + ``` + + #### 使用 Pixi 进行开发(推荐) + + 对于开发,Kornia 使用 [pixi](https://pixi.sh) 进行快速的 Python 包管理和环境管理。项目包含一个 `pixi.toml` 配置文件用于可重现的依赖管理。 + + ```bash + # 安装 pixi(如果尚未安装) + curl -fsSL https://pixi.sh/install.sh | bash + + # 安装依赖并设置开发环境 + pixi install + + # 运行测试 + pixi run test + + # 用于 CUDA 开发 + pixi run -e cuda install + pixi run -e cuda test-cuda + ``` + + 这将设置一个包含所有依赖的完整开发环境。有关依赖管理和可用任务的更多详细信息,请参阅 [CONTRIBUTING.md](CONTRIBUTING.md)。 + + #### 通过源码安装(从GIT自动下载最新代码): + + ```bash + pip install git+https://github.com/kornia/kornia + ``` +
+ + +## 例子 + +可以尝试通过这些 [教程](https://kornia.github.io/tutorials/) 来学习和使用这个库。 + + + +:triangular_flag_on_post: **Updates** +- :white_check_mark: 现已通过 [Gradio](https://github.com/gradio-app/gradio) 将Kornia集成进 [Huggingface Spaces](https://huggingface.co/spaces). 可以尝试 [Gradio 在线Demo](https://huggingface.co/spaces/akhaliq/Kornia-LoFTR). + +## 引用 + +如果您在与研究相关的文档中使用 Kornia,您可以引用我们的论文。更多信息可以在 [CITATION](https://github.com/kornia/kornia/blob/main/CITATION.md) 看到。 + + ```bibtex + @inproceedings{eriba2019kornia, + author = {E. Riba, D. Mishkin, D. Ponsa, E. Rublee and G. Bradski}, + title = {Kornia: an Open Source Differentiable Computer Vision Library for PyTorch}, + booktitle = {Winter Conference on Applications of Computer Vision}, + year = {2020}, + url = {https://arxiv.org/pdf/1910.02190.pdf} + } + ``` + +## 贡献 +我们感谢所有的贡献者为改进和提升 Kornia 所作出的努力。您可以直接修复一个已知的BUG而无需进一步讨论;如果您想要添加一个任何新的或者扩展功能,请务必先通过提交一个Issue来与我们讨论。详情请阅读 [贡献指南](https://github.com/kornia/kornia/blob/main/CONTRIBUTING.md)。开源项目的参与者请务必了解如下 [规范](https://github.com/kornia/kornia/blob/main/CODE_OF_CONDUCT.md)。 + +### AI 政策 + +Kornia 接受 AI 辅助的代码,但严格拒绝提交者仅作为代理的 AI 生成贡献。所有贡献者必须是每一行代码的**唯一责任作者**。在提交 pull request 之前,请查看我们的 [AI 政策](AI_POLICY.md)。主要要求包括: + +- **验证证据**:PR 必须包含本地测试日志以证明代码已执行 +- **事前讨论**:所有 PR 在实施前必须在 Discord 或通过 GitHub issue 进行讨论 +- **库引用**:实现必须基于现有库引用(PyTorch、OpenCV 等) +- **使用现有工具**:使用现有的 `kornia` 工具,而不是重新发明轮子 +- **解释能力**:您必须能够解释您提交的任何代码 + +自动化 AI 审查工具(例如 GitHub Copilot)将根据这些政策检查 PR。完整详情请参阅 [AI_POLICY.md](AI_POLICY.md)。 + +## 社区 +- **论坛:** 讨论代码实现,学术研究等。[GitHub Forums](https://github.com/kornia/kornia/discussions) +- **GitHub Issues:** bug reports, feature requests, install issues, RFCs, thoughts, etc. [OPEN](https://github.com/kornia/kornia/issues/new/choose) +- **Slack:** 加入我们的Slack社区,与我们的核心贡献者保持联系。 [JOIN HERE](https://join.slack.com/t/kornia/shared_invite/zt-csobk21g-2AQRi~X9Uu6PLMuUZdvfjA) +- 常见信息请访问我们的网站 www.kornia.org + +## 中文社区 +扫描下方的二维码可关注 Kornia 的官方交流QQ群(679683070)以及Kornia知乎账号。 + +
+ + +
+ +我们会在 Kornia 交流社区为大家 + +- 📢 更新 Kornia 的最新动态 +- 📘 进行更高效的答疑解惑以及意见反馈 +- 💻 提供与行业大牛的充分交流的平台 diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000..bff0a5a --- /dev/null +++ b/TESTING.md @@ -0,0 +1,278 @@ +# Testing Guide + +This document explains how to run the Kornia test suite, what the common pytest flags are, and how to handle recurring classes of test failures. + +## Running Tests + +Kornia uses [pixi](https://pixi.sh) to manage the test environment. + +```bash +# All tests, default device (cpu) and dtype (float32) +pixi run test + +# Specific file +pixi run test tests/geometry/test_boxes.py + +# Specific device and dtype +pixi run test tests/ --device=cuda --dtype=float32 + +# All devices and dtypes +pixi run test tests/ --device=all --dtype=all + +# Skip slow tests (default); include them with --runslow +pixi run test tests/ --runslow + +# Quick tests (excludes jit, grad, nn markers) +pixi run test-quick +``` + +### CLI Options + +| Option | Env var | Default | Description | +|---|---|---|---| +| `--device` | `KORNIA_TEST_DEVICE` | `cpu` | `cpu`, `cuda`, `mps`, `tpu`, or `all` | +| `--dtype` | `KORNIA_TEST_DTYPE` | `float32` | `float32`, `float64`, `float16`, `bfloat16`, or `all` | +| `--runslow` | `KORNIA_TEST_RUNSLOW` | off | Include `@pytest.mark.slow` tests | +| `--tf32` | `KORNIA_TEST_TF32` | off | Enable TF32 mode (see below) | +| `--optimizer` | `KORNIA_TEST_OPTIMIZER` | `inductor` | `torch.compile` backend for dynamo tests | +| `--isolate-half-precision` | `KORNIA_TEST_ISOLATE_HALF` | off | Run float16/bfloat16 CUDA tests each in a fresh `subprocess.run` process (no shared CUDA state) | + +## Test Structure + +All tests inherit from `testing.base.BaseTester` and implement a standard set of methods: + +```python +from testing.base import BaseTester + +class TestMyFunction(BaseTester): + def test_smoke(self, device, dtype): ... # basic run, all arg combinations + def test_exception(self, device, dtype): ... # error paths + def test_cardinality(self, device, dtype): ... # output shapes + def test_feature(self, device, dtype): ... # correctness / numerical accuracy + def test_gradcheck(self, device): ... # gradient checking via self.gradcheck() + def test_dynamo(self, device, dtype, torch_optimizer): ... # torch.compile compat +``` + +The `device` and `dtype` fixtures are injected automatically from the CLI options. Use `self.assert_close()` for tensor comparisons — it automatically selects tolerances appropriate for the dtype. + +## Markers + +| Marker | Meaning | +|---|---| +| `@pytest.mark.slow` | Long-running test; skipped unless `--runslow` is passed | +| `@pytest.mark.grad` | Gradient-check test | +| `@pytest.mark.jit` | TorchScript test | +| `@pytest.mark.nn` | Module-level test | +| `@pytest.mark.tf32` | Known to fail under TF32 (see section below); xfail unless `--tf32` | + +--- + +## Known Sources of Test Failures + +### 1. TF32 (TensorFloat-32) Precision + +**What it is.** `torch.set_float32_matmul_precision("high")` enables TF32 mode for CUDA matrix multiplications (`torch.bmm`, `torch.mm`, etc.). TF32 truncates float32 inputs to a 10-bit mantissa before the multiply-accumulate, giving roughly float16 mantissa precision for those ops. This is the default when `torch.compile` is in use and is enabled by many deep-learning frameworks for throughput. + +**Effect on tests.** Numerically sensitive tests that compare float32 outputs of matrix operations against hardcoded expected values (or against a CPU reference) can fail because the accumulated rounding error exceeds the test tolerance. + +**In Kornia's test suite.** TF32 is **off by default**. Pass `--tf32` (or set `KORNIA_TEST_TF32=true`) to enable it. Tests that are known to be sensitive to TF32 are marked `@pytest.mark.tf32`; without `--tf32` they are marked `xfail` so the suite stays green. + +**Fixing a TF32-sensitive test.** Prefer fixing the test data over relaxing tolerances: + +- Use integer-valued coordinates — all integers up to 2047 are exactly representable in TF32 (10-bit mantissa, so $n < 2^{10}$; integers up to 2047 via powers-of-two representation). +- Restrict inputs to ranges that keep intermediate values well within the TF32-exact region (e.g. pixel coordinates within image bounds rather than ±1500). +- For camera/geometry tests, avoid near-zero depth values and use realistic intrinsic matrices (e.g. `fx=500, cx=256`) rather than fully random ones. +- Only relax `atol`/`rtol` **as a last resort**, and only when the new tolerance is still below 0.01. + +**Example: 3D box transforms.** Float coordinates like `z=283.162` fall in the TF32 range [256, 512) where the representable step is 0.25. Rounding `283.162 → 283.25` then computing `2×283.25+1 = 567.5` instead of the expected `567.324` gives an error of 0.176 — which fails with `atol=1e-4` but is not meaningful. The fix is to use integer coordinates (`z=284`) which are TF32-exact. + +--- + +### 2. Device-Dependent PRNG + +**What it is.** `torch.rand(..., device='cuda')` uses a different random number generator (Philox) than the CPU (Mersenne Twister). Even with the same seed set by `torch.manual_seed(n)`, the two devices produce **different sequences**. + +**Effect on tests.** Tests that: +1. Generate random tensors directly on a non-CPU device, AND +2. Compare against hardcoded expected values computed on CPU + +will fail on CUDA/MPS even though the code is correct. + +**In Kornia's test suite.** This affects augmentation tests with seeded expected values (`TestRandomRGBShift`, `TestRandomMixUpGen`), color roundtrip tests (`TestLuvToRgb`), and any test that runs `torch.rand(..., device=device)` without a device-independent strategy. + +**Fixes (in order of preference):** + +1. **Generate on CPU, move to device.** This is fully device-agnostic: + ```python + torch.manual_seed(42) + data = torch.rand(3, 4, 5).to(device=device, dtype=dtype) + ``` + +2. **Skip for non-CPU when checking specific values.** Follow the existing pattern used in augmentation generator tests: + ```python + if device.type != "cpu": + pytest.skip("Random number sequences differ between CPU and non-CPU devices") + ``` + +3. **Test properties instead of exact values** (e.g., check that output is in [0, 1] rather than matching a specific tensor). + +**Note.** `torch.manual_seed` seeds all devices in PyTorch ≥ 1.8, but the sequences still differ per device because the underlying algorithms differ. + +--- + +### 3. CUDA Non-Determinism in Backward + +**What it is.** Some CUDA kernels use `atomicAdd` for scatter and reduction operations. The order of floating-point additions is non-deterministic across runs, producing slightly different gradient values each time backward is called. + +**Effect on tests.** `torch.autograd.gradcheck` calls backward twice with the same inputs and checks that the results are bit-identical (`nondet_tol=0.0` by default). If the op uses atomics, gradcheck raises `GradcheckError: Backward is not reentrant`. + +**Affected operations.** Histogram-based orientation estimators (`LAFOrienter`), ALIKED backbone (scatter/pool ops), and any custom op that uses `torch.scatter_add` or `atomicAdd` on CUDA. + +**Fix.** Pass `nondet_tol` to gradcheck: +```python +# In a test using self.gradcheck(): +self.gradcheck(fn, inputs, rtol=1e-3, atol=1e-3, nondet_tol=1e-3) + +# In a test calling torch.autograd.gradcheck() directly: +torch.autograd.gradcheck(fn, inputs, eps=1e-4, atol=1e-3, rtol=1e-3, + fast_mode=True, nondet_tol=1e-3) +``` + +A value of `1e-3` is usually sufficient; it should not exceed `atol`. + +--- + +### 4. Test-Order Dependencies (Full Suite vs. Isolation) + +**What it is.** A test can pass in isolation but fail when preceded by other tests, because some global state was mutated by an earlier test. + +**Common sources:** +- **CUDA RNG state**: unseeded `torch.rand(..., device='cuda')` draws from the CUDA RNG, whose state depends on all prior CUDA random operations in the process. +- **`torch.set_float32_matmul_precision`**: if any test (or the conftest warmup) sets this to `"high"`, all subsequent CUDA matmuls use TF32. +- **`torch.use_deterministic_algorithms`**: a test enabling deterministic mode affects all later tests. +- **Model caches / lazy initialisation**: some feature extractors load weights on first call and cache them globally. + +**Diagnosis.** If a test fails in the full suite but passes in isolation, run it with `--randomly-seed=last` (if `pytest-randomly` is installed) to reproduce the ordering, or prefix the failing test with the suspected culprit and check if the failure disappears. + +**Fix.** Always seed RNG state explicitly in tests that compare against reference values, and prefer generating random data on CPU: +```python +torch.manual_seed(0) +data = torch.rand(B, C, H, W).to(device=device, dtype=dtype) +``` + +--- + +### 5. Float32 Numerical Precision in Geometry/Camera Tests + +**What it is.** Operations like camera projection, LuV color conversion, and homography estimation involve divisions and non-linear functions. In float32, the roundtrip error can be significant for extreme inputs. + +**Common anti-patterns (and fixes):** + +| Anti-pattern | Problem | Fix | +|---|---|---| +| Depth in `[-500, 500]` | Near-zero depth → `1/z` blow-up | Restrict depth to `[1, 500]` | +| Pixel coords in `[-1500, 1500]` | Far outside image; large TF32 rounding | Use `[0, W) × [0, H)` | +| Fully random `K` matrix | Unrealistic intrinsics | Use `fx=500, cx=256` or similar | +| Fully random rotation matrix | May not be a valid rotation | Use `axis_angle_to_rotation_matrix` | +| `torch.rand(B, N, 2)` for homography points | Random degenerate configs | Use `create_random_homography` from `testing.geometry.create` | + +--- + +### 6. SVD Numerical Stability (float32 on CUDA) + +**What it is.** `torch.linalg.svd` on float32 CUDA tensors can produce inaccurate singular values for ill-conditioned matrices. This affects anything that uses `_torch_svd_cast` internally: stereo camera reprojection, fundamental/essential matrix estimation, etc. + +**Fix (implemented in kornia).** `kornia.core.utils._torch_svd_cast` automatically promotes float32 inputs to float64 before SVD (except on MPS where float64 is unsupported), then casts the result back. This matches the existing behaviour of `_torch_solve_cast`. + +If you write a new function that calls SVD, use `_torch_svd_cast` rather than calling `torch.linalg.svd` directly. + +--- + +### 7. Half-Precision dtypes (float16 / bfloat16) + +**What it is.** float16 and bfloat16 have limited support across PyTorch and kornia: + +- **bfloat16**: Many kornia functions explicitly reject it. In addition, many CUDA kernels lack bfloat16 implementations (`svd_cuda`, `linalg_eigh_cuda`, `cdist_cuda`, `lu_factor_cublas`, `geqrf_cuda`, etc.). +- **float16**: PyTorch's `linalg` routines (`linalg.inv`, `linalg.eigh`, `linalg.svd`, …) do not accept float16 on CPU (`RuntimeError: Low precision dtypes not supported`). On CUDA, many kernels trigger device-side asserts for float16 inputs. + +**Testing strategy: isolated runs.** Half-precision tests live alongside their float32/float64 counterparts in the same directories and files. They are **not** run in combined (`--dtype=all`) invocations on CUDA; instead, half-precision and standard-precision suites are run as separate, isolated pytest invocations: + +```bash +# Standard CI — all devices, float32 and float64 only +pixi run test tests/ --dtype=float32,float64 + +# Half-precision — run separately, per directory or file +pytest tests/color/ --dtype=float16,bfloat16 +pytest tests/geometry/ --dtype=float16,bfloat16 --device=cuda +``` + +Keeping the runs separate means a half-precision failure or CUDA context corruption cannot affect the float32/float64 results. + +**CUDA device-side asserts and test contamination.** CUDA kernel errors are *asynchronous*: a failing kernel logs the error but continues execution until the next host–device synchronisation point. If that sync happens inside a *different* (passing) test, that test fails spuriously. Once a device-side assert fires, the CUDA context is permanently broken for the process lifetime. + +The root `conftest.py` contains two autouse fixtures to handle this: + +- **`skip_half_precision_on_cuda`** — *skips* float16 and bfloat16 tests on CUDA when tests are run in combined mode. Skipping means no CUDA kernel is launched, so no assert can be triggered. On CPU/MPS/TPU, tests run as normal (they may fail). + +- **`cuda_device_assert_guard`** — synchronises the CUDA device *before* each CUDA test. If the context is already corrupted by a previous test, the current test is *skipped* rather than allowed to fail spuriously. After each CUDA test, a second synchronisation drains the queue so that any async error surfaces in teardown of the test that caused it, not at the start of the next one. + +**Running half-precision tests across a whole directory.** Use `--isolate-half-precision`. Each float16/bfloat16 CUDA test is run in a completely fresh Python process via `subprocess.run`, so a device-side assert in one test cannot affect any other test — there is no shared CUDA state at all: + +```bash +# Whole directory, fully isolated — results reported normally (pass/fail per test) +pytest tests/color/ --device=cuda --dtype=bfloat16 --isolate-half-precision +pytest tests/geometry/ --device=cuda --dtype=all --isolate-half-precision + +# Via pixi tasks +pixi run test-half # float16 + bfloat16, CPU +pixi run test-cuda-half # float16 + bfloat16, CUDA, with isolation +``` + +Without `--isolate-half-precision`, float16/bfloat16 CUDA tests are **skipped** (safe default for combined runs). + +**See also.** `docs/source/get-started/precision.rst` for the per-module half-precision support table. + +### 8. MPS (Apple Silicon) Limitations + +**What MPS is.** The `mps` device uses Apple's Metal Performance Shaders backend. Run tests against it with `--device=mps`. + +**Known unsupported operations.** Several operations are not implemented in the MPS backend and raise a `RuntimeError` at runtime: + +| Operation | Error | +|---|---| +| `float64` (double precision) | `TypeError: Cannot convert a MPS Tensor to float64 dtype` | +| `complex128` (cdouble) | `NotImplementedError: … not implemented for 'ComplexDouble'` | +| `F.grid_sample` with `padding_mode="border"` on 2D | `RuntimeError: MPS: Unsupported Border padding mode` | +| `F.grid_sample` with `mode="nearest"` on 5-D (3D volumes) | `RuntimeError: grid_sampler_3d: Unsupported Nearest interpolation` | +| `torch.autocast("mps")` | Converts output to float16 instead of preserving original dtype | + +**How Kornia handles these automatically.** The test infrastructure in `conftest.py` and `testing/base.py` skips known-unsupported test classes at collection time so you don't need per-test guards for the common cases: + +- `test_gradcheck[mps*]` — skipped automatically (`gradcheck` requires float64) +- `*[mps*cdtype1*]` — skipped automatically (parametrized `torch.cdouble` tests) +- `test_autocast[mps*]` — skipped automatically (MPS autocast changes dtype) + +The `padding_mode="border"` issue in `F.grid_sample` (2D) is worked around in the implementation (`kornia/feature/laf.py`) by clamping the sampling grid to `[-1, 1]` and using `padding_mode="zeros"`, which is mathematically equivalent. + +**Writing new tests that work on MPS.** Follow these rules: + +1. **Never create `float64` tensors on the device in non-gradcheck tests.** Use the `dtype` fixture, or if the algorithm requires double precision, skip explicitly: + ```python + if device.type == "mps": + pytest.skip("MPS does not support float64") + ``` +2. **`self.gradcheck()` is safe.** `BaseTester.gradcheck()` automatically skips on MPS devices. +3. **Avoid `padding_mode="border"` in 2D `grid_sample` on MPS.** Use the clamped-zeros workaround or check `device.type == "mps"`. +4. **3D `grid_sample` with `mode="nearest"` is unsupported on MPS.** Do not silently fall back to bilinear (it would corrupt segmentation masks); instead raise or skip the test. +5. **Device comparison.** `tensor.device` on MPS returns `device(type='mps', index=0)`, not `device(type='mps')`. The test fixture provides `torch.device("mps:0")` so `tensor.device == device` comparisons work correctly. + +--- + +## Writing Robust Tests + +- **Seed the RNG** when the test compares against reference values: `torch.manual_seed(seed)`. +- **Generate random inputs on CPU** then move to device, to avoid device-specific RNG sequences. +- **Use realistic inputs**: positive depths, in-bounds pixel coordinates, valid rotation matrices, and sensible intrinsic matrices. +- **Avoid hardcoding CUDA expected values** computed from CPU runs — they will differ. +- **Use `nondet_tol`** in `gradcheck` for ops that use CUDA atomics. +- **Check the error magnitude** before relaxing tolerances: only relax `atol`/`rtol` if the maximum observed error is well below 0.01, and document why. diff --git a/benchmarks/augmentation/2d_geometric_test.py b/benchmarks/augmentation/2d_geometric_test.py new file mode 100644 index 0000000..db78a4e --- /dev/null +++ b/benchmarks/augmentation/2d_geometric_test.py @@ -0,0 +1,204 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch + +from kornia.augmentation import ( + CenterCrop, + PadTo, + RandomAffine, + RandomCrop, + RandomElasticTransform, + RandomErasing, + RandomFisheye, + RandomHorizontalFlip, + RandomPerspective, + RandomResizedCrop, + RandomRotation, + RandomShear, + RandomThinPlateSpline, + RandomTranslate, + RandomVerticalFlip, + Resize, +) + + +def test_aug_2d_centercrop(benchmark, device, dtype, torch_optimizer, shape): + w_target = h_target = 64 + data = torch.rand(*shape, device=device, dtype=dtype) + aug = CenterCrop((h_target, w_target), p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == (*shape[:-2], h_target, w_target) + + +def test_aug_2d_padto(benchmark, device, dtype, torch_optimizer, shape): + w_target = h_target = 256 + data = torch.rand(*shape, device=device, dtype=dtype) + aug = PadTo((h_target, w_target)) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == (*shape[:-2], h_target, w_target) + + +def test_aug_2d_affine(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomAffine(degrees=45, translate=0.25, scale=(0.9, 1.1), shear=1.25, p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_crop(benchmark, device, dtype, torch_optimizer, shape): + w_target = h_target = 64 + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomCrop((h_target, w_target), pad_if_needed=True, p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == (*shape[:-2], h_target, w_target) + + +def test_aug_2d_elastic_transform(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomElasticTransform(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_erasing(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomErasing(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_fisheye(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + center_x = center_y = torch.tensor([-0.3, 0.3], device=device, dtype=dtype) + gamma = torch.tensor([0.9, 1.0], device=device, dtype=dtype) + aug = RandomFisheye(center_x, center_y, gamma, p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_horizontal_flip(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomHorizontalFlip(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_perspective(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomPerspective(0.5, p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_resized_crop(benchmark, device, dtype, torch_optimizer, shape): + w_target = h_target = 64 + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomResizedCrop((h_target, w_target), p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == (*shape[:-2], h_target, w_target) + + +def test_aug_2d_rotation(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomRotation(45, p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_shear(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomShear((-5.0, 2.0, 5.0, 10.0), p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_thin_plate_spline(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomThinPlateSpline(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_translate(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomTranslate((-0.2, 0.2), (-0.1, 0.1), p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_vertical_flip(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomVerticalFlip(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_resize(benchmark, device, dtype, torch_optimizer, shape): + w_target = h_target = 64 + data = torch.rand(*shape, device=device, dtype=dtype) + aug = Resize((h_target, w_target), p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == (*shape[:-2], h_target, w_target) diff --git a/benchmarks/augmentation/2d_intensity_test.py b/benchmarks/augmentation/2d_intensity_test.py new file mode 100644 index 0000000..48151f2 --- /dev/null +++ b/benchmarks/augmentation/2d_intensity_test.py @@ -0,0 +1,435 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.augmentation import ( + ColorJiggle, + ColorJitter, + Denormalize, + Normalize, + RandomAutoContrast, + RandomBoxBlur, + RandomBrightness, + RandomChannelDropout, + RandomChannelShuffle, + RandomClahe, + RandomContrast, + RandomEqualize, + RandomGamma, + RandomGaussianBlur, + RandomGaussianIllumination, + RandomGaussianNoise, + RandomGrayscale, + RandomHue, + RandomInvert, + RandomLinearCornerIllumination, + RandomLinearIllumination, + RandomMedianBlur, + RandomMotionBlur, + RandomPlanckianJitter, + RandomPlasmaBrightness, + RandomPlasmaContrast, + RandomPlasmaShadow, + RandomPosterize, + RandomRain, + RandomRGBShift, + RandomSaltAndPepperNoise, + RandomSaturation, + RandomSharpness, + RandomSnow, + RandomSolarize, +) + + +def test_aug_2d_collor_jiggle(benchmark, device, dtype, torch_optimizer, shape): + if shape[1] != 3: + pytest.skip("Skipping because input should be rgb") + data = torch.rand(*shape, device=device, dtype=dtype) + aug = ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_collor_jitter(benchmark, device, dtype, torch_optimizer, shape): + if shape[1] != 3: + pytest.skip("Skipping because input should be rgb") + data = torch.rand(*shape, device=device, dtype=dtype) + aug = ColorJitter(0.1, 0.1, 0.1, 0.1, p=1.0) + op = torch_optimizer(aug) + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_denormalize(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = Denormalize(0.0, 1.0, p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_auto_contrast(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomAutoContrast(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_box_blur(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomBoxBlur(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_brightness(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomBrightness((0.1, 1), p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_channel_dropout(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomChannelDropout(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_channel_shuffle(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomChannelShuffle(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_clahe(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomClahe((10, 40), p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_contrast(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomContrast((0.1, 1), p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_equalize(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomEqualize(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_gamma(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomGamma((0.0, 1.0), (0.0, 1.0), p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_gaussian_blur(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomGaussianBlur(3, (1.6, 1.7), p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_gaussian_illumination(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomGaussianIllumination(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_gaussian_noise(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomGaussianNoise(0.0, 1.0, p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_grayscale(benchmark, device, dtype, torch_optimizer, shape): + if shape[1] != 3: + pytest.skip("Skipping because input should be rgb") + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomGrayscale(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_hue(benchmark, device, dtype, torch_optimizer, shape): + if shape[1] != 3: + pytest.skip("Skipping because input should be rgb") + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomHue((0.0, 0.5), p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_invert(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomInvert(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +# TODO(joao.amorim): figure out dynamo issue with it +# def test_aug_2d_jpeg(benchmark, device, dtype, torch_optimizer, shape): +# data = torch.rand(*shape, device=device, dtype=dtype) +# aug = RandomJPEG(p=1.0) +# op = torch_optimizer(aug) +# +# actual = benchmark(op, input=data) +# +# assert actual.shape == shape + + +def test_aug_2d_linear_corner_illumination(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomLinearCornerIllumination(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_linear_illumination(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomLinearIllumination(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_median_blur(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomMedianBlur(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_motion_blur(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomMotionBlur((3, 3), 45.0, 5.5, p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_normalize(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = Normalize(25.0, 2.5, p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_plackian_jitter(benchmark, device, dtype, torch_optimizer, shape): + if shape[1] != 3: + pytest.skip("Skipping because input should be rgb") + + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomPlanckianJitter(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_plasma_briggtness(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomPlasmaBrightness(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_plasma_contrast(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomPlasmaContrast(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_plasma_shadow(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomPlasmaShadow(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_posterize(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomPosterize(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_rain(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomRain(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_rgb_shift(benchmark, device, dtype, torch_optimizer, shape): + if shape[1] != 3: + pytest.skip("Skipping because input should be rgb") + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomRGBShift(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_salt_and_peper_noise(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomSaltAndPepperNoise(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_saturation(benchmark, device, dtype, torch_optimizer, shape): + if shape[1] != 3: + pytest.skip("Skipping because input should be rgb") + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomSaturation((0.0, 2.0), p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_sharpness(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomSharpness(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_snow(benchmark, device, dtype, torch_optimizer, shape): + if shape[1] != 3: + pytest.skip("Skipping because input should be rgb") + + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomSnow(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape + + +def test_aug_2d_solarize(benchmark, device, dtype, torch_optimizer, shape): + data = torch.rand(*shape, device=device, dtype=dtype) + aug = RandomSolarize(p=1.0) + op = torch_optimizer(aug) + + actual = benchmark(op, input=data) + + assert actual.shape == shape diff --git a/benchmarks/augmentation/conftest.py b/benchmarks/augmentation/conftest.py new file mode 100644 index 0000000..a957c22 --- /dev/null +++ b/benchmarks/augmentation/conftest.py @@ -0,0 +1,33 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from itertools import product + +import pytest + + +@pytest.fixture +def shape(B, C, H, W): + return (B, C, H, W) + + +def pytest_generate_tests(metafunc): + B = [1, 5] + C = [1, 3] + H = W = [128] # , 237, 512] + params = list(product(B, C, H, W)) + metafunc.parametrize("B, C, H, W", params) diff --git a/benchmarks/color/gray_test.py b/benchmarks/color/gray_test.py new file mode 100644 index 0000000..62379de --- /dev/null +++ b/benchmarks/color/gray_test.py @@ -0,0 +1,34 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.color import grayscale_to_rgb + + +@pytest.mark.parametrize("B", [1, 5]) +@pytest.mark.parametrize("C", [1]) +@pytest.mark.parametrize("H", [128, 237, 512]) +@pytest.mark.parametrize("W", [128, 237, 512]) +def test_grayscale_to_rgb(benchmark, device, dtype, torch_optimizer, B, C, H, W): + data = torch.rand(B, C, H, W, device=device, dtype=dtype) + op = torch_optimizer(grayscale_to_rgb) + + actual = benchmark(op, image=data) + + assert actual.shape == (B, 3, H, W) diff --git a/benchmarks/color/xyz_test.py b/benchmarks/color/xyz_test.py new file mode 100644 index 0000000..5a9188c --- /dev/null +++ b/benchmarks/color/xyz_test.py @@ -0,0 +1,49 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.color import rgb_to_xyz, xyz_to_rgb + + +@pytest.mark.parametrize("B", [1, 8, 32]) +@pytest.mark.parametrize("C", [3]) +@pytest.mark.parametrize("H", [128, 256, 512]) +@pytest.mark.parametrize("W", [128, 256, 512]) +def test_rgb_to_xyz_benchmark(benchmark, device, dtype, torch_optimizer, B, C, H, W): + """Benchmark the rgb_to_xyz function.""" + + data = torch.rand(B, C, H, W, device=device, dtype=dtype) + op = torch_optimizer(rgb_to_xyz) + actual = benchmark(op, data) + + assert actual.shape == (B, 3, H, W) + + +@pytest.mark.parametrize("B", [1, 8, 32]) +@pytest.mark.parametrize("C", [3]) +@pytest.mark.parametrize("H", [128, 256, 512]) +@pytest.mark.parametrize("W", [128, 256, 512]) +def test_xyz_to_rgb_benchmark(benchmark, device, dtype, torch_optimizer, B, C, H, W): + """Benchmark the xyz_to_rgb function.""" + + data = torch.rand(B, C, H, W, device=device, dtype=dtype) + op = torch_optimizer(xyz_to_rgb) + actual = benchmark(op, image=data) + + assert actual.shape == (B, 3, H, W) diff --git a/benchmarks/color/yuv_test.py b/benchmarks/color/yuv_test.py new file mode 100644 index 0000000..5f53fc9 --- /dev/null +++ b/benchmarks/color/yuv_test.py @@ -0,0 +1,49 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.color import rgb_to_yuv, yuv_to_rgb + + +@pytest.mark.parametrize("B", [1, 8, 32]) +@pytest.mark.parametrize("C", [3]) +@pytest.mark.parametrize("H", [128, 256, 512]) +@pytest.mark.parametrize("W", [128, 256, 512]) +def test_rgb_to_yuv_benchmark(benchmark, device, dtype, torch_optimizer, B, C, H, W): + """Benchmark the rgb_to_yuv function.""" + + data = torch.rand(B, C, H, W, device=device, dtype=dtype) + op = torch_optimizer(rgb_to_yuv) + actual = benchmark(op, data) + + assert actual.shape == (B, 3, H, W) + + +@pytest.mark.parametrize("B", [1, 8, 32]) +@pytest.mark.parametrize("C", [3]) +@pytest.mark.parametrize("H", [128, 256, 512]) +@pytest.mark.parametrize("W", [128, 256, 512]) +def test_yuv_to_rgb_benchmark(benchmark, device, dtype, torch_optimizer, B, C, H, W): + """Benchmark the yuv_to_rgb function.""" + + data = torch.rand(B, C, H, W, device=device, dtype=dtype) + op = torch_optimizer(yuv_to_rgb) + actual = benchmark(op, image=data) + + assert actual.shape == (B, 3, H, W) diff --git a/benchmarks/feature/scale_space_detector.py b/benchmarks/feature/scale_space_detector.py new file mode 100644 index 0000000..ea51530 --- /dev/null +++ b/benchmarks/feature/scale_space_detector.py @@ -0,0 +1,659 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Benchmark a fully-configurable feature pipeline on Oxford-Affine-style sequences. + +An Oxford-Affine sequence directory contains:: + + img1.png img2.png ... imgN.png + H1to2p H1to3p ... H1toNp (3x3 homography ground truth, whitespace-separated) + +The pipeline can be either a ``ScaleSpaceDetector``-based combination (freely composable +response function, subpix module, descriptor, orientation estimator and affine-shape +estimator) or one of the end-to-end learned detectors (ALIKED, DISK, DeDoDe). + +Output columns per image pair (img1 vs imgK): + +* ``error [px]`` -- mean corner reprojection error vs ground-truth homography (lower is better). +* ``inliers [#]`` -- RANSAC inlier count (higher is better). +* ``time [ms]`` -- detect + describe + match wall-clock time (lower is better). + +Usage examples:: + + # default (AdaptiveQuadInterp3d + DoG + SIFT) + python benchmarks/feature/scale_space_detector.py --seq /data/graf + + # swap descriptor + python benchmarks/feature/scale_space_detector.py --seq /data/graf --desc hardnet + + # Hessian + AffNet + OriNet + python benchmarks/feature/scale_space_detector.py --seq /data/graf \\ + --resp hessian --aff affnet --ori orinet + + # end-to-end ALIKED (ignores --resp/--subpix/--desc/--ori/--aff) + python benchmarks/feature/scale_space_detector.py --seq /data/graf --method aliked + + # one named sequence inside a root folder + python benchmarks/feature/scale_space_detector.py --root /data/oxford-affine --seq graf + + # all sequences inside a root folder + python benchmarks/feature/scale_space_detector.py --root /data/oxford-affine --device cuda +""" + +from __future__ import annotations + +import argparse +import time +from pathlib import Path +from typing import Callable, Dict, List, Tuple, Union + +import cv2 +import numpy as np +import torch +from torch import nn + +import kornia.feature as KF +from kornia.feature.integrated import KeyNetAffNetHardNet, KeyNetHardNet +from kornia.feature.responses import BlobDoG, BlobDoGSingle, BlobHessian, CornerGFTT, CornerHarris +from kornia.geometry import RANSAC +from kornia.geometry.subpix import ( + AdaptiveQuadInterp3d, + ConvQuadInterp3d, + ConvSoftArgmax3d, + IterativeQuadInterp3d, +) +from kornia.geometry.transform import ScalePyramid + +# --------------------------------------------------------------------------- +# Metric +# --------------------------------------------------------------------------- + + +def get_MAE_imgcorners(h: int, w: int, H_gt, H_est) -> float: + """Mean corner reprojection error of H_est vs H_gt (pixels). + + Example:: + + H_gt = np.loadtxt(Hgt) + img1 = K.image_to_tensor(cv2.imread(f1, 0), False) / 255. + img2 = K.image_to_tensor(cv2.imread(f2, 0), False) / 255. + h, w = img1.size(2), img1.size(3) + H_out = matchImages(img1, img2) + MAE = get_MAE_imgcorners(h, w, H_gt, H_out.detach().cpu().numpy()) + """ + pts = np.float32([[0, 0], [0, h - 1], [w - 1, h - 1], [w - 1, 0]]).reshape(-1, 1, 2) + dst = cv2.perspectiveTransform(pts, H_est).squeeze(1) + dst_GT = cv2.perspectiveTransform(pts, H_gt).squeeze(1) + return float(np.abs(dst - dst_GT).sum(axis=1).mean()) + + +# --------------------------------------------------------------------------- +# I/O helpers +# --------------------------------------------------------------------------- + + +def load_gray(path: str, device: torch.device) -> torch.Tensor: + img = cv2.imread(path, cv2.IMREAD_GRAYSCALE) + if img is None: + raise FileNotFoundError(f"Cannot read: {path}") + return torch.from_numpy(img).float().div(255.0).unsqueeze(0).unsqueeze(0).to(device) + + +def gray_to_rgb(t: torch.Tensor) -> torch.Tensor: + return t.expand(-1, 3, -1, -1) + + +def find_pairs(seq: Path): + pairs = [] + for k in range(2, 10): + img_p = seq / f"img{k}.png" + h_p = seq / f"H1to{k}p" + if img_p.exists() and h_p.exists(): + pairs.append((k, img_p, h_p)) + return pairs + + +# --------------------------------------------------------------------------- +# Component registries +# --------------------------------------------------------------------------- + +RESP_REGISTRY: dict[str, tuple] = { + "dog": (BlobDoG, True, True), + "dog_single": (lambda: BlobDoGSingle(1.0, 1.6), False, True), + "hessian": (BlobHessian, False, True), + "harris": (lambda: CornerHarris(k=0.04), False, False), + "gftt": (CornerGFTT, False, False), +} + +SUBPIX_REGISTRY: dict[str, Callable] = { + "adaptive": lambda: AdaptiveQuadInterp3d(strict_maxima_bonus=0.0, allow_scale_steps=True), + "conv": lambda: ConvQuadInterp3d(strict_maxima_bonus=0.0), + "iterative": lambda: IterativeQuadInterp3d(strict_maxima_bonus=0.0), + "soft": lambda: ConvSoftArgmax3d((3, 3, 3), (1, 1, 1), (1, 1, 1), normalized_coordinates=False, output_value=True), +} + +DESC_REGISTRY: dict[str, tuple] = { + "sift": (lambda: KF.SIFTDescriptor(32, rootsift=True), 32), + "hardnet": (lambda: KF.HardNet(pretrained=True), 32), + "hardnet8": (lambda: KF.HardNet8(pretrained=True), 32), + "hynet": (lambda: KF.HyNet(pretrained=True), 32), + "sosnet": (lambda: KF.SOSNet(pretrained=True), 32), + "tfeat": (lambda: KF.TFeat(pretrained=True), 32), + "mkd": (lambda: KF.MKDDescriptor(32), 32), +} + +ORI_REGISTRY: dict[str, Callable] = { + "none": KF.PassLAF, + "lap": lambda: KF.LAFOrienter(32, angle_detector=KF.PatchDominantGradientOrientation(32)), + "orinet": lambda: KF.LAFOrienter(32, angle_detector=KF.OriNet(pretrained=True)), +} + +AFF_REGISTRY: dict[str, Callable] = { + "none": KF.PassLAF, + "patch": lambda: KF.LAFAffineShapeEstimator(32), + "affnet": lambda: KF.LAFAffNetShapeEstimator(pretrained=True), +} + + +# --------------------------------------------------------------------------- +# Extractor wrappers (unified: img -> (kp N x 2, desc N x D)) +# --------------------------------------------------------------------------- + + +class ScaleSpaceExtractor(nn.Module): + def __init__(self, detector, desc_module, ori_module, aff_module, patch_size: int): + super().__init__() + self.detector, self.desc = detector, desc_module + self.ori, self.aff = ori_module, aff_module + self.patch_size = patch_size + + @torch.no_grad() + def forward(self, img: torch.Tensor): + t_det0 = time.perf_counter() + lafs, _ = self.detector(img) + if img.device.type == "cuda": + torch.cuda.synchronize() + det_ms = (time.perf_counter() - t_det0) * 1000 + lafs = self.aff(lafs, img) + lafs = self.ori(lafs, img) + patches = KF.extract_patches_from_pyramid(img, lafs, self.patch_size) + B, N, C, H, W = patches.shape + desc = self.desc(patches.view(B * N, C, H, W)).view(B, N, -1) + return KF.get_laf_center(lafs)[0], desc[0], det_ms + + +class ALIKEDExtractor(nn.Module): + def __init__(self, model): + super().__init__() + self.model = model + + @torch.no_grad() + def forward(self, img: torch.Tensor): + f = self.model(gray_to_rgb(img))[0] + return f.keypoints, f.descriptors, None + + +class DISKExtractor(nn.Module): + def __init__(self, model, n: int): + super().__init__() + self.model, self.n = model, n + + @torch.no_grad() + def forward(self, img: torch.Tensor): + f = self.model(gray_to_rgb(img), n=self.n, pad_if_not_divisible=True)[0] + return f.keypoints, f.descriptors, None + + +class DeDoDEExtractor(nn.Module): + def __init__(self, model, n: int): + super().__init__() + self.model, self.n = model, n + + @torch.no_grad() + def forward(self, img: torch.Tensor): + kp, _, desc = self.model(gray_to_rgb(img), n=self.n) + return kp[0], desc[0], None + + +class DoGHardNet(nn.Module): + def __init__(self, nf: int): + super().__init__() + from kornia_moons.feature import OpenCVDetectorWithAffNetKornia + + kornia_cv2dogaffnet = OpenCVDetectorWithAffNetKornia( + cv2.SIFT_create(nf, edgeThreshold=-1, contrastThreshold=-1), make_upright=True + ) + self.det = kornia_cv2dogaffnet + self.desc = KF.HardNet(pretrained=True) + self.ori = KF.LAFOrienter(32, angle_detector=KF.OriNet(pretrained=True)) + + @torch.no_grad() + def forward(self, img: torch.Tensor): + lafs, _ = self.det(img) + lafs = self.ori(lafs, img) + patches = KF.extract_patches_from_pyramid(img, lafs, 32) + B, N, C, H, W = patches.shape + desc = self.desc(patches.view(B * N, C, H, W)).view(B, N, -1) + return KF.get_laf_center(lafs)[0], desc[0], None + + +class CV2SIFT(nn.Module): + def __init__(self, nf: int): + super().__init__() + self.det = cv2.SIFT_create(nf, edgeThreshold=-1, contrastThreshold=-1) + + @torch.no_grad() + def forward(self, img: torch.Tensor): + kpts, desc = self.det.detectAndCompute((255 * img.cpu().numpy().squeeze()).astype(np.uint8), None) + return torch.from_numpy(np.array([(x.pt[0], x.pt[1]) for x in kpts])), torch.from_numpy(desc), None + + +class KeyNetExtractor(nn.Module): + def __init__(self, n: int, ori: str, aff: str, device: torch.device, compile_model: bool = False): + super().__init__() + if aff != "none": + self.feat = KeyNetAffNetHardNet(num_features=n, upright=(ori == "none")).to(device) + else: + self.feat = KeyNetHardNet(num_features=n, upright=(ori == "none")).to(device) + if compile_model: + det = self.feat.detector + # model and nms run at 6 different image sizes → dynamic=True avoids recompilation + det.model = torch.compile(det.model, dynamic=True) + det.nms = torch.compile(det.nms, dynamic=True) + # aff/ori/descriptor NOT compiled: extract_patches_from_pyramid (laf.py) contains + # an .item() call inside a while-loop that creates graph breaks, causing extra + # sub-graphs that need additional warmup and show as a spike on the first eval pair. + + @torch.no_grad() + def forward(self, img: torch.Tensor): + lafs, _, desc = self.feat(img) + return KF.get_laf_center(lafs)[0], desc[0], None + + +# --------------------------------------------------------------------------- +# Factory +# --------------------------------------------------------------------------- + + +def build_extractor( + method: str, + resp: str, + subpix: str, + desc: str, + ori: str, + aff: str, + device: torch.device, + nf: int, + compile_modules: Union[bool, List[str]] = False, +) -> nn.Module: + if method == "scalespace": + resp_factory, ssr, minima = RESP_REGISTRY[resp] + subpix_mod = SUBPIX_REGISTRY[subpix]() + detector = KF.ScaleSpaceDetector( + num_features=nf, + resp_module=resp_factory(), + subpix_module=subpix_mod, + scale_pyr_module=ScalePyramid(3, 1.6, 32, double_image=True), + scale_space_response=ssr, + minima_are_also_good=minima, + compile_modules=["subpix", "resp", "scale_pyr"] if compile_modules else [], + ).to(device) + desc_factory, patch_size = DESC_REGISTRY[desc] + return ScaleSpaceExtractor( + detector, + desc_factory().to(device), + ORI_REGISTRY[ori]().to(device), + AFF_REGISTRY[aff]().to(device), + patch_size=patch_size, + ) + if method == "aliked": + return ALIKEDExtractor(KF.ALIKED.from_pretrained("aliked-n16rot", max_num_keypoints=nf).to(device)) + if method == "disk": + return DISKExtractor(KF.DISK.from_pretrained("depth", device=device), n=nf) + if method == "keynet": + return KeyNetExtractor(n=nf, ori=ori, aff=aff, device=device, compile_model=bool(compile_modules)) + if method == "opencv_sift_affnet": + return DoGHardNet(nf).to(device) + if method == "opencv_sift": + return CV2SIFT(nf).to(device) + if method == "dedode": + return DeDoDEExtractor( + KF.DeDoDe.from_pretrained(detector_weights="L-upright", descriptor_weights="B-upright").to(device), n=nf + ) + raise ValueError(f"Unknown method: {method!r}") + + +def make_label(method: str, resp: str, subpix: str, desc: str, ori: str, aff: str) -> str: + if method != "scalespace": + return method + parts = [f"resp={resp}", f"subpix={subpix}", f"desc={desc}"] + if ori != "none": + parts.append(f"ori={ori}") + if aff != "none": + parts.append(f"aff={aff}") + return " ".join(parts) + + +# --------------------------------------------------------------------------- +# Matching pipeline +# --------------------------------------------------------------------------- + + +@torch.no_grad() +def match_pair(img1, img2, extractor, ransac): + t0 = time.perf_counter() + kp1, desc1, det_ms1 = extractor(img1) + kp2, desc2, det_ms2 = extractor(img2) + _, idxs = KF.match_snn(desc1, desc2, 0.85) + ms = (time.perf_counter() - t0) * 1000 + det_ms = (det_ms1 + det_ms2) if (det_ms1 is not None and det_ms2 is not None) else None + if idxs.shape[0] < 4: + return None, ms, det_ms, 0 + H_est, mask = ransac(kp1[idxs[:, 0]], kp2[idxs[:, 1]]) + n_inliers = int(mask.sum().item()) if mask is not None else 0 + return H_est.cpu().numpy(), ms, det_ms, n_inliers + + +# --------------------------------------------------------------------------- +# Evaluation + printing +# --------------------------------------------------------------------------- + + +def eval_sequence(seq: Path, extractor, ransac, device) -> dict: + img1 = load_gray(str(seq / "img1.png"), device) + h, w = img1.shape[2], img1.shape[3] + pairs = find_pairs(seq) + if not pairs: + raise RuntimeError(f"No valid pairs in {seq}") + rows = [] + for k, img_path, h_path in pairs: + img2 = load_gray(str(img_path), device) + H_gt = np.loadtxt(str(h_path)) + H, ms, det_ms, ni = match_pair(img1, img2, extractor, ransac) + mae = get_MAE_imgcorners(h, w, H_gt, H) if H is not None else float("nan") + rows.append({"k": k, "mae": mae, "ni": ni, "ms": ms, "det_ms": det_ms}) + return {"seq": seq.name, "h": h, "w": w, "rows": rows} + + +def _col(rows, key): + return [r[key] for r in rows] + + +def print_sequence_table(stats: dict, label: str) -> None: + rows = stats["rows"] + has_det = rows[0]["det_ms"] is not None + print(f"\n-- {stats['seq']} ({stats['h']}x{stats['w']}) --") + print(f" method : {label}") + if has_det: + print(f"\n{'pair':<6} {'error [px]':>12} {'inliers [#]':>12} {'det [ms]':>10} {'time [ms]':>10}") + print("-" * 56) + for r in rows: + print(f"1<>{r['k']:<3} {r['mae']:>12.1f} {r['ni']:>12} {r['det_ms']:>10.1f} {r['ms']:>10.1f}") + print("-" * 56) + print( + f"{'mean':<6} {np.nanmean(_col(rows, 'mae')):>12.1f}" + f" {np.mean(_col(rows, 'ni')):>12.1f}" + f" {np.nanmean(_col(rows, 'det_ms')):>10.1f}" + f" {np.mean(_col(rows, 'ms')):>10.1f}" + ) + else: + print(f"\n{'pair':<6} {'error [px]':>12} {'inliers [#]':>12} {'time [ms]':>10}") + print("-" * 44) + for r in rows: + print(f"1<>{r['k']:<3} {r['mae']:>12.1f} {r['ni']:>12} {r['ms']:>10.1f}") + print("-" * 44) + print( + f"{'mean':<6} {np.nanmean(_col(rows, 'mae')):>12.1f}" + f" {np.mean(_col(rows, 'ni')):>12.1f}" + f" {np.mean(_col(rows, 'ms')):>10.1f}" + ) + + +def print_summary(all_stats, label: str) -> None: + print("\n" + "=" * 44) + print(f"OVERALL method={label}") + print("=" * 44) + + def agg(key): + return np.nanmean([np.nanmean([r[key] for r in s["rows"]]) for s in all_stats]) + + print(f" error [px] : {agg('mae'):.1f}") + print(f" inliers [#] : {agg('ni'):.1f}") + has_det = all_stats[0]["rows"][0]["det_ms"] is not None + if has_det: + print(f" det [ms] : {agg('det_ms'):.1f}") + print(f" time [ms] : {agg('ms'):.1f}") + + +# --------------------------------------------------------------------------- +# Detection-only batch speed benchmark +# --------------------------------------------------------------------------- + + +def _get_bench_module(extractor: nn.Module) -> Tuple[nn.Module, str]: + """Return (module_to_time, label) for the speed benchmark. + + For pipeline extractors, isolates the detector so aff/ori/desc are excluded. + For end-to-end models (ALIKED, DISK, DeDoDe) the full forward is timed. + """ + if isinstance(extractor, ScaleSpaceExtractor): + return extractor.detector, "detection only" + if isinstance(extractor, KeyNetExtractor): + return extractor.feat.detector, "detection only" + return extractor, "full forward" + + +def _time_fn(fn: Callable, n_iter: int, dev: torch.device) -> float: + """Return mean wall-clock time in ms over n_iter calls.""" + if dev.type == "cuda": + torch.cuda.synchronize() + t0 = time.perf_counter() + for _ in range(n_iter): + fn() + if dev.type == "cuda": + torch.cuda.synchronize() + return (time.perf_counter() - t0) / n_iter * 1000 + + +def run_speed_benchmark( + extractor: nn.Module, + img: torch.Tensor, + batch_sizes: Tuple[int, ...] = (1, 4, 8), + n_iter_gpu: int = 20, + n_iter_cpu: int = 5, + warmup: int = 3, +) -> None: + """Print a batch-size x device timing table. + + Args: + extractor: the extractor built by :func:`build_extractor`. + img: single-image tensor ``(1, C, H, W)`` on any device (moved internally). + batch_sizes: batch sizes to benchmark. + n_iter_gpu: number of timed iterations on GPU. + n_iter_cpu: number of timed iterations on CPU. + warmup: warm-up iterations (not timed). + """ + mod, what = _get_bench_module(extractor) + + try: + orig_dev = next(mod.parameters()).device + except StopIteration: + orig_dev = img.device + + dev_map: Dict[str, torch.device] = {"cpu": torch.device("cpu")} + if torch.cuda.is_available(): + dev_map["gpu"] = torch.device("cuda") + + col_w = 14 + print(f"\n-- Speed benchmark ({what}, ms / call) --") + print(f"{'bs':<6}" + "".join(f"{k:>{col_w}}" for k in dev_map)) + print("-" * (6 + col_w * len(dev_map))) + + for bs in batch_sizes: + row = f"{bs:<6}" + for _, dev in dev_map.items(): + n_iter = n_iter_gpu if dev.type == "cuda" else n_iter_cpu + mod.to(dev) + img_b = img[0:1].expand(bs, -1, -1, -1).contiguous().to(dev) + try: + with torch.no_grad(): + for _ in range(warmup): + mod(img_b) + ms = _time_fn(lambda: mod(img_b), n_iter, dev) # noqa: B023 + row += f"{ms:>{col_w - 3}.1f} ms" + except Exception: + row += f"{'N/A':>{col_w}}" + print(row) + + mod.to(orig_dev) + print() + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + g = p.add_argument_group("Sequences") + g.add_argument("--seq", metavar="DIR", action="append", default=[]) + g.add_argument( + "--root", + metavar="DIR", + default=None, + help="Base folder. Alone: enumerates all subdirs with img1.png. With --seq name: resolves to root/name.", + ) + + g = p.add_argument_group("Method") + g.add_argument( + "--method", + default="scalespace", + choices=["scalespace", "aliked", "disk", "dedode", "keynet", "opencv_sift_affnet", "opencv_sift"], + ) + g.add_argument( + "--resp", + default="dog", + choices=list(RESP_REGISTRY), + metavar="RESP", + help=f"[scalespace only] choices: {list(RESP_REGISTRY)}", + ) + g.add_argument( + "--subpix", + default="adaptive", + choices=list(SUBPIX_REGISTRY), + metavar="SUBPIX", + help=f"[scalespace only] choices: {list(SUBPIX_REGISTRY)}", + ) + g.add_argument( + "--desc", + default="sift", + choices=list(DESC_REGISTRY), + metavar="DESC", + help=f"[scalespace only] choices: {list(DESC_REGISTRY)}", + ) + g.add_argument( + "--ori", + default="none", + choices=list(ORI_REGISTRY), + metavar="ORI", + help=f"[scalespace only] none=upright; choices: {list(ORI_REGISTRY)}", + ) + g.add_argument( + "--aff", + default="none", + choices=list(AFF_REGISTRY), + metavar="AFF", + help=f"[scalespace only] none=circular; choices: {list(AFF_REGISTRY)}", + ) + + g = p.add_argument_group("Shared") + g.add_argument("--nf", metavar="N", type=int, default=4096) + g.add_argument( + "--compile", + action="store_true", + help="torch.compile the extractor (if supported by the method and PyTorch version)", + ) + g.add_argument("--device", metavar="DEV", default=None) + g.add_argument("--warmup", metavar="N", type=int, default=1) + g.add_argument( + "--speed-bench", + action="store_true", + help="Run detection-only speed benchmark (bs=1,4,8 on cpu+gpu) after the matching eval.", + ) + return p.parse_args() + + +def main() -> None: + args = parse_args() + device = torch.device(args.device or ("cuda" if torch.cuda.is_available() else "cpu")) + root = Path(args.root) if args.root else None + if args.seq: + # Resolve each --seq name: use as-is if it exists, else try root/name + seqs = [] + for s in args.seq: + p = Path(s) + if not p.exists() and root is not None: + p = root / s + seqs.append(p) + elif root is not None: + # --root only: enumerate all subdirectories that look like sequences + seqs = sorted(d for d in root.iterdir() if d.is_dir() and (d / "img1.png").exists()) + else: + seqs = [] + if not seqs: + raise SystemExit("No sequences. Use --seq DIR or --root DIR or --root DIR --seq name.") + + label = make_label(args.method, args.resp, args.subpix, args.desc, args.ori, args.aff) + print(f"device: {device} nf: {args.nf} sequences: {len(seqs)}") + print(f" method : {label}") + + extractor = build_extractor( + args.method, + args.resp, + args.subpix, + args.desc, + args.ori, + args.aff, + device, + args.nf, + compile_modules=args.compile, + ) + ransac = RANSAC("homography", inl_th=2.0, max_iter=10, batch_size=8196, confidence=0.9999, seed=3407) + + first_img1 = load_gray(str(seqs[0] / "img1.png"), device) + first_pairs = find_pairs(seqs[0]) + if first_pairs: + first_img2 = load_gray(str(first_pairs[0][1]), device) + for _ in range(args.warmup): + match_pair(first_img1, first_img2, extractor, ransac) + + all_stats = [] + for seq in seqs: + stats = eval_sequence(seq, extractor, ransac, device) + all_stats.append(stats) + print_sequence_table(stats, label) + + if len(all_stats) > 1: + print_summary(all_stats, label) + + if args.speed_bench: + bench_img = load_gray(str(seqs[0] / "img1.png"), torch.device("cpu")) + run_speed_benchmark(extractor, bench_img) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/geometry/depth_to_normals.py b/benchmarks/geometry/depth_to_normals.py new file mode 100644 index 0000000..c07da13 --- /dev/null +++ b/benchmarks/geometry/depth_to_normals.py @@ -0,0 +1,115 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Benchmark for kornia.geometry.depth.depth_to_normals across image sizes. + +Usage: + python benchmarks/geometry/depth_to_normals.py + python benchmarks/geometry/depth_to_normals.py --cuda +""" + +from __future__ import annotations + +import argparse +import datetime +import platform +import shutil +import subprocess +import time + +import torch + +from kornia.geometry.depth import depth_to_normals + +# ───────────────────────────────────────────────────────────────────────────── +# Helpers +# ───────────────────────────────────────────────────────────────────────────── + + +def _sync(device: str) -> None: + if device == "cuda": + torch.cuda.synchronize() + + +def bench(fn, *args, warmup: int = 5, reps: int = 20, device: str = "cpu", label: str = "") -> float: + for _ in range(warmup): + fn(*args) + _sync(device) + t0 = time.perf_counter() + for _ in range(reps): + fn(*args) + _sync(device) + ms = (time.perf_counter() - t0) / reps * 1000 + print(f" {label:<64s}: {ms:8.3f} ms") + return ms + + +def _print_env() -> None: + date = datetime.datetime.now(tz=datetime.UTC).strftime("%Y-%m-%d %H:%M:%S UTC") + git = shutil.which("git") or "git" + try: + commit = subprocess.check_output([git, "rev-parse", "--short", "HEAD"], text=True).strip() + except Exception: + commit = "unknown" + cpu = platform.processor() or platform.machine() + print(f" date : {date}") + print(f" commit : {commit}") + print(f" cpu : {cpu}") + if torch.cuda.is_available(): + print(f" gpu : {torch.cuda.get_device_name(0)}") + + +# ───────────────────────────────────────────────────────────────────────────── +# Benchmark +# ───────────────────────────────────────────────────────────────────────────── + + +def run(device: str) -> None: + print(f"\n{'=' * 72}") + print(f" DEVICE : {device.upper()}") + print(f"{'=' * 72}") + + K_base = torch.eye(3, device=device, dtype=torch.float32) + K_base[0, 0] = K_base[1, 1] = 500.0 + K_base[0, 2] = K_base[1, 2] = 320.0 + + configs = [ + ("B=1 H=64 W=64 ", 1, 64, 64), + ("B=1 H=256 W=256 ", 1, 256, 256), + ("B=4 H=480 W=640 ", 4, 480, 640), + ("B=1 H=720 W=1280", 1, 720, 1280), + ] + + print(f"\n {'config':<20s} {'depth_to_normals':>20s}") + print(f" {'-' * 20} {'-' * 20}") + for label, B, H, W in configs: + K = K_base.unsqueeze(0).expand(B, -1, -1).contiguous() + depth = torch.rand(B, 1, H, W, device=device, dtype=torch.float32).add_(0.1) + bench(depth_to_normals, depth, K, device=device, label=label) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cuda", action="store_true") + args = parser.parse_args() + + _print_env() + run("cpu") + if args.cuda: + if torch.cuda.is_available(): + run("cuda") + else: + print("\nWarning: --cuda requested but CUDA is not available.") diff --git a/benchmarks/geometry/linalg_epipolar_distort.py b/benchmarks/geometry/linalg_epipolar_distort.py new file mode 100644 index 0000000..e42b7f5 --- /dev/null +++ b/benchmarks/geometry/linalg_epipolar_distort.py @@ -0,0 +1,203 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Benchmark for linalg, epipolar projection/triangulation, and calibration distortion functions. + +Usage: + python benchmarks/geometry/linalg_epipolar_distort.py + python benchmarks/geometry/linalg_epipolar_distort.py --cuda +""" + +from __future__ import annotations + +import argparse +import datetime +import platform +import shutil +import subprocess +import time + +import torch + +from kornia.geometry.calibration.distort import distort_points, tilt_projection +from kornia.geometry.epipolar.projection import ( + KRt_from_projection, + depth_from_point, + projection_from_KRt, + projections_from_fundamental, + scale_intrinsics, +) +from kornia.geometry.epipolar.triangulation import triangulate_points +from kornia.geometry.linalg import ( + batched_dot_product, + compose_transformations, + euclidean_distance, + inverse_transformation, + point_line_distance, + relative_transformation, + transform_points, +) + +# ───────────────────────────────────────────────────────────────────────────── +# Helpers +# ───────────────────────────────────────────────────────────────────────────── + + +def _sync(device: str) -> None: + if device == "cuda": + torch.cuda.synchronize() + + +def bench(fn, *args, warmup: int = 10, reps: int = 50, device: str = "cpu", label: str = "") -> float: + for _ in range(warmup): + fn(*args) + _sync(device) + t0 = time.perf_counter() + for _ in range(reps): + fn(*args) + _sync(device) + ms = (time.perf_counter() - t0) / reps * 1000 + print(f" {label:<60s}: {ms:8.4f} ms") + return ms + + +def _print_env() -> None: + date = datetime.datetime.now(tz=datetime.UTC).strftime("%Y-%m-%d %H:%M:%S UTC") + git = shutil.which("git") or "git" + try: + commit = subprocess.check_output([git, "rev-parse", "--short", "HEAD"], text=True).strip() + except Exception: + commit = "unknown" + cpu = platform.processor() or platform.machine() + print(f" date : {date}") + print(f" commit : {commit}") + print(f" cpu : {cpu}") + if torch.cuda.is_available(): + print(f" gpu : {torch.cuda.get_device_name(0)}") + + +# ───────────────────────────────────────────────────────────────────────────── +# Benchmarks +# ───────────────────────────────────────────────────────────────────────────── + + +def bench_linalg(device: str) -> None: + print(f"\n--- linalg device={device} ---") + + configs = [ + ("B=1 ", 1), + ("B=16 ", 16), + ("B=256", 256), + ] + + for label, B in configs: + T = torch.eye(4, device=device).unsqueeze(0).expand(B, -1, -1).contiguous() + pts = torch.rand(B, 1000, 3, device=device) + pts_flat = torch.rand(B, 1000, 4, device=device) + lines = torch.rand(B, 1000, 3, device=device) + x = torch.rand(B, 1000, 8, device=device) + y = torch.rand(B, 1000, 8, device=device) + + print(f"\n {label}") + bench(compose_transformations, T, T, device=device, label="compose_transformations") + bench(inverse_transformation, T, device=device, label="inverse_transformation") + bench(relative_transformation, T, T, device=device, label="relative_transformation") + bench(transform_points, T, pts, device=device, label="transform_points") + bench(point_line_distance, pts_flat[..., :3], lines, device=device, label="point_line_distance") + bench(batched_dot_product, x, y, device=device, label="batched_dot_product") + bench(euclidean_distance, x, y, device=device, label="euclidean_distance") + + +def bench_epipolar(device: str) -> None: + print(f"\n--- epipolar projection / triangulation device={device} ---") + + configs = [ + ("B=1 N=100 ", 1, 100), + ("B=8 N=1K ", 8, 1000), + ("B=32 N=1K ", 32, 1000), + ] + + K_base = torch.eye(3, device=device) + K_base[0, 0] = K_base[1, 1] = 500.0 + K_base[0, 2] = K_base[1, 2] = 320.0 + + for label, B, N in configs: + K = K_base.unsqueeze(0).expand(B, -1, -1).contiguous() + R = torch.eye(3, device=device).unsqueeze(0).expand(B, -1, -1).contiguous() + t = torch.zeros(B, 3, 1, device=device) + X = torch.rand(B, N, 3, device=device).add_(1.0) + P1 = torch.cat([R, t], dim=-1) # B x 3 x 4 + P2 = torch.cat([R, t + 0.1], dim=-1) + pts1 = torch.rand(B, N, 2, device=device) + pts2 = torch.rand(B, N, 2, device=device) + F = torch.rand(B, 3, 3, device=device) + P_proj = torch.rand(B, 3, 4, device=device) + + print(f"\n {label}") + bench(projection_from_KRt, K, R, t, device=device, label="projection_from_KRt") + bench(scale_intrinsics, K, 2.0, device=device, label="scale_intrinsics") + bench(depth_from_point, R, t, X, device=device, label="depth_from_point") + bench(projections_from_fundamental, F, device=device, label="projections_from_fundamental") + bench(triangulate_points, P1, P2, pts1, pts2, device=device, label="triangulate_points") + bench(KRt_from_projection, P_proj, device=device, label="KRt_from_projection") + + +def bench_distort(device: str) -> None: + print(f"\n--- calibration distortion device={device} ---") + + K_base = torch.eye(3, device=device) + K_base[0, 0] = K_base[1, 1] = 500.0 + K_base[0, 2] = K_base[1, 2] = 320.0 + dist_base = torch.tensor([0.1, -0.05, 0.001, 0.001, 0.02, 0.01, -0.005, 0.002], device=device) + + configs = [ + ("B=1 N=1K ", 1, 1_000), + ("B=1 N=100K", 1, 100_000), + ("B=32 N=10K ", 32, 10_000), + ] + + for label, B, N in configs: + K_b = K_base.unsqueeze(0).expand(B, -1, -1).contiguous() + dist_b = dist_base.unsqueeze(0).expand(B, -1).contiguous() + pts2 = torch.rand(B, N, 2, device=device).mul_(640.0) + taux = torch.zeros(B, device=device) + tauy = torch.zeros(B, device=device) + + print(f"\n {label}") + bench(distort_points, pts2, K_b, dist_b, device=device, label="distort_points") + bench(tilt_projection, taux, tauy, device=device, label="tilt_projection") + + +def run(device: str) -> None: + sep = "=" * 72 + print(f"\n{sep}\n DEVICE: {device.upper()}\n{sep}") + bench_linalg(device) + bench_epipolar(device) + bench_distort(device) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cuda", action="store_true") + args = parser.parse_args() + + _print_env() + run("cpu") + if args.cuda: + if torch.cuda.is_available(): + run("cuda") + else: + print("\nWarning: --cuda requested but CUDA is not available.") diff --git a/benchmarks/geometry/pointcloud.py b/benchmarks/geometry/pointcloud.py new file mode 100644 index 0000000..f5930f7 --- /dev/null +++ b/benchmarks/geometry/pointcloud.py @@ -0,0 +1,282 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Benchmark for kornia.geometry point cloud operations on CPU and CUDA. + +Covers: + - transform_points + - project_points / unproject_points + - convert_points_to/from_homogeneous + - depth_to_3d, depth_to_3d_v2 (uncached vs cached grid) + - depth_to_normals + - warp_frame_depth + +Usage: + python benchmarks/geometry/pointcloud.py # CPU only + python benchmarks/geometry/pointcloud.py --cuda # CPU + CUDA + python benchmarks/geometry/pointcloud.py --compile # include torch.compile variants + python benchmarks/geometry/pointcloud.py --cuda --compile +""" + +from __future__ import annotations + +import argparse +import datetime +import functools +import platform +import shutil +import subprocess +import time + +import torch + +from kornia.geometry.camera import project_points, unproject_points +from kornia.geometry.conversions import convert_points_from_homogeneous, convert_points_to_homogeneous +from kornia.geometry.depth import depth_to_3d, depth_to_3d_v2, depth_to_normals, unproject_meshgrid, warp_frame_depth +from kornia.geometry.linalg import transform_points + +# ───────────────────────────────────────────────────────────────────────────── +# Helpers +# ───────────────────────────────────────────────────────────────────────────── + + +def _sync(device: str) -> None: + if device == "cuda": + torch.cuda.synchronize() + + +def bench(fn, *args, warmup: int = 5, reps: int = 20, device: str = "cpu", label: str = "") -> float: + """Return mean wall-clock time in milliseconds.""" + for _ in range(warmup): + fn(*args) + _sync(device) + t0 = time.perf_counter() + for _ in range(reps): + fn(*args) + _sync(device) + ms = (time.perf_counter() - t0) / reps * 1000 + print(f" {label:<66s}: {ms:8.3f} ms") + return ms + + +def _print_env() -> None: + date = datetime.datetime.now(tz=datetime.UTC).strftime("%Y-%m-%d %H:%M:%S UTC") + git = shutil.which("git") or "git" + try: + commit = subprocess.check_output([git, "rev-parse", "--short", "HEAD"], text=True).strip() + except Exception: + commit = "unknown" + cpu = platform.processor() or platform.machine() + print(f" date : {date}") + print(f" commit : {commit}") + print(f" cpu : {cpu}") + if torch.cuda.is_available(): + print(f" gpu : {torch.cuda.get_device_name(0)}") + + +# ───────────────────────────────────────────────────────────────────────────── +# transform_points +# ───────────────────────────────────────────────────────────────────────────── + + +def bench_transform_points(device: str, dtype: torch.dtype = torch.float32, compile_: bool = False) -> None: + print(f"\n--- transform_points device={device} dtype={dtype} ---") + + fn = transform_points + fn_c = torch.compile(fn) if compile_ else None + + configs = [ + ("B=1 N=1K single transform", 1, 1_000, True), + ("B=8 N=10K single transform", 8, 10_000, True), + ("B=32 N=100K single transform", 32, 100_000, True), + ("B=1 N=1K per-sample transform", 1, 1_000, False), + ("B=8 N=10K per-sample transform", 8, 10_000, False), + ("B=32 N=100K per-sample transform", 32, 100_000, False), + ] + + for label, B, N, single_T in configs: + pts = torch.randn(B, N, 3, device=device, dtype=dtype) + T = torch.eye(4, device=device, dtype=dtype).unsqueeze(0) + if not single_T: + T = T.expand(B, -1, -1).contiguous() + + bench(fn, T, pts, device=device, label=label) + if compile_: + bench(fn_c, T, pts, device=device, label=f"{label} (compiled)") + + +# ───────────────────────────────────────────────────────────────────────────── +# project / unproject +# ───────────────────────────────────────────────────────────────────────────── + + +def bench_project_unproject(device: str, dtype: torch.dtype = torch.float32, compile_: bool = False) -> None: + print(f"\n--- project_points / unproject_points device={device} dtype={dtype} ---") + + K_base = torch.eye(3, device=device, dtype=dtype) + K_base[0, 0] = K_base[1, 1] = 500.0 + K_base[0, 2] = K_base[1, 2] = 320.0 + + proj_fn = project_points + unproj_fn = unproject_points + if compile_: + proj_fn = torch.compile(proj_fn) + unproj_fn = torch.compile(unproj_fn) + + configs = [ + ("B=1 N=1K ", 1, 1_000), + ("B=8 N=10K ", 8, 10_000), + ("B=32 N=100K", 32, 100_000), + ] + + for label, B, N in configs: + K_b = K_base.unsqueeze(0).expand(B, -1, -1) + pts3 = torch.rand(B, N, 3, device=device, dtype=dtype).add_(0.5) + pts2 = torch.rand(B, N, 2, device=device, dtype=dtype).mul_(640.0) + depth = torch.ones(B, N, 1, device=device, dtype=dtype) + + bench(proj_fn, pts3, K_b, device=device, label=f"{label} project_points") + bench(unproj_fn, pts2, depth, K_b, device=device, label=f"{label} unproject_points") + + +# ───────────────────────────────────────────────────────────────────────────── +# homogeneous conversions +# ───────────────────────────────────────────────────────────────────────────── + + +def bench_homogeneous_conversions(device: str, dtype: torch.dtype = torch.float32) -> None: + print(f"\n--- homogeneous conversions device={device} dtype={dtype} ---") + + for label, shape, fn in [ + ("N=1M 3-D → homogeneous", (1_000_000, 3), convert_points_to_homogeneous), + ("N=1M 4-D → euclidean", (1_000_000, 4), convert_points_from_homogeneous), + ("B=32 N=100K 3-D → homogeneous", (32, 100_000, 3), convert_points_to_homogeneous), + ("B=32 N=100K 4-D → euclidean", (32, 100_000, 4), convert_points_from_homogeneous), + ]: + x = torch.randn(*shape, device=device, dtype=dtype) + bench(fn, x, device=device, label=label) + + +# ───────────────────────────────────────────────────────────────────────────── +# depth_to_3d / depth_to_3d_v2 / depth_to_normals +# ───────────────────────────────────────────────────────────────────────────── + + +def bench_depth_functions(device: str, dtype: torch.dtype = torch.float32, compile_: bool = False) -> None: + print(f"\n--- depth_to_3d / depth_to_3d_v2 / depth_to_normals device={device} dtype={dtype} ---") + + K_base = torch.eye(3, device=device, dtype=dtype) + K_base[0, 0] = K_base[1, 1] = 500.0 + K_base[0, 2] = K_base[1, 2] = 320.0 + + configs = [ + ("B=1 H=64 W=64 ", 1, 64, 64), + ("B=1 H=256 W=256 ", 1, 256, 256), + ("B=4 H=480 W=640 ", 4, 480, 640), + ("B=1 H=720 W=1280", 1, 720, 1280), + ] + + for label, B, H, W in configs: + K_b = K_base.unsqueeze(0).expand(B, -1, -1).contiguous() + depth4 = torch.rand(B, 1, H, W, device=device, dtype=dtype).add_(0.1) + depth3 = depth4.squeeze(1) + + bench(depth_to_3d, depth4, K_b, device=device, label=f"{label} depth_to_3d") + bench(depth_to_3d_v2, depth3, K_b, device=device, label=f"{label} depth_to_3d_v2 (no cache)") + + grid = unproject_meshgrid(H, W, K_b, device=device, dtype=dtype) + bench( + functools.partial(depth_to_3d_v2, xyz_grid=grid), + depth3, + K_b, + device=device, + label=f"{label} depth_to_3d_v2 (cached grid)", + ) + + bench(depth_to_normals, depth4, K_b, device=device, label=f"{label} depth_to_normals") + + if compile_: + fn_c = torch.compile(depth_to_3d_v2) + bench( + functools.partial(fn_c, xyz_grid=grid), + depth3, + K_b, + device=device, + label=f"{label} depth_to_3d_v2 (cached + compiled)", + ) + + +# ───────────────────────────────────────────────────────────────────────────── +# warp_frame_depth +# ───────────────────────────────────────────────────────────────────────────── + + +def bench_warp_frame_depth(device: str, dtype: torch.dtype = torch.float32) -> None: + print(f"\n--- warp_frame_depth device={device} dtype={dtype} ---") + + K_base = torch.eye(3, device=device, dtype=dtype) + K_base[0, 0] = K_base[1, 1] = 500.0 + K_base[0, 2] = K_base[1, 2] = 320.0 + + configs = [ + ("B=1 C=3 H=256 W=256 ", 1, 3, 256, 256), + ("B=4 C=3 H=480 W=640 ", 4, 3, 480, 640), + ("B=1 C=3 H=720 W=1280", 1, 3, 720, 1280), + ] + + for label, B, C, H, W in configs: + K_b = K_base.unsqueeze(0).expand(B, -1, -1).contiguous() + depth = torch.rand(B, 1, H, W, device=device, dtype=dtype).add_(0.1) + image = torch.rand(B, C, H, W, device=device, dtype=dtype) + T = torch.eye(4, device=device, dtype=dtype).unsqueeze(0).expand(B, -1, -1).contiguous() + + bench(warp_frame_depth, image, depth, T, K_b, device=device, label=label) + + +# ───────────────────────────────────────────────────────────────────────────── +# Entry point +# ───────────────────────────────────────────────────────────────────────────── + + +def run_all(device: str, compile_: bool = False) -> None: + sep = "=" * 72 + print(f"\n{sep}") + print(f" DEVICE : {device.upper()}") + print(f" compile: {compile_}") + print(sep) + bench_transform_points(device, compile_=compile_) + bench_project_unproject(device, compile_=compile_) + bench_homogeneous_conversions(device) + bench_depth_functions(device, compile_=compile_) + bench_warp_frame_depth(device) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--cuda", action="store_true", help="also run on CUDA") + parser.add_argument("--compile", action="store_true", dest="compile_", help="include torch.compile variants") + args = parser.parse_args() + + _print_env() + run_all("cpu", compile_=args.compile_) + if args.cuda: + if torch.cuda.is_available(): + run_all("cuda", compile_=args.compile_) + else: + print("\nWarning: --cuda requested but CUDA is not available.") diff --git a/benchmarks/geometry/project_distort.py b/benchmarks/geometry/project_distort.py new file mode 100644 index 0000000..188a129 --- /dev/null +++ b/benchmarks/geometry/project_distort.py @@ -0,0 +1,158 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Benchmark for project/unproject_points and calibration distortion functions. + +Usage: + python benchmarks/geometry/project_distort.py + python benchmarks/geometry/project_distort.py --cuda +""" + +from __future__ import annotations + +import argparse +import datetime +import platform +import shutil +import subprocess +import time + +import torch + +from kornia.geometry.calibration.distort import distort_points +from kornia.geometry.calibration.undistort import undistort_points +from kornia.geometry.camera import project_points, unproject_points +from kornia.geometry.conversions import denormalize_points_with_intrinsics, normalize_points_with_intrinsics + +# ───────────────────────────────────────────────────────────────────────────── +# Helpers +# ───────────────────────────────────────────────────────────────────────────── + + +def _sync(device: str) -> None: + if device == "cuda": + torch.cuda.synchronize() + + +def bench(fn, *args, warmup: int = 10, reps: int = 50, device: str = "cpu", label: str = "") -> float: + for _ in range(warmup): + fn(*args) + _sync(device) + t0 = time.perf_counter() + for _ in range(reps): + fn(*args) + _sync(device) + ms = (time.perf_counter() - t0) / reps * 1000 + print(f" {label:<60s}: {ms:8.3f} ms") + return ms + + +def _print_env() -> None: + date = datetime.datetime.now(tz=datetime.UTC).strftime("%Y-%m-%d %H:%M:%S UTC") + git = shutil.which("git") or "git" + try: + commit = subprocess.check_output([git, "rev-parse", "--short", "HEAD"], text=True).strip() + except Exception: + commit = "unknown" + cpu = platform.processor() or platform.machine() + print(f" date : {date}") + print(f" commit : {commit}") + print(f" cpu : {cpu}") + if torch.cuda.is_available(): + print(f" gpu : {torch.cuda.get_device_name(0)}") + + +# ───────────────────────────────────────────────────────────────────────────── +# Benchmarks +# ───────────────────────────────────────────────────────────────────────────── + + +def bench_project_unproject(device: str) -> None: + print(f"\n--- project_points / unproject_points device={device} ---") + + K_base = torch.eye(3, device=device) + K_base[0, 0] = K_base[1, 1] = 500.0 + K_base[0, 2] = K_base[1, 2] = 320.0 + + configs = [ + ("B=1 N=1K ", 1, 1_000), + ("B=8 N=10K ", 8, 10_000), + ("B=32 N=100K", 32, 100_000), + ] + + for label, B, N in configs: + K_b = K_base.unsqueeze(0).expand(B, -1, -1) + pts3 = torch.rand(B, N, 3, device=device).add_(0.5) + pts2 = torch.rand(B, N, 2, device=device).mul_(640.0) + pts2_norm = normalize_points_with_intrinsics(pts2, K_b) + depth = torch.ones(B, N, 1, device=device) + + print(f"\n {label}") + bench(project_points, pts3, K_b, device=device, label="project_points") + bench(unproject_points, pts2, depth, K_b, device=device, label="unproject_points") + bench(normalize_points_with_intrinsics, pts2, K_b, device=device, label="normalize_points_with_intrinsics") + bench( + denormalize_points_with_intrinsics, + pts2_norm, + K_b, + device=device, + label="denormalize_points_with_intrinsics", + ) + + +def bench_distort_undistort(device: str) -> None: + print(f"\n--- distort_points / undistort_points device={device} ---") + + K_base = torch.eye(3, device=device) + K_base[0, 0] = K_base[1, 1] = 500.0 + K_base[0, 2] = K_base[1, 2] = 320.0 + dist_base = torch.tensor([0.1, -0.05, 0.001, 0.001, 0.02, 0.01, -0.005, 0.002], device=device) + + configs = [ + ("B=1 N=1K ", 1, 1_000), + ("B=1 N=100K", 1, 100_000), + ("B=32 N=10K ", 32, 10_000), + ] + + for label, B, N in configs: + K_b = K_base.unsqueeze(0).expand(B, -1, -1).contiguous() + dist_b = dist_base.unsqueeze(0).expand(B, -1).contiguous() + pts2 = torch.rand(B, N, 2, device=device).mul_(640.0) + + print(f"\n {label}") + bench(distort_points, pts2, K_b, dist_b, device=device, label="distort_points") + bench(undistort_points, pts2, K_b, dist_b, device=device, label="undistort_points (5 iters)") + + +def run(device: str) -> None: + sep = "=" * 72 + print(f"\n{sep}\n DEVICE: {device.upper()}\n{sep}") + bench_project_unproject(device) + bench_distort_undistort(device) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cuda", action="store_true") + args = parser.parse_args() + + _print_env() + run("cpu") + if args.cuda: + if torch.cuda.is_available(): + run("cuda") + else: + print("\nWarning: --cuda requested but CUDA is not available.") diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..23c6dc0 --- /dev/null +++ b/conftest.py @@ -0,0 +1,686 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import subprocess +import sys +import time +from functools import partial +from itertools import product + +import numpy as np +import pytest +import torch + +try: + from pytest import TestReport # public since pytest 7.x +except ImportError: # pragma: no cover + from _pytest.reports import TestReport # type: ignore[no-redef] + +import kornia + +try: + import torch._dynamo + + _backends_non_experimental = torch._dynamo.list_backends() +except ImportError: + _backends_non_experimental = [] + + +WEIGHTS_CACHE_DIR = "weights/" + + +def get_test_devices() -> dict[str, torch.device]: + """Create a dictionary with the devices to test the source code. + + CUDA devices will be tested only if the current hardware supports it. + + Returns: + Dictionary mapping device names to torch.device objects. + """ + devices: dict[str, torch.device] = {"cpu": torch.device("cpu")} + + if torch.cuda.is_available(): + devices["cuda"] = torch.device("cuda:0") + + if kornia.core.utils.xla_is_available(): + import torch_xla.core.xla_model as xm + + devices["tpu"] = xm.xla_device() + + if hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): + devices["mps"] = torch.device("mps:0") + + return devices + + +def get_test_dtypes() -> dict[str, torch.dtype]: + """Create a dictionary with the dtypes to test. + + Returns: + Dictionary mapping dtype names to torch.dtype objects. + """ + return { + "bfloat16": torch.bfloat16, + "float16": torch.float16, + "float32": torch.float32, + "float64": torch.float64, + } + + +# setup the devices to test the source code + +TEST_DEVICES: dict[str, torch.device] = get_test_devices() +TEST_DTYPES: dict[str, torch.dtype] = get_test_dtypes() +TEST_OPTIMIZER_BACKEND = {"", None, "jit", *_backends_non_experimental} +# Combinations of device and dtype to be excluded from testing. +# Example: DEVICE_DTYPE_BLACKLIST = {('cpu', 'float16')} +DEVICE_DTYPE_BLACKLIST: set[tuple[str, ...]] = set() + + +@pytest.fixture() +def device(device_name) -> torch.device: + """Return device for testing, skipping if device is unavailable.""" + if device_name not in TEST_DEVICES: + pytest.skip(f"Device '{device_name}' is not available on this system") + return TEST_DEVICES[device_name] + + +@pytest.fixture() +def dtype(dtype_name) -> torch.dtype: + """Return dtype for testing.""" + return TEST_DTYPES[dtype_name] + + +@pytest.fixture() +def torch_optimizer(optimizer_backend): + """Return torch optimizer based on backend selection. + + Args: + optimizer_backend: The optimization backend ('jit', 'inductor', etc.) + + Returns: + A function that optimizes/compiles torch modules or functions. + """ + if not optimizer_backend: + return lambda x: x + + if optimizer_backend == "jit": + return torch.jit.script + + torch._dynamo.reset() + return partial(torch.compile, backend=optimizer_backend) + + +def _parse_test_option(config, option: str, all_values: dict | set) -> list[str]: + """Parse a test option from CLI, expanding 'all' to full list.""" + raw_value = config.getoption(option) + if raw_value == "all": + return list(all_values.keys()) if isinstance(all_values, dict) else list(all_values) + return raw_value.split(",") + + +def pytest_generate_tests(metafunc) -> None: + """Generate test parametrization based on fixtures and CLI options.""" + # Build list of (fixture_name, values) for fixtures that are used + params: list[tuple[str, list]] = [] + + if "device_name" in metafunc.fixturenames: + params.append(("device_name", _parse_test_option(metafunc.config, "--device", TEST_DEVICES))) + if "dtype_name" in metafunc.fixturenames: + params.append(("dtype_name", _parse_test_option(metafunc.config, "--dtype", TEST_DTYPES))) + if "optimizer_backend" in metafunc.fixturenames: + params.append(("optimizer_backend", _parse_test_option(metafunc.config, "--optimizer", TEST_OPTIMIZER_BACKEND))) + + if not params: + return + + # Single parameter: pass values directly (not as tuples) + if len(params) == 1: + name, values = params[0] + metafunc.parametrize(name, values) + return + + # Multiple parameters: generate combinations and filter blacklisted ones + names = ",".join(name for name, _ in params) + values = [v for _, v in params] + combinations = [combo for combo in product(*values) if combo[:2] not in DEVICE_DTYPE_BLACKLIST] + metafunc.parametrize(names, combinations) + + +def pytest_collection_modifyitems(config, items): + """Collect test options.""" + # Deselect dynamo/compile tests when no optimizer is specified + # Check environment variable directly (not config option which has default "inductor") + optimizer_env = os.environ.get("KORNIA_TEST_OPTIMIZER", "").strip() + if not optimizer_env: + # Filter out tests with "dynamo" or "compile" in their name + items[:] = [item for item in items if "dynamo" not in item.name.lower() and "compile" not in item.name.lower()] + + # MPS does not support float64; gradcheck requires float64 — skip all gradcheck tests on MPS + skip_mps_gradcheck = pytest.mark.skip(reason="gradcheck requires float64 which is not supported on MPS") + for item in items: + if "gradcheck" in item.name.lower() and "[mps" in item.nodeid: + item.add_marker(skip_mps_gradcheck) + + # MPS does not support complex128 (cdouble); skip tests parametrized with it + skip_mps_cdouble = pytest.mark.skip(reason="MPS does not support complex128 (cdouble)") + for item in items: + if "[mps" in item.nodeid and "cdtype1" in item.nodeid: + item.add_marker(skip_mps_cdouble) + + # MPS autocast uses float16 and does not preserve original dtype — skip autocast tests on MPS + skip_mps_autocast = pytest.mark.skip(reason="MPS autocast changes dtype to float16, not supported the same way") + for item in items: + if "autocast" in item.name.lower() and "[mps" in item.nodeid: + item.add_marker(skip_mps_autocast) + + tf32_enabled = config.getoption("--tf32") + + if not config.getoption("--runslow"): + skip_slow = pytest.mark.skip(reason="need --runslow option to run") + for item in items: + if "slow" in item.keywords: + item.add_marker(skip_slow) + + # Tests marked @pytest.mark.tf32 are known to produce incorrect results when + # TF32 is active (torch.set_float32_matmul_precision("high")). When running + # without --tf32 (the default), mark them xfail so the suite stays green. + # When --tf32 is explicitly passed, run them normally so the failures are visible. + if not tf32_enabled: + xfail_tf32 = pytest.mark.xfail( + reason=( + "This test is sensitive to TF32 (TensorFloat-32) precision reduction in CUDA matrix " + "multiplications. Run with --tf32 to reproduce the failure." + ), + strict=False, + ) + for item in items: + if "tf32" in item.keywords: + item.add_marker(xfail_tf32) + + +def pytest_addoption(parser): + """Add options with environment variable fallbacks. + + Environment variables (for CI/pixi integration): + KORNIA_TEST_DEVICE: Device to test on (default: cpu) + KORNIA_TEST_DTYPE: Data type to test (default: float32) + KORNIA_TEST_OPTIMIZER: Optimizer backend (default: inductor) + KORNIA_TEST_RUNSLOW: Run slow tests (default: false) + KORNIA_TEST_TF32: Enable TF32 (TensorFloat-32) mode for CUDA matrix multiplications (default: false) + """ + parser.addoption( + "--device", + action="store", + default=os.environ.get("KORNIA_TEST_DEVICE", "cpu"), + help="Device to run tests on (env: KORNIA_TEST_DEVICE)", + ) + parser.addoption( + "--dtype", + action="store", + default=os.environ.get("KORNIA_TEST_DTYPE", "float32"), + help="Data type for tests (env: KORNIA_TEST_DTYPE)", + ) + parser.addoption( + "--optimizer", + action="store", + default=os.environ.get("KORNIA_TEST_OPTIMIZER", "inductor"), + help="Optimizer backend (env: KORNIA_TEST_OPTIMIZER)", + ) + parser.addoption( + "--runslow", + action="store_true", + default=os.environ.get("KORNIA_TEST_RUNSLOW", "false").lower() == "true", + help="Run slow tests (env: KORNIA_TEST_RUNSLOW)", + ) + parser.addoption( + "--tf32", + action="store_true", + default=os.environ.get("KORNIA_TEST_TF32", "false").lower() == "true", + help=( + "Enable TF32 (TensorFloat-32) mode for CUDA matrix multiplications " + "(torch.set_float32_matmul_precision('high')). " + "Tests marked @pytest.mark.tf32 are expected to fail under TF32 and are skipped unless this flag is set. " + "(env: KORNIA_TEST_TF32)" + ), + ) + parser.addoption( + "--isolate-half-precision", + action="store_true", + default=os.environ.get("KORNIA_TEST_ISOLATE_HALF", "false").lower() == "true", + help=( + "Run float16/bfloat16 CUDA tests in fresh subprocesses via subprocess.run. " + "Each test gets its own Python process with no shared CUDA state, so a " + "device-side assert cannot contaminate subsequent tests. " + "Without this flag, float16/bfloat16 CUDA tests are skipped. " + "(env: KORNIA_TEST_ISOLATE_HALF)" + ), + ) + + +def _setup_torch_compile() -> None: + """Warm up torch.compile to reduce first-run latency in tests.""" + print("Setting up torch compile...") + + def _dummy_fn(x, y): + return (x + y).sum() + + class _DummyModule(torch.nn.Module): + def forward(self, x): + return (x**2).sum() + + torch.compile(_dummy_fn) + torch.compile(_DummyModule()) + + +def pytest_sessionstart(session): + """Start pytest session.""" + # Enable TF32 only when explicitly requested via --tf32 / KORNIA_TEST_TF32=true. + # + # TF32 (TensorFloat-32) truncates float32 inputs to a 10-bit mantissa before + # CUDA matrix multiplications (torch.bmm, torch.mm, etc.), giving ~float16 + # mantissa precision for those ops. This can cause test failures for + # numerically sensitive operations even though the same test passes without TF32. + # By default we run with full float32 precision so that tests are deterministic + # and correct. Use --tf32 to reproduce the behaviour of torch.compile("high") + # and to run the @pytest.mark.tf32-marked tests. + if session.config.getoption("--tf32"): + torch.set_float32_matmul_precision("high") + + # Skip torch.compile warmup in subprocess mode — it adds startup overhead and + # pollutes the captured output used for failure reporting in pytest_runtest_protocol. + if not os.environ.get("KORNIA_TEST_IN_SUBPROCESS"): + try: + _setup_torch_compile() + except RuntimeError as ex: + if "not yet supported for torch.compile" not in str( + ex + ) and "Dynamo is not supported on Python 3.12+" not in str(ex): + raise ex + + os.makedirs(WEIGHTS_CACHE_DIR, exist_ok=True) + torch.hub.set_dir(WEIGHTS_CACHE_DIR) + # For HuggingFace model caching + os.environ["HF_HOME"] = WEIGHTS_CACHE_DIR + + +def _get_env_info() -> dict[str, dict[str, str]]: + if not hasattr(torch.utils, "collect_env"): + return {} + + run_lmb = torch.utils.collect_env.run + separator = ":" + br = "\n" + + def _get_key_value(v: str): + parts = v.split(separator) + return parts[0].strip(), parts[-1].strip() + + def _get_cpu_info() -> dict[str, str]: + cpu_info = {} + cpu_str = torch.utils.collect_env.get_cpu_info(run_lmb) + if not cpu_str: + return {} + + for data in cpu_str.split(br): + key, value = _get_key_value(data) + cpu_info[key] = value + + return cpu_info + + def _get_gpu_info() -> dict[str, str]: + gpu_info = {} + gpu_str = torch.utils.collect_env.get_gpu_info(run_lmb) + + if not gpu_str: + return {} + + for data in gpu_str.split(br): + key, value = _get_key_value(data) + gpu_info[key] = value + + return gpu_info + + return { + "cpu": _get_cpu_info(), + "gpu": _get_gpu_info(), + "nvidia": torch.utils.collect_env.get_nvidia_driver_version(run_lmb), + "gcc": torch.utils.collect_env.get_gcc_version(run_lmb), + } + + +def pytest_report_header(config): + """Return report header.""" + try: + import accelerate + + accelerate_info = f"accelerate-{accelerate.__version__}" + except ImportError: + accelerate_info = "`accelerate` not found" + + try: + import kornia_rs + + rs_version = kornia_rs.__version__ + except ImportError: + rs_version = "not found" + + try: + import onnx + + onnx_version = onnx.__version__ + except ImportError: + onnx_version = "not found" + + env_info = _get_env_info() + cached_weights = os.listdir(WEIGHTS_CACHE_DIR) if os.path.exists(WEIGHTS_CACHE_DIR) else [] + if "cpu" in env_info: + desired_cpu_info = ["Model name", "Architecture", "CPU(s)", "Thread(s) per core", "CPU max MHz", "CPU min MHz"] + cpu_info = "cpu info:\n" + "\n".join( + f"\t- {i}: {env_info['cpu'][i]}" for i in desired_cpu_info if i in env_info["cpu"] + ) + else: + cpu_info = "" + gpu_info = f"gpu info: {env_info['gpu']}" if "gpu" in env_info else "" + gcc_info = f"gcc info: {env_info['gcc']}" if "gcc" in env_info else "" + + return f""" +{cpu_info} +{gpu_info} +main deps: + - kornia-{kornia.__version__} + - torch-{torch.__version__} + - commit: {torch.version.git_version} + - cuda: {torch.version.cuda} + - nvidia-driver: {env_info["nvidia"] if "nvidia" in env_info else None} +x deps: + - {accelerate_info} +dev deps: + - kornia_rs-{rs_version} + - onnx-{onnx_version} +{gcc_info} +available optimizers: {TEST_OPTIMIZER_BACKEND} +model weights cached: {cached_weights} +""" + + +def _extract_failure_output(output: str) -> str: + """Return just the FAILURES/ERRORS section from pytest stdout, or full output as fallback.""" + import re + + m = re.search(r"^=+ (FAILURES|ERRORS) =+", output, re.MULTILINE) + if m: + return output[m.start() :].strip() + return output.strip() + + +def _is_subprocess_isolated_test(item) -> bool: + """Return True if this test should be run in a fresh subprocess. + + Checks that: + - ``--isolate-half-precision`` is set + - we are NOT already inside a subprocess (``KORNIA_TEST_IN_SUBPROCESS`` env var) + - the test is parametrised with a half-precision dtype on CUDA + """ + if os.environ.get("KORNIA_TEST_IN_SUBPROCESS"): + return False + if not item.config.getoption("--isolate-half-precision", default=False): + return False + callspec = getattr(item, "callspec", None) + if callspec is None: + return False + params = callspec.params + if params.get("dtype_name") not in ("float16", "bfloat16"): + return False + if params.get("device_name") != "cuda": + return False + return True + + +def pytest_runtest_protocol(item, nextitem): + """Run float16/bfloat16 CUDA tests in a fresh subprocess for true isolation. + + ``pytest-forked`` uses ``fork()``, which copies the parent's CUDA context handle + into the child. A device-side assert in the child corrupts the *same* underlying + GPU state the parent holds — so the isolation is illusory for CUDA float16. + + This hook uses ``subprocess.run`` instead, which spawns a completely independent + Python interpreter with no shared CUDA state. The child's result (pass / fail / + skip) is parsed and reported back into the parent's session as a synthetic + ``TestReport``. + """ + if not _is_subprocess_isolated_test(item): + return None # use the default protocol + + item.ihook.pytest_runtest_logstart(nodeid=item.nodeid, location=item.location) + + # Forward device/dtype so pytest_generate_tests in the subprocess produces the + # same parametrisation as the parent — without these the [cuda-float16] nodeid + # can't be found because the subprocess defaults to [cpu-float32]. + params = item.callspec.params + device_name = params.get("device_name", "cpu") + dtype_name = params.get("dtype_name", "float32") + cmd = [ + sys.executable, + "-m", + "pytest", + item.nodeid, + "--no-header", + "--tb=short", + "-q", + "--color=no", + f"--device={device_name}", + f"--dtype={dtype_name}", + ] + if item.config.getoption("--runslow"): + cmd.append("--runslow") + if item.config.getoption("--tf32"): + cmd.append("--tf32") + optimizer_backend = params.get("optimizer_backend") + if optimizer_backend: + cmd.append(f"--optimizer={optimizer_backend}") + + env = {**os.environ, "KORNIA_TEST_IN_SUBPROCESS": "1"} + t0 = time.monotonic() + proc = subprocess.run( # noqa: S603 + cmd, capture_output=True, text=True, cwd=str(item.config.rootdir), env=env, check=False + ) + duration = time.monotonic() - t0 + output = (proc.stdout + proc.stderr).strip() + + # exit code 5 → no tests collected (test was deselected or already parametrised away) + if proc.returncode == 5: + outcome: str = "skipped" + longrepr = ("", 0, "subprocess: no tests collected") + elif proc.returncode == 0: + # Distinguish a genuine pass from a skipped test + if "passed" not in output and "skipped" in output: + skip_line = next( + (ln.strip() for ln in output.splitlines() if "SKIP" in ln.upper()), "skipped in subprocess" + ) + outcome = "skipped" + longrepr = ("", 0, skip_line) + else: + outcome = "passed" + longrepr = None + else: + outcome = "failed" + longrepr = _extract_failure_output(output) + + def _report(when: str, out: str, rep_longrepr, dur: float = 0.0) -> TestReport: + return TestReport( + nodeid=item.nodeid, + location=item.location, + keywords=dict(item.keywords), + outcome=out, + longrepr=rep_longrepr, + when=when, + duration=dur, + ) + + for rep in [ + _report("setup", "passed", None), + _report("call", outcome, longrepr, duration), + _report("teardown", "passed", None), + ]: + item.ihook.pytest_runtest_logreport(report=rep) + + item.ihook.pytest_runtest_logfinish(nodeid=item.nodeid, location=item.location) + return True + + +@pytest.fixture(autouse=True) +def skip_half_precision_on_cuda(request): + """Skip float16/bfloat16 CUDA tests unless running inside a subprocess. + + CUDA device-side asserts are asynchronous: a failing half-precision kernel does not + raise immediately but corrupts the CUDA context until the next synchronisation point, + which may be inside a completely different (float32) test. Once triggered, the + context is permanently broken for the process — all subsequent CUDA ops fail. + + Default behaviour (no flag): float16/bfloat16 CUDA tests are *skipped*. + + With ``--isolate-half-precision`` (or ``KORNIA_TEST_ISOLATE_HALF=true``): each + float16/bfloat16 CUDA test is intercepted by ``pytest_runtest_protocol`` *before* + any fixture runs and executed in a fresh ``subprocess.run`` process. This fixture + only runs inside those subprocesses (where ``KORNIA_TEST_IN_SUBPROCESS=1`` is set) + and exits immediately so the test proceeds normally. + + Usage:: + + pytest tests/color/ --device=cuda --dtype=bfloat16 --isolate-half-precision + pytest tests/ --device=cuda --dtype=all --isolate-half-precision + """ + # Inside a subprocess spawned by pytest_runtest_protocol — run the test normally. + if os.environ.get("KORNIA_TEST_IN_SUBPROCESS"): + return + + if "dtype" not in request.fixturenames: + return + dtype = request.getfixturevalue("dtype") + if dtype not in (torch.bfloat16, torch.float16): + return + if "device" not in request.fixturenames: + return + + try: + device = request.getfixturevalue("device") + except pytest.FixtureLookupError: + return + + if device.type != "cuda": + return + + if not request.config.getoption("--isolate-half-precision"): + dtype_name = "bfloat16" if dtype == torch.bfloat16 else "float16" + pytest.skip( + f"{dtype_name} on CUDA: skipped by default to prevent device-side assert contamination. " + "Run with --isolate-half-precision to execute in isolated subprocesses." + ) + + +@pytest.fixture(autouse=True) +def cuda_device_assert_guard(request): + """Guard against CUDA device-side assert contamination between tests. + + Active only when running inside a subprocess (``KORNIA_TEST_IN_SUBPROCESS=1``) + or with ``--isolate-half-precision``, so regular float32 CI is not slowed + down by the extra host-device synchronisations. + + This fixture synchronises CUDA before each test; if the context is already + corrupted the test is skipped rather than allowed to fail spuriously. + After each test a second synchronisation drains the queue so any async + device-side assert surfaces in the test that caused it, not the next one. + If a device-side assert is detected in the post-test sync the test is + failed (not silently passed) so asynchronous errors are always visible. + """ + in_subprocess = os.environ.get("KORNIA_TEST_IN_SUBPROCESS") + isolate = request.config.getoption("--isolate-half-precision", default=False) + if not (in_subprocess or isolate): + yield + return + + if "device" not in request.fixturenames: + yield + return + + try: + device = request.getfixturevalue("device") + except pytest.FixtureLookupError: + yield + return + + if device.type != "cuda": + yield + return + + # Pre-test: verify the CUDA context is healthy. + try: + torch.cuda.synchronize(device) + except RuntimeError: + pytest.skip("CUDA context corrupted by a device-side assert in a previous test; run this test in isolation") + + yield + + # Post-test: drain the CUDA queue so any async device-side assert surfaces here, + # in the test that caused it, rather than at the start of the next test. + # Fail the test if a device-side assert is detected so it is not silently passed. + try: + torch.cuda.synchronize(device) + except RuntimeError as exc: + torch.cuda.empty_cache() + pytest.fail(f"CUDA device-side assert triggered during this test: {exc}") + + +@pytest.fixture(autouse=True) +def add_doctest_deps(doctest_namespace): + """Add dependencies for doctests.""" + doctest_namespace["np"] = np + doctest_namespace["torch"] = torch + doctest_namespace["kornia"] = kornia + + +# Test data commit hashes from kornia/data_test repository +_DATA_TEST_SHA = { + "loftr": "cb8f42bf28b9f347df6afba5558738f62a11f28a", + "adalam": "f7d8da661701424babb64850e03c5e8faec7ea62", + "disk": "8b98f44abbe92b7a84631ed06613b08fee7dae14", + "xfeat": "279e95e411f2d3926953dea3842347242190f4da", +} + +# URLs for test data files +_TEST_DATA_URLS: dict[str, str] = { + "loftr_homo": f"https://github.com/kornia/data_test/blob/{_DATA_TEST_SHA['loftr']}/loftr_outdoor_and_homography_data.pt?raw=true", + "loftr_fund": f"https://github.com/kornia/data_test/blob/{_DATA_TEST_SHA['loftr']}/loftr_indoor_and_fundamental_data.pt?raw=true", + "adalam_idxs": f"https://github.com/kornia/data_test/blob/{_DATA_TEST_SHA['adalam']}/adalam_test.pt?raw=true", + "lightglue_idxs": f"https://github.com/kornia/data_test/blob/{_DATA_TEST_SHA['adalam']}/adalam_test.pt?raw=true", + "disk_outdoor": f"https://github.com/kornia/data_test/blob/{_DATA_TEST_SHA['disk']}/knchurch_disk.pt?raw=true", + "xfeat_outdoor": f"https://github.com/kornia/data_test/blob/{_DATA_TEST_SHA['xfeat']}/xfeat_reference.pt?raw=true", + "dexined": "https://cmp.felk.cvut.cz/~mishkdmy/models/DexiNed_BIPED_10.pth", +} + + +@pytest.fixture(scope="session") +def data(request): + """Load test data from remote URL. + + Use with @pytest.mark.parametrize("data", ["loftr_homo"], indirect=True) + """ + if request.param not in _TEST_DATA_URLS: + raise ValueError(f"Unknown test data: {request.param}. Available: {list(_TEST_DATA_URLS.keys())}") + return torch.hub.load_state_dict_from_url(_TEST_DATA_URLS[request.param], map_location=torch.device("cpu")) diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..98a840b --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,5 @@ +build/doctest +build/html/.buildinfo +build/html/objects.inv +source/tutorials/* +source/_static/img/*.png diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..4786ee8 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,27 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = kornia +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +clean: + @echo "Removing everything under 'build'.." + @rm -rf $(BUILDDIR)/html/ $(BUILDDIR)/doctrees + +images: + python3 generate_examples.py diff --git a/docs/generate_examples.py b/docs/generate_examples.py new file mode 100644 index 0000000..b9cda52 --- /dev/null +++ b/docs/generate_examples.py @@ -0,0 +1,751 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import importlib +import math +import os +from pathlib import Path +from typing import Optional + +import cv2 +import matplotlib as mpl +import matplotlib.pyplot as plt +import numpy as np +import requests +import torch +import torch.nn.functional as F +from kornia_moons.feature import visualize_LAF + +import kornia as K + +mpl.use("Agg") + + +def download_tutorials_examples(download_infos: dict[str, str], directory: Path): + URL_BASE = "https://raw.githubusercontent.com/kornia/tutorials/master/" + for filename, path in download_infos.items(): + url = URL_BASE + path + # perform request + response = requests.get(url, timeout=60).content + + path = directory / filename + with open(path, "wb") as fp: + fp.write(response) + + +def read_img_from_url(url: str, resize_to: Optional[tuple[int, int]] = None, **resize_kwargs) -> torch.Tensor: + # perform request + response = requests.get(url, timeout=60).content + # convert to array of ints + nparr = np.frombuffer(response, np.uint8) + # convert to image array and resize + img: np.ndarray = cv2.imdecode(nparr, cv2.IMREAD_UNCHANGED)[..., :3] + # convert the image to a tensor + img_t: torch.Tensor = K.image.image_to_tensor(img, keepdim=False) # 1xCxHXW + img_t = img_t.float() / 255.0 + if resize_to is None: + img_t = K.geometry.resize(img_t, 184, **resize_kwargs) + else: + img_t = K.geometry.resize(img_t, resize_to, **resize_kwargs) + return img_t + + +def transparent_pad(src: torch.Tensor, shape: tuple[int, int]) -> torch.Tensor: + """Apply a transparent pad to src (centerized) to match with shape (h, w)""" + w_pad = abs(int(src.shape[-1] - shape[-1]) // 2) + h_pad = abs(int(src.shape[-2] - shape[-2]) // 2) + return F.pad(K.color.rgb_to_rgba(src, 1.0), (w_pad, w_pad, h_pad, h_pad), "constant", 0.0) + + +def draw_bbox_kpts(imgs: torch.Tensor, bboxes: torch.Tensor, keypoints: torch.Tensor) -> torch.Tensor: + rectangle = torch.zeros(imgs.shape[0], imgs.shape[1], 4) + rectangle[..., 0] = bboxes[..., 0] # x1 + rectangle[..., 1] = bboxes[..., 1] # y1 + rectangle[..., 2] = bboxes[..., 0] + bboxes[..., -2] # x2 + rectangle[..., 3] = bboxes[..., 1] + bboxes[..., -1] # y2 + color = torch.tensor([1, 0, 0]).repeat(imgs.shape[0], imgs.shape[1], 1) + imgs_draw = K.image.draw_rectangle(imgs, rectangle, color=color) + + rectangle2 = torch.zeros(imgs.shape[0], imgs.shape[1], 4) + for n in range(keypoints.shape[-2]): + rectangle2[..., n, 0] = keypoints[..., n, 0] - 2 + rectangle2[..., n, 1] = keypoints[..., n, 1] - 2 + rectangle2[..., n, 2] = keypoints[..., n, 0] + 2 + rectangle2[..., n, 3] = keypoints[..., n, 1] + 2 + color = torch.tensor([0, 0, 1]).repeat(imgs.shape[0], imgs.shape[1], 1) + imgs_draw = K.utils.draw_rectangle(imgs_draw, rectangle2, color=color, fill=True) + + return imgs_draw + + +def main(): + # Download the tutorial examples for the main docs + # Note: Training API examples (image_classifier, object_detection, semantic_segmentation) removed + # as they depend on kornia.x which has been removed + URLS_TUTORIALS_EXAMPLES = {} + + OUTPUT_PATH_SCRIPTS = Path(__file__).absolute().parent / "source/_static/scripts/" + + os.makedirs(OUTPUT_PATH_SCRIPTS, exist_ok=True) + print(f"Downloading script examples from kornia/tutorials. Saving into the path {OUTPUT_PATH_SCRIPTS}.") + download_tutorials_examples(URLS_TUTORIALS_EXAMPLES, OUTPUT_PATH_SCRIPTS) + + # load the images + BASE_IMAGE_URL1: str = "https://raw.githubusercontent.com/kornia/data/main/panda.jpg" # augmentation + BASE_IMAGE_URL2: str = "https://raw.githubusercontent.com/kornia/data/main/simba.png" # color + BASE_IMAGE_URL3: str = "https://raw.githubusercontent.com/kornia/data/main/girona.png" # enhance + BASE_IMAGE_URL4: str = "https://raw.githubusercontent.com/kornia/data/main/baby_giraffe.png" # morphology + BASE_IMAGE_URL5: str = "https://raw.githubusercontent.com/kornia/data/main/persistencia_memoria.jpg" # filters + BASE_IMAGE_URL6: str = "https://raw.githubusercontent.com/kornia/data/main/delorean.png" # geometry + hash1 = "8b98f44abbe92b7a84631ed06613b08fee7dae14" + BASE_IMAGEOUTDOOR_URL7: str = f"https://github.com/kornia/data_test/raw/{hash1}/knchurch_disk.pt" # image matching + BASE_IMAGEOUTDOOR_URL8: str = ( # Response functions + "https://github.com/kornia/data/raw/main/kornia_banner_pixie.png" + ) + MASK_IMAGE_URL2: str = "https://raw.githubusercontent.com/kornia/data/main/simba_mask.png" + + OUTPUT_PATH = Path(__file__).absolute().parent / "source/_static/img" + + os.makedirs(OUTPUT_PATH, exist_ok=True) + print(f"Pointing images to path {OUTPUT_PATH}.") + img1 = read_img_from_url(BASE_IMAGE_URL1) + img2 = read_img_from_url(BASE_IMAGE_URL2, img1.shape[-2:]) + img3 = read_img_from_url(BASE_IMAGE_URL3, img1.shape[-2:]) + img4 = read_img_from_url(BASE_IMAGE_URL4) + img5 = read_img_from_url(BASE_IMAGE_URL5, (234, 320)) + img6 = read_img_from_url(BASE_IMAGE_URL6) + img_kornia = read_img_from_url(BASE_IMAGEOUTDOOR_URL8) + + # Read the masks as (B, H, W) + mask2 = read_img_from_url(MASK_IMAGE_URL2, img1.shape[-2:], interpolation="nearest") + mask2 = mask2.median(dim=1)[0] + + # TODO: make this more generic for modules out of kornia.augmentation + # Dictionary containing the transforms to generate the sample images: + # Key: Name of the transform class. + # Value: (parameters, num_samples, seed) + mod = importlib.import_module("kornia.augmentation") + augmentations_list: dict = { + "CenterCrop": ((184, 184), 1, 2018), + "ColorJiggle": ((0.3, 0.3, 0.3, 0.3), 2, 2018), + "ColorJitter": ((0.3, 0.3, 0.3, 0.3), 2, 2022), + "PadTo": (((220, 450),), 1, 2022), + "RandomAffine": (((-15.0, 20.0), (0.1, 0.1), (0.7, 1.3), 20), 2, 2019), + "RandomBoxBlur": (((7, 7),), 1, 2020), + "RandomBrightness": (((0.0, 1.0),), 2, 2022), + "RandomContrast": (((0.0, 1.0),), 2, 2022), + "RandomCrop": ((img1.shape[-2:], (50, 50)), 2, 2020), + "RandomChannelDropout": ((), 1, 2020), + "RandomChannelShuffle": ((), 1, 2020), + "RandomElasticTransform": (((63, 63), (32, 32), (2.0, 2.0)), 2, 2018), + "RandomEqualize": ((), 1, 2020), + "RandomErasing": (((0.2, 0.4), (0.3, 1 / 0.3)), 2, 2017), + "RandomFisheye": ((torch.tensor([-0.3, 0.3]), torch.tensor([-0.3, 0.3]), torch.tensor([0.9, 1.0])), 2, 2020), + "RandomGamma": (((0.0, 1.0),), 2, 2022), + "RandomGaussianBlur": (((3, 3), (0.1, 2.0)), 1, 2020), + "RandomGaussianIllumination": (((0.5, 0.5), (0.5, 0.5), (0.5, 0.5), (-1.0, 1.0)), 2, 2021), + "RandomGaussianNoise": ((0.0, 0.05), 1, 2020), + "RandomGrayscale": ((), 1, 2020), + "RandomHue": (((-0.5, 0.5),), 2, 2022), + "RandomHorizontalFlip": ((), 1, 2020), + "RandomInvert": ((), 1, 2020), + "RandomJPEG": (((1.0, 5.0),), 1, 2024), + "RandomLinearCornerIllumination": (((0.5, 0.5), (-1.0, 1.0)), 2, 2021), + "RandomLinearIllumination": (((0.5, 0.5), (-1.0, 1.0)), 2, 2021), + "RandomMedianBlur": (((3, 3),), 1, 2023), + "RandomMotionBlur": ((7, 35.0, 0.5), 2, 2020), + "RandomPerspective": ((0.2,), 2, 2020), + "RandomPlanckianJitter": ((), 2, 2022), + "RandomPlasmaShadow": (((0.2, 0.5),), 2, 2022), + "RandomPlasmaBrightness": ((), 2, 2022), + "RandomPlasmaContrast": ((), 2, 2022), + "RandomPosterize": (((1, 4),), 2, 2016), + "RandomResizedCrop": ((img1.shape[-2:], (1.0, 2.0), (1.0, 2.0)), 2, 2020), + "RandomRotation": ((45.0,), 2, 2019), + "RandomSaltAndPepperNoise": (((0.05, 0.5), (0.1, 0.7)), 2, 2024), + "RandomSaturation": (((0.5, 5.0),), 2, 2022), + "RandomSharpness": ((16.0,), 1, 2019), + "RandomSolarize": ((0.2, 0.2), 2, 2019), + "RandomVerticalFlip": ((), 1, 2020), + "RandomThinPlateSpline": ((), 1, 2020), + "RandomJigsaw": ((), 2, 2020), + } + + # ITERATE OVER THE TRANSFORMS + for aug_name, (args, num_samples, seed) in augmentations_list.items(): + img_in = img1.repeat(num_samples, 1, 1, 1) + # dynamically create the class instance + cls = getattr(mod, aug_name) + try: + aug = cls(*args, p=1.0) + except TypeError: + aug = cls(*args) + + # set seed + torch.manual_seed(seed) + if aug_name == "RandomJigsaw": # make sure the image is dividable + img_in = K.geometry.resize(img_in, (1020, 500)) + elif aug_name == "RandomJPEG": + img_in = img_in[..., :176, :] + # apply the augmentation to the image and concat + out = aug(img_in) + + # save ori image to concatenate into the out image + ori = img_in[0] + if aug_name == "CenterCrop": + # Convert to RGBA, and center the output image with transparent pad + out = transparent_pad(out, tuple(img1[-2:].shape)) + ori = K.color.rgb_to_rgba(ori, 1.0) # To match the dims + elif aug_name == "PadTo": + # Convert to RGBA, and center the original image with transparent pad + ori = transparent_pad(img_in[0], tuple(out.shape[-2:])) + out = K.color.rgb_to_rgba(out, 1.0) # To match the dims + + out = torch.cat([ori, *(out[i] for i in range(out.size(0)))], dim=-1) + # save the output image + out_np = K.image.tensor_to_image((out * 255.0).byte()) + cv2.imwrite(str(OUTPUT_PATH / f"{aug_name}.png"), out_np) + sig = f"{aug_name}({', '.join([str(a) for a in args])}, p=1.0)" + print(f"Generated image example for {aug_name}. {sig}") + + mix_augmentations_list = {"RandomMixUpV2": ((), 2, 20), "RandomCutMixV2": ((), 2, 2019), "PatchMix": ((), 2, 2024)} + # ITERATE OVER THE TRANSFORMS + for aug_name, (args, _, seed) in mix_augmentations_list.items(): + img_in = torch.cat([img1, img2]) + # dynamically create the class instance + cls = getattr(mod, aug_name) + aug = cls(*args, p=1.0) + # set seed + torch.manual_seed(seed) + # apply the augmentation to the image and concat + # PatchMix returns (B, C, H, W); index [0] to get (C, H, W) for cat + if aug_name == "PatchMix": + img_aug = aug(img_in)[0] + else: + img_aug, _ = aug(img_in, torch.tensor([0, 1])) + + output = torch.cat([img_in[0], img_in[1], img_aug], dim=-1) + # save the output image + out_np = K.utils.tensor_to_image((output * 255.0).byte()) + cv2.imwrite(str(OUTPUT_PATH / f"{aug_name}.png"), out_np) + sig = f"{aug_name}({', '.join([str(a) for a in args])}, p=1.0)" + print(f"Generated image example for {aug_name}. {sig}") + + mask_augmentations_list = {"RandomTransplantation": (([0],), 0)} + # ITERATE OVER THE TRANSFORMS + for aug_name, (args, seed) in mask_augmentations_list.items(): + img_in = torch.cat([img1, img2]) + mask_in = torch.cat([torch.zeros_like(mask2), mask2]) + + # dynamically create the class instance + cls = getattr(mod, aug_name) + aug = cls(*args, p=1.0) + # set seed + torch.manual_seed(seed) + # apply the augmentation to the image and concat + img_aug, _ = aug(img_in, mask_in) + + output = torch.cat([img_in[0], img_in[1], img_aug[0]], dim=-1) + # save the output image + out_np = K.utils.tensor_to_image((output * 255.0).byte()) + cv2.imwrite(str(OUTPUT_PATH / f"{aug_name}.png"), out_np) + sig = f"{aug_name}({', '.join([str(a) for a in args])}, p=1.0)" + print(f"Generated image example for {aug_name}. {sig}") + + # Containers + aug_container_list = { + "AugmentationSequential": ( + { + "args": ( + K.augmentation.ColorJitter(0.1, 0.1, 0.1, 0.1, p=1.0), + K.augmentation.RandomAffine(360, [0.1, 0.1], [0.7, 1.2], [30.0, 50.0], p=1.0), + K.augmentation.RandomPerspective(0.5, p=1.0), + ), + "data_keys": ["input", "bbox_xywh", "keypoints"], + }, + ( + torch.tensor([[[125, 5, 115, 80]]], dtype=torch.float32), # bbox + torch.tensor([[[166, 42], [197, 42]]], dtype=torch.float32), # keypoints + ), + 2, + 2023, + ), + "PatchSequential": ( + { + "args": ( + K.augmentation.ColorJitter(0.2, 0.1, 0.1, 0.1, p=1), + K.augmentation.RandomAffine(10, [0.1, 0.2], [0.7, 1.2], [0.0, 15.0], p=1), + K.augmentation.RandomPerspective(0.3, p=1), + K.augmentation.RandomSolarize(0.01, 0.05, p=0.6), + ), + "grid_size": (2, 2), + "same_on_batch": False, + "patchwise_apply": False, + }, + (), + 2, + 2023, + ), + } + for aug_name, (args, labels, num_samples, seed) in aug_container_list.items(): + img_in = img1.repeat(num_samples, 1, 1, 1) + cls = getattr(mod, aug_name) + tfms = args.pop("args") + augs = cls(*tfms, **args) + + # set seed + torch.manual_seed(seed) + if aug_name == "PatchSequential": + out = augs(img_in) + inp = img_in + else: + labels = (labels[0].expand(num_samples, -1, -1), labels[1].expand(num_samples, -1, -1)) + + out = augs(img_in, *labels) + out = draw_bbox_kpts(out[0], out[1].int(), out[2].int()) + inp = draw_bbox_kpts(img_in, labels[0].int(), labels[1].int()) + output = torch.cat([inp[0], *(out[i] for i in range(out.size(0)))], dim=-1) + + # save the output image + out_np = K.utils.tensor_to_image((output * 255.0).byte()) + cv2.imwrite(str(OUTPUT_PATH / f"{aug_name}.png"), out_np) + sig = f"{aug_name}({', '.join([str(a) for a in args])}, p=1.0)" + print(f"Generated image example for {aug_name}. {sig}") + + # ------------------------------------------------------------------------------------ + mod = importlib.import_module("kornia.color") + color_transforms_list: dict = { + "grayscale_to_rgb": ((), 3), + "rgb_to_bgr": ((), 1), + "rgb_to_grayscale": ((), 1), + "rgb_to_hsv": ((), 1), + "rgb_to_hls": ((), 1), + "rgb_to_luv": ((), 1), + "rgb_to_lab": ((), 1), + # "rgb_to_rgba": ((1.,), 1), + "rgb_to_xyz": ((), 1), + "rgb_to_ycbcr": ((), 1), + "rgb_to_yuv": ((), 1), + "rgb_to_linear_rgb": ((), 1), + "apply_colormap": ((K.color.ColorMap("autumn", 256),), 1), + "ApplyColorMap": ((K.color.ColorMap("winter", 256),), 1), + } + # ITERATE OVER THE TRANSFORMS + for fn_name, (args, _) in color_transforms_list.items(): + # import function and apply + fn = getattr(mod, fn_name) + if fn_name == "grayscale_to_rgb": + out = fn(K.color.rgb_to_grayscale(img2), *args) + elif fn_name == "apply_colormap": + gray_image = (K.color.rgb_to_grayscale(img2) * 255.0).round() + out = K.color.rgb_to_bgr(fn(gray_image, *args)) + elif fn_name == "ApplyColorMap": + gray_image = (K.color.rgb_to_grayscale(img2) * 255.0).round() + out = K.color.rgb_to_bgr(fn(*args)(gray_image)) + else: + out = fn(img2, *args) + # perform normalization to visualize + if fn_name == "rgb_to_lab": + out = out[:, :1] / 100.0 + elif fn_name == "rgb_to_hsv": + out[:, :1] = out[:, :1] / 2 * math.pi + elif fn_name == "rgb_to_luv": + out = out[:, :1] / 116.0 + # repeat channels for grayscale + if out.shape[1] != 3: + out = out.repeat(1, 3, 1, 1) + # save the output image + if fn_name in ("grayscale_to_rgb", "apply_colormap", "ApplyColorMap"): + out = torch.cat( + [K.color.rgb_to_grayscale(img2[0]).repeat(3, 1, 1), *(out[i] for i in range(out.size(0)))], dim=-1 + ) + else: + out = torch.cat([img2[0], *(out[i] for i in range(out.size(0)))], dim=-1) + out_np = K.image.tensor_to_image((out * 255.0).byte()) + cv2.imwrite(str(OUTPUT_PATH / f"{fn_name}.png"), out_np) + sig = f"{fn_name}({', '.join([str(a) for a in args])})" + print(f"Generated image example for {fn_name}. {sig}") + + # korna.color.colormap + colormaps_list = {"AUTUMN": (256,)} + bar_img_gray = torch.arange(0, 256).repeat(1, 40, 1) # 1x1x40x256 + bar_img = K.color.grayscale_to_rgb(bar_img_gray) + # ITERATE OVER THE COLORMAPS + for colormap_name, args in colormaps_list.items(): + cm = K.color.ColorMap(base=colormap_name, num_colors=args[0]) + out = K.color.rgb_to_bgr(K.color.apply_colormap(bar_img_gray, cm))[0] + + out = torch.cat([bar_img, out], dim=-1) + + out_np = K.image.tensor_to_image((out * 255.0).byte()) + cv2.imwrite(str(OUTPUT_PATH / f"{colormap_name}.png"), out_np) + sig = f"{colormap_name}({', '.join([str(a) for a in args])})" + print(f"Generated image example for {colormap_name}. {sig}") + + # Plot for all ColorMaps (ColorMapType.png) + height_image = 40 + num_colors = 256 + num_columns = 3 + + # 1 x height_image x num_colors + input_tensor = torch.arange(start=0, end=num_colors, step=1).unsqueeze(0).repeat(1, height_image, 1) + input_tensor = input_tensor.to("cpu").to(torch.float32) + + # Get colormap list + colormap_list = K.color.ColorMapType.list() + num_colormaps = len(colormap_list) + # Calculate number of rows needed + num_rows = (num_colormaps + num_columns - 1) // num_columns + + # Create figure and axis objects + fig, axes = plt.subplots(num_rows, num_columns, figsize=(12, 8)) + for i, ax in enumerate(axes.flat): + if i < num_colormaps: + cmap = K.color.ColorMap(base=colormap_list[i], num_colors=num_colors) + res = K.color.ApplyColorMap(colormap=cmap)(input_tensor)[0] + ax.imshow(res.permute(1, 2, 0).numpy()) + ax.set_title(colormap_list[i], fontsize=12) + ax.axis("off") + else: + fig.delaxes(ax) + fig.tight_layout() + fig.savefig(os.path.join(OUTPUT_PATH, "ColorMapType.png"), dpi=300) + + # korna.enhance module + mod = importlib.import_module("kornia.enhance") + transforms: dict = { + "adjust_brightness": ((torch.tensor([0.25, 0.5]),), 2), + "adjust_contrast": ((torch.tensor([0.65, 0.5]),), 2), + "adjust_gamma": ((torch.tensor([0.85, 0.75]), 2.0), 2), + "adjust_hue": ((torch.tensor([-math.pi / 4, math.pi / 4]),), 2), + "adjust_saturation": ((torch.tensor([1.0, 2.0]),), 2), + "solarize": ((torch.tensor([0.8, 0.5]), torch.tensor([-0.25, 0.25])), 2), + "posterize": ((torch.tensor([4, 2]),), 2), + "sharpness": ((torch.tensor([1.0, 2.5]),), 2), + "equalize": ((), 1), + "invert": ((), 1), + "equalize_clahe": ((), 1), + "add_weighted": ((0.75, 0.25, 2.0), 1), + "jpeg_codec_differentiable": ((torch.tensor([50]),), 1), + } + # ITERATE OVER THE TRANSFORMS + for fn_name, (args, num_samples) in transforms.items(): + img_in = img3.repeat(num_samples, 1, 1, 1) + if fn_name == "jpeg_codec_differentiable": + img_in = img_in[..., :176, :] + if fn_name == "add_weighted": + args_in = (img_in, args[0], img2, args[1], args[2]) + else: + args_in = (img_in, *args) + # import function and apply + fn = getattr(mod, fn_name) + out = fn(*args_in) + # save the output image + out = torch.cat([img_in[0], *(out[i] for i in range(out.size(0)))], dim=-1) + out_np = K.image.tensor_to_image((out * 255.0).byte()) + cv2.imwrite(str(OUTPUT_PATH / f"{fn_name}.png"), out_np) + sig = f"{fn_name}({', '.join([str(a) for a in args])})" + print(f"Generated image example for {fn_name}. {sig}") + + # korna.morphology module + mod = importlib.import_module("kornia.morphology") + kernel = torch.tensor([[0, 1, 0], [1, 1, 1], [0, 1, 0]]) + transforms: dict = { + "dilation": ((kernel,), 1), + "erosion": ((kernel,), 1), + "opening": ((kernel,), 1), + "closing": ((kernel,), 1), + "gradient": ((kernel,), 1), + "top_hat": ((kernel,), 1), + "bottom_hat": ((kernel,), 1), + } + # ITERATE OVER THE TRANSFORMS + for fn_name, (args, num_samples) in transforms.items(): + img_in = img4.repeat(num_samples, 1, 1, 1) + args_in = (img_in, *args) + # import function and apply + # import pdb;pdb.set_trace() + fn = getattr(mod, fn_name) + out = fn(*args_in) + # save the output image + out = torch.cat([img_in[0], *(out[i] for i in range(out.size(0)))], dim=-1) + out_np = K.image.tensor_to_image((out * 255.0).byte()) + cv2.imwrite(str(OUTPUT_PATH / f"{fn_name}.png"), out_np) + sig = f"{fn_name}({', '.join([str(a) for a in args])})" + print(f"Generated image example for {fn_name}. {sig}") + + # korna.filters module + mod = importlib.import_module("kornia.filters") + kernel = torch.tensor([[0, 1, 0], [1, 1, 1], [0, 1, 0]]) + transforms: dict = { + "bilateral_blur": (((11, 11), 0.1, (3, 3)), 1), + "joint_bilateral_blur": (((11, 11), 0.1, (3, 3)), 1), + "box_blur": (((5, 5),), 1), + "median_blur": (((5, 5),), 1), + "gaussian_blur2d": (((5, 5), (1.5, 1.5)), 1), + "guided_blur": (((5, 5), 0.01), 1), + "motion_blur": ((5, 90.0, 1.0), 1), + "max_blur_pool2d": ((5,), 1), + "blur_pool2d": ((5,), 1), + "unsharp_mask": (((5, 5), (1.5, 1.5)), 1), + "laplacian": ((5,), 1), + "sobel": ((), 1), + "spatial_gradient": ((), 1), + "canny": ((), 1), + } + # ITERATE OVER THE TRANSFORMS + for fn_name, (args, num_samples) in transforms.items(): + img_in = img5.repeat(num_samples, 1, 1, 1) + if fn_name == "joint_bilateral_blur": + guide = K.geometry.resize(img2.repeat(num_samples, 1, 1, 1), img_in.shape[-2:]) + args_in = (img_in, guide, *args) + elif fn_name == "guided_blur": + args_in = (img_in, img_in, *args) + else: + args_in = (img_in, *args) + # import function and apply + fn = getattr(mod, fn_name) + out = fn(*args_in) + if fn_name in ("max_blur_pool2d", "blur_pool2d"): + out = K.geometry.resize(out, img_in.shape[-2:]) + if fn_name == "canny": + out = out[1].repeat(1, 3, 1, 1) + if isinstance(out, torch.Tensor): + out = out.clamp(min=0.0, max=1.0) + if fn_name in ("laplacian", "sobel", "spatial_gradient", "canny"): + out = K.enhance.normalize_min_max(out) + if fn_name == "spatial_gradient": + out = out.permute(2, 1, 0, 3, 4).squeeze() + if fn_name == "joint_bilateral_blur": + out = torch.cat([args_in[1], out], dim=-1) + # save the output image + out = torch.cat([img_in[0], *(out[i] for i in range(out.size(0)))], dim=-1) + out_np = K.image.tensor_to_image((out * 255.0).byte()) + cv2.imwrite(str(OUTPUT_PATH / f"{fn_name}.png"), out_np) + sig = f"{fn_name}({', '.join([str(a) for a in args])})" + print(f"Generated image example for {fn_name}. {sig}") + + # kornia.filters.in_range + mod = importlib.import_module("kornia.filters") + transforms: dict = { + "in_range": (((0.314, 0.2, 0.2), (0.47, 1.0, 1.0), True), 1), + } + # ITERATE OVER THE TRANSFORMS + for fn_name, (args, _) in transforms.items(): + img_hsv = K.color.rgb_to_hsv(img1) + h, s, v = torch.split(img_hsv, split_size_or_sections=1, dim=1) + h = h / (2 * torch.pi) + img_hsv = torch.cat((h, s, v), dim=1) + args_in = (img_hsv, *args) + fn = getattr(mod, fn_name) + mask = fn(*args_in) + filtered = img1 * mask + mask = mask.repeat(1, img1.shape[1], 1, 1) + # save the output image + out = torch.cat([img1[0], mask[0], filtered[0]], dim=-1) + out_np = K.image.tensor_to_image((out * 255.0).byte()) + cv2.imwrite(str(OUTPUT_PATH / f"{fn_name}.png"), out_np) + sig = f"{fn_name}({', '.join([str(a) for a in args])})" + print(f"Generated image example for {fn_name}. {sig}") + + # korna.geometry.transform module + mod = importlib.import_module("kornia.geometry.transform") + h, w = img6.shape[-2:] + + def _get_tps_args(): + src = torch.tensor([[[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, -1.0], [0.0, 0.0]]]).repeat(2, 1, 1) # Bx5x2 + dst = src + torch.distributions.Uniform(-0.2, 0.2).rsample((2, 5, 2)) + kernel, affine = K.geometry.transform.get_tps_transform(dst, src) + return src, kernel, affine + + transforms: dict = { + "warp_affine": ( + ( + K.geometry.transform.get_affine_matrix2d( + translations=torch.zeros(2, 2), + center=(torch.tensor([w, h]) / 2).repeat(2, 1), + scale=torch.distributions.Uniform(0.5, 1.5).rsample((2, 2)), + angle=torch.tensor([-25.0, 25.0]), + )[:, :2, :3], + (h, w), + ), + 2, + ), + "remap": ( + ( + *(K.geometry.create_meshgrid(h, w, normalized_coordinates=True) - 0.25).unbind(-1), + "bilinear", + "zeros", + True, + True, + ), + 1, + ), + "warp_image_tps": ((_get_tps_args()), 2), + "rotate": ((torch.tensor([-15.0, 25.0]),), 2), + "translate": ((torch.tensor([[10.0, -15], [50.0, -25.0]]),), 2), + "scale": ((torch.tensor([[0.5, 1.25], [1.0, 1.5]]),), 2), + "shear": ((torch.tensor([[0.1, -0.2], [-0.2, 0.1]]),), 2), + "rot180": ((), 1), + "hflip": ((), 1), + "vflip": ((), 1), + "resize": (((120, 220),), 1), + "rescale": ((0.5,), 1), + "elastic_transform2d": ((torch.rand(1, 2, h, w) * 2 - 1, (63, 63), (32, 32), (4.0, 4.0)), 1), + "pyrdown": ((), 1), + "pyrup": ((), 1), + "build_pyramid": ((3,), 1), + "build_laplacian_pyramid": ((3,), 1), + } + # ITERATE OVER THE TRANSFORMS + for fn_name, (args, num_samples) in transforms.items(): + img_in = img6.repeat(num_samples, 1, 1, 1) + args_in = (img_in, *args) + # import function and apply + fn = getattr(mod, fn_name) + out = fn(*args_in) + if fn_name in ("resize", "rescale", "pyrdown", "pyrup"): + h_new, w_new = out.shape[-2:] + out = torch.nn.functional.pad(out, (0, (w - w_new), 0, (h - h_new))) + if fn_name == "build_pyramid": + _out = [] + for pyr in out[1:]: + h_new, w_new = pyr.shape[-2:] + out_tmp = torch.nn.functional.pad(pyr, (0, (w - w_new), 0, (h - h_new))) + _out.append(out_tmp) + out = torch.cat(_out) + + if fn_name == "build_laplacian_pyramid": + h_, w_ = out[0].shape[-2:] + _out = [out[0]] + for pyr in out[1:]: + h_new, w_new = pyr.shape[-2:] + out_tmp = torch.nn.functional.pad(pyr, (0, (w_ - w_new), 0, (h_ - h_new))) + print(out_tmp.size()) + _out.append(out_tmp) + out = torch.cat(_out) + + # save the output image + if fn_name != "build_laplacian_pyramid": + out = torch.cat([img_in[0], *(out[i] for i in range(out.size(0)))], dim=-1) + else: + out = torch.cat([*(out[i] for i in range(out.size(0)))], dim=-1) + out_np = K.image.tensor_to_image((out * 255.0).byte()) + cv2.imwrite(str(OUTPUT_PATH / f"{fn_name}.png"), out_np) + sig = f"{fn_name}({', '.join([str(a) for a in args])})" + print(f"Generated image example for {fn_name}. {sig}") + + # Image Matching and local features + img_matching_data = torch.hub.load_state_dict_from_url(BASE_IMAGEOUTDOOR_URL7, map_location=torch.device("cpu")) + img_outdoor = img_matching_data["img2"] + print("Generating local feature detections ") + disk = K.feature.DISK.from_pretrained("depth") + with torch.no_grad(): + disk_feat = disk(img_outdoor)[0] + xy = disk_feat.keypoints.detach().cpu().numpy() + cur_fname = str(OUTPUT_PATH / "disk_outdoor_depth.jpg") + plt.figure() + plt.imshow(K.tensor_to_image(img_outdoor)) + plt.scatter(xy[:, 0], xy[:, 1], 3, color="lime") + plt.title('DISK("depth") keypoints') + plt.savefig(cur_fname) + plt.close() + + kah = K.feature.KeyNetAffNetHardNet(512).eval() + with torch.no_grad(): + lafs, _resps, _descs = kah(K.color.rgb_to_grayscale(img_outdoor)) + fig1, ax = visualize_LAF(img_outdoor, lafs, color="lime", return_fig_ax=True, draw_ori=False) + ax.set_title("KeyNetAffNet 512 LAFs") + cur_fname = str(OUTPUT_PATH / "keynet_affnet.jpg") + fig1.savefig(cur_fname) + plt.close() + + keynet = K.feature.KeyNetDetector(True, 512).eval() + with torch.no_grad(): + lafs, _resps = keynet(K.color.rgb_to_grayscale(img_outdoor)) + xy = K.feature.get_laf_center(lafs).detach().cpu().numpy().reshape(-1, 2) + cur_fname = str(OUTPUT_PATH / "keynet.jpg") + plt.figure() + plt.imshow(K.tensor_to_image(img_outdoor)) + plt.scatter(xy[:, 0], xy[:, 1], 3, color="lime") + plt.title("KeyNet 512 keypoints") + plt.savefig(cur_fname) + plt.close() + + # korna.feature module + mod = importlib.import_module("kornia.feature") + responses: list = [ + "harris_response", + "gftt_response", + "hessian_response", + "dog_response_single", + "KeyNet", + "DISK", + "ALIKED", + "XFeat", + ] + # ITERATE OVER THE TRANSFORMS + for fn_name in responses: + # import function and apply + img_in = K.color.rgb_to_grayscale(img_kornia) + if fn_name == "KeyNet": + fn = K.feature.KeyNet(True) + out = fn(img_in) + elif fn_name == "DISK": + fn = K.feature.DISK.from_pretrained("depth") + h, w = img_outdoor.shape[2:] + pd_h = 32 - h % 32 if h % 32 > 0 else 0 + pd_w = 32 - w % 32 if w % 32 > 0 else 0 + img_in = torch.nn.functional.pad(img_outdoor, (0, pd_w, 0, pd_h), value=0.0) + out, _ = fn.heatmap_and_dense_descriptors(img_in) + out = K.color.grayscale_to_rgb(out) + img_in = K.color.rgb_to_bgr(img_in) + elif fn_name == "ALIKED": + fn = K.feature.ALIKED.from_pretrained() + with torch.no_grad(): + _, out = fn.extract_dense_map(img_outdoor) + out = K.color.grayscale_to_rgb(out) + img_in = K.color.rgb_to_bgr(img_outdoor) + elif fn_name == "XFeat": + fn = K.feature.XFeat.from_pretrained() + with torch.no_grad(): + preprocessed, _, _ = fn._preprocess_tensor(img_outdoor) + _, K1, _ = fn.net(preprocessed) + out = K.feature.XFeat._get_kpts_heatmap(K1) + # upsample heatmap from preprocessed size back to original image size + out = torch.nn.functional.interpolate(out, img_outdoor.shape[-2:], mode="bilinear", align_corners=False) + out = K.color.grayscale_to_rgb(out) + img_in = K.color.rgb_to_bgr(img_outdoor) + else: + fn = getattr(mod, fn_name) + out = fn(img_in) + + out = out - out.min() + out = out / (1e-8 + out.max()) + + # save the output image + out = torch.cat([img_in[0], *(out[i] for i in range(out.size(0)))], dim=-1) + out_np = K.image.tensor_to_image((out * 255.0).byte()) + cv2.imwrite(str(OUTPUT_PATH / f"{fn_name}.png"), out_np) + sig = f"{fn_name}({', '.join([str(a) for a in args])})" + print(f"Generated image example for response function {fn_name}") + + +if __name__ == "__main__": + main() diff --git a/docs/source/_static/css/main.css b/docs/source/_static/css/main.css new file mode 100644 index 0000000..92dc2a0 --- /dev/null +++ b/docs/source/_static/css/main.css @@ -0,0 +1,13 @@ +.sidebar-logo { + max-width: 30%; +} + +/* TODO: make sure it works for all the browsers */ +body{ + font-family: 'Montserrat', sans-serif; +} + +/* The section headers in the toc tree */ +.sidebar-scroll p.caption{ + background-color: #4d59ff; +} diff --git a/docs/source/_static/image_registration.py b/docs/source/_static/image_registration.py new file mode 100644 index 0000000..417e017 --- /dev/null +++ b/docs/source/_static/image_registration.py @@ -0,0 +1,55 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +import cv2 +import imageio +import torch + +import kornia as K +import kornia.geometry as KG + + +def load_timg(file_name): + """Loads the image with OpenCV and converts to torch.Tensor.""" + assert os.path.isfile(file_name), f"Invalid file {file_name}" # nosec + # load image with OpenCV + img = cv2.imread(file_name, cv2.IMREAD_COLOR) + # convert image to torch tensor + tensor = K.image_to_tensor(img, None).float() / 255.0 + return K.color.bgr_to_rgb(tensor) + + +registrator = KG.ImageRegistrator("similarity") + +img1 = K.resize(load_timg("/Users/oldufo/datasets/stewart/MR-CT/CT.png"), (400, 600)) +img2 = K.resize(load_timg("/Users/oldufo/datasets/stewart/MR-CT/MR.png"), (400, 600)) +model, intermediate = registrator.register(img1, img2, output_intermediate_models=True) + +video_writer = imageio.get_writer("medical_registration.gif", fps=2) + +timg_dst_first = img1.clone() +timg_dst_first[0, 0, :, :] = img2[0, 0, :, :] +video_writer.append_data(K.tensor_to_image((timg_dst_first * 255.0).byte())) + +with torch.no_grad(): + for m in intermediate: + timg_dst = KG.homography_warp(img1, m, img2.shape[-2:]) + timg_dst[0, 0, :, :] = img2[0, 0, :, :] + video_writer.append_data(K.tensor_to_image((timg_dst_first * 255.0).byte())) +video_writer.close() diff --git a/docs/source/_static/img/hakuna_matata.gif b/docs/source/_static/img/hakuna_matata.gif new file mode 100644 index 0000000..1e8d04e Binary files /dev/null and b/docs/source/_static/img/hakuna_matata.gif differ diff --git a/docs/source/_static/img/kornia_logo.svg b/docs/source/_static/img/kornia_logo.svg new file mode 100644 index 0000000..a4613ba --- /dev/null +++ b/docs/source/_static/img/kornia_logo.svg @@ -0,0 +1,108 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/source/_static/img/kornia_logo_only_dark.svg b/docs/source/_static/img/kornia_logo_only_dark.svg new file mode 100644 index 0000000..1a7c8f8 --- /dev/null +++ b/docs/source/_static/img/kornia_logo_only_dark.svg @@ -0,0 +1 @@ + diff --git a/docs/source/_static/img/kornia_logo_only_light.svg b/docs/source/_static/img/kornia_logo_only_light.svg new file mode 100644 index 0000000..f80aa37 --- /dev/null +++ b/docs/source/_static/img/kornia_logo_only_light.svg @@ -0,0 +1 @@ + diff --git a/docs/source/_static/img/registration.gif b/docs/source/_static/img/registration.gif new file mode 100644 index 0000000..1a700c6 Binary files /dev/null and b/docs/source/_static/img/registration.gif differ diff --git a/docs/source/_static/js/custom.js b/docs/source/_static/js/custom.js new file mode 100644 index 0000000..03f76e1 --- /dev/null +++ b/docs/source/_static/js/custom.js @@ -0,0 +1,42 @@ +// Based on https://github.com/huggingface/transformers/blob/master/docs/source/_static/js/custom.js +// TODO (Jian): probably update this by the latest https://buttons.github.io/, which added dark and light mode to align furo theme. + + +function addGithubButton() { + const div = ` + + `; + document.querySelector(".sidebar-brand").insertAdjacentHTML('afterend', div); +} + +/*! + * github-buttons v2.2.10 + * (c) 2019 なつき + * @license BSD-2-Clause + */ +/** + * modified to run programmatically + */ +function parseGithubButtons (){"use strict";var e=window.document,t=e.location,o=window.encodeURIComponent,r=window.decodeURIComponent,n=window.Math,a=window.HTMLElement,i=window.XMLHttpRequest,l="https://unpkg.com/github-buttons@2.2.10/dist/buttons.html",c=i&&i.prototype&&"withCredentials"in i.prototype,d=c&&a&&a.prototype.attachShadow&&!a.prototype.attachShadow.prototype,s=function(e,t,o){e.addEventListener?e.addEventListener(t,o):e.attachEvent("on"+t,o)},u=function(e,t,o){e.removeEventListener?e.removeEventListener(t,o):e.detachEvent("on"+t,o)},h=function(e,t,o){var r=function(n){return u(e,t,r),o(n)};s(e,t,r)},f=function(e,t,o){var r=function(n){if(t.test(e.readyState))return u(e,"readystatechange",r),o(n)};s(e,"readystatechange",r)},p=function(e){return function(t,o,r){var n=e.createElement(t);if(o)for(var a in o){var i=o[a];null!=i&&(null!=n[a]?n[a]=i:n.setAttribute(a,i))}if(r)for(var l=0,c=r.length;l'},eye:{width:16,height:16,path:''},star:{width:14,height:16,path:''},"repo-forked":{width:10,height:16,path:''},"issue-opened":{width:14,height:16,path:''},"cloud-download":{width:16,height:16,path:''}},w={},x=function(e,t,o){var r=p(e.ownerDocument),n=e.appendChild(r("style",{type:"text/css"}));n.styleSheet?n.styleSheet.cssText=m:n.appendChild(e.ownerDocument.createTextNode(m));var a,l,d=r("a",{className:"btn",href:t.href,target:"_blank",innerHTML:(a=t["data-icon"],l=/^large$/i.test(t["data-size"])?16:14,a=(""+a).toLowerCase().replace(/^octicon-/,""),{}.hasOwnProperty.call(v,a)||(a="mark-github"),'"),"aria-label":t["aria-label"]||void 0},[" ",r("span",{},[t["data-text"]||""])]);/\.github\.com$/.test("."+d.hostname)?/^https?:\/\/((gist\.)?github\.com\/[^\/?#]+\/[^\/?#]+\/archive\/|github\.com\/[^\/?#]+\/[^\/?#]+\/releases\/download\/|codeload\.github\.com\/)/.test(d.href)&&(d.target="_top"):(d.href="#",d.target="_self");var u,h,g,x,y=e.appendChild(r("div",{className:"widget"+(/^large$/i.test(t["data-size"])?" lg":"")},[d]));/^(true|1)$/i.test(t["data-show-count"])&&"github.com"===d.hostname&&(u=d.pathname.replace(/^(?!\/)/,"/").match(/^\/([^\/?#]+)(?:\/([^\/?#]+)(?:\/(?:(subscription)|(fork)|(issues)|([^\/?#]+)))?)?(?:[\/?#]|$)/))&&!u[6]?(u[2]?(h="/repos/"+u[1]+"/"+u[2],u[3]?(x="subscribers_count",g="watchers"):u[4]?(x="forks_count",g="network"):u[5]?(x="open_issues_count",g="issues"):(x="stargazers_count",g="stargazers")):(h="/users/"+u[1],g=x="followers"),function(e,t){var o=w[e]||(w[e]=[]);if(!(o.push(t)>1)){var r=b(function(){for(delete w[e];t=o.shift();)t.apply(null,arguments)});if(c){var n=new i;s(n,"abort",r),s(n,"error",r),s(n,"load",function(){var e;try{e=JSON.parse(n.responseText)}catch(e){return void r(e)}r(200!==n.status,e)}),n.open("GET",e),n.send()}else{var a=this||window;a._=function(e){a._=null,r(200!==e.meta.status,e.data)};var l=p(a.document)("script",{async:!0,src:e+(/\?/.test(e)?"&":"?")+"callback=_"}),d=function(){a._&&a._({meta:{}})};s(l,"load",d),s(l,"error",d),l.readyState&&f(l,/de|m/,d),a.document.getElementsByTagName("head")[0].appendChild(l)}}}.call(this,"https://api.github.com"+h,function(e,t){if(!e){var n=t[x];y.appendChild(r("a",{className:"social-count",href:t.html_url+"/"+g,target:"_blank","aria-label":n+" "+x.replace(/_count$/,"").replace("_"," ").slice(0,n<2?-1:void 0)+" on GitHub"},[r("b"),r("i"),r("span",{},[(""+n).replace(/\B(?=(\d{3})+(?!\d))/g,",")])]))}o&&o(y)})):o&&o(y)},y=window.devicePixelRatio||1,C=function(e){return(y>1?n.ceil(n.round(e*y)/y*2)/2:n.ceil(e))||0},F=function(e,t){e.style.width=t[0]+"px",e.style.height=t[1]+"px"},k=function(t,r){if(null!=t&&null!=r)if(t.getAttribute&&(t=function(e){for(var t={href:e.href,title:e.title,"aria-label":e.getAttribute("aria-label")},o=["icon","text","size","show-count"],r=0,n=o.length;r`_ + +.. youtube:: hzQroGp5FSQ + +Using our API you easily detect faces in images as shown below: + +.. code-block:: python + + # select the device + device = torch.device('cpu') + if args.cuda and torch.cuda.is_available(): + device = torch.device('cuda:0') + + # load the image and scale + img_raw = cv2.imread(args.image_file, cv2.IMREAD_COLOR) + img_raw = scale_image(img_raw, args.image_size) + + # preprocess + img = K.image_to_tensor(img_raw, keepdim=False).to(device) + img = K.color.bgr_to_rgb(img.float()) + + # create the detector and find the faces ! + face_detection = FaceDetector().to(device) + + with torch.no_grad(): + dets = face_detection(img) + dets = [FaceDetectorResult(o) for o in dets[0]] + + +Play yourself with the detector and generate new images with this `tutorial `_. diff --git a/docs/source/applications/image_augmentations.rst b/docs/source/applications/image_augmentations.rst new file mode 100644 index 0000000..326c096 --- /dev/null +++ b/docs/source/applications/image_augmentations.rst @@ -0,0 +1,143 @@ +Image Augmentation +================== + +Image Augmentation is a data augmentation method that generates more training data +from the existing training samples. Image Augmentation is especially useful in domains +where training data is limited or expensive to obtain like in biomedical applications. + +.. image:: https://github.com/kornia/data/raw/main/girona_aug.png + :align: center + +Learn more: `https://paperswithcode.com/task/image-augmentation `_ + +Kornia Augmentations +-------------------- + +Kornia leverages differentiable and GPU image data augmentation through the module `kornia.augmentation `_ +by implementing the functionality to be easily used with `torch.nn.Sequential `_ +and other advanced containers such as +:py:class:`~kornia.augmentation.container.AugmentationSequential`, +:py:class:`~kornia.augmentation.container.ImageSequential`, +:py:class:`~kornia.augmentation.container.PatchSequential` and +:py:class:`~kornia.augmentation.container.VideoSequential`. + +Our augmentations package is highly inspired by torchvision augmentation APIs while our intention is to not replace it. +Kornia is a library that aligns better to OpenCV functionalities enforcing floating operators to guarantees a better precision +without any float -> uint8 conversions plus on device acceleration. + +However, we provide the following guide to migrate kornia <-> torchvision. Please, checkout the `Colab: Kornia Playground `_. + +.. code-block:: python + + import kornia.augmentation as K + import torch.nn as nn + + transform = nn.Sequential( + K.RandomAffine(360), + K.ColorJiggle(0.2, 0.3, 0.2, 0.3) + ) + + +Best Practices 1: Image Augmentation +++++++++++++++++++++++++++++++++++++ + +Kornia augmentations provides simple on-device augmentation framework with the support of various syntax sugars +(e.g. return transformation matrix, inverse geometric transform). Therefore, we provide advanced augmentation +container :py:class:`~kornia.augmentation.container.AugmentationSequential` to ease the pain of building augmenation pipelines. This API would also provide predefined routines +for automating the processing of masks, bounding boxes, and keypoints. + +.. code-block:: python + + import kornia.augmentation as K + + aug = K.AugmentationSequential( + K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + K.RandomAffine(360, [0.1, 0.1], [0.7, 1.2], [30., 50.], p=1.0), + K.RandomPerspective(0.5, p=1.0), + data_keys=["input", "bbox", "keypoints", "mask"], # Just to define the future input here. + return_transform=False, + same_on_batch=False, + ) + # forward the operation + out_tensors = aug(img_tensor, bbox, keypoints, mask) + # Inverse the operation + out_tensor_inv = aug.inverse(*out_tensor) + +.. image:: https://discuss.pytorch.org/uploads/default/optimized/3X/2/4/24bb0f4520f547d3a321440293c1d44921ecadf8_2_690x119.jpeg + +From left to right: the original image, the transformed image, and the inversed image. + + +Best Practices 2: Video Augmentation +++++++++++++++++++++++++++++++++++++ + +Video data is a special case of 3D volumetric data that contains both spatial and temporal information, which can be referred as 2.5D than 3D. +In most applications, augmenting video data requires a static temporal dimension to have the same augmentations are performed for each frame. +Thus, :py:class:`~kornia.augmentation.container.VideoSequential` can be used to do such trick as same as `nn.Sequential`. +Currently, :py:class:`~kornia.augmentation.container.VideoSequential` supports data format like :math:`(B, C, T, H, W)` and :math:`(B, T, C, H, W)`. + +.. code-block:: python + + import kornia.augmentation as K + + transform = K.VideoSequential( + K.RandomAffine(360), + K.RandomGrayscale(p=0.5), + K.RandomAffine(p=0.5) + data_format="BCTHW", + same_on_frame=True + ) + +.. image:: https://user-images.githubusercontent.com/17788259/101993516-4625ca80-3c89-11eb-843e-0b87dca6e2b8.png + + +Customization ++++++++++++++ + +Kornia augmentation implementations have two additional parameters compare to TorchVision, +``return_transform`` and ``same_on_batch``. The former provides the ability of undoing one geometry +transformation while the latter can be used to control the randomness for a batched transformation. +To enable those behaviour, you may simply set the flags to True. + +.. code-block:: python + + import kornia.augmentation as K + + class MyAugmentationPipeline(nn.Module): + def __init__(self) -> None: + super(MyAugmentationPipeline, self).__init__() + self.aff = K.RandomAffine( + 360, return_transform=True, same_on_batch=True + ) + self.jit = K.ColorJiggle(0.2, 0.3, 0.2, 0.3, same_on_batch=True) + + def forward(self, input): + input, transform = self.aff(input) + input, transform = self.jit((input, transform)) + return input, transform + +Example for semantic segmentation using low-level randomness control: + +.. code-block:: python + + import kornia.augmentation as K + + class MyAugmentationPipeline(nn.Module): + def __init__(self) -> None: + super(MyAugmentationPipeline, self).__init__() + self.aff = K.RandomAffine(360) + self.jit = K.ColorJiggle(0.2, 0.3, 0.2, 0.3) + + def forward(self, input, mask): + assert input.shape == mask.shape, + f"Input shape should be consistent with mask shape, " + f"while got {input.shape}, {mask.shape}" + + aff_params = self.aff.forward_parameters(input.shape) + input = self.aff(input, aff_params) + mask = self.aff(mask, aff_params) + + jit_params = self.jit.forward_parameters(input.shape) + input = self.jit(input, jit_params) + mask = self.jit(mask, jit_params) + return input, mask diff --git a/docs/source/applications/image_denoising.rst b/docs/source/applications/image_denoising.rst new file mode 100644 index 0000000..1b035eb --- /dev/null +++ b/docs/source/applications/image_denoising.rst @@ -0,0 +1,8 @@ +Image Denoising +=============== + + + +.. raw:: html + + diff --git a/docs/source/applications/image_matching.rst b/docs/source/applications/image_matching.rst new file mode 100644 index 0000000..0083cc2 --- /dev/null +++ b/docs/source/applications/image_matching.rst @@ -0,0 +1,24 @@ +Image Matching +============== + +Image matching is a process of finding pixel and region correspondences between two images of the same scene. +Such correspondences are useful for 3D reconstruction of the scene and relative camera pose estimation. +It is also known as "Wide baseline stereo" and you can read more about it at `Wide Baseline Stereo Blog `_ + +We provide many modules and functions for the image matching: from building blocks like `local feature detectors `_, `descriptors `_, +`descriptor matching `_, `geometric model estimation `_ + +However we recommend to start with high-level API, such as :py:class:`~kornia.feature.LoFTR` you can use to find correspondence between two images. + +.. code:: python + + from kornia.feature import LoFTR + + matcher = LoFTR(pretrained="outdoor") + input = {"image0": img1, "image1": img2} + correspondences_dict = matcher(input) + + +.. image:: https://raw.githubusercontent.com/kornia/data/main/matching/matching_loftr.jpg + +You also can go through or full tutorial using Colab found `here `_. diff --git a/docs/source/applications/image_registration.rst b/docs/source/applications/image_registration.rst new file mode 100644 index 0000000..837afe3 --- /dev/null +++ b/docs/source/applications/image_registration.rst @@ -0,0 +1,33 @@ +Image Registration +================== + +Image registration is the process of transforming different sets of data into one coordinate system. Data may be multiple photographs, data from different sensors, times, depths, or viewpoints. It is used in computer vision, medical imaging, and compiling and analyzing images and data from satellites. Registration is necessary in order to be able to compare or integrate the data obtained from these different measurements. + +Learn more: `https://paperswithcode.com/task/image-registration `_ + +.. youtube:: Re1q6vRfZac + +We provide the :py:class:`~kornia.geometry.transform.image_registrator.ImageRegistrator` API from which you can leverage to align automatically two images by a process of direct optimisation using the PyTorch Autograd differentiability. + +.. code:: python + + from kornia.geometry import ImageRegistrator + img_src = torch.rand(1, 1, 32, 32) + img_dst = torch.rand(1, 1, 32, 32) + registrator = ImageRegistrator('similarity') + homo = registrator.register(img_src, img_dst) + +Then, if you want to perform a more sophisticated process: + +.. literalinclude:: ../_static/image_registration.py + +To reproduce the same results as in the showed video you can go through or full tutorial using Colab found `here `_ . + +Interactive Demo +---------------- +.. raw:: html + + + + +Visit the `image registration demo on the Hugging Face Spaces `_. diff --git a/docs/source/applications/image_stitching.rst b/docs/source/applications/image_stitching.rst new file mode 100644 index 0000000..e0bcbb6 --- /dev/null +++ b/docs/source/applications/image_stitching.rst @@ -0,0 +1,31 @@ +Image Stitching +============================ + +Image stitching is the process of combining multiple images with overlapping fields of view to produce a segmented panorama. Here, we provide :py:class:`~kornia.contrib.image_stitching.ImageStitcher` to easily stitch a number of images. + +.. image:: https://raw.githubusercontent.com/kornia/data/main/matching/stitch_before.png + +Learn more: https://paperswithcode.com/task/image-stitching/ + +.. code:: python + + from kornia.contrib import ImageStitcher + + matcher = KF.LoFTR(pretrained='outdoor') + IS = ImageStitcher(matcher, estimator='ransac').cuda() + # NOTE: it would require a large CPU memory if many images. + with torch.no_grad(): + out = IS(*imgs) + +.. image:: https://raw.githubusercontent.com/kornia/data/main/panorama/out_panorama.jpg + +Explore with your data: https://colab.research.google.com/github/kornia/tutorials/blob/master/source/image_stitching.ipynb + + +Interactive Demo +---------------- +.. raw:: html + + + +Visit the demo on `Hugging Face Spaces `_. diff --git a/docs/source/applications/intro.rst b/docs/source/applications/intro.rst new file mode 100644 index 0000000..3729e39 --- /dev/null +++ b/docs/source/applications/intro.rst @@ -0,0 +1,23 @@ +Computer Vision Algorithms +========================== + +Kornia provides a bottom to top granularity for **CURATED** Computer Vision algorithms. + +In this section, we showcase our high-level API in terms of abstraction for common Computer Vision algorithms +that can be used across different domains such as Robotics, Industrial applications or for the AR/VR industry. + +.. tip:: + + Expect to see in the future a selection of the top performing algorithms in the following sub-areas: + + - Super Resolution + - Deep Edge detection + - Stereo and Optical flow and camera calibration + - Neural Rendering + - Semantic and Panoptic segmentation + - Object Detection and Tracking + - Image classification + +.. admonition:: We are looking for contributors !! + + If you have any suggestion, proposal or just want to give us a hand - join our `Slack `_ diff --git a/docs/source/applications/visual_prompting.rst b/docs/source/applications/visual_prompting.rst new file mode 100644 index 0000000..7711b6b --- /dev/null +++ b/docs/source/applications/visual_prompting.rst @@ -0,0 +1,56 @@ +Visual Prompting +================ + +.. image:: https://kornia.github.io/tutorials/nbs/image_prompter_files/figure-html/cell-34-output-1.png + :width: 20% + +Visual Prompting is the task of streamlining computer vision processes by harnessing the power of prompts, +inspired by the breakthroughs of text prompting in NLP. This innovative approach involves using a few visual +prompts to swiftly convert an unlabeled dataset into a deployed model, significantly reducing development time +for both individual projects and enterprise solutions. + +By leveraging large pre-trained vision transformers, Visual Prompting not only eliminates the need for extensive +data labeling but also facilitates the "teaching" of smaller AI systems. + + +How Kornia leverages Visual Prompting ? +--------------------------------------- + +Kornia leverages the Visual Prompting task through the :code:`VisualPrompter`` API, which integrates powerful models like +the Segment Anything Model (SAM) into its computer vision toolkit. By incorporating SAM and the VisualPrompter API, +developers can harness the efficiency of Visual Prompting for faster segmentation tasks and improved computer vision workflows. This seamless integration allows users to utilize pre-trained vision transformers, significantly reducing manual data labeling efforts and enabling the "teaching" of smaller AI systems. As a result, Kornia users can take advantage of the versatility and adaptability offered by Visual Prompting, unlocking new possibilities for various computer vision applications. + +How to use with Kornia +---------------------- + +.. code-block:: python + + from kornia.io import load_image, ImageLoadType + from kornia.contrib.visual_prompter import VisualPrompter + + # load an image + image = load_image('./example.jpg', ImageLoadType.RGB32, device) + + # Load the prompter + prompter = VisualPrompter() + + # set the image: This will preprocess the image and already generate the embeddings of it + prompter.set_image(image) + + # Generate the prompts + keypoints = Keypoints(torch.tensor([[[500, 375]]], device=device, dtype=torch.float32)) # BxNx2 + + # For the keypoints label: 1 indicates a foreground point; 0 indicates a background point + keypoints_labels = torch.tensor([[1]], device=device) # BxN + + # Runs the prediction with the kypoints prompts + prediction = prompter.predict( + keypoints=keypoints, + keypoints_labels=keypoints_labels, + multimask_output=True, + ) + +You also can go through or full tutorial using Colab found `here `_. + + +Integration with other libraries, fineturning and more examples soon. diff --git a/docs/source/augmentation.auto.rst b/docs/source/augmentation.auto.rst new file mode 100644 index 0000000..40f9f53 --- /dev/null +++ b/docs/source/augmentation.auto.rst @@ -0,0 +1,48 @@ +Automatic Augmentation Methods +============================== + +.. meta:: + :name: description + :content: "The Automatic Augmentation Methods module in Kornia provides common data augmentation policies like AutoAugment, RandAugment, and TrivialAugment to improve the accuracy of image classification models. It also includes methods for augmentation search." + +.. currentmodule:: kornia.augmentation.auto + +Augmentation Policy +------------------- + +This module contains common data augmentation policies that can improve the accuracy of image classification models. + +.. autoclass:: AutoAugment + + .. automethod:: get_transformation_matrix + + .. automethod:: forward_parameters + + .. automethod:: forward + + .. automethod:: inverse + +.. autoclass:: RandAugment + + .. automethod:: get_transformation_matrix + + .. automethod:: forward_parameters + + .. automethod:: forward + + .. automethod:: inverse + +.. autoclass:: TrivialAugment + + .. automethod:: get_transformation_matrix + + .. automethod:: forward_parameters + + .. automethod:: forward + + .. automethod:: inverse + +Augmentation Search Methods +--------------------------- + +WIP. This module contains common data augmentation search methods. diff --git a/docs/source/augmentation.base.rst b/docs/source/augmentation.base.rst new file mode 100644 index 0000000..3acb38a --- /dev/null +++ b/docs/source/augmentation.base.rst @@ -0,0 +1,142 @@ +Base Classes +============ + +.. meta:: + :name: description + :content: "The Base Classes module in Kornia provides foundational classes for creating new image transformations. It supports rigid (e.g., affine) and non-rigid (e.g., cut-out) augmentations, with predefined routines for sampling, applying, and reversing transformations." + +.. currentmodule:: kornia.augmentation + +This is the base class for creating a new transform on top the predefined routine of `kornia.augmentation`. +Specifically, an any given augmentation can be recognized as either rigid (e.g. affine transformations that +manipulate images with standard transformation matrice), or non-rigid (e.g. cut out a random area). At +image-level, Kornia supports rigid transformation like `GeometricAugmentationBase2D` that modifies the geometric +location of image pixels and `IntensityAugmentationBase2D` that preserves the pixel locations, as well as +generic `AugmentationBase2D` that allows higher freedom for customized augmentation design. + + +The Predefined Augmentation Routine +----------------------------------- + +Kornia augmentation follows the simplest `sample-apply` routine for all the augmentations. + +- `sample`: Kornia aims at flexible tensor-level augmentations that augment all images in a tensor with + different augmentations and probabilities. The sampling operation firstly samples a suite of random + parameters. Then all the sampled augmentation state (parameters) is stored + inside `_param` of the augmentation, the users can hereby reproduce the same augmentation results. +- `apply`: With generated or passed parameters, the augmentation will be performed accordingly. + Apart from performing image tensor operations, Kornia also supports inverse operations that + to revert the transform operations. Meanwhile, other data modalities (`datakeys` in Kornia) like + masks, keypoints, and bounding boxes. Such features are better supported with `AugmentationSequential`. + Notably, the augmentation pipeline for rigid operations are implemented already without further efforts. + For non-rigid operations, the user may implement customized inverse and data modality operations, e.g. + `apply_mask_transform` for applying transformations on mask tensors. + + +Custom Augmentation Classes +--------------------------- + +For rigid transformations, `IntensityAugmentationBase2D` and `GeometricAugmentationBase2D` are sharing the exact same logic +apart from the transformation matrix computations. Namely, the intensity augmentation always results in +identity transformation matrices, without changing the geometric location for each pixel. + +If it is a rigid geometric operation, `compute_transformation` and `apply_transform` need to be implemented, as well as +`compute_inverse_transformation` and `inverse_transform` to compute its inverse. + +.. autoclass:: GeometricAugmentationBase2D + + .. automethod:: compute_transformation + .. automethod:: apply_transform + .. automethod:: compute_inverse_transformation + .. automethod:: inverse_transform + + +For `IntensityAugmentationBase2D`, the user only needs to override `apply_transform`. + +.. autoclass:: IntensityAugmentationBase2D + + .. automethod:: apply_transform + +A minimal example to create your own rigid geometric augmentations with the following snippet: + + +.. code-block:: python + + import torch + import kornia as K + + from kornia.augmentation import GeometricAugmentationBase2D + from kornia.augmentation import random_generator as rg + + + class MyRandomTransform(GeometricAugmentationBase2D): + + def __init__( + self, + factor=(0., 1.), + same_on_batch: bool = False, + p: float = 1.0, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self._param_generator = rg.PlainUniformGenerator((factor, "factor", None, None)) + + def compute_transformation(self, input, params): + # a simple identity transformation example + factor = params["factor"].to(input) * 0. + 1 + return K.eyelike(input, 3) * factor + + def apply_transform( + self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Tensor: + factor = params["factor"].to(input) + return input * factor + + +For non-rigid augmentations, the user may implement the `apply_transform*` and `apply_non_transform*` APIs +to meet the needs. Specifically, `apply_transform*` applies to the elements of a tensor that need to be transformed, +while `apply_non_transform*` applies to the elements of a tensor that are skipped from augmentation. For example, +a crop operation may change the tensor size partially, while we need to resize the rest to maintain the whole tensor +as an integrated one with the same size. + + +.. autoclass:: AugmentationBase2D + + .. automethod:: apply_transform + .. automethod:: apply_non_transform + .. automethod:: apply_transform_mask + .. automethod:: apply_non_transform_mask + .. automethod:: apply_transform_box + .. automethod:: apply_non_transform_box + .. automethod:: apply_transform_keypoint + .. automethod:: apply_non_transform_keypoint + .. automethod:: apply_transform_class + .. automethod:: apply_non_transform_class + + +The similar logic applies to 3D augmentations as well. + + +Some Further Notes +------------------ + +Probabilities +^^^^^^^^^^^^^ +Kornia supports two types of randomness for element-level randomness `p` and batch-level randomness `p_batch`, +as in `_BasicAugmentationBase`. Under the hood, operations like `crop`, `resize` are implemented with a fixed +element-level randomness of `p=1` that only maintains batch-level randomness. + + +Random Generators +^^^^^^^^^^^^^^^^^ +For automatically generating the corresponding ``__repr__`` with full customized parameters, you may need to +implement ``_param_generator`` by inheriting ``RandomGeneratorBase`` for generating random parameters and +put all static parameters inside ``self.flags``. You may take the advantage of ``PlainUniformGenerator`` to +generate simple uniform parameters with less boilerplate code. + + +Random Reproducibility +^^^^^^^^^^^^^^^^^^^^^^ +Plain augmentation base class without the functionality of transformation matrix calculations. +By default, the random computations will be happened on CPU with ``torch.get_default_dtype()``. +To change this behaviour, please use ``set_rng_device_and_dtype``. diff --git a/docs/source/augmentation.container.rst b/docs/source/augmentation.container.rst new file mode 100644 index 0000000..8530232 --- /dev/null +++ b/docs/source/augmentation.container.rst @@ -0,0 +1,149 @@ +Augmentation Containers +======================= + +.. meta:: + :name: description + :content: "The Augmentation Containers module in Kornia provides advanced frameworks for building augmentation pipelines. It includes classes like AugmentationSequential, ManyToManyAugmentationDispatcher, and VideoSequential for managing data formats such as images, videos, and temporal data. It also supports processing masks, bounding boxes, and keypoints in augmentation workflows." + +.. currentmodule:: kornia.augmentation.container + +The classes in this section are containers for augmenting different data formats (e.g. images, videos). + + +Augmentation Sequential +----------------------- + +Kornia augmentations provides simple on-device augmentation framework with the support of various syntax sugars +(e.g. return transformation matrix, inverse geometric transform). Therefore, we provide advanced augmentation +container to ease the pain of building augmenation pipelines. This API would also provide predefined routines +for automating the processing of masks, bounding boxes, and keypoints. + +.. autoclass:: AugmentationSequential + + .. automethod:: forward + + .. automethod:: inverse + + +Augmentation Dispatchers +------------------------ +Kornia supports two types of augmentation dispatching, namely many-to-many and many-to-one. The former wraps +different augmentations into one group and allows user to input multiple inputs in align with the number of +augmentations. The latter aims at performing different augmentations for one input that to obtain a list of +various transformed data. + +.. autoclass:: ManyToManyAugmentationDispather + + .. automethod:: forward + + +.. autoclass:: ManyToOneAugmentationDispather + + .. automethod:: forward + + + +ImageSequential +--------------- + +Kornia augmentations provides simple on-device augmentation framework with the support of various syntax sugars +(e.g. return transformation matrix, inverse geometric transform). Additionally, ImageSequential supports the +mix usage of both image processing and augmentation modules. + +.. autoclass:: ImageSequential + + .. automethod:: forward + +Differences Between ImageSequential and AugmentationSequential +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``ImageSequential`` and ``AugmentationSequential`` are both pipeline containers +in Kornia, but they're designed for fundamentally different data handling +scenarios. Understanding when to use each prevents common pitfalls in vision +pipelines. + +**Use ``AugmentationSequential`` when:** + +- The task requires synchronized transformations across multiple related tensors + (images, masks, bounding boxes, keypoints). +- Spatial correspondence must be maintained between inputs and targets, as in + semantic segmentation or object detection workflows. +- Multiple data formats need to be handled automatically with consistent random + parameter sampling across all targets. + +**Use ``ImageSequential`` when:** + +- The pipeline only processes image tensors without auxiliary spatial targets. +- The workflow combines augmentation modules with general image processing + operations (gaussian blur, edge detection, color transforms). +- A lightweight container is preferred without the overhead of multi-target + synchronization logic. + +Example using ``ImageSequential``:: + + import torch + import kornia.augmentation as K + from kornia.augmentation.container import ImageSequential + from kornia.filters import gaussian_blur2d + + img = torch.rand(1, 3, 256, 256) + + seq = ImageSequential( + K.RandomHorizontalFlip(p=1.0), + gaussian_blur2d, # arbitrary differentiable ops can be inserted + ) + + out = seq(img) + +Example using ``AugmentationSequential`` with synchronized transforms:: + + import torch + import kornia.augmentation as K + + img = torch.rand(1, 3, 256, 256) + mask = torch.rand(1, 1, 256, 256) + + aug = K.AugmentationSequential( + K.RandomResizedCrop((128, 128), p=1.0), + data_keys=["input", "mask"], + ) + + img_out, mask_out = aug(img, mask) + # identical random parameters applied to both tensors + +The core distinction: ``AugmentationSequential`` guarantees that random +augmentation parameters are shared across all specified data keys, maintaining +geometric consistency. ``ImageSequential`` applies operations independently to +single image tensors without multi-target awareness. + + +PatchSequential +--------------- + +.. autoclass:: PatchSequential + + .. automethod:: forward + + +Video Data Augmentation +----------------------- + +Video data is a special case of 3D volumetric data that contains both spatial and temporal information, which can be referred as 2.5D than 3D. +In most applications, augmenting video data requires a static temporal dimension to have the same augmentations are performed for each frame. +Thus, `VideoSequential` can be used to do such trick as same as `nn.Sequential`. +Currently, `VideoSequential` supports data format like :math:`(B, C, T, H, W)` and :math:`(B, T, C, H, W)`. + +.. code-block:: python + + import kornia.augmentation as K + + transform = K.VideoSequential( + K.RandomAffine(360), + K.ColorJiggle(0.2, 0.3, 0.2, 0.3), + data_format="BCTHW", + same_on_frame=True + ) + +.. autoclass:: VideoSequential + + .. automethod:: forward diff --git a/docs/source/augmentation.module.rst b/docs/source/augmentation.module.rst new file mode 100644 index 0000000..0a5a1d1 --- /dev/null +++ b/docs/source/augmentation.module.rst @@ -0,0 +1,124 @@ +Image Augmentations +=================== + +.. meta:: + :name: description + :content: "The Image Augmentations module in Kornia provides a wide range of 2D and 3D data augmentation transforms. It includes intensity-based augmentations, geometric transformations, mix-based augmentations, and normalization operations for both 2D and 3D image tensors. Key functions include random color shifts, rotations, cropping, elastic transformations, and more." + +.. currentmodule:: kornia.augmentation + +Transforms2D +------------ + +Set of operators to perform data augmentation on 2D image tensors. + +Intensity +~~~~~~~~~ + +.. autoclass:: ColorJiggle +.. autoclass:: ColorJitter +.. autoclass:: RandomAutoContrast +.. autoclass:: RandomBoxBlur +.. autoclass:: RandomBrightness +.. autoclass:: RandomChannelDropout +.. autoclass:: RandomChannelShuffle +.. autoclass:: RandomClahe +.. autoclass:: RandomContrast +.. autoclass:: RandomEqualize +.. autoclass:: RandomDissolving +.. autoclass:: RandomGamma +.. autoclass:: RandomGaussianBlur +.. autoclass:: RandomGaussianIllumination +.. autoclass:: RandomGaussianNoise +.. autoclass:: RandomGrayscale +.. autoclass:: RandomHue +.. autoclass:: RandomInvert +.. autoclass:: RandomJPEG +.. autoclass:: RandomLinearCornerIllumination +.. autoclass:: RandomLinearIllumination +.. autoclass:: RandomMedianBlur +.. autoclass:: RandomMotionBlur +.. autoclass:: RandomPlanckianJitter +.. autoclass:: RandomPlasmaBrightness +.. autoclass:: RandomPlasmaContrast +.. autoclass:: RandomPlasmaShadow +.. autoclass:: RandomPosterize +.. autoclass:: RandomRain +.. autoclass:: RandomRGBShift +.. autoclass:: RandomSaltAndPepperNoise +.. autoclass:: RandomSaturation +.. autoclass:: RandomSharpness +.. autoclass:: RandomSnow +.. autoclass:: RandomSolarize + + +Geometric +~~~~~~~~~ + +.. autoclass:: CenterCrop +.. autoclass:: PadTo +.. autoclass:: RandomAffine +.. autoclass:: RandomCrop +.. autoclass:: RandomElasticTransform +.. autoclass:: RandomErasing +.. autoclass:: RandomFisheye +.. autoclass:: RandomHorizontalFlip +.. autoclass:: RandomPerspective +.. autoclass:: RandomResizedCrop +.. autoclass:: RandomRotation90 +.. autoclass:: RandomRotation +.. autoclass:: RandomShear +.. autoclass:: RandomThinPlateSpline +.. autoclass:: RandomVerticalFlip + + +Mix +~~~ + +.. autoclass:: RandomCutMixV2 +.. autoclass:: RandomJigsaw +.. autoclass:: RandomMixUpV2 +.. autoclass:: RandomMosaic +.. autoclass:: RandomTransplantation + +Transforms3D +------------ + +Set of operators to perform data augmentation on 3D volumetric tensors. + +Geometric +~~~~~~~~~ + +.. autoclass:: CenterCrop3D +.. autoclass:: RandomAffine3D +.. autoclass:: RandomCrop3D +.. autoclass:: RandomDepthicalFlip3D +.. autoclass:: RandomHorizontalFlip3D +.. autoclass:: RandomRotation3D +.. autoclass:: RandomVerticalFlip3D + +Intensity +~~~~~~~~~ + +.. autoclass:: RandomEqualize3D +.. autoclass:: RandomMotionBlur3D + +Mix +~~~ + +.. autoclass:: RandomTransplantation3D + +Normalizations +-------------- + +Normalization operations are shape-agnostic for both 2D and 3D tensors. + +.. autoclass:: Denormalize +.. autoclass:: Normalize + +Image Resize +------------ + +.. autoclass:: LongestMaxSize +.. autoclass:: Resize +.. autoclass:: SmallestMaxSize diff --git a/docs/source/augmentation.rst b/docs/source/augmentation.rst new file mode 100644 index 0000000..3613648 --- /dev/null +++ b/docs/source/augmentation.rst @@ -0,0 +1,76 @@ +kornia.augmentation +=================== + +.. meta:: + :name: description + :content: "The Augmentation module in Kornia provides high-level data augmentation functionalities for computer vision tasks, including random rotations, affine transformations, color intensities, image noise distortion, and more. It supports batch processing, device compatibility, and backpropagation. Additionally, users can retrieve transformation details for more flexibility in complex pipelines." + +This module implements in a high level logic. The main features of this module, and similar to the rest of the +library, is that can it perform data augmentation routines in a batch mode, using any supported device, +and can be used for backpropagation. Some of the available functionalities which are worth to mention are the +following: random rotations; affine and perspective transformations; several random color intensities transformations, +image noise distortion, motion blurring, and many of the different differentiable data augmentation policies. +In addition, we include a novel feature which is not found in other augmentations frameworks, +which allows the user to retrieve the applied transformation or chained transformations after each +call e.g. the generated random rotation matrix which can be used later to undo the image transformation +itself, or to be applied to additional metadata such as the label images for semantic segmentation, +in bounding boxes or landmark keypoints for object detection tasks. It gives the user the flexibility to +perform complex data augmentations pipelines. + +Interactive Demo +~~~~~~~~~~~~~~~~ +.. raw:: html + + + +Benchmark +--------- + +.. table:: Here is a benchmark performed on `Google Colab `_ + K80 GPU with different libraries and batch sizes. This benchmark shows + strong GPU augmentation speed acceleration brought by Kornia data augmentations. The image size is fixed to 224x224 and the + unit is milliseconds (ms). + + +--------------------------------+-----------------+-----------------+-----------------------------------------------------+ + | Libraries | TorchVision | Albumentations | Kornia (GPU) | + +--------------------------------+-----------------+-----------------+-----------------+-----------------+-----------------+ + | Batch Size | 1 | 1 | 1 | 32 | 128 | + +================================+=================+=================+=================+=================+=================+ + | RandomPerspective | 4.88±1.82 | 4.68±3.60 | 4.74±2.84 | 0.37±2.67 | 0.20±27.00 | + +--------------------------------+-----------------+-----------------+-----------------+-----------------+-----------------+ + | ColorJiggle | 4.40±2.88 | 3.58±3.66 | 4.14±3.85 | 0.90±24.68 | 0.83±12.96 | + +--------------------------------+-----------------+-----------------+-----------------+-----------------+-----------------+ + | RandomAffine | 3.12±5.80 | 2.43±7.11 | 3.01±7.80 | 0.30±4.39 | 0.18±6.30 | + +--------------------------------+-----------------+-----------------+-----------------+-----------------+-----------------+ + | RandomVerticalFlip | 0.32±0.08 | 0.34±0.16 | 0.35±0.82 | 0.02±0.13 | 0.01±0.35 | + +--------------------------------+-----------------+-----------------+-----------------+-----------------+-----------------+ + | RandomHorizontalFlip | 0.32±0.08 | 0.34±0.18 | 0.31±0.59 | 0.01±0.26 | 0.01±0.37 | + +--------------------------------+-----------------+-----------------+-----------------+-----------------+-----------------+ + | RandomRotate | 1.82±4.70 | 1.59±4.33 | 1.58±4.44 | 0.25±2.09 | 0.17±5.69 | + +--------------------------------+-----------------+-----------------+-----------------+-----------------+-----------------+ + | RandomCrop | 4.09±3.41 | 4.03±4.94 | 3.84±3.07 | 0.16±1.17 | 0.08±9.42 | + +--------------------------------+-----------------+-----------------+-----------------+-----------------+-----------------+ + | RandomErasing | 2.31±1.47 | 1.89±1.08 | 2.32±3.31 | 0.44±2.82 | 0.57±9.74 | + +--------------------------------+-----------------+-----------------+-----------------+-----------------+-----------------+ + | RandomGrayscale | 0.41±0.18 | 0.43±0.60 | 0.45±1.20 | 0.03±0.11 | 0.03±7.10 | + +--------------------------------+-----------------+-----------------+-----------------+-----------------+-----------------+ + | RandomResizedCrop | 4.23±2.86 | 3.80±3.61 | 4.07±2.67 | 0.23±5.27 | 0.13±8.04 | + +--------------------------------+-----------------+-----------------+-----------------+-----------------+-----------------+ + | CenterCrop | 2.93±1.29 | 2.81±1.38 | 2.88±2.34 | 0.13±2.20 | 0.07±9.41 | + +--------------------------------+-----------------+-----------------+-----------------+-----------------+-----------------+ + + +.. currentmodule:: kornia.augmentation + +.. toctree:: + + augmentation.auto + augmentation.base + augmentation.container + augmentation.module diff --git a/docs/source/color.rst b/docs/source/color.rst new file mode 100644 index 0000000..bc8520d --- /dev/null +++ b/docs/source/color.rst @@ -0,0 +1,235 @@ +kornia.color +============ + +.. meta:: + :name: description + :content: "The Color module in Kornia provides a variety of functions for color space conversions, including RGB, HLS, HSV, Lab, and more. It also offers utilities for color maps and Bayer RAW processing." + +.. currentmodule:: kornia.color + +The functions in this section perform various color space conversions. + +.. note:: + Check a tutorial for color space conversions `here `__. + + +Grayscale +--------- + +.. tip:: + Learn more: https://en.wikipedia.org/wiki/Grayscale + +.. autofunction:: rgb_to_grayscale +.. autofunction:: bgr_to_grayscale +.. autofunction:: grayscale_to_rgb +.. autofunction:: apply_colormap + :noindex: + +.. autoclass:: GrayscaleToRgb +.. autoclass:: RgbToGrayscale +.. autoclass:: BgrToGrayscale +.. autoclass:: ApplyColorMap + :noindex: + +RGB +--- + +.. tip:: + Learn more: https://en.wikipedia.org/wiki/RGB_color_model + +.. autofunction:: rgb_to_bgr +.. autofunction:: bgr_to_rgb + +.. autofunction:: rgb_to_linear_rgb +.. autofunction:: linear_rgb_to_rgb + +.. autofunction:: rgb_to_rgb255 +.. autofunction:: rgb255_to_rgb +.. autofunction:: rgb255_to_normals +.. autofunction:: normals_to_rgb255 + +.. autoclass:: RgbToBgr +.. autoclass:: BgrToRgb + +.. autoclass:: LinearRgbToRgb +.. autoclass:: RgbToLinearRgb +.. autoclass:: Rgb255ToRgb +.. autoclass:: RgbToRgb255 +.. autoclass:: Rgb255ToNormals +.. autoclass:: NormalsToRgb255 + + +RGBA +---- + +.. tip:: + Learn more: https://en.wikipedia.org/wiki/RGBA_color_model + +.. autofunction:: bgr_to_rgba +.. autofunction:: rgb_to_rgba +.. autofunction:: rgba_to_rgb +.. autofunction:: rgba_to_bgr + +.. autoclass:: RgbToRgba +.. autoclass:: BgrToRgba +.. autoclass:: RgbaToRgb +.. autoclass:: RgbaToBgr + +HLS +--- + +.. tip:: + Learn more: https://en.wikipedia.org/wiki/HSL_and_HSV + +.. autofunction:: rgb_to_hls +.. autofunction:: hls_to_rgb + +.. autoclass:: RgbToHls +.. autoclass:: HlsToRgb + +HSV +--- + +.. tip:: + Learn more: https://en.wikipedia.org/wiki/HSL_and_HSV + +.. autofunction:: rgb_to_hsv +.. autofunction:: hsv_to_rgb + +.. autoclass:: RgbToHsv +.. autoclass:: HsvToRgb + +LUV +--- + +.. tip:: + Learn more: https://en.wikipedia.org/wiki/CIELUV + +.. autofunction:: rgb_to_luv +.. autofunction:: luv_to_rgb + +.. autoclass:: RgbToLuv +.. autoclass:: LuvToRgb + +Lab +--- + +.. tip:: + Learn more: https://en.wikipedia.org/wiki/CIELAB_color_space + +.. autofunction:: rgb_to_lab +.. autofunction:: lab_to_rgb + +.. autoclass:: RgbToLab +.. autoclass:: LabToRgb + +YCbCr +----- + +.. tip:: + Learn more: https://en.wikipedia.org/wiki/YCbCr + +.. autofunction:: rgb_to_ycbcr +.. autofunction:: ycbcr_to_rgb + +.. autoclass:: YcbcrToRgb +.. autoclass:: RgbToYcbcr + +YUV +--- + +.. tip:: + Learn more: https://en.wikipedia.org/wiki/YUV + +.. autofunction:: rgb_to_yuv +.. autofunction:: yuv_to_rgb + +.. autoclass:: RgbToYuv +.. autoclass:: YuvToRgb + +YUV420 +------ + +.. tip:: + Learn more: https://en.wikipedia.org/wiki/YUV + +.. tip:: + Learn more: https://en.wikipedia.org/wiki/Chroma_subsampling + +.. autofunction:: rgb_to_yuv420 +.. autofunction:: yuv420_to_rgb + +.. autoclass:: RgbToYuv420 +.. autoclass:: Yuv420ToRgb + +YUV422 +------ + +.. tip:: + Learn more: https://en.wikipedia.org/wiki/YUV + +.. tip:: + Learn more: https://en.wikipedia.org/wiki/Chroma_subsampling + +.. autofunction:: rgb_to_yuv422 +.. autofunction:: yuv422_to_rgb + +.. autoclass:: RgbToYuv422 +.. autoclass:: Yuv422ToRgb + +XYZ +--- + +.. tip:: + Learn more: https://en.wikipedia.org/wiki/CIELUV + +.. autofunction:: rgb_to_xyz +.. autofunction:: xyz_to_rgb + +.. autoclass:: RgbToXyz +.. autoclass:: XyzToRgb + +Bayer RAW +--------- + +.. tip:: + Learn more: https://en.wikipedia.org/wiki/Bayer_filter + +.. autoclass:: CFA + :members: + :undoc-members: + +.. autofunction:: rgb_to_raw +.. autofunction:: raw_to_rgb +.. autofunction:: raw_to_rgb_2x2_downscaled + +.. autoclass:: RawToRgb +.. autoclass:: RgbToRaw +.. autoclass:: RawToRgb2x2Downscaled + +Sepia +----- + +.. autoclass:: Sepia +.. autofunction:: sepia + + +Color Maps +---------- + +.. autoclass:: ColorMap +.. autoclass:: RGBColor + + +Color maps available: + +.. autoclass:: ColorMapType + :members: + :undoc-members: + :member-order: bysource + + +Functions and modules to use the color maps: + +.. autofunction:: apply_colormap +.. autoclass:: ApplyColorMap diff --git a/docs/source/community/bibliography.rst b/docs/source/community/bibliography.rst new file mode 100644 index 0000000..0d21440 --- /dev/null +++ b/docs/source/community/bibliography.rst @@ -0,0 +1,4 @@ +Bibliography +============ + +.. bibliography:: diff --git a/docs/source/community/chinese.rst b/docs/source/community/chinese.rst new file mode 100644 index 0000000..05322f2 --- /dev/null +++ b/docs/source/community/chinese.rst @@ -0,0 +1,13 @@ +Kornia 社区 +=============== + +欢迎加入 Kornia 社区!扫描下方的二维码可关注 Kornia 团队的 `官方交流 QQ 群 `_. + +.. image:: https://github.com/kornia/kornia/raw/main/docs/source/_static/img/cn_community_qq.jpg + :height: 700 + :alt: QQ group + + +.. image:: https://github.com/kornia/kornia/raw/main/docs/source/_static/img/cn_community_zhihu.jpg + :height: 700 + :alt: Zhihu diff --git a/docs/source/community/contribute.rst b/docs/source/community/contribute.rst new file mode 100644 index 0000000..b534261 --- /dev/null +++ b/docs/source/community/contribute.rst @@ -0,0 +1,28 @@ +Contribute to Kornia +==================== + +Everyone is welcomed to get involved with the project. There are different ways in how you can put your two cents: + +1. Ask/Answer questions in the #kornia tag in `Kornia Discuss `_: + - Please, don't use GitHub issues for Q&A. + +2. Report bugs through GitHub issues: + - Do a quick search first to see whether others reported a similar issue. + - In case you find an unreported bug, please open a new ticket. + - Try to provide as much information as possible. + +3. Join our Slack `[HERE] `_ + +4. Fix a bug or develop a feature from the roadmap: + - We will always have an open ticket showing the current roadmap. + - Pick an unassigned feature (or potentially propose new one) or an open bug ticket. + - Follow the instructions from `Developing Kornia `_ to set up your development environment and start coding. + - Checkout our coding conventions. See more details below. + - Run the test framework locally and make sure all works as expected before sending a pull request. + - Open a Pull Request, get the green light from the CI and get your code merged. + +5. Donate resources to the project through `GitHub Sponsor `_ or `Open Collective `_ ! + +For more information about our development process, see our `Developers corner `_. + +**Happy coding !** diff --git a/docs/source/community/faqs.rst b/docs/source/community/faqs.rst new file mode 100644 index 0000000..c699e0a --- /dev/null +++ b/docs/source/community/faqs.rst @@ -0,0 +1,23 @@ +Frequently Asked Questions +========================== + +This document contains frequently asked questions. + +Kornia relation to Pytorch Geometry/Geometric +--------------------------------------------- + +This project started as a small differentiable geometric computer +vision package called `PyTorch Geometry `_ +released during the PyTorch devcon 2018 (see the presented +`poster `_). +The project evolved to a more generic computer vision library and due to the naming +conflict between `Pytorch Geometric `_ +we decided to rename the whole package and focus to more generic vision functionalities. + +Kornia relation to Other Computer Vision Projects +------------------------------------------------- + +The project mimics some of the functionalities found in OpenCV. Even though +the project is backed up by the `OpenCV.org `_, there is no +intention at all to merge in any form both projects. *Kornia* is meant to +provide differentiable operators to train nets, while *OpenCV* scope is inference. diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..f12a9d5 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,338 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import builtins +import importlib.util +import inspect +import os +import sys +from datetime import UTC, datetime + +# Monkey-patch for PyTorch compatibility with sphinx_autodoc_typehints +# Newer versions of PyTorch removed torch.jit.annotations.compiler_flag +import torch.jit.annotations + +if not hasattr(torch.jit.annotations, "compiler_flag"): + torch.jit.annotations.compiler_flag = None + +# To add an evnironment variable +builtins.__sphinx_build__ = True + +# --- Patch sphinx_autodoc_defaultargs to not crash on torchscript/pybind11 callables --- +# --- Patch sphinx_autodoc_defaultargs to not crash on torchscript/pybind11 callables --- +try: + import sphinx_autodoc_defaultargs + + _orig_process_docstring = sphinx_autodoc_defaultargs.process_docstring + + def _safe_process_docstring(app, what, name, obj, options, lines): + try: + return _orig_process_docstring(app, what, name, obj, options, lines) + except ValueError as e: + msg = str(e).lower() + if "no signature found for builtin" in msg or "pybind11" in msg: + return # leave docstring unchanged + raise + + sphinx_autodoc_defaultargs.process_docstring = _safe_process_docstring + +except (ModuleNotFoundError, ImportError): + # Optional dependency not installed in some environments. + sphinx_autodoc_defaultargs = None + +except AttributeError: + # Extension API changed; don't patch. + sphinx_autodoc_defaultargs = None + + +# readthedocs generated the whole documentation in an isolated environment +# by cloning the git repo. Thus, any on-the-fly operation will not effect +# on the resulting documentation. We therefore need to import and run the +# corresponding code here. +spec = importlib.util.spec_from_file_location("generate_examples", "../generate_examples.py") +generate_examples = importlib.util.module_from_spec(spec) +spec.loader.exec_module(generate_examples) + +# Pre-generate the example images +generate_examples.main() + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +current_path = os.path.abspath(os.path.join(__file__, "..", "..", "..")) +sys.path.append(current_path) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.mathjax", + "sphinx.ext.napoleon", + "sphinx_autodoc_typehints", + "sphinx_autodoc_defaultargs", + "sphinx_copybutton", + "sphinx.ext.linkcode", + "sphinx.ext.githubpages", + "sphinxcontrib.bibtex", + "sphinxcontrib.gtagjs", + "sphinxcontrib.youtube", + "sphinx_design", + "notfound.extension", +] + +# substitutes the default values +docstring_default_arg_substitution = "Default: " +autodoc_preserve_defaults = True + +bibtex_bibfiles = ["references.bib"] +napoleon_use_ivar = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = [".rst", ".ipynb"] + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = "Kornia" +author = f"{project} developers" +copyright = f"{datetime.now(tz=UTC).year}, {author}" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. + +# version = 'master (' + kornia.__version__ + ' )' +version = "" + +if "READTHEDOCS" not in os.environ: + # if developing locally, use kornia.__version__ as version + from kornia import __version__ + + version = __version__ + +# release = 'master' +release = version + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = "en" + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ["_build", ".ipynb_checkpoints"] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "friendly" +pygments_dark_style = "monokai" + +html_theme = "furo" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# TODO(jian): make to work with https://docs.kornia.org +html_baseurl = "https://kornia.readthedocs.io/en/latest/" + +# Changing sidebar title to Kornia +html_title = "Kornia" + +html_theme_options = { + # 'analytics_id': 'G-RKS4WFXVHJ', # Unsupported by furo theme + "light_logo": "img/kornia_logo_only_light.svg", + "dark_logo": "img/kornia_logo_only_dark.svg", + "sidebar_hide_name": True, + "navigation_with_keys": True, + "light_css_variables": { + "color-sidebar-background": "#3980F5", + "color-sidebar-background-border": "#3980F5", + "color-sidebar-caption-text": "white", + "color-sidebar-link-text--top-level": "white", + "color-sidebar-link-text": "white", + "sidebar-caption-font-size": "normal", + "color-sidebar-item-background--hover": " #5dade2", + }, + "dark_css_variables": { + "color-sidebar-background": "#1a1c1e", + "color-sidebar-background-border": "#1a1c1e", + "color-sidebar-caption-text": "white", + "color-sidebar-link-text--top-level": "white", + }, + # "announcement": """ + # + # + # Star Kornia on GitHub + # + # """, +} + +# html_logo = '_static/img/kornia_logo.svg' +# html_logo = '_static/img/kornia_logo_only.png' +html_favicon = "_static/img/kornia_logo_favicon.png" + +# Config the `sphinxcontrib.gtagjs` extension +# NOTE: if this didn't work, we can remove the extension itself +gtagjs_ids = [ + "G-YSCFZB2WDV", # Shouldn't be necessary if the readthedocs autoinjection work +] + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] +html_extra_path = [] + +# Output file base name for HTML help builder. +htmlhelp_basename = "Kornia" +html_css_files = ["css/main.css"] +html_js_files = [ + "js/custom.js", + ("https://gradio.s3-us-west-2.amazonaws.com/4.38.1/gradio.js", {"defer": "defer", "type": "module"}), + "https://cdnjs.cloudflare.com/ajax/libs/iframe-resizer/4.3.2/iframeResizer.min.js", +] + +# Configure viewcode extension. +# based on https://github.com/readthedocs/sphinx-autoapi/issues/202 +rtd_version = os.environ.get("READTHEDOCS_VERSION") +if rtd_version and rtd_version not in {"latest", "stable"}: + code_ref = rtd_version +else: + code_ref = "main" + +code_url = f"https://github.com/kornia/kornia/blob/{code_ref}" + + +def linkcode_resolve(domain, info): + if domain != "py": + return None + + modname = info.get("module") + fullname = info.get("fullname") + if not modname or not fullname: + return None + + try: + mod = importlib.import_module(modname) + except (ImportError, ModuleNotFoundError): + return None + + obj = mod + for part in fullname.split("."): + try: + obj = getattr(obj, part) + except AttributeError: + return None + + obj = inspect.unwrap(obj) + + try: + fn = inspect.getsourcefile(obj) + src, start = inspect.getsourcelines(obj) + except (TypeError, OSError, ValueError): + return None + + if not fn: + return None + + fn = os.path.abspath(fn).replace("\\", "/") + marker = "/kornia/" + idx = fn.rfind(marker) + if idx == -1: + return None + + file_rel = fn[idx + 1 :] # -> "kornia/....py" + end = start + len(src) - 1 + return f"{code_url}/{file_rel}#L{start}-L{end}" + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [(master_doc, "kornia.tex", "Kornia", "manual")] + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(master_doc, "Kornia", "Kornia Documentation", [author], 1)] + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "kornia", + "Kornia Documentation", + author, + "Kornia", + "Differentiable Computer Vision in Pytorch.", + "Miscellaneous", + ) +] + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + "python": ("https://docs.python.org/3/", None), + "numpy": ("http://numpy.org/doc/stable/", None), + "torch": ("http://pytorch.org/docs/stable/", None), +} + +# mock these modules and won't try to actually import them +autodoc_mock_imports = ["boxmot", "segmentation_models_pytorch"] diff --git a/docs/source/contrib.rst b/docs/source/contrib.rst new file mode 100644 index 0000000..9a1f3a9 --- /dev/null +++ b/docs/source/contrib.rst @@ -0,0 +1,208 @@ +kornia.contrib +============== + +.. meta:: + :name: description + :content: "The Kornia.contrib module provides models and utilities for deep learning applications, including base models and the EfficientViT architecture. The module offers configurable, efficient Vision Transformer (ViT) models and tools to load and utilize pre-trained checkpoints, designed to be integrated into deep learning pipelines with PyTorch." + +.. currentmodule:: kornia.contrib + +Models +------ + +Base +^^^^ +.. autoclass:: kornia.models.base.ModelBase + :members: + :undoc-members: + +EfficientViT +^^^^^^^^^^^^ + +.. autoclass:: kornia.models.efficient_vit.EfficientViT + :members: from_config, forward, load_checkpoint + :undoc-members: + :special-members: __init__, + + +.. autoclass:: kornia.models.efficient_vit.EfficientViTConfig + :members: + :undoc-members: + +Backbones +^^^^^^^^^ + +.. autoclass:: kornia.models.efficient_vit.backbone.EfficientViTBackbone + :members: + :undoc-members: + +.. autofunction:: kornia.models.efficient_vit.backbone.efficientvit_backbone_b0 +.. autofunction:: kornia.models.efficient_vit.backbone.efficientvit_backbone_b1 +.. autofunction:: kornia.models.efficient_vit.backbone.efficientvit_backbone_b2 +.. autofunction:: kornia.models.efficient_vit.backbone.efficientvit_backbone_b3 + +.. autoclass:: kornia.models.efficient_vit.backbone.EfficientViTLargeBackbone + :members: + :undoc-members: + +.. autofunction:: kornia.models.efficient_vit.backbone.efficientvit_backbone_l0 +.. autofunction:: kornia.models.efficient_vit.backbone.efficientvit_backbone_l1 +.. autofunction:: kornia.models.efficient_vit.backbone.efficientvit_backbone_l2 +.. autofunction:: kornia.models.efficient_vit.backbone.efficientvit_backbone_l3 + +Structures +^^^^^^^^^^ + +.. _anchor SegmentationResults: +.. autoclass:: kornia.models.structures.SegmentationResults + :members: + :undoc-members: + +.. autoclass:: kornia.models.structures.Prompts + :members: + :undoc-members: + +VisualPrompter +-------------- + +.. autoclass:: kornia.contrib.visual_prompter.VisualPrompter + :members: set_image, reset_image, compile, predict, preprocess_image, preprocess_prompts + +Edge Detection +-------------- + +.. autoclass:: EdgeDetector + +Face Detection +-------------- + +.. autoclass:: FaceDetector + +.. autoclass:: FaceKeypoint + :members: + :undoc-members: + +.. autoclass:: FaceDetectorResult + :members: + :undoc-members: + +Interactive Demo +^^^^^^^^^^^^^^^^ +.. raw:: html + + + +Visit the `Kornia face detection demo on the Hugging Face Spaces +`_. + +Object Detection +---------------- + +.. autoclass:: kornia.contrib.object_detection.BoundingBoxDataFormat + :members: + :undoc-members: + :member-order: bysource + +.. autoclass:: kornia.contrib.object_detection.BoundingBox + :members: + :undoc-members: + +.. autoclass:: kornia.contrib.object_detection.ObjectDetectorResult + :members: + :undoc-members: + +.. autoclass:: kornia.contrib.object_detection.ObjectDetector + :members: + :undoc-members: + :special-members: __init__, + +.. autoclass:: kornia.contrib.object_detection.ResizePreProcessor + :members: + :undoc-members: + +.. autofunction:: kornia.contrib.object_detection.results_from_detections + +Real-Time Detection Transformer (RT-DETR) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. autoclass:: kornia.models.rt_detr.RTDETRModelType + :members: + :undoc-members: + :member-order: bysource + +.. autoclass:: kornia.models.rt_detr.RTDETRConfig + :members: + :undoc-members: + +.. autoclass:: kornia.models.rt_detr.RTDETR + :members: from_config, forward, load_checkpoint + :undoc-members: + :special-members: __init__, + +.. autoclass:: kornia.models.rt_detr.DETRPostProcessor + :members: + :undoc-members: + +Image Segmentation +------------------ +.. autofunction:: connected_components + +Segment Anything (SAM) +^^^^^^^^^^^^^^^^^^^^^^ + +.. autoclass:: kornia.models.sam.SamModelType + :members: + :undoc-members: + :member-order: bysource + +.. autoclass:: kornia.models.sam.SamConfig + :members: + :undoc-members: + +.. autoclass:: kornia.models.sam.Sam + :members: from_config, forward, load_checkpoint + :undoc-members: + :special-members: __init__, + +Image Patches +------------- + +.. autofunction:: compute_padding +.. autofunction:: extract_tensor_patches +.. autofunction:: combine_tensor_patches + +.. autoclass:: ExtractTensorPatches +.. autoclass:: CombineTensorPatches + +Image Classification +-------------------- + +.. autoclass:: kornia.models.vit.VisionTransformer + :members: +.. autoclass:: kornia.models.vit_mobile.MobileViT +.. autoclass:: TinyViT + :members: + +Image Stitching +--------------- + +.. autoclass:: ImageStitcher + +Lambda +------ + +.. autoclass:: Lambda + +Distance Transform +------------------ + +.. autofunction:: distance_transform +.. autofunction:: diamond_square + +.. autoclass:: DistanceTransform + +KMeans +------ + +.. autoclass:: KMeans + :members: diff --git a/docs/source/core.rst b/docs/source/core.rst new file mode 100644 index 0000000..a538831 --- /dev/null +++ b/docs/source/core.rst @@ -0,0 +1,12 @@ +kornia.core +=========== + +.. meta:: + :name: description + :content: "The kornia.core module in Kornia provides foundational classes and utilities for tensor manipulation. Key classes like TensorWrapper allow for enhanced handling of image tensors with support for various operations and transformations in computer vision tasks." + +.. currentmodule:: kornia.core + +.. autoclass:: TensorWrapper + :members: + :undoc-members: diff --git a/docs/source/data/epipolar_geometry.svg.png b/docs/source/data/epipolar_geometry.svg.png new file mode 100644 index 0000000..dd4f113 Binary files /dev/null and b/docs/source/data/epipolar_geometry.svg.png differ diff --git a/docs/source/data/lie.png b/docs/source/data/lie.png new file mode 100644 index 0000000..818d95f Binary files /dev/null and b/docs/source/data/lie.png differ diff --git a/docs/source/data/lie_ops.png b/docs/source/data/lie_ops.png new file mode 100644 index 0000000..9fbfd51 Binary files /dev/null and b/docs/source/data/lie_ops.png differ diff --git a/docs/source/data/pinhole_model.png b/docs/source/data/pinhole_model.png new file mode 100644 index 0000000..e423d64 Binary files /dev/null and b/docs/source/data/pinhole_model.png differ diff --git a/docs/source/enhance.rst b/docs/source/enhance.rst new file mode 100644 index 0000000..3221fa9 --- /dev/null +++ b/docs/source/enhance.rst @@ -0,0 +1,99 @@ +kornia.enhance +============== + +.. meta:: + :name: description + :content: "The Kornia.enhance module provides a suite of image enhancement functions including brightness, contrast, hue, saturation adjustments, as well as normalization and equalization techniques. It also features advanced transformations like ZCA whitening and differentiable JPEG codec. Explore interactive demos on Hugging Face Spaces." + +.. currentmodule:: kornia.enhance + +The functions in this section perform normalisations and intensity transformations. + +Adjustment +---------- + +.. autofunction:: add_weighted +.. autofunction:: adjust_brightness +.. autofunction:: adjust_contrast +.. autofunction:: adjust_contrast_with_mean_subtraction +.. autofunction:: adjust_gamma +.. autofunction:: adjust_hue +.. autofunction:: adjust_saturation +.. autofunction:: adjust_sigmoid +.. autofunction:: adjust_log +.. autofunction:: invert +.. autofunction:: posterize +.. autofunction:: sharpness +.. autofunction:: solarize + +Interactive Demo +~~~~~~~~~~~~~~~~ +.. raw:: html + + + +Visit the demo on `Hugging Face Spaces `__. + +Equalization +------------ + +.. autofunction:: equalize +.. autofunction:: equalize_clahe +.. autofunction:: equalize3d + +Interactive Demo + +.. raw:: html + + + +Visit the demo on `Hugging Face Spaces `__. + + +.. autofunction:: histogram +.. autofunction:: histogram2d +.. autofunction:: image_histogram2d + +Normalizations +-------------- + +.. autofunction:: normalize +.. autofunction:: normalize_min_max +.. autofunction:: denormalize +.. autofunction:: zca_mean +.. autofunction:: zca_whiten +.. autofunction:: linear_transform + +Codec +----- + +.. autofunction:: jpeg_codec_differentiable + + +Modules +------- + +.. autoclass:: Normalize +.. autoclass:: Denormalize +.. autoclass:: ZCAWhitening + :members: + +.. autoclass:: AdjustBrightness +.. autoclass:: AdjustContrast +.. autoclass:: AdjustSaturation +.. autoclass:: AdjustHue +.. autoclass:: AdjustGamma +.. autoclass:: AdjustSigmoid +.. autoclass:: AdjustLog +.. autoclass:: AddWeighted + +.. autoclass:: Invert + +.. autoclass:: JPEGCodecDifferentiable + + +ZCA Whitening Interactive Demo +------------------------------ +.. raw:: html + + diff --git a/docs/source/feature.rst b/docs/source/feature.rst new file mode 100644 index 0000000..070117c --- /dev/null +++ b/docs/source/feature.rst @@ -0,0 +1,299 @@ +Local Features and Image Matching +================================= + +.. meta:: + :name: description + :content: "The Local Features and Image Matching module in Kornia offers tools to detect, describe, and match local features in images. It includes various detectors, descriptors, and matching algorithms based on classical methods and deep learning models like Harris, SIFT, KeyNet, DISK, and LoFTR. The module also provides tools for working with local affine frames (LAFs), including affine shape and orientation estimation." + +This module provides a set of tools to detect and describe local features in images. The module is designed to be +compatible with the PyTorch ecosystem and provides a set of models and differentiable operations to be used in deep learning +pipelines. + +The module is divided into three main components: + +1. **Detectors**: These are models that are used to detect keypoints in images. The module provides a set of detectors that + are based on different algorithms such as Harris, GFTT, Hessian, and DoG. The module also provides a set of detectors that + are based on deep learning models such as KeyNet, DISK and DeDoDe. +2. **Descriptors**: These are models that are used to describe the local features detected by the detectors. The module + provides a set of descriptors that are based on different algorithms such as SIFT, HardNet, and TFeat. The module also + provides a set of descriptors that are based on deep learning models such as HyNet, SOSNet, and LAFDescriptor. +3. **Matching**: These are models that are used to match the local features detected and described by the detectors and + descriptors. The module provides a set of matching algorithms such as nearest neighbor, mutual nearest neighbor, and + geometrically aware matching. Besides this, the module also contains AdaLAM hancrafted and LightGlue learned matchers. + Finally, the module provides LoFTR - detector-less semi-dense image matching model. + +Besides this, the module also provides a set of tools to work with local affine frames (LAF) such as extracting patches, +normalizing, denormalizing, and rotating LAFs. The module also provides a set of models to estimate the affine shape of +LAFs such as LAFAffineShapeEstimator and PatchAffineShapeEstimator. The module also provides a set of models to estimate +the orientation of LAFs such as OriNet and LAFOrienter. + + +Finally, kind of addition, module contains a DeFMO model for the task of video frame interpolation, specifically high speed objects debluring. + +Benchmarks and recommendations +-------------------------------- + +The following table shows the performance of the different models on `IMC2021 benchmark `_ . + + +.. list-table:: IMC2021 Benchmark, 8000 features + :widths: 50 50 50 50 50 + :header-rows: 1 + + * - Feature name + - Stereo mAA @ 10 degrees, PhotoTourism. + - Multiview mAA @ 10 degrees, PhotoTourism. + - Stereo mAA @ 10 degrees, PragueParks. + - Multiview mAA @ 10 degrees, PragueParks. + * - DISK-LightGlue + - 0.6184 + - 0.7741 + - 0.6116 + - 0.4988 + * - LoFTR + - 0.6090 + - 0.7609 + - 0.7546 + - 0.4711 + * - OpenCV-DoG-HardNet-LightGlue + - 0.5850 + - 0.7587 + - 0.6525 + - 0.4973 + * - OpenCV-DoG-AffNet-HardNet8-AdaLAM + - 0.5502 + - 0.7522 + - 0.5998 + - 0.4712 + * - Upright SIFT (OpenCV) + - 0.5122 + - 0.6849 + - 0.6060 + - 0.4439 + + +.. list-table:: IMC2021 Benchmark, 2048 features + :widths: 50 50 50 50 50 + :header-rows: 1 + + * - Feature name + - Stereo mAA @ 10 degrees, PhotoTourism. + - Multiview mAA @ 10 degrees, PhotoTourism. + - Stereo mAA @ 10 degrees, PragueParks. + - Multiview mAA @ 10 degrees, PragueParks. + * - DISK-LightGlue + - 0.5720 + - 0.7543 + - 0.5099 + - 0.4565 + * - OpenCV-DoG-HardNet-LightGlue + - 0.3954 + - 0.6272 + - 0.5157 + - 0.4456 + * - Upright SIFT (OpenCV) + - 0.3827 + - 0.5545 + - 0.4136 + - 0.3607 + +LoFTR works the best for indoor scenes, whereas DISK and DeDoDe + LightGlue work the best for outdoor scenes. +The DeDoDe and speed benchmarks are coming soon. +For some other use-cases you may want to use SIFT, or SIFT + HardNet + LightGlue, e.g. for remote sensing or medical imaging. + + +.. currentmodule:: kornia.feature + +Detectors +--------- + +.. autofunction:: gftt_response +.. autofunction:: harris_response +.. autofunction:: hessian_response +.. autofunction:: dog_response +.. autofunction:: dog_response_single +.. autoclass:: BlobHessian +.. autoclass:: CornerGFTT +.. autoclass:: CornerHarris +.. autoclass:: BlobDoG +.. autoclass:: BlobDoGSingle +.. autoclass:: KeyNet +.. autoclass:: MultiResolutionDetector + :members: forward, remove_borders, detect_features_on_single_level, detect + + +.. autoclass:: ScaleSpaceDetector + :members: forward + +.. autoclass:: KeyNetDetector + :members: forward + +Descriptors +----------- + +.. autoclass:: DenseSIFTDescriptor +.. autoclass:: SIFTDescriptor +.. autoclass:: MKDDescriptor +.. autoclass:: HardNet +.. autoclass:: HardNet8 +.. autoclass:: HyNet +.. autoclass:: TFeat +.. autoclass:: SOSNet +.. autoclass:: LAFDescriptor + :members: forward + +.. autoclass:: SOLD2 + :members: forward + +.. autofunction:: get_laf_descriptors + + + +Local Features (Detector and Descriptors together) +--------------------------------------------------- + + +.. autoclass:: LocalFeature + :members: forward + +.. autoclass:: SOLD2_detector + :members: forward + +.. autoclass:: ALIKED + :members: forward, from_pretrained, forward_laf + +.. autoclass:: ALIKEDFeatures + :undoc-members: + :members: n, to + +.. autoclass:: DeDoDe + :members: forward, from_pretrained, describe, detect + +.. autoclass:: DISK + :members: forward, from_pretrained, heatmap_and_dense_descriptors + +.. autoclass:: XFeat + :members: forward, from_pretrained, detectAndCompute, detectAndComputeDense, match_xfeat, match_xfeat_star + +.. autoclass:: XFeatModel + :members: forward + +.. autoclass:: InterpolateSparse2d + :members: forward + +.. autoclass:: DISKFeatures + :undoc-members: + :members: x, y, to + +.. autoclass:: SIFTFeature + :members: forward + +.. autoclass:: SIFTFeatureScaleSpace + :members: forward + +.. autoclass:: GFTTAffNetHardNet + :members: forward + +.. autoclass::HesAffNetHardNet + :members: forward + +.. autoclass:: KeyNetAffNetHardNet + :members: forward + +.. autoclass:: KeyNetHardNet + :members: forward + + +Matching +-------- + +.. autofunction:: match_nn +.. autofunction:: match_mnn +.. autofunction:: match_snn +.. autofunction:: match_smnn +.. autofunction:: match_fginn +.. autofunction:: match_adalam + +.. autoclass:: DescriptorMatcher + :members: forward + +.. autoclass:: GeometryAwareDescriptorMatcher + :members: forward + +.. autoclass:: LocalFeatureMatcher + :members: forward + +.. autoclass:: LightGlueMatcher + :members: forward + +.. autoclass:: LightGlue + :members: forward + +.. autoclass:: LoFTR + :members: forward + +Interactive Demo +~~~~~~~~~~~~~~~~ +.. raw:: html + + + +.. autoclass:: OnnxLightGlue + :members: forward + + +Local Affine Frames (LAF) +------------------------- + +.. autofunction:: extract_patches_from_pyramid +.. autofunction:: extract_patches_simple +.. autofunction:: normalize_laf +.. autofunction:: denormalize_laf +.. autofunction:: laf_to_boundary_points +.. autofunction:: ellipse_to_laf +.. autofunction:: make_upright +.. autofunction:: scale_laf +.. autofunction:: get_laf_scale +.. autofunction:: get_laf_center +.. autofunction:: rotate_laf +.. autofunction:: get_laf_orientation +.. autofunction:: set_laf_orientation +.. autofunction:: laf_from_center_scale_ori +.. autofunction:: laf_is_inside_image +.. autofunction:: laf_to_three_points +.. autofunction:: laf_from_three_points +.. autofunction:: KORNIA_CHECK_LAF +.. autofunction:: perspective_transform_lafs + +.. autoclass:: PassLAF + :members: forward + +.. autoclass:: PatchAffineShapeEstimator + :members: forward + +.. autoclass:: LAFAffineShapeEstimator + :members: forward + +.. autoclass:: LAFOrienter + :members: forward + +.. autoclass:: PatchDominantGradientOrientation + :members: forward + +.. autoclass:: OriNet + :members: forward + +.. autoclass:: LAFAffNetShapeEstimator + :members: forward + +Layers +------ + +.. autoclass:: FilterResponseNorm2d +.. autoclass:: TLU + +Other +------ + +.. autoclass:: DeFMO + :members: forward diff --git a/docs/source/filters.rst b/docs/source/filters.rst new file mode 100644 index 0000000..a5a0ba1 --- /dev/null +++ b/docs/source/filters.rst @@ -0,0 +1,101 @@ +kornia.filters +============== + +.. meta:: + :name: description + :content: "The Kornia filters module provides various image filtering operations such as blurring, edge detection, and noise reduction. It includes functions for bilateral, Gaussian, motion, median, and unsharp mask filtering, as well as pooling operations for blurring. These operations are designed to be differentiable and can be integrated seamlessly into deep learning pipelines." + +.. currentmodule:: kornia.filters + +The functions in this sections perform various image filtering operations. + +Blurring +-------- + +.. autofunction:: bilateral_blur +.. autofunction:: blur_pool2d +.. autofunction:: box_blur +.. autofunction:: gaussian_blur2d +.. autofunction:: guided_blur +.. autofunction:: joint_bilateral_blur +.. autofunction:: max_blur_pool2d +.. autofunction:: median_blur +.. autofunction:: motion_blur +.. autofunction:: unsharp_mask + +Interactive Demo +~~~~~~~~~~~~~~~~ +.. raw:: html + + + +Visit the `Kornia image filtering demo on the Hugging Face Spaces +`_. + +Edge detection +-------------- + +.. autofunction:: canny +.. autofunction:: laplacian +.. autofunction:: sobel +.. autofunction:: spatial_gradient +.. autofunction:: spatial_gradient3d + + +.. autoclass:: Laplacian +.. autoclass:: Sobel +.. autoclass:: Canny +.. autoclass:: SpatialGradient +.. autoclass:: SpatialGradient3d + + +Interactive Demo +~~~~~~~~~~~~~~~~ +.. raw:: html + + + +Visit the `Kornia edge detector demo on the Hugging Face Spaces +`_. + +Segmentation +-------------- + +.. autofunction:: in_range +.. autoclass:: InRange +.. autofunction:: otsu_threshold +.. autoclass:: OtsuThreshold + +Filtering API +------------- + +.. autofunction:: filter2d +.. autofunction:: filter2d_separable +.. autofunction:: filter3d + +Kernels +------- + +.. autofunction:: get_gaussian_kernel1d +.. autofunction:: get_gaussian_erf_kernel1d +.. autofunction:: get_gaussian_discrete_kernel1d +.. autofunction:: get_gaussian_kernel2d +.. autofunction:: get_hanning_kernel1d +.. autofunction:: get_hanning_kernel2d +.. autofunction:: get_laplacian_kernel1d +.. autofunction:: get_laplacian_kernel2d +.. autofunction:: get_motion_kernel2d + +Module +------ + +.. autoclass:: BilateralBlur +.. autoclass:: BlurPool2D +.. autoclass:: BoxBlur +.. autoclass:: MaxBlurPool2D +.. autoclass:: MedianBlur +.. autoclass:: GaussianBlur2d +.. autoclass:: GuidedBlur +.. autoclass:: JointBilateralBlur +.. autoclass:: MotionBlur +.. autoclass:: UnsharpMask diff --git a/docs/source/geometry.bbox.rst b/docs/source/geometry.bbox.rst new file mode 100644 index 0000000..79661bc --- /dev/null +++ b/docs/source/geometry.bbox.rst @@ -0,0 +1,11 @@ +kornia.geometry.bbox +==================== + +.. meta:: + :name: description + :content: "The kornia.geometry.bbox module in Kornia provides useful functions for manipulating 2D and 3D bounding boxes. It includes tools for creating, transforming, and operating on bounding boxes, making it easier to handle spatial data in computer vision tasks." + +Module with useful functionalities for 2D and 3D bounding boxes manipulation. + +.. automodule:: kornia.geometry.bbox + :members: diff --git a/docs/source/geometry.boxes.rst b/docs/source/geometry.boxes.rst new file mode 100644 index 0000000..132bd51 --- /dev/null +++ b/docs/source/geometry.boxes.rst @@ -0,0 +1,11 @@ +kornia.geometry.boxes +===================== + +.. meta:: + :name: description + :content: "The kornia.geometry.boxes module in Kornia provides essential functions for manipulating 2D and 3D bounding boxes. It offers utilities for the creation, transformation, and handling of bounding boxes, which are crucial for spatial data processing in computer vision applications." + +Module with useful functionalities for 2D and 3D bounding boxes manipulation. + +.. automodule:: kornia.geometry.boxes + :members: diff --git a/docs/source/geometry.calibration.rst b/docs/source/geometry.calibration.rst new file mode 100644 index 0000000..d154876 --- /dev/null +++ b/docs/source/geometry.calibration.rst @@ -0,0 +1,130 @@ +kornia.geometry.calibration +=========================== + +.. meta:: + :name: description + :content: "The kornia.geometry.calibration module provides essential functions for camera calibration, including lens distortion modeling, undistortion, and Perspective-n-Point (PnP) solutions. This module supports advanced camera calibration techniques for both pinhole and distorted models, aiding in accurate 3D point projection, distortion correction, and camera pose estimation." + +.. currentmodule:: kornia.geometry.calibration + +Module with useful functionalities for camera calibration. + +The pinhole model is an ideal projection model that not considers lens distortion for the projection of a 3D point :math:`(X, Y, Z)` onto the image plane. To model the distortion of a projected 2D pixel point :math:`(u,v)` with the linear pinhole model, we need first to estimate the normalized 2D points coordinates :math:`(\bar{u}, \bar{v})`. For that, we can use the calibration matrix :math:`\mathbf{K}` with the following expression + +.. math:: + \begin{align} + \begin{bmatrix} + \bar{u}\\ + \bar{v}\\ + 1 + \end{bmatrix} = \mathbf{K}^{-1} \begin{bmatrix} + u \\ + v \\ + 1 + \end{bmatrix} \enspace, + \end{align} + +which is equivalent to directly using the internal parameters: focals :math:`f_u, f_v` and principal point :math:`(u_0, v_0)` to estimated the normalized coordinates + +.. math:: + \begin{equation} + \bar{u} = (u - u_0)/f_u \enspace, \\ + \bar{v} = (v - v_0)/f_v \enspace. + \end{equation} + +The normalized distorted point :math:`(\bar{u}_d, \bar{v}_d)` is given by + +.. math:: + \begin{align} + \begin{bmatrix} + \bar{u}_d\\ + \bar{v}_d + \end{bmatrix} = \dfrac{1+k_1r^2+k_2r^4+k_3r^6}{1+k_4r^2+k_5r^4+k_6r^6} \begin{bmatrix} + \bar{u}\\ + \bar{v} + \end{bmatrix} + + \begin{bmatrix} + 2p_1\bar{u}\bar{v} + p_2(r^2 + 2\bar{u}^2) + s_1r^2 + s_2r^4\\ + 2p_2\bar{u}\bar{v} + p_1(r^2 + 2\bar{v}^2) + s_3r^2 + s_4r^4 + \end{bmatrix} \enspace, + \end{align} + +where :math:`r = \bar{u}^2 + \bar{v}^2`. With this model we consider radial :math:`(k_1, k_2, k_3, k_4, k_4, k_6)`, tangential :math:`(p_1, p_2)`, and thin prism :math:`(s_1, s_2, s_3, s_4)` distortion. If we want to consider tilt distortion :math:`(\tau_x, \tau_y)`, we need an additional step where we estimate a point :math:`(\bar{u}'_d, \bar{v}'_d)` + +.. math:: + \begin{align} + \begin{bmatrix} + \bar{u}'_d\\ + \bar{v}'_d\\ + 1 + \end{bmatrix} = \begin{bmatrix} + \mathbf{R}_{33}(\tau_x, \tau_y) & 0 & -\mathbf{R}_{13}(\tau_x, \tau_y)\\ + 0 & \mathbf{R}_{33}(\tau_x, \tau_y) & -\mathbf{R}_{23}(\tau_x, \tau_y)\\ + 0 & 0 & 1 + \end{bmatrix} + \mathbf{R}(\tau_x, \tau_y) \begin{bmatrix} + \bar{u}_d \\ + \bar{v}_d \\ + 1 + \end{bmatrix} \enspace, + \end{align} + +where :math:`\mathbf{R}(\tau_x, \tau_y)` is a 3D rotation matrix defined by an :math:`X` and :math:`Y` rotation given by the angles :math:`\tau_x` and :math:`\tau_y`. Furthermore, :math:`\mathbf{R}_{ij}(\tau_x, \tau_y)` represent the :math:`i`-th row and :math:`j`-th column from :math:`\mathbf{R}(\tau_x, \tau_y)` matrix. + +.. math:: + \begin{align} + \mathbf{R}(\tau_x, \tau_y) = + \begin{bmatrix} + \cos \tau_y & 0 & -\sin \tau_y \\ + 0 & 1 & 0 \\ + \sin \tau_y & 0 & \cos \tau_y + \end{bmatrix} + \begin{bmatrix} + 1 & 0 & 0 \\ + 0 & \cos \tau_x & \sin \tau_x \\ + 0 & -\sin \tau_x & \cos \tau_x + \end{bmatrix} \enspace. + \end{align} + + +Finally, we just need to come back to the original (unnormalized) pixel space. For that we can use the intrinsic matrix + +.. math:: + \begin{align} + \begin{bmatrix} + u_d\\ + v_d\\ + 1 + \end{bmatrix} = \mathbf{K} \begin{bmatrix} + \bar{u}'_d\\ + \bar{v}'_d\\ + 1 + \end{bmatrix} \enspace, + \end{align} + + +which is equivalent to + +.. math:: + \begin{equation} + u_d = f_u \bar{u}'_d + u_0 \enspace, \\ + v_d = f_v \bar{v}'_d + v_0 \enspace. + \end{equation} + +Undistortion +------------ + +To compensate for lens distortion a set of 2D points, i.e., to estimate the undistorted coordinates for a given set of distorted points, we need to inverse the previously explained distortion model. For the case of undistorting an image, instead of estimating the undistorted location for each pixel, we distort each pixel in the destination image (final undistorted image) to match them with the input image. We finally interpolate the intensity values at each pixel. + +.. autofunction:: undistort_image + +.. autofunction:: undistort_points + +.. autofunction:: distort_points + +.. autofunction:: tilt_projection + +Perspective-n-Point (PnP) +------------------------- + +.. autofunction:: solve_pnp_dlt diff --git a/docs/source/geometry.camera.perspective.rst b/docs/source/geometry.camera.perspective.rst new file mode 100644 index 0000000..45d8514 --- /dev/null +++ b/docs/source/geometry.camera.perspective.rst @@ -0,0 +1,12 @@ +Perspective Camera +------------------ + +.. meta:: + :name: description + :content: "The kornia.geometry.camera.perspective module provides functions for 3D point projection and unprojection in perspective camera models. These functions are essential for transforming 3D world coordinates into 2D image plane coordinates and vice versa, enabling advanced computer vision applications like depth estimation and 3D scene reconstruction." + +.. currentmodule:: kornia.geometry.camera.perspective + + +.. autofunction:: project_points +.. autofunction:: unproject_points diff --git a/docs/source/geometry.camera.pinhole.rst b/docs/source/geometry.camera.pinhole.rst new file mode 100644 index 0000000..160690d --- /dev/null +++ b/docs/source/geometry.camera.pinhole.rst @@ -0,0 +1,79 @@ +Pinhole Camera +-------------- + +.. meta:: + :name: description + :content: "The kornia.geometry.camera.pinhole module provides functions and data structures to describe the projection of 3D scene space onto a 2D image plane using the Pinhole Camera model. This model is widely used in computer vision for tasks like camera calibration and 3D scene reconstruction, utilizing the projection of 3D points via perspective transformations." + +.. currentmodule:: kornia.geometry.camera.pinhole + +In this module we have all the functions and data structures needed to describe the projection of a 3D scene space onto a 2D image plane. + +In computer vision, we can map between the 3D world and a 2D image using *projective geometry*. The module implements the simplest camera model, the **Pinhole Camera**, which is the most basic model for general projective cameras from the finite cameras group. + +The Pinhole Camera model is shown in the following figure: + +.. image:: data/pinhole_model.png + +Using this model, a scene view can be formed by projecting 3D points into the image plane using a perspective transformation. + +.. math:: + s \; m' = K [R|t] M' + +or + +.. math:: + s \begin{bmatrix} u \\ v \\ 1\end{bmatrix} = + \begin{bmatrix} + f_x & 0 & u_0 \\ + 0 & f_y & v_0 \\ + 0 & 0 & 1 + \end{bmatrix} + \begin{bmatrix} + r_{11} & r_{12} & r_{13} & t_1 \\ + r_{21} & r_{22} & r_{23} & t_2 \\ + r_{31} & r_{32} & r_{33} & t_3 + \end{bmatrix} + \begin{bmatrix} + X \\ + Y \\ + Z \\ + 1 + \end{bmatrix} + +where: + * :math:`M'` is a 3D point in space with coordinates :math:`[X,Y,Z]^T` expressed in an Euclidean coordinate frame known as the *world coordinate system*. + * :math:`m'` is the projection of the 3D point :math:`M'` onto the *image plane* with coordinates :math:`[u,v]^T` expressed in pixel units. + * :math:`K` is the *camera calibration matrix*, also referred as the intrinsic matrix. + * :math:`C` is the *principal point offset* with coordinates :math:`[u_0, v_0]^T` at the origin in the image plane. + * :math:`fx, fy` are the focal lengths expressed in pixel units. + +The camera rotation and translation are expressed in terms of an Euclidean coordinate frame known as the *world coordinate system*. These terms are usually expressed by the joint rotation-translation matrix :math:`[R|t]` which is also known as the extrinsic matrix. It is used to describe the camera pose around a static scene and transforms the coordinates of a 3D point :math:`(X,Y,Z)` from the *world coordinate system* to the *camera coordinate system*. + +The :class:`PinholeCamera` expects the *intrinsic matrices* and the *extrinsic matrices* +to be of shape `(B, 4, 4)` such that each *intrinsic matrix* has the following format: + +.. math:: + \begin{bmatrix} + f_x & 0 & u_0 & 0\\ + 0 & f_y & v_0 & 0\\ + 0 & 0 & 1 & 0 \\ + 0 & 0 & 0 & 1 + \end{bmatrix} + +And each *extrinsic matrix* has the following format: + +.. math:: + \begin{bmatrix} + r_{11} & r_{12} & r_{13} & t_1 \\ + r_{21} & r_{22} & r_{23} & t_2 \\ + r_{31} & r_{32} & r_{33} & t_3 \\ + 0 & 0 & 0 & 1 + \end{bmatrix} + + +.. autoclass:: PinholeCamera + :members: + +.. autofunction:: cam2pixel +.. autofunction:: pixel2cam diff --git a/docs/source/geometry.camera.rst b/docs/source/geometry.camera.rst new file mode 100644 index 0000000..72b44e7 --- /dev/null +++ b/docs/source/geometry.camera.rst @@ -0,0 +1,37 @@ +kornia.geometry.camera +====================== + +.. meta:: + :name: description + :content: "The kornia.geometry.camera module provides a variety of functions for handling camera projections and distortions. It includes support for projecting 3D points to a 2D image plane, both with perspective and orthographic projections, as well as distortion models like affine and Kannala-Brandt. This module enables robust camera calibration and 3D scene transformations in computer vision applications." + +.. currentmodule:: kornia.geometry.camera + +Projections +----------- + +.. autofunction:: project_points_z1 +.. autofunction:: unproject_points_z1 +.. autofunction:: dx_project_points_z1 + +.. autofunction:: project_points_orthographic +.. autofunction:: unproject_points_orthographic +.. autofunction:: dx_project_points_orthographic + +Distortion +---------- + +.. autofunction:: distort_points_affine +.. autofunction:: undistort_points_affine +.. autofunction:: dx_distort_points_affine + +.. autofunction:: distort_points_kannala_brandt +.. autofunction:: undistort_points_kannala_brandt +.. autofunction:: dx_distort_points_kannala_brandt + +.. toctree:: + :maxdepth: 2 + + geometry.camera.pinhole + geometry.camera.perspective + geometry.camera.stereo diff --git a/docs/source/geometry.camera.stereo.rst b/docs/source/geometry.camera.stereo.rst new file mode 100644 index 0000000..560a9d3 --- /dev/null +++ b/docs/source/geometry.camera.stereo.rst @@ -0,0 +1,164 @@ +Stereo Camera +------------- + +.. meta:: + :name: description + :content: "The kornia.geometry.camera.stereo module provides functionality for working with a horizontal stereo camera setup. It includes the StereoCamera class, which allows for the conversion of disparity maps into 3D point clouds using the stereo rectification model. This module leverages camera calibration matrices and disparity information to reproject pixels from 2D to 3D space, facilitating tasks such as depth estimation and 3D reconstruction." + +.. currentmodule:: kornia.geometry.camera.stereo + +In this module we provide the :class:`StereoCamera` that contains functionality for working with a horizontal stereo camera setup. + +The horizontal stereo camera setup is assumed to be calibrated and rectified such that the setup can be described by two camera matrices: + +The *left rectified camera matrix*: + +.. math:: + P_0 = \begin{bmatrix} + fx & 0 & cx & 0 \\ + 0 & fy & cy & 0 \\ + 0 & 0 & 1 & 0 + \end{bmatrix} + +The *right rectified camera matrix*: + +.. math:: + P_1 = \begin{bmatrix} + fx & 0 & cx & tx * fx \\ + 0 & fy & cy & 0 \\ + 0 & 0 & 1 & 0 + \end{bmatrix} + +where: + * :math:`fx` is the focal length in the x-direction in pixels. + * :math:`fy` is the focal length in the y-direction in pixels. + * :math:`cx` is the x-coordinate of the principal point in pixels. + * :math:`cy` is the y-coordinate of the principal point in pixels. + * :math:`tx` is the horizontal baseline in metric units. + +These camera matrices are obtained by calibrating your stereo camera setup which can be done `in OpenCV `_. + +The :class:`StereoCamera` allows you to convert disparity maps to the real world 3D geometry represented by a point cloud. + +This is done by forming the :math:`Q` matrix. + +Using the pinhole camera model to project :math:`[X Y Z 1]` in world coordinates to :math:`uv` pixels in the left and right camera frame respectively: + +.. math:: + + \begin{bmatrix} + u \\ + v \\ + 1 + \end{bmatrix} = P_0 * + \begin{bmatrix} + X \\ + Y \\ + Z \\ + 1 + \end{bmatrix} \\ + \begin{bmatrix} + u-d \\ + v \\ + 1 + \end{bmatrix} = P_1 * + \begin{bmatrix} + X \\ + Y \\ + Z \\ + 1 + \end{bmatrix} + +Where :math:`d` is the disparity between pixels in left and right image. + +Combining these two expressions let us write it as one matrix multiplication + +.. math:: + \begin{bmatrix} + u \\ + v \\ + u-d \\ + 1 + \end{bmatrix} = + \begin{bmatrix} + fx & 0 & cx_{left} & 0 \\ + 0 & fy & cy & 0 \\ + fx & 0 & cx_{right} & fx * tx \\ + 0 & 0 & 1 & 0 + \end{bmatrix} + \begin{bmatrix} + X \\ + Y \\ + Z \\ + 1 + \end{bmatrix} + +Now subtract the first from the third row and invert the expression and you'll get: + +.. math:: + \begin{bmatrix} + u \\ + v \\ + d \\ + 1 + \end{bmatrix} = + \begin{bmatrix} + fy * tx & 0 & 0 & -fy * cx * tx \\ + 0 & fx * tx & 0 & -fx * cy * tx \\ + 0 & 0 & 0 & fx * fy * tx \\ + 0 & 0 & -fy & fy * (cx_{left} -cx_{right}) + \end{bmatrix} + \begin{bmatrix} + X \\ + Y \\ + Z \\ + 1 + \end{bmatrix} + +Where :math:`Q` is + +.. math:: + + Q = \begin{bmatrix} + fy * tx & 0 & 0 & -fy * cx * tx \\ + 0 & fx * tx & 0 & -fx * cy * tx \\ + 0 & 0 & 0 & fx * fy * tx \\ + 0 & 0 & -fy & fy * (cx_{left} -cx_{right}) + \end{bmatrix} + +Notice here that the x-coordinate for the principal point in the left and right camera :math:`cx` might differ, which is being taken into account here. + +Assuming :math:`fx = fy` you can further reduce this to: + +.. math:: + Q = \begin{bmatrix} + 1 & 0 & 0 & -cx \\ + 0 & 1 & 0 & -cy \\ + 0 & 0 & 0 & fx \\ + 0 & 0 & -1/tx & (cx_{left} -cx_{right} / tx) + \end{bmatrix} + +But we'll use the general :math:`Q` matrix. + +Using the :math:`Q` matrix we can obtain the 3D points by: + +.. math:: + \begin{bmatrix} + X \\ + Y \\ + Z \\ + W + \end{bmatrix} = Q * + \begin{bmatrix} + u \\ + v \\ + disparity(y, v) \\ + z + \end{bmatrix} + +.. autoclass:: StereoCamera + :members: + + .. automethod:: __init__ + +.. autofunction:: reproject_disparity_to_3D diff --git a/docs/source/geometry.conversions.rst b/docs/source/geometry.conversions.rst new file mode 100644 index 0000000..1136d82 --- /dev/null +++ b/docs/source/geometry.conversions.rst @@ -0,0 +1,83 @@ +kornia.geometry.conversions +================================== + +.. meta:: + :name: description + :content: "The kornia.geometry.conversions module provides a collection of functions for converting between various geometric representations, such as angles, coordinates, homographies, quaternions, rotation matrices, and Euler angles. This module includes utilities for transforming points between homogeneous and non-homogeneous coordinates, normalizing and denormalizing pixel coordinates, as well as handling pose transformations and quaternion operations for 3D geometry tasks." + +.. currentmodule:: kornia.geometry.conversions + +Angles +------ + +.. autofunction:: rad2deg +.. autofunction:: deg2rad +.. autofunction:: pol2cart +.. autofunction:: cart2pol +.. autofunction:: angle_to_rotation_matrix + +Coordinates +----------- + +.. autofunction:: convert_points_from_homogeneous +.. autofunction:: convert_points_to_homogeneous +.. autofunction:: convert_affinematrix_to_homography +.. autofunction:: denormalize_pixel_coordinates +.. autofunction:: normalize_pixel_coordinates +.. autofunction:: denormalize_pixel_coordinates3d +.. autofunction:: normalize_pixel_coordinates3d +.. autofunction:: normalize_points_with_intrinsics +.. autofunction:: denormalize_points_with_intrinsics + + +Homography +---------- + +.. autofunction:: normalize_homography +.. autofunction:: denormalize_homography +.. autofunction:: normalize_homography3d + +Quaternion +---------- + +.. autofunction:: quaternion_to_axis_angle +.. autofunction:: quaternion_to_rotation_matrix +.. autofunction:: quaternion_log_to_exp +.. autofunction:: quaternion_exp_to_log +.. autofunction:: normalize_quaternion + +Vector +------ + +.. autofunction:: vector_to_skew_symmetric_matrix + +Rotation Matrix +--------------- + +.. autofunction:: rotation_matrix_to_axis_angle +.. autofunction:: rotation_matrix_to_quaternion + +Axis Angle +---------- + +.. autofunction:: axis_angle_to_quaternion +.. autofunction:: axis_angle_to_rotation_matrix + +Euler Angles +------------ + +.. autofunction:: quaternion_from_euler +.. autofunction:: euler_from_quaternion + +Pose +---------- + +.. autofunction:: Rt_to_matrix4x4 +.. autofunction:: matrix4x4_to_Rt +.. autofunction:: worldtocam_to_camtoworld_Rt +.. autofunction:: camtoworld_to_worldtocam_Rt +.. autofunction:: camtoworld_graphics_to_vision_4x4 +.. autofunction:: camtoworld_vision_to_graphics_4x4 +.. autofunction:: camtoworld_graphics_to_vision_Rt +.. autofunction:: camtoworld_vision_to_graphics_Rt +.. autofunction:: ARKitQTVecs_to_ColmapQTVecs diff --git a/docs/source/geometry.depth.rst b/docs/source/geometry.depth.rst new file mode 100644 index 0000000..cff5214 --- /dev/null +++ b/docs/source/geometry.depth.rst @@ -0,0 +1,16 @@ +kornia.geometry.depth +===================== + +.. meta:: + :name: description + :content: "The kornia.geometry.depth module provides functions for working with depth-related transformations in 3D vision tasks. Key functionalities include computing depth from disparity, converting depth maps to 3D points, obtaining surface normals from depth data, and unprojecting depth data from mesh grids. Additionally, the module supports depth-based image warping and working with depth through plane equations, enabling advanced geometric operations in computer vision." + +.. currentmodule:: kornia.geometry.depth + +.. autofunction:: depth_from_disparity +.. autofunction:: depth_to_3d +.. autofunction:: depth_to_3d_v2 +.. autofunction:: unproject_meshgrid +.. autofunction:: depth_to_normals +.. autofunction:: depth_from_plane_equation +.. autofunction:: warp_frame_depth diff --git a/docs/source/geometry.epipolar.rst b/docs/source/geometry.epipolar.rst new file mode 100644 index 0000000..90e0205 --- /dev/null +++ b/docs/source/geometry.epipolar.rst @@ -0,0 +1,64 @@ +kornia.geometry.epipolar +======================== + +.. meta:: + :name: description + :content: "The kornia.geometry.epipolar module provides essential tools for working with epipolar geometry, crucial in tasks like Structure from Motion (SfM). It includes functions for computing the essential and fundamental matrices, decomposing them, and deriving relative camera motion. The module also offers various metrics for evaluating epipolar constraints, triangulating 3D points, and handling projection transformations. Additionally, it supports functions for normalizing points and transformations, calculating epipolar distances, and generating camera intrinsics." + +.. currentmodule:: kornia.geometry.epipolar + +Module with useful functionalities for epipolar geometry used by Structure from Motion + +.. image:: data/epipolar_geometry.svg.png + + +Essential +--------- +.. autofunction:: find_essential +.. autofunction:: essential_from_fundamental +.. autofunction:: essential_from_Rt +.. autofunction:: decompose_essential_matrix +.. autofunction:: motion_from_essential +.. autofunction:: motion_from_essential_choose_solution +.. autofunction:: relative_camera_motion + +Fundamental +----------- + +.. autofunction:: find_fundamental +.. autofunction:: fundamental_from_essential +.. autofunction:: fundamental_from_projections +.. autofunction:: compute_correspond_epilines +.. autofunction:: normalize_points +.. autofunction:: normalize_transformation +.. autofunction:: get_perpendicular +.. autofunction:: get_closest_point_on_epipolar_line + + +Metrics +------- + +.. autofunction:: sampson_epipolar_distance +.. autofunction:: symmetrical_epipolar_distance +.. autofunction:: left_to_right_epipolar_distance +.. autofunction:: right_to_left_epipolar_distance + +Projection +---------- + +.. autofunction:: projection_from_KRt +.. autofunction:: projections_from_fundamental +.. autofunction:: intrinsics_like +.. autofunction:: scale_intrinsics +.. autofunction:: random_intrinsics + +Numeric +------- + +.. autofunction:: cross_product_matrix + + +Triangulation +------------- + +.. autofunction:: triangulate_points diff --git a/docs/source/geometry.grid.rst b/docs/source/geometry.grid.rst new file mode 100644 index 0000000..086427b --- /dev/null +++ b/docs/source/geometry.grid.rst @@ -0,0 +1,11 @@ +kornia.geometry.grid +==================== + +.. meta:: + :name: description + :content: "The kornia.geometry.grid module provides functions for generating coordinate grids for images. It includes functions for creating 2D and 3D mesh grids with optional coordinate normalization, making it easy to work with grid-based transformations and sampling operations in computer vision tasks." + +.. currentmodule:: kornia.geometry.grid + +.. autofunction:: create_meshgrid +.. autofunction:: create_meshgrid3d diff --git a/docs/source/geometry.homography.rst b/docs/source/geometry.homography.rst new file mode 100644 index 0000000..52cd22b --- /dev/null +++ b/docs/source/geometry.homography.rst @@ -0,0 +1,18 @@ +kornia.geometry.homography +========================== + +.. meta:: + :name: description + :content: "The kornia.geometry.homography module provides essential tools for manipulating and working with homographies, which describe the transformation between two images of the same scene from different viewpoints. The module includes a variety of functions for computing, applying, and manipulating homographies, making it useful for tasks such as image stitching, object tracking, and perspective warping. The module also offers an interactive demo to explore homography warping in real time." + +Module with useful functionalities for homographies manipulation. + +.. automodule:: kornia.geometry.homography + :members: + + +Interactive Demo +---------------- +.. raw:: html + + diff --git a/docs/source/geometry.keypoints.rst b/docs/source/geometry.keypoints.rst new file mode 100644 index 0000000..ecd7ff2 --- /dev/null +++ b/docs/source/geometry.keypoints.rst @@ -0,0 +1,17 @@ +kornia.geometry.keypoints +========================= + +.. meta:: + :name: description + :content: "The kornia.geometry.keypoints module offers essential functionalities for the manipulation of 2D and 3D keypoints. This module provides tools for working with keypoints in both 2D and 3D space, enabling tasks such as feature detection, matching, and transformation. The module includes the `Keypoints` and `Keypoints3D` classes, which facilitate the management and processing of keypoint data, making it useful for applications like object recognition, tracking, and 3D reconstruction." + +Module with useful functionalities for 2D and 3D keypoints manipulation. + +.. autoclass:: kornia.geometry.keypoints.Keypoints + :members: + :undoc-members: + + +.. autoclass:: kornia.geometry.keypoints.Keypoints3D + :members: + :undoc-members: diff --git a/docs/source/geometry.liegroup.rst b/docs/source/geometry.liegroup.rst new file mode 100644 index 0000000..dd7fdcf --- /dev/null +++ b/docs/source/geometry.liegroup.rst @@ -0,0 +1,64 @@ +kornia.geometry.liegroup +========================== + +.. meta:: + :name: description + :content: "The kornia.geometry.liegroup module provides mathematical tools and operations for working with Lie groups and Lie algebras, which are fundamental in many areas of robotics and computer vision. Lie groups describe smooth manifolds that satisfy group axioms, while Lie algebras represent the tangent space at the identity element of these groups. This module includes key classes for common Lie groups like `SO2`, `SO3`, `SE2`, and `SE3`, and provides functions for performing operations like the exponential and logarithmic maps, which connect Lie algebras and Lie groups." + +.. currentmodule:: kornia.geometry.liegroup + +The Lie group encompasses the concepts of `group` and `smooth manifold` in a unique body. + +A group is a non-empty set with an operation that satisfies the following constraints: the operation is associative, has an identity element, +and every element of the set has an inverse element. + +See more: `Group `_ + +A Lie group :math:`G` is a smooth manifold whose elements satisfy the group axioms.You can visualize the idea of manifold like a curved, smooth (hyper)-surface, with no edges or +spikes, embedded in a space of higher dimension. + +See more: `Manifold `_ + +In robotics, we say that our state vector evolves on this surface, that is, the manifold describes or is defined by the constraints imposed on +the state. + +lie algebra +=========== + +.. image:: data/lie.png + +If :math:`M` is the manifold that represents a lie group, the tangent space at the identity is called the Lie algebra of :math:`M`. +The Lie algebra :math:`m` is a vector space. As such, its elements can be identified with vectors in :math:`R^d`, whose +dimension :math:`d` is the number of degrees of freedom of :math:`M`. For example :math:`d` would be 3 in the case of lie group :math:`SO3` + +lie group and lie algebra +========================== + +Every Lie group has an associated Lie algebra. We relate the +Lie group with its Lie algebra through the following facts + +#. The Lie algebra :math:`m` is a vector space. As such, its elements can be identified with vectors in :math:`R^d`, whose + dimension :math:`d` is the number of degrees of freedom of :math:`M`. + +#. The exponential map, `exp` : :math:`m` → :math:`M`, exactly converts elements of the Lie algebra into elements of the group. + The `log` map is the inverse operation. + +.. image:: data/lie_ops.png + +Reference: `Micro lie theory `_ + +.. autoclass:: So3 + :members: + :special-members: + +.. autoclass:: Se3 + :members: + :special-members: + +.. autoclass:: So2 + :members: + :special-members: + +.. autoclass:: Se2 + :members: + :special-members: diff --git a/docs/source/geometry.linalg.rst b/docs/source/geometry.linalg.rst new file mode 100644 index 0000000..cb53b46 --- /dev/null +++ b/docs/source/geometry.linalg.rst @@ -0,0 +1,18 @@ +kornia.geometry.linalg +====================== + +.. meta:: + :name: description + :content: "The kornia.geometry.linalg module offers essential linear algebra utilities for geometric computations in computer vision and robotics. It includes functions for transforming points, calculating transformations, computing distances, and performing vector operations. Key functions include relative transformation, inverse transformation, point-line distance, squared norm, dot product, and Euclidean distance. These operations are fundamental for tasks such as geometric transformations, distance computations, and vector manipulations." + +.. currentmodule:: kornia.geometry.linalg + + +.. autofunction:: relative_transformation +.. autofunction:: compose_transformations +.. autofunction:: inverse_transformation +.. autofunction:: transform_points +.. autofunction:: point_line_distance +.. autofunction:: squared_norm +.. autofunction:: batched_dot_product +.. autofunction:: euclidean_distance diff --git a/docs/source/geometry.line.rst b/docs/source/geometry.line.rst new file mode 100644 index 0000000..cf1ffd1 --- /dev/null +++ b/docs/source/geometry.line.rst @@ -0,0 +1,25 @@ +kornia.geometry.line +==================== + +.. meta:: + :name: description + :content: "The kornia.geometry.line module provides functionality for working with lines and line segments in geometric space. It includes classes such as ParametrizedLine for line representation, Hyperplane for handling high-dimensional planes, and functions like fit_line for fitting lines to data points. This module is essential for tasks like line segment matching and line fitting in computer vision and geometric analysis." + +.. currentmodule:: kornia.geometry.line + +.. autoclass:: ParametrizedLine + :members: + :special-members: __init__ + +.. autofunction:: fit_line + +.. autoclass:: Hyperplane + + +Interactive Demo +---------------- + +.. raw:: html + + + diff --git a/docs/source/geometry.pointcloud.rst b/docs/source/geometry.pointcloud.rst new file mode 100644 index 0000000..7d31a1b --- /dev/null +++ b/docs/source/geometry.pointcloud.rst @@ -0,0 +1,11 @@ +kornia.geometry.pointcloud +========================== + +.. meta:: + :name: description + :content: "The kornia.geometry.pointcloud module provides functionality for loading and saving point clouds in PLY format. It includes functions for reading and writing 3D point cloud data, making it easy to work with point cloud files in computer vision and geometric processing tasks." + +.. currentmodule:: kornia.geometry.pointcloud + +.. autofunction:: save_pointcloud_ply +.. autofunction:: load_pointcloud_ply diff --git a/docs/source/geometry.quaternion.rst b/docs/source/geometry.quaternion.rst new file mode 100644 index 0000000..956639a --- /dev/null +++ b/docs/source/geometry.quaternion.rst @@ -0,0 +1,12 @@ +kornia.geometry.quaternion +========================== + +.. meta:: + :name: description + :content: "The kornia.geometry.quaternion module provides tools for working with quaternions, a mathematical concept widely used in 3D geometry and computer vision. The Quaternion class allows for quaternion manipulation, including conversion between different representations like axis-angle and rotation matrices. This module is essential for operations involving 3D rotations and transformations." + +.. currentmodule:: kornia.geometry.quaternion + +.. autoclass:: Quaternion + :members: + :special-members: diff --git a/docs/source/geometry.ransac.rst b/docs/source/geometry.ransac.rst new file mode 100644 index 0000000..c7e495a --- /dev/null +++ b/docs/source/geometry.ransac.rst @@ -0,0 +1,13 @@ +kornia.geometry.ransac +====================== + +.. meta:: + :name: description + :content: "The kornia.geometry.ransac module implements the RANSAC (Random Sample Consensus) algorithm for robust fitting of models in the presence of outliers. The RANSAC class allows for efficient outlier rejection and model estimation, which is crucial in tasks such as stereo vision, homography estimation, and 3D reconstruction. This module is valuable for geometric computer vision problems." + +.. currentmodule:: kornia.geometry.ransac + +Module with RANSAC + +.. autoclass:: RANSAC + :members: forward diff --git a/docs/source/geometry.rst b/docs/source/geometry.rst new file mode 100644 index 0000000..404cea7 --- /dev/null +++ b/docs/source/geometry.rst @@ -0,0 +1,69 @@ +kornia.geometry +=============== + +.. meta:: + :name: description + :content: "The Kornia.geometry module provides essential geometric transformations for computer vision tasks, including 2D and 3D image manipulation. It includes submodules for image transforms, camera models, coordinate conversions, linear algebra operations, and depth map processing, supporting a wide range of geometric operations for accurate spatial transformations and 3D reconstructions." + +Geometric image transformations is another key ingredient in computer vision to manipulate images. +Since geometry operations are typically performed in 2D or 3D, we provide several algorithms to work +with both cases. This module, the original core of the library, consists of the following submodules: +transforms, camera, conversions, linalg and depth. We next describe each of them: + +- `transforms`: The module provides low level interfaces to manipulate 2D images, with routines for Rotating, + Scaling, Translating, Shearing; Cropping functions in several modalities such as central crops, + crop and resize; Flipping transformations in the vertical and horizontal axis; Resizing operations; + Functions to warp tensors given affine or perspective transformations, + and utilities to compute the transformation matrices to perform the mentioned operations. +- `camera`: A set of routines specific to different types of camera representations such as Pinhole + or Orthographic models containing functionalities such as projecting and unprojecting points from the + camera to a world frame. +- `conversions`: Routines to perform conversions between angle representation such as + radians to degrees, coordinates normalization, and homogeneous to euclidean. Moreover, we include advanced + conversions for 3D geometry representations such as Quaternion, Axis-Angle, Rotation Matrix, or Rodrigues + formula. +- `linalg`: Functions to perform general rigid-body homogeneous transformations. We include implementations to + transform points between frames and for homogeneous transformations, manipulation such as composition, + inverse and to compute relative poses. +- `depth`: A set of layers to manipulate depth maps such as how to compute 3D point clouds given depth maps and + calibrated cameras; compute surface normals per pixel and warp tensor frames given calibrated cameras setup. + + +:Resources: + + **align_corners** + + align_corners is a switch that widely offered in PyTorch geometric transform functions. + Here is a simple illustration showing how a 4x4 image is upsampled to 8x8, made by + `bkkm16 `_. + + .. image:: https://user-images.githubusercontent.com/4803565/110627988-df8a4d00-81a2-11eb-8e13-06d3f7b09ef1.png + + - `align_corners=True`, pixels are arranged as a grid of points. Points at the corners are aligned. + - `align_corners=False`, pixels are arranged as 1x1 areas. Area boundaries, rather than their centers, are aligned. + + +.. currentmodule:: kornia.geometry + +.. toctree:: + :maxdepth: 3 + + geometry.bbox + geometry.boxes + geometry.keypoints + geometry.calibration + geometry.camera + geometry.conversions + geometry.depth + geometry.epipolar + geometry.grid + geometry.homography + geometry.liegroup + geometry.linalg + geometry.line + geometry.pointcloud + geometry.quaternion + geometry.solvers + geometry.subpix + geometry.transform + geometry.ransac diff --git a/docs/source/geometry.solvers.rst b/docs/source/geometry.solvers.rst new file mode 100644 index 0000000..841a570 --- /dev/null +++ b/docs/source/geometry.solvers.rst @@ -0,0 +1,24 @@ +kornia.geometry.solvers +======================= + +.. meta:: + :name: description + :content: "The kornia.geometry.solvers module provides various solvers and optimizers for geometric problems. It includes polynomial solvers for quadratic and cubic equations, as well as functions for multiplying polynomials of different degrees and converting determinants into polynomial forms. These tools are essential for handling geometric transformations, optimizations, and other computational geometry tasks in computer vision." + +.. currentmodule:: kornia.geometry.solvers + +Module containing various geometrical solvers/optimizers. + +Homogeneous Solvers +------------------- + +.. autofunction:: null_vector_3x4 + +Polynomial Solvers +------------------ + +.. autofunction:: solve_quadratic +.. autofunction:: solve_cubic +.. autofunction:: multiply_deg_one_poly +.. autofunction:: multiply_deg_two_one_poly +.. autofunction:: determinant_to_polynomial diff --git a/docs/source/geometry.subpix.rst b/docs/source/geometry.subpix.rst new file mode 100644 index 0000000..93cc9ff --- /dev/null +++ b/docs/source/geometry.subpix.rst @@ -0,0 +1,75 @@ +kornia.geometry.subpix +====================== + +.. meta:: + :name: description + :content: "The kornia.geometry.subpix module provides functionalities for extracting coordinates with sub-pixel accuracy. It includes convolutional methods like soft argmax and quadratic interpolation for precise 2D and 3D coordinate extraction. Additionally, it offers spatial softmax and expectation techniques, as well as non-maximum suppression (NMS) for 2D and 3D data, making it ideal for tasks requiring high-resolution spatial localization in computer vision." + +.. currentmodule:: kornia.geometry.subpix + +Module with useful functionalities to extract coordinates sub-pixel accuracy. + +Convolutional +------------- + +.. autofunction:: conv_soft_argmax2d +.. autofunction:: conv_soft_argmax3d +.. autofunction:: conv_quad_interp3d +.. autofunction:: iterative_quad_interp3d + +.. tip:: + + :class:`AdaptiveQuadInterp3d` (the default subpix module in + :class:`~kornia.feature.ScaleSpaceDetector`) automatically picks the faster + backend based on the input device: + + * **CUDA** → :func:`conv_quad_interp3d` — batched gather+solve, 1.5–2× faster on GPU. + * **CPU** → :func:`iterative_quad_interp3d` — processes only NMS maxima directly, + no dilation overhead. + + Both backends produce numerically identical results (max difference < 2 × 10\ :sup:`-6`). + + .. code-block:: python + + import torch + from kornia.geometry.subpix import AdaptiveQuadInterp3d + from kornia.feature import ScaleSpaceDetector + from kornia.feature.responses import BlobDoG + from kornia.geometry.transform import ScalePyramid + + detector = ScaleSpaceDetector( + num_features=2000, + resp_module=BlobDoG(), + # default — auto-selects conv on CUDA, patch on CPU: + subpix_module=AdaptiveQuadInterp3d(strict_maxima_bonus=0.0), + scale_pyr_module=ScalePyramid(3, 1.6, 32, double_image=True), + scale_space_response=True, + minima_are_also_good=True, + ) + +Spatial +------- + +.. autofunction:: spatial_softmax2d +.. autofunction:: spatial_expectation2d +.. autofunction:: spatial_soft_argmax2d +.. autofunction:: render_gaussian2d + +Non Maxima Suppression +---------------------- + +.. autofunction:: nms2d +.. autofunction:: nms3d +.. autofunction:: nms3d_minmax + +Module +------ + +.. autoclass:: SpatialSoftArgmax2d +.. autoclass:: ConvSoftArgmax2d +.. autoclass:: ConvSoftArgmax3d +.. autoclass:: AdaptiveQuadInterp3d +.. autoclass:: ConvQuadInterp3d +.. autoclass:: IterativeQuadInterp3d +.. autoclass:: NonMaximaSuppression2d +.. autoclass:: NonMaximaSuppression3d diff --git a/docs/source/geometry.transform.rst b/docs/source/geometry.transform.rst new file mode 100644 index 0000000..694a554 --- /dev/null +++ b/docs/source/geometry.transform.rst @@ -0,0 +1,109 @@ +kornia.geometry.transform +========================= + +.. meta:: + :name: description + :content: "The kornia.geometry.transform module provides a comprehensive set of functions for performing various geometric transformations on 2D and 3D images. It includes operators for warping, affine, and perspective transformations, as well as tools for resizing, scaling, rotating, and shearing images. Additionally, it supports advanced image registration techniques, pyramid building, and non-rigid transformations such as elastic transforms. This module is essential for tasks requiring image manipulation and alignment in computer vision and image processing workflows." + +.. currentmodule:: kornia.geometry.transform + +The functions in this section perform various geometrical transformations of 2D images. + + +Warp operators +-------------- + +.. autofunction:: warp_perspective +.. autofunction:: warp_perspective3d +.. autofunction:: warp_affine +.. autofunction:: warp_affine3d +.. autofunction:: warp_image_tps +.. autofunction:: warp_points_tps +.. autofunction:: warp_grid +.. autofunction:: warp_grid3d +.. autofunction:: remap + +Image 2d transforms +------------------- + +.. autofunction:: affine +.. autofunction:: rotate +.. autofunction:: translate +.. autofunction:: scale +.. autofunction:: shear +.. autofunction:: hflip +.. autofunction:: vflip +.. autofunction:: rot180 +.. autofunction:: resize +.. autofunction:: rescale +.. autofunction:: elastic_transform2d +.. autofunction:: pyrdown +.. autofunction:: pyrup +.. autofunction:: build_pyramid +.. autofunction:: build_laplacian_pyramid + +Matrix transformations +---------------------- + +.. autofunction:: get_perspective_transform +.. autofunction:: get_perspective_transform3d +.. autofunction:: get_projective_transform +.. autofunction:: get_rotation_matrix2d +.. autofunction:: get_shear_matrix2d +.. autofunction:: get_shear_matrix3d +.. autofunction:: get_affine_matrix2d +.. autofunction:: get_affine_matrix3d +.. autofunction:: invert_affine_transform +.. autofunction:: projection_from_Rt +.. autofunction:: get_tps_transform + +Crop operators +-------------- + +.. autofunction:: crop_by_indices +.. autofunction:: crop_by_boxes +.. autofunction:: center_crop +.. autofunction:: crop_and_resize + +Module +------ + + + +.. autoclass:: Rotate +.. autoclass:: Translate +.. autoclass:: Scale +.. autoclass:: Shear +.. autoclass:: PyrDown +.. autoclass:: PyrUp +.. autoclass:: ScalePyramid +.. autoclass:: Hflip +.. autoclass:: Vflip +.. autoclass:: Rot180 +.. autoclass:: Resize +.. autoclass:: Rescale +.. autoclass:: Affine +.. autoclass:: HomographyWarper + + +Image registration +------------------ + +.. image:: _static/img/registration.gif + :width: 400 + :alt: Image registration with ImageRegistrator module + +.. automodule:: kornia.geometry.transform.image_registrator + :members: + +Interactive Demo +---------------- + +.. raw:: html + + + + +.. raw:: html + + diff --git a/docs/source/get-started/about.rst b/docs/source/get-started/about.rst new file mode 100644 index 0000000..e17d553 --- /dev/null +++ b/docs/source/get-started/about.rst @@ -0,0 +1,36 @@ +About +===== + +Cite us +------- + +Below you can find the different academic publication derived from Kornia. If you use Kornia in your +work, do not hesitate to cite us :) + +.. code-block:: bash + + @inproceedings{eriba2020kornia, + author = {E. Riba, D. Mishkin, J. Shi, D. Ponsa, F. Moreno-Noguer and G. Bradski}, + title = {A survey on Kornia: an Open Source Differentiable Computer Vision Library for PyTorch}, + year = {2020}, + } + +.. code-block:: bash + + @inproceedings{eriba2019kornia, + author = {E. Riba, D. Mishkin, D. Ponsa, E. Rublee and G. Bradski}, + title = {Kornia: an Open Source Differentiable Computer Vision Library for PyTorch}, + booktitle = {Winter Conference on Applications of Computer Vision}, + year = {2020}, + url = {https://arxiv.org/pdf/1910.02190.pdf} + } + +.. code-block:: bash + + @misc{Arraiy2018, + author = {E. Riba, M. Fathollahi, W. Chaney, E. Rublee and G. Bradski}, + title = {torchgeometry: when PyTorch meets geometry}, + booktitle = {PyTorch Developer Conference}, + year = {2018}, + url = {https://drive.google.com/file/d/1xiao1Xj9WzjJ08YY_nYwsthE-wxfyfhG/view?usp=sharing} + } diff --git a/docs/source/get-started/governance.rst b/docs/source/get-started/governance.rst new file mode 100644 index 0000000..2143f84 --- /dev/null +++ b/docs/source/get-started/governance.rst @@ -0,0 +1,70 @@ +Kornia AI Organization +====================== + +The Kornia AI organization +-------------------------- + +Kornia AI is the non-profit organization registered in Spain that maintains the Kornia library +and its related projects. + +Kornia AI is a community-driven project with the goal to accelerate research, development and deployment of Computer Vision solutions +and is the organization behind the Kornia library and its ecosystem projects. + +1. The association focuses on studying and disseminating knowledge in artificial intelligence (AI), computer vision, and free software, and aims to democratize access to these technologies. +2. It fosters an inclusive community for various stakeholders to promote innovation, development, and knowledge sharing in AI-related fields. +3. The organization facilitates discussions on ethical, philosophical, and practical aspects of AI, and organizes various activities to achieve its objectives. +4. Operating on a non-profit basis, the association prohibits using its name or symbols for profit-making activities, but encourages the use of public AI goods by third parties. + +What we do ? +------------ + +- We maintain the Kornia library and its ecosystem projects such a Limbus, kornia-rs +- We help to organize events such as workshops, meetups related to Computer Vision and robotics. +- We promote the use of Kornia and its ecosystem projects in the industry and academia. +- We promote Computer Vision and robotics in the social media and other channels. + +How to contribute ? +------------------- + +- Make a donation to the organization via: + - `Open Collective `_ + - `Github sponsors `_ +- Join the community and help us to maintain the library and its ecosystem projects. + - `Kornia Slack `_ +- Help us to organize events such as workshops, meetups related to Computer Vision and robotics. +- Help us to write blog posts, tutorials, and other educational material to promote the use of Kornia and its ecosystem projects in the industry and academia. + +Membership +---------- + +The membership is open to anyone who is interested in the mission of the organization and is willing to contribute to its activities. +The membership is free of charge and is granted by the board of directors and can be revoked at any time if the member is not complying with the organization's rules or [Code of Conduct](https://github.com/kornia/kornia/blob/main/CODE_OF_CONDUCT.md). + +Governance +---------- + +The governance of the project is based on the following principles: + +Kornia adopts a governance structure by an altruist group of maintainers driven by the community requests. Kornia’s design philosophy sits on the inspiration from existing Computer Vision libraries such as OpenCV or TorchVision and mimics the same behaviour so that the user can experience a soft transition between frameworks. + +Project Maintainers +------------------- + +- Luis Ferraz (`lferraz `__) +- João Gustavo (`johnnv1 `__) +- Dmytro Mishkin (`ducha-aiki `__) +- Edgar Riba (`edgarriba `__) +- Jian Shi (`shijianjian `__) + +Project Contributors and alumnis +-------------------------------- + +**Consider to contribute to the project and become a contributor! and add your name here!** + +- Christie Jacob (`cjpurackal `__) +- Ceyda Cinarel (`cceyda `__) +- Daniel Koguciuk (`dkoguciuk `__) +- Duc Nguyen (`justanhduc `__) +- Gonzalo Romero-García (`Manza12 `__) + +Check the full list of contributors in the `Github contributors page `_. diff --git a/docs/source/get-started/highlights.rst b/docs/source/get-started/highlights.rst new file mode 100644 index 0000000..0dec098 --- /dev/null +++ b/docs/source/get-started/highlights.rst @@ -0,0 +1,44 @@ +Highlighted Features +==================== + +At Kornia, we are dedicated to pushing the boundaries of computer vision by providing a robust, efficient, and versatile toolkit. Our library is built on the powerful PyTorch backend, leveraging its efficiency and auto-differentiation capabilities to deliver high-performance solutions for a wide range of vision tasks. + + +Accessible AI Models +-------------------- + +We are excited to announce our latest advancement: a new initiative designed to seamlessly integrate lightweight AI models into Kornia. This addition is crafted to empower developers, researchers, and enthusiasts to harness the full potential of accessible AI, simplifying complex vision tasks and accelerating innovation. + +We have curated a selection of lightweight AI models, including YuNet, Loftr, and SAM, optimized for performance and efficiency. These models offer efficient computations that do not require expensive GPUs, making cutting-edge AI accessible to everyone. We welcome the whole community of developers and researchers who are passionate about advancing computer vision, throwing PRs for your lightning fast models! + + +Classic Operators +----------------- + +.. image:: https://github.com/kornia/data/raw/main/kornia_paper_mosaic.png + :align: center + +At a granular level, Kornia is a library that consists of the following components: + ++-----------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| **Component** | **Description** | ++-----------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| `kornia `_ | a Differentiable Computer Vision library like OpenCV, with strong GPU support | ++-----------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| `kornia.augmentation `_| a module to perform data augmentation in the GPU | ++-----------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| `kornia.color `_ | a set of routines to perform color space conversions | ++-----------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| `kornia.contrib `_ | a compilation of user contrib and experimental operators | ++-----------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| `kornia.feature `_ | a module to perform feature detection | ++-----------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| `kornia.filters `_ | a module to perform image filtering and edge detection | ++-----------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| `kornia.geometry `_ | a geometric computer vision library to perform image transformations, 3D linear algebra and conversions using different camera models | ++-----------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| `kornia.losses `_ | a stack of loss functions to solve different vision tasks | ++-----------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| `kornia.morphology `_ | a module to perform morphological operations | ++-----------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ ++-----------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/docs/source/get-started/installation.rst b/docs/source/get-started/installation.rst new file mode 100644 index 0000000..c6f7088 --- /dev/null +++ b/docs/source/get-started/installation.rst @@ -0,0 +1,32 @@ +Installation +============ + +To install *Kornia*, you can do it in two different ways: using the provided `PyPi +`_ wheels or directly from source. + +.. note:: + *Kornia only has as a dependency Pytorch.* + +1. From pip: + +.. code:: bash + + pip install kornia + +2. From source: + +.. code:: bash + + python setup.py install + +3. From source using pip: + +.. code:: bash + + pip install git+https://github.com/kornia/kornia + +Once you succeeded installing *Kornia* check you can import: + +.. code:: bash + + python -c "import kornia; print(kornia.__version__)" diff --git a/docs/source/get-started/introduction.rst b/docs/source/get-started/introduction.rst new file mode 100644 index 0000000..10f7473 --- /dev/null +++ b/docs/source/get-started/introduction.rst @@ -0,0 +1,26 @@ +What is Kornia library ? +======================== + +Kornia is a differentiable library that allows classical computer vision to be integrated into deep learning models. + +It consists of a set of routines and differentiable modules to solve generic computer vision problems. +At its core, the package uses PyTorch as its main backend both for efficiency and to take advantage of +the reverse-mode auto-differentiation to define and compute the gradient of complex functions. + +.. image:: https://raw.githubusercontent.com/kornia/kornia/main/docs/source/_static/img/hakuna_matata.gif + :align: center + +The library is composed by a subset of packages containing operators that can be inserted +within neural networks to train models to perform image transformations, epipolar geometry, depth estimation, +and low level image processing such as filtering and edge detection that operate directly on tensors. + +Why Kornia ? +------------ + +With *Kornia* we fill the gap between classical and deep computer vision that implements +standard and advanced vision algorithms for AI: + +1. **Computer Vision:** Kornia fills the gap between Classical and Deep computer Vision. +2. **Differentiable:** We leverage the Computer Vision 2.0 paradigm. +3. **Open Source:** Our libraries and initiatives are always according to the community needs. +4. **PyTorch:** At our core we use PyTorch and its Autograd engine for its efficiency. diff --git a/docs/source/get-started/multi-framework-support.rst b/docs/source/get-started/multi-framework-support.rst new file mode 100644 index 0000000..79035a2 --- /dev/null +++ b/docs/source/get-started/multi-framework-support.rst @@ -0,0 +1,119 @@ +.. raw:: html + + +
+ +
+
+
+ +Multi-Framework Support +======================= + +Kornia can now be used with `TensorFlow `_, `JAX `_, +and `Numpy `_ thanks to an integration with `Ivy `_. + +This can be accomplished using the following functions, which are now part of the Kornia api: + +* :code:`kornia.to_tensorflow()` + +* :code:`kornia.to_jax()` + +* :code:`kornia.to_numpy()` + +Here's an example of using kornia with TensorFlow: + +.. code:: python + + import kornia + import tensorflow as tf + + tf_kornia = kornia.to_tensorflow() + + rgb_image = tf.random.normal((1, 3, 224, 224)) + gray_image = tf_kornia.color.rgb_to_grayscale(rgb_image) + +So what's happening here? Let's break it down. + +#. Transpiling kornia to TensorFlow + + This line lazily transpiles everything in the kornia api to TensorFlow, and creates a new module for this transpiled version of kornia. + Because the transpilation happens lazily, no function or class will be transpiled until it's actually called. + + .. code-block:: python + + tf_kornia = kornia.to_tensorflow() + +#. Calling a TF kornia function + + We can now call any kornia function (or class) with TF arguments. However, this function will be very slow relative to + the original function - as the function is being transpiled during this step. + + .. code-block:: python + + rgb_image = tf.random.normal((1, 3, 224, 224)) + gray_image = tf_kornia.color.rgb_to_grayscale(rgb_image) # slow + +#. Subsequent function calls + + The good news is any calls of the function after the initial call will be much faster, as it has already been transpiled, + and should approximately match the speed of the original kornia function. + + .. code-block:: python + + gray_image = tf_kornia.color.rgb_to_grayscale(rgb_image) # fast + +#. Transpilations in different Python sessions + + You may be wondering if you'll have to wait for these long initial transpilations to take place each time you start a + new Python session? The good news is that when a transpilation occurs, Ivy will save the generated source code in the + local directory, so if the same transpilation is ever attempted again from within the same directory, it will be + immediately retrieved and used. + + +Kornia can be used with JAX and NumPy in the same way: + +.. code:: python + + import kornia + import numpy as np + + np_kornia = kornia.to_numpy() + + rgb_image = np.random.normal(size=(1, 3, 224, 224)) + gray_image = np_kornia.color.rgb_to_grayscale(rgb_image) + + +.. code:: python + + import kornia + import jax + + jax_kornia = kornia.to_jax() + + rgb_image = jax.random.normal(jax.random.key(42), shape=(1, 3, 224, 224)) + gray_image = jax_kornia.color.rgb_to_grayscale(rgb_image) + + +Limitations +----------- + +* Converting Kornia to TensorFlow or JAX works for functions, classes and trainable modules; converting to NumPy supports functions and classes, but not trainable modules. + +* Transpilation does not currently work with custom kernels, such as flash attention. + +* Certain stateful classes cannot currently be transpiled, such as optimizers (torch.optim.Adam, etc.), trainers, and data loaders. + +* Compatibility with native compilers (*jax.jit* and *tf.function*) is somewhat limited with transpiled versions of Kornia, + particularly compared with *torch.compile* on standard Kornia. Improving compatibility with these is one of the key areas of + focus for the current development of Ivy. + + +From the Ivy Team +----------------- + +We hope you find using Kornia with TensorFlow, JAX and NumPy useful! Ivy is still very much under development, +so if you find any issues/bugs, feel free to raise an issue on the `ivy `_ repository. +We'd also really appreciate a star, if you'd like to show your support! + +To learn more about Ivy, we recommend taking a look through our `documentation `_. diff --git a/docs/source/get-started/precision.rst b/docs/source/get-started/precision.rst new file mode 100644 index 0000000..c24b00b --- /dev/null +++ b/docs/source/get-started/precision.rst @@ -0,0 +1,202 @@ +Half-Precision Support +====================== + +This page documents which kornia modules support half-precision floating-point dtypes +(``torch.float16`` and ``torch.bfloat16``) and what limitations to expect. + +.. list-table:: Half-Precision Support by Module + :header-rows: 1 + :widths: 28 14 14 44 + + * - Module + - float16 + - bfloat16 + - Notes + * - ``kornia.color`` + - ⚠️ Partial + - ⚠️ Partial + - Most color space conversions work for both half-precision dtypes. + FFT-based operations may fail on CUDA. + * - ``kornia.filters`` + - ⚠️ Partial + - ⚠️ Partial + - Basic convolution-based filters (Gaussian, Sobel, Median, Box) work + for both dtypes. FFT-based operations (``fft_conv``) may fail on CUDA. + * - ``kornia.enhance`` + - ⚠️ Partial + - ⚠️ Partial + - Histogram equalization, CLAHE, gamma correction, and ZCA whitening work + for both dtypes. ZCA linalg ops go through ``_torch_svd_cast`` / + ``_torch_inverse_cast`` which promote to float32 before computing. + * - ``kornia.morphology`` + - ✅ Yes + - ✅ Yes + - Uses only convolution and pooling; no dtype restrictions. + * - ``kornia.augmentation`` + - ⚠️ Partial + - ⚠️ Partial + - Both dtypes are accepted by ``validate_tensor``. Most ops work; + precision-sensitive transforms (e.g. affine with large rotations) may + produce inaccurate results at half precision. + * - ``kornia.geometry.transform`` + - ⚠️ Partial + - ⚠️ Partial + - Affine, homography, resize, and warp operations use ``_torch_inverse_cast`` + / ``_torch_solve_cast`` which promote to float32 and cast back; + both dtypes work. + * - ``kornia.geometry.camera`` + - ⚠️ Partial + - ⚠️ Partial + - Pinhole camera model and most projection ops work for both dtypes. + ``StereoCamera`` accepts both float16 and bfloat16. + * - ``kornia.geometry.calibration`` + - ❌ No + - ❌ No + - ``solve_pnp_dlt()`` explicitly checks that inputs are ``float32`` or + ``float64`` and raises otherwise. + * - ``kornia.geometry.epipolar`` + - ⚠️ Partial + - ⚠️ Partial + - SVD and solve operations use ``_torch_svd_cast`` / ``_torch_solve_cast`` + / ``_torch_inverse_cast``; both dtypes work via casting to float32. + * - ``kornia.geometry.homography`` + - ⚠️ Partial + - ⚠️ Partial + - Uses ``_torch_svd_cast``; both dtypes are promoted to float32 before SVD + and the result is cast back. + * - ``kornia.geometry.liegroup`` + - ⚠️ Partial + - ⚠️ Partial + - Most rotation/translation operations (SO2, SO3, SE2, SE3) work for both + dtypes via cast helpers. A few code paths may still fail. + * - ``kornia.geometry.solvers`` + - ⚠️ Partial + - ⚠️ Partial + - RANSAC-based solvers use ``_torch_solve_cast``; both dtypes are promoted + before the solve and the result is cast back. + * - ``kornia.geometry.subpix`` + - ⚠️ Partial + - ⚠️ Partial + - Soft-argmax and weighted softmax work for both dtypes. + Precision-sensitive ops may produce inaccurate results. + * - ``kornia.losses`` + - ⚠️ Partial + - ⚠️ Partial + - Photometric losses (SSIM, PSNR, MS-SSIM) work for both dtypes. + Losses based on linalg operations (Hausdorff, etc.) may not. + * - ``kornia.feature`` + - ⚠️ Partial + - ⚠️ Partial + - Local feature detectors and descriptors (SIFT, HardNet, DISK, DeDoDe) + work for inference. Feature *matching* uses a manual ``cdist`` fallback + for both half-precision dtypes on CUDA. + * - ``kornia.metrics`` + - ⚠️ Partial + - ⚠️ Partial + - Simple pixel-level metrics work for both dtypes. Metrics involving linalg + operations may not. + * - ``kornia.models`` + - ⚠️ Partial + - ⚠️ Partial + - Conv-based models work for both dtypes. Attention-based models (e.g. + VLMs, ViTs) may have internal dtype mismatches. + +Legend +------ + +- ✅ **Yes** — Works correctly; results are accurate at the given precision. +- ⚠️ **Partial** — Some operations work; others fail at runtime or produce inaccurate results due to limited numerical range/precision. +- ❌ **No** — Not supported; raises a ``RuntimeError`` or ``TypeError`` at runtime (explicit dtype check in the implementation). + +Test Results +------------ + +Measured on commit ``6131e98`` (2026-03-21), full test suite (no ``--runslow``). +Pass% = passed ÷ (passed + failed); skipped and xfailed tests are excluded. + +.. list-table:: + :header-rows: 1 + :widths: 32 10 10 10 10 + + * - Run + - Passed + - Failed + - Skipped + - Pass% + * - CPU float32 *(baseline)* + - 7647 + - 3 + - 3269 + - **99.9%** + * - CUDA float32 *(baseline)* + - 7634 + - 3 + - 3280 + - **99.9%** + * - CPU float16 + - 6866 + - 747 + - 3306 + - **90.1%** + * - CPU bfloat16 + - 6838 + - 812 + - 3269 + - **89.3%** + * - CUDA float16 *(KORNIA_TEST_IN_SUBPROCESS=1)* + - 6727 + - 643 + - 3556 + - **91.3%** + * - CUDA bfloat16 *(KORNIA_TEST_IN_SUBPROCESS=1)* + - 6695 + - 713 + - 3518 + - **90.4%** + +.. note:: + + CUDA half-precision tests are measured using ``KORNIA_TEST_IN_SUBPROCESS=1`` + which bypasses the ``skip_half_precision_on_cuda`` fixture. Each test then + runs in the same process but with the ``cuda_device_assert_guard`` fixture + synchronising CUDA before and after each test. For full isolation the current + implementation uses ``subprocess.run`` for true process isolation; a fresh + ``--isolate-half-precision`` flag spawns each test in a fresh ``subprocess.run`` + process with no shared CUDA state. + +Test Suite Behaviour +-------------------- + +Half-precision tests live in the same directories and files as their +float32/float64 counterparts. They are run as **separate, isolated pytest +invocations** rather than being mixed into a combined ``--dtype=all`` run. +This prevents a CUDA device-side assert in a half-precision test from +corrupting the CUDA context and causing unrelated float32 tests to fail. + +.. code-block:: bash + + # Standard precision — default CI + pixi run test tests/ --dtype=float32,float64 + + # Half-precision — run in isolation, per directory + pytest tests/color/ --dtype=float16,bfloat16 + pytest tests/geometry/ --dtype=float16,bfloat16 --device=cuda + +Two autouse fixtures in the root ``conftest.py`` enforce safe behaviour: + +- **``skip_half_precision_on_cuda``** — skips float16/bfloat16 tests on CUDA + in combined runs so no half-precision kernel is ever launched (and therefore + no device-side assert can fire). +- **``cuda_device_assert_guard``** — synchronises CUDA before and after each + CUDA test to catch async device-side assert errors in the test that caused + them, not in the next one. If the context is already corrupted, the test + is skipped rather than allowed to fail spuriously. + +With ``--isolate-half-precision``, each float16/bfloat16 CUDA test is +intercepted by a custom ``pytest_runtest_protocol`` hook and executed in a +completely fresh Python process via ``subprocess.run``. There is no shared +CUDA context between tests, so a device-side assert in one test cannot affect +any other. + +See ``TESTING.md`` in the repository root for a full description of the +contamination mechanism and fixture implementation. diff --git a/docs/source/image.rst b/docs/source/image.rst new file mode 100644 index 0000000..186de2b --- /dev/null +++ b/docs/source/image.rst @@ -0,0 +1,59 @@ +kornia.image +============ + +.. meta:: + :name: description + :content: "The kornia.image module offers a high-level API designed for processing images in computer vision tasks. It provides functionalities for handling image size, pixel formats, channel orders, and image layouts, streamlining the manipulation of images in deep learning workflows. With a user-friendly interface, this module simplifies image data preprocessing and handling for various computer vision and machine learning tasks." + +Module to provide a high level API to process images. + +.. currentmodule:: kornia.image + +.. autoclass:: ImageSize + :members: + :undoc-members: + +.. autoclass:: PixelFormat + :members: + :undoc-members: + +.. autoclass:: ChannelsOrder + :members: + :undoc-members: + +.. autoclass:: ImageLayout + :members: + :undoc-members: + +.. autoclass:: Image + :members: + :undoc-members: + +Drawing +------- + +.. autofunction:: draw_line +.. autofunction:: draw_rectangle +.. autofunction:: draw_convex_polygon +.. autofunction:: draw_point2d + +Image Conversion +---------------- + +.. autofunction:: tensor_to_image +.. autofunction:: image_to_tensor +.. autofunction:: image_list_to_tensor +.. autoclass:: ImageToTensor + +Image Printing +-------------- + +.. autofunction:: image_to_string +.. autofunction:: print_image + +Utilities +--------- + +.. autofunction:: make_grid +.. autofunction:: perform_keep_shape_image +.. autofunction:: perform_keep_shape_video diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..9569832 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,177 @@ + +.. image:: https://github.com/kornia/data/raw/main/kornia_banner_pixie.png + :align: center + +State-of-the-art and curated Computer Vision algorithms for AI. + +Kornia AI is on the mission to leverage and democratize the next generation of Computer Vision tools and Deep Learning libraries +within the context of an Open Source community. + +.. code:: python + + >>> import kornia.geometry as K + >>> registrator = K.ImageRegistrator('similarity') + >>> model = registrator.register(img1, img2) + +Ready to use with state-of-the art Deep Learning models: + +DexiNed edge detection model. + +.. code-block:: python + + image = kornia.utils.sample.get_sample_images()[0][None] + model = DexiNedBuilder.build() + model.save(image) + +RTDETRDetector for object detection. + +.. code-block:: python + + image = kornia.utils.sample.get_sample_images()[0][None] + model = RTDETRDetectorBuilder.build() + model.save(image) + +BoxMotTracker for object tracking. + +.. code-block:: python + + import kornia + image = kornia.utils.sample.get_sample_images()[0][None] + model = BoxMotTracker() + for i in range(4): + model.update(image) + model.save(image) + +Vision Transformer for image classification. + +.. code:: python + + >>> import torch.nn as nn + >>> from kornia.models.vit import VisionTransformer + >>> classifier = nn.Sequential( + ... VisionTransformer(image_size=224, patch_size=16), + ... nn.Linear(768, 1000), # Example: 768 is the default hidden_dim, 1000 is num_classes + ... ) + >>> logits = classifier(img) # BxN + >>> scores = logits.argmax(-1) # B + +Multi-framework support +----------------------- + +You can now use Kornia with `NumPy `_, `TensorFlow `_, and `JAX `_. + +.. code:: python + + >>> import kornia + >>> tf_kornia = kornia.to_tensorflow() + +.. raw:: html + +

+ Powered by + +

+ +

+ +Join the community +------------------ + +- Join our social network communities with 1.8k+ members: + - `Twitter `_: we share recent research and news with our community. + - `Slack `_: come to us and chat with our engineers and mentors to get support and resolve your questions. +- Subscribe to our `YouTube channel `_ to get the latest video demos. + +---- + +.. toctree:: + :caption: GET STARTED + :hidden: + + get-started/introduction + get-started/highlights + get-started/installation + get-started/precision + get-started/about + Tutorials + get-started/multi-framework-support + OpenCV AI Kit + get-started/governance + +.. toctree:: + :caption: API REFERENCE + :maxdepth: 2 + :hidden: + + augmentation + color + contrib + core + enhance + feature + filters + geometry + sensors + io + image + losses + models + metrics + morphology + onnx + tracking + +.. toctree:: + :caption: KORNIA APPLICATIONS + :hidden: + + applications/intro + applications/visual_prompting + applications/face_detection + applications/image_augmentations + applications/image_matching + applications/image_stitching + applications/image_registration + applications/image_denoising + +.. toctree:: + :caption: KORNIA MODELS + :hidden: + + models/efficient_vit + models/rt_detr + models/segment_anything + models/mobile_sam + models/yunet + models/vit + models/vit_mobile + models/tiny_vit + models/loftr + models/defmo + models/hardnet + models/affnet + models/sold2 + models/dexined + +.. toctree:: + :caption: SUPPORT + :hidden: + + Issue tracker + Slack community + LibreCV community + Twitter @kornia_foss + community/chinese + Kornia Youtube + Kornia LinkedIn + Kornia AI + +.. toctree:: + :caption: COMMUNITY + :hidden: + + community/contribute + community/faqs + community/bibliography diff --git a/docs/source/io.rst b/docs/source/io.rst new file mode 100644 index 0000000..3a21f44 --- /dev/null +++ b/docs/source/io.rst @@ -0,0 +1,50 @@ +kornia.io +========= + +.. meta:: + :name: description + :content: "The Kornia.io package provides utilities to load and save image data efficiently. It integrates with `kornia_rs`, a low-level Rust implementation for computer vision, and supports the DLPack protocol to reduce memory footprint. This package is designed for Linux platforms and requires PyTorch 1.10.0 or higher." + +.. currentmodule:: kornia.io + +Package to load and save image data. + +The package internally implements `kornia_rs `_ which contains a low level implementation +for Computer Vision in the `Rust `_ language. In addition, we implement the `DLPack `_ protocol +natively in Rust to reduce the memory footprint during the decoding and types conversion. + +.. tip:: + You need to ``pip install kornia_rs`` to use this package. For now we only support Linux platforms. + Contact us or sponsor the project for more support (mac, win, rust, c++, video and camera). See: + `https://opencollective.com/kornia `_ + +.. note:: + The package needs at least PyTorch 1.10.0 installed. + +.. code-block:: python + + import kornia as K + from kornia.io import ImageLoadType + from kornia.core import Tensor + + img: Tensor = K.io.load_image(file_path, ImageLoadType.UNCHANGED, device="cuda") + # will load CxHxW / in the original format in "cuda" + + img: Tensor = K.io.load_image(file_path, ImageLoadType.RGB8, device="cpu") + # will load 3xHxW / in torch.uint in range [0,255] in "cpu" + + img: Tensor = K.io.load_image(file_path, ImageLoadType.GRAY8, device="cuda") + # will load 1xHxW / in torch.uint8 in range [0,255] in "cuda" + + img: Tensor = K.io.load_image(file_path, ImageLoadType.GRAY32, device="cpu") + # will load 1xHxW / in torch.float32 in range [0,1] in "cpu" + + img: Tensor = K.io.load_image(file_path, ImageLoadType.RGB32, device="cuda") + # will load 3xHxW / in torch.float32 in range [0,1] in "cuda" + +.. autofunction:: load_image +.. autofunction:: write_image + +.. autoclass:: ImageLoadType + :members: + :undoc-members: diff --git a/docs/source/losses.rst b/docs/source/losses.rst new file mode 100644 index 0000000..03b9c07 --- /dev/null +++ b/docs/source/losses.rst @@ -0,0 +1,62 @@ +kornia.losses +============= + +.. meta:: + :name: description + :content: "The kornia.losses module offers a comprehensive collection of loss functions for computer vision tasks, including image reconstruction, semantic segmentation, distribution-based losses, and morphological losses. With a wide range of loss types such as SSIM, PSNR, focal loss, and dice loss, this module enables efficient optimization for deep learning models across various domains, enhancing training for tasks like image restoration, segmentation, and object detection." + +.. currentmodule:: kornia.losses + +Reconstruction +-------------- + +.. autofunction:: ssim_loss +.. autofunction:: ssim3d_loss +.. autofunction:: psnr_loss +.. autofunction:: total_variation +.. autofunction:: inverse_depth_smoothness_loss +.. autofunction:: charbonnier_loss +.. autofunction:: welsch_loss +.. autofunction:: cauchy_loss +.. autofunction:: geman_mcclure_loss + +.. autoclass:: SSIMLoss +.. autoclass:: SSIM3DLoss +.. autoclass:: MS_SSIMLoss +.. autoclass:: TotalVariation +.. autoclass:: PSNRLoss +.. autoclass:: InverseDepthSmoothnessLoss +.. autoclass:: CharbonnierLoss +.. autoclass:: WelschLoss +.. autoclass:: CauchyLoss +.. autoclass:: GemanMcclureLoss + +Semantic Segmentation +--------------------- + +.. autofunction:: kornia.losses.one_hot +.. autofunction:: binary_focal_loss_with_logits +.. autofunction:: focal_loss +.. autofunction:: dice_loss +.. autofunction:: tversky_loss +.. autofunction:: lovasz_hinge_loss +.. autofunction:: lovasz_softmax_loss + +.. autoclass:: BinaryFocalLossWithLogits +.. autoclass:: DiceLoss +.. autoclass:: TverskyLoss +.. autoclass:: FocalLoss +.. autoclass:: LovaszHingeLoss +.. autoclass:: LovaszSoftmaxLoss + +Distributions +------------- + +.. autofunction:: js_div_loss_2d +.. autofunction:: kl_div_loss_2d + +Morphology +---------- + +.. autoclass:: HausdorffERLoss +.. autoclass:: HausdorffERLoss3D diff --git a/docs/source/metrics.rst b/docs/source/metrics.rst new file mode 100644 index 0000000..58b481c --- /dev/null +++ b/docs/source/metrics.rst @@ -0,0 +1,54 @@ +kornia.metrics +============== + +.. meta:: + :name: description + :content: "The kornia.metrics module provides a variety of metrics to evaluate the performance of deep learning models in computer vision tasks. It includes metrics for classification, segmentation, detection, image quality, and optical flow. With functions such as accuracy, mean IoU, PSNR, and AEPE, this module facilitates efficient monitoring and evaluation of models during training, making it a valuable tool for model performance assessment." + +.. currentmodule:: kornia.metrics + +Module containing metrics for training networks + +Classification +-------------- + +.. autofunction:: accuracy + +Segmentation +------------ + +.. autofunction:: confusion_matrix +.. autofunction:: mean_iou + +Detection +--------- + +.. autofunction:: mean_average_precision +.. autofunction:: mean_iou_bbox + +Image Quality +------------- + +.. autofunction:: psnr +.. autofunction:: ssim +.. autofunction:: ssim3d +.. autoclass:: SSIM +.. autoclass:: SSIM3D + +Optical Flow +------------- + +.. autofunction:: aepe +.. autoclass:: AEPE + +Stereo +------ + +.. autofunction:: mean_absolute_disparity_error +.. autofunction:: root_mean_squared_disparity_error +.. autofunction:: mean_bad_pixel_error + +Monitoring +---------- + +.. autoclass:: AverageMeter diff --git a/docs/source/models.rst b/docs/source/models.rst new file mode 100644 index 0000000..cf6fac7 --- /dev/null +++ b/docs/source/models.rst @@ -0,0 +1,140 @@ +Models Overview +=============== + +.. meta:: + :name: description + :content: "The Kornia models overview provides detailed information about key built-in models for computer vision tasks, including real-time object detection (RT-DETR), edge detection (DexiNed), segmentation (UNet, DeepLabV3), and multi-object tracking (BoxMotTracker). It offers comprehensive documentation on each model, including methods, parameters, and example usage to streamline the integration of these models into computer vision workflows." + + +This section covers several of Kornia's built-in models for key computer vision tasks. Each model is documented with its respective API and example usage. + +.. _RTDETRDetectorBuilder: + +RTDETRDetectorBuilder +--------------------- + +The `RTDETRDetectorBuilder` class is a builder for constructing a detection model based on the RT-DETR architecture, which is designed for real-time object detection. It is capable of detecting multiple objects within an image and provides efficient inference suitable for real-world applications. + +**Key Methods:** + +- `build`: Constructs and returns an instance of the RTDETR detection model. +- `save`: Saves the processed image or results after applying the detection model. + +.. autoclass:: kornia.contrib.object_detection.RTDETRDetectorBuilder + :members: + :undoc-members: + :show-inheritance: + + .. rubric:: Example + + The following code demonstrates how to use `RTDETRDetectorBuilder` to detect objects in an image: + + .. code-block:: python + + import kornia + image = kornia.utils.sample.get_sample_images()[0][None] + model = kornia.contrib.object_detection.RTDETRDetectorBuilder.build() + model.save(image) + +.. _EdgeDetectorBuilder: + +EdgeDetectorBuilder +------------------- + +The `EdgeDetectorBuilder` class implements a state-of-the-art edge detection model based on DexiNed, which excels at detecting fine-grained edges in images. This model is well-suited for tasks like medical imaging, object contour detection, and more. + +**Key Methods:** + +- `build`: Builds and returns an instance of the DexiNed edge detection model. +- `save`: Saves the detected edges for further processing or visualization. + +.. autoclass:: kornia.contrib.edge_detection.EdgeDetectorBuilder + :members: + :undoc-members: + :show-inheritance: + + .. rubric:: Example + + The following code shows how to use the `EdgeDetectorBuilder` to detect edges in an image: + + .. code-block:: python + + import kornia + image = kornia.utils.sample.get_sample_images()[0][None] + model = kornia.contrib.edge_detection.EdgeDetectorBuilder.build() + model.save(image) + +.. _SegmentationModels: + +SegmentationModelsBuilder +------------------------- + +The `SegmentationModelsBuilder` class offers a flexible API for implementing and running various segmentation models. It supports a variety of architectures such as UNet, FPN, and others, making it highly adaptable for tasks like semantic segmentation, instance segmentation, and more. + +**Key Methods:** + +- `__init__`: Initializes a segmentation model based on the chosen architecture (e.g., UNet, DeepLabV3, etc.). +- `forward`: Runs inference on an input tensor and returns segmented output. + +**Parameters:** + +- `model_name`: (str) Name of the segmentation architecture to use, e.g., `"Unet"`, `"DeepLabV3"`. +- `classes`: (int) The number of output classes for segmentation. + +.. autoclass:: kornia.models.segmentation.segmentation_models.SegmentationModelsBuilder + :members: + :undoc-members: + :show-inheritance: + + .. rubric:: Example + + Here's an example of how to use `SegmentationModelsBuilder` for binary segmentation: + + .. code-block:: python + + import kornia + input_tensor = kornia.utils.sample.get_sample_images()[0][None] + model = kornia.models.segmentation.segmentation_models.SegmentationModelsBuilder.build() + segmented_output = model(input_tensor) + print(segmented_output.shape) + +.. _BoxMotTracker: + +BoxMotTracker +------------- + +The `BoxMotTracker` class is used for multi-object tracking in video streams. It is designed to track bounding boxes of objects across multiple frames, supporting various tracking algorithms for object detection and tracking continuity. + +**Key Methods:** + +- `__init__`: Initializes the multi-object tracker. +- `update`: Updates the tracker with a new image frame. +- `save`: Saves the tracked object data or visualization for post-processing. + +**Parameters:** + +- `max_lost`: (int) The maximum number of frames where an object can be lost before it is removed from the tracker. + +.. autoclass:: kornia.contrib.boxmot_tracker.BoxMotTracker + :members: + :undoc-members: + :show-inheritance: + + .. rubric:: Example + + The following example demonstrates how to track objects across multiple frames using `BoxMotTracker`: + + .. code-block:: python + + import kornia + image = kornia.utils.sample.get_sample_images()[0][None] + model = kornia.contrib.boxmot_tracker.BoxMotTracker() + for i in range(4): + model.update(image) # Update the tracker with new frames + model.save(image) # Save the tracking result + +--- + +.. note:: + + This documentation provides detailed information about each model class, its methods, and usage examples. For further details on individual methods and arguments, refer to the respective code documentation. diff --git a/docs/source/models/affnet.rst b/docs/source/models/affnet.rst new file mode 100644 index 0000000..3ef5dc6 --- /dev/null +++ b/docs/source/models/affnet.rst @@ -0,0 +1,23 @@ +Affnet (detection) +.................. + +.. card:: + :link: https://paperswithcode.com/paper/repeatability-is-not-enough-learning-affine + + **Affnet: Repeatability Is Not Enough: Learning Affine Regions via Discriminability** + ^^^ + **Abstract:** A method for learning local affine-covariant regions is presented. We show that maximizing geometric repeatability does not lead to local regions, a.k.a features,that are reliably matched and this necessitates descriptor-based learning. We explore factors that influence such learning and registration: the loss function, descriptor type, geometric parametrization and the trade-off between matchability and geometric accuracy and propose a novel hard negative-constant loss function for learning of affine regions. The affine shape estimator -- AffNet -- trained with the hard negative-constant loss outperforms the state-of-the-art in bag-of-words image retrieval and wide baseline stereo. The proposed training process does not require precisely geometrically aligned patches. + + **Tasks:** Image Retrieval + + **Datasets:** Oxford5k, HPatches + + **Conference:** ECCV 2018 + + **Licence:** MIT + + +++ + **Authors:** Dmytro Mishkin, Filip Radenovic, Jiri Matas + +.. image:: https://raw.githubusercontent.com/ducha-aiki/affnet/master/imgs/graf16HesAffNet.jpg + :align: center diff --git a/docs/source/models/defmo.rst b/docs/source/models/defmo.rst new file mode 100644 index 0000000..a0862e2 --- /dev/null +++ b/docs/source/models/defmo.rst @@ -0,0 +1,22 @@ +DeFMO (video) +............. + +.. card:: + :link: https://paperswithcode.com/paper/defmo-deblurring-and-shape-recovery-of-fast + + **DeFMO: Deblurring and Shape Recovery of Fast Moving Objects** + ^^^ + **Abstract:** Objects moving at high speed appear significantly blurred when captured with cameras. The blurry appearance is especially ambiguous when the object has complex shape or texture. In such cases, classical methods, or even humans, are unable to recover the object's appearance and motion. We propose a method that, given a single image with its estimated background, outputs the object's appearance and position in a series of sub-frames as if captured by a high-speed camera (i.e. temporal super-resolution). The proposed generative model embeds an image of the blurred object into a latent space representation, disentangles the background, and renders the sharp appearance. Inspired by the image formation model, we design novel self-supervised loss function terms that boost performance and show good generalization capabilities. The proposed DeFMO method is trained on a complex synthetic dataset, yet it performs well on real-world data from several datasets. DeFMO outperforms the state of the art and generates high-quality temporal super-resolution frames. + + **Tasks:** Deblurring, Object Tracking, Super-Resolution, Video Super-Resolution. + + **Datasets:** Falling Objects. + + **Conference:** CVPR 2021 + + **Licence:** Apache-2.0 + + +++ + **Authors:** Denys Rozumnyi, Martin R. Oswald, Vittorio Ferrari, Jiri Matas, Marc Pollefeys + +.. youtube:: pmAynZvaaQ4 diff --git a/docs/source/models/dexined.rst b/docs/source/models/dexined.rst new file mode 100644 index 0000000..5b7a049 --- /dev/null +++ b/docs/source/models/dexined.rst @@ -0,0 +1,26 @@ +.. _dexined_model: + +Dexined (edge detection) +------------------------ + +.. card:: + :link: https://www.computer.org/csdl/proceedings-article/wacv/2020/09093290/1jPbjFHmwi4 + + **Dense Extreme Inception Network for Edge Detection** + ^^^ + **Abstract:** Edge detection is the basis of many computer vision applications. State of the art predominantly relies on deep learning with two decisive factors: dataset content and network's architecture. Most of the publicly available datasets are not curated for edge detection tasks. Here, we offer a solution to this constraint. First, we argue that edges, contours and boundaries, despite their overlaps, are three distinct visual features requiring separate benchmark datasets. To this end, we present a new dataset of edges. Second, we propose a novel architecture, termed Dense Extreme Inception Network for Edge Detection (DexiNed), that can be trained from scratch without any pre-trained weights. DexiNed outperforms other algorithms in the presented dataset. It also generalizes well to other datasets without any fine-tuning. The higher quality of DexiNed is also perceptually evident thanks to the sharper and finer edges it outputs. + + **Tasks:** Edge Detection + + **Datasets:** BSD500, BIPED, MDBD + + **Journal:** 2020 IEEE Winter Conference on Applications of Computer Vision (WACV) + + **Licence:** MIT + + +++ + **Authors:** X. Soria and E. Riba and A. Sappa + + +.. image:: https://github.com/xavysp/DexiNed/raw/master/figs/DexiNed_banner.png + :align: center diff --git a/docs/source/models/efficient_vit.rst b/docs/source/models/efficient_vit.rst new file mode 100644 index 0000000..82d789e --- /dev/null +++ b/docs/source/models/efficient_vit.rst @@ -0,0 +1,16 @@ +EfficientViT +============ + +.. card:: + :link: https://arxiv.org/abs/2205.14756 + + **EfficientViT: Multi-Scale Linear Attention for High-Resolution Dense Prediction** + ^^^ + **Abstract:** High-resolution dense prediction enables many appealing real-world applications, such as computational photography, autonomous driving, etc. However, the vast computational cost makes deploying state-of-the-art high-resolution dense prediction models on hardware devices difficult. This work presents EfficientViT, a new family of high-resolution vision models with novel multi-scale linear attention. Unlike prior high-resolution dense prediction models that rely on heavy softmax attention, hardware-inefficient large-kernel convolution, or complicated topology structure to obtain good performances, our multi-scale linear attention achieves the global receptive field and multi-scale learning (two desirable features for high-resolution dense prediction) with only lightweight and hardware-efficient operations. As such, EfficientViT delivers remarkable performance gains over previous state-of-the-art models with significant speedup on diverse hardware platforms, including mobile CPU, edge GPU, and cloud GPU. Without performance loss on Cityscapes, our EfficientViT provides up to 13.9x and 6.2x GPU latency reduction over SegFormer and SegNeXt, respectively. For super-resolution, EfficientViT delivers up to 6.4x speedup over Restormer while providing 0.11dB gain in PSNR. For Segment Anything, EfficientViT delivers similar zero-shot image segmentation quality as ViT-Huge with 84x higher throughput on GPU. Code: this https URL. + + **Tasks:** Classification, Segmentation, Detection + + **Licence:** Apache 2.0 + + +++ + **Authors:** Han Cai, Junyan Li, Muyan Hu, Chuang Gan, Song Han diff --git a/docs/source/models/hardnet.rst b/docs/source/models/hardnet.rst new file mode 100644 index 0000000..5855efc --- /dev/null +++ b/docs/source/models/hardnet.rst @@ -0,0 +1,23 @@ +Hardnet (descriptor) +.................... + +.. card:: + :link: https://paperswithcode.com/paper/working-hard-to-know-your-neighbors-margins + + **Hardnet: Working hard to know your neighbor's margins: Local descriptor learning loss** + ^^^ + **Abstract:** We introduce a novel loss for learning local feature descriptors which is inspired by the Lowe's matching criterion for SIFT. We show that the proposed loss that maximizes the distance between the closest positive and closest negative patch in the batch is better than complex regularization methods; it works well for both shallow and deep convolution network architectures. Applying the novel loss to the L2Net CNN architecture results in a compact descriptor -- it has the same dimensionality as SIFT (128) that shows state-of-art performance in wide baseline stereo, patch verification and instance retrieval benchmarks. It is fast, computing a descriptor takes about 1 millisecond on a low-end GPU + + **Tasks:** Image Retrieval, Patch Matching + + **Datasets:** Oxford5k, HPatches, Oxford-Affine + + **Conference:** NeurIPS 2017 + + **Licence:** MIT + + +++ + **Authors:** Anastasiya Mishchuk, Dmytro Mishkin, Filip Radenovic, Jiri Matas + +.. image:: https://raw.githubusercontent.com/DagnyT/hardnet/master/img/hardnet_hpatches.png + :align: center diff --git a/docs/source/models/loftr.rst b/docs/source/models/loftr.rst new file mode 100644 index 0000000..c396046 --- /dev/null +++ b/docs/source/models/loftr.rst @@ -0,0 +1,23 @@ +LoFTR (matching) +................ + +.. card:: + :link: https://paperswithcode.com/paper/loftr-detector-free-local-feature-matching + + **LoFTR: Detector-Free Local Feature Matching with Transformers** + ^^^ + **Abstract:** We present a novel method for local image feature matching. Instead of performing image feature detection, description, and matching sequentially, we propose to first establish pixel-wise dense matches at a coarse level and later refine the good matches at a fine level. In contrast to dense methods that use a cost volume to search correspondences, we use self and cross attention layers in Transformer to obtain feature descriptors that are conditioned on both images. The global receptive field provided by Transformer enables our method to produce dense matches in low-texture areas, where feature detectors usually struggle to produce repeatable interest points. The experiments on indoor and outdoor datasets show that LoFTR outperforms state-of-the-art methods by a large margin. LoFTR also ranks first on two public benchmarks of visual localization among the published methods. + + **Tasks:** Local Feature Matching, Visual Localisation + + **Datasets:** ScanNet, HPatches, MegaDepth, InLoc + + **Conference:** CVPR 2021 + + **Licence:** Apache-2.0 + + +++ + **Authors:** Jiaming Sun*, Zehong Shen*, Yu'ang Wang*, Hujun Bao, Xiaowei Zhou + +.. image:: https://raw.githubusercontent.com/zju3dv/LoFTR/master/assets/loftr-github-demo.gif + :align: center diff --git a/docs/source/models/mobile_sam.rst b/docs/source/models/mobile_sam.rst new file mode 100644 index 0000000..c0828a3 --- /dev/null +++ b/docs/source/models/mobile_sam.rst @@ -0,0 +1,45 @@ +Faster Segment Anything (MobileSAM) +=================================== + +.. card:: + :link: https://arxiv.org/abs/2306.14289 + + **Faster Segment Anything: Towards Lightweight SAM for Mobile Applications** + ^^^ + **Abstract:** Segment Anything Model (SAM) has attracted significant attention due to its impressive zero-shot transfer performance and high versatility for numerous vision applications (like image editing with fine-grained control). Many of such applications need to be run on resource-constraint edge devices, like mobile phones. In this work, we aim to make SAM mobile-friendly by replacing the heavyweight image encoder with a lightweight one. A naive way to train such a new SAM as in the original SAM paper leads to unsatisfactory performance, especially when limited training sources are available. We find that this is mainly caused by the coupled optimization of the image encoder and mask decoder, motivated by which we propose decoupled distillation. Concretely, we distill the knowledge from the heavy image encoder (ViT-H in the original SAM) to a lightweight image encoder, which can be automatically compatible with the mask decoder in the original SAM. The training can be completed on a single GPU within less than one day, and the resulting lightweight SAM is termed MobileSAM which is more than 60 times smaller yet performs on par with the original SAM. For inference speed, With a single GPU, MobileSAM runs around 10ms per image: 8ms on the image encoder and 4ms on the mask decoder. With superior performance, our MobileSAM is around 5 times faster than the concurrent FastSAM and 7 times smaller, making it more suitable for mobile applications. Moreover, we show that MobileSAM can run relatively smoothly on CPU. The code for our project is provided at https://github.com/ChaoningZhang/MobileSAM, with a demo showing that MobileSAM can run relatively smoothly on CPU. + + **Tasks:** Segmentation + + **Datasets:** SA-1B + + **Licence:** Apache 2.0 + + +++ + **Authors:** Chaoning Zhang, Dongshen Han, Yu Qiao, Jung Uk Kim, Sung-Ho Bae, Seungkyu Lee, Choong Seon Hong + + +Usage +~~~~~ + +MobileSAM is integrated directly into our Segment-Anything implementation. Once you have loaded MobileSAM, you can use it just like how you use SAM. + +.. code:: python + + import torch + from kornia.models.sam import SamConfig + from kornia.contrib.visual_prompter import VisualPrompter + + image = torch.randn(3, 512, 512) + prompter = VisualPrompter(SamConfig("mobile_sam", pretrained=True)) + prompter.set_image(image) + + keypoints = Keypoints(torch.tensor([[[500.0, 375.0]]])) # BxNx2 + labels = torch.tensor([[1]], device=device) # BxN + + prediction = prompter.predict( + keypoints=keypoints, + keypoints_labels=labels, + multimask_output=False, + ) + +For more information on how to use SAM and :py:class:`kornia.contrib.visual_prompter.VisualPrompter`, refer to :doc:`segment_anything`. diff --git a/docs/source/models/rt_detr.rst b/docs/source/models/rt_detr.rst new file mode 100644 index 0000000..ee56f55 --- /dev/null +++ b/docs/source/models/rt_detr.rst @@ -0,0 +1,54 @@ +Real-Time Detection Transformer (RT-DETR) +========================================= + +.. code-block:: python + + from kornia.io import load_image + from kornia.contrib.object_detection import RTDETRDetectorBuilder + + input_img = load_image(img_path)[None] # Load image to BCHW + + # NOTE: available models: 'rtdetr_r18vd', 'rtdetr_r34vd', 'rtdetr_r50vd_m', 'rtdetr_r50vd', 'rtdetr_r101vd'. + # NOTE: recommended image scales: [480, 512, 544, 576, 608, 640, 640, 640, 672, 704, 736, 768, 800] + detector = RTDETRDetectorBuilder.build("rtdetr_r18vd", image_size=640) + + # get the output boxes + boxes = detector(input_img) + + # draw the bounding boxes on the images directly. + output = detector.draw(input_img, output_type="pil") + output[0].save("Kornia-RTDETR-output.png") + + # convert the whole model to ONNX directly + detector.to_onnx("RTDETR-640.onnx", image_size=640) + + +.. card:: + :link: https://arxiv.org/abs/2304.08069 + + **RT-DETR** + ^^^ + **Abstract:** Recently, end-to-end transformer-based detectors (DETRs) have achieved remarkable performance. + However, the issue of the high computational cost of DETRs has not been effectively addressed, limiting their + practical application and preventing them from fully exploiting the benefits of no post-processing, such as + non-maximum suppression (NMS). In this paper, we first analyze the influence of NMS in modern real-time object + detectors on inference speed, and establish an end-to-end speed benchmark. To avoid the inference delay caused + by NMS, we propose a Real-Time DEtection TRansformer (RT-DETR), the first real-time end-to-end object detector + to our best knowledge. Specifically, we design an efficient hybrid encoder to efficiently process multi-scale + features by decoupling the intra-scale interaction and cross-scale fusion, and propose IoU-aware query selection + to improve the initialization of object queries. In addition, our proposed detector supports flexibly adjustment + of the inference speed by using different decoder layers without the need for retraining, which facilitates the + practical application of real-time object detectors. Our RT-DETR-L achieves 53.0% AP on COCO val2017 and 114 FPS + on T4 GPU, while RT-DETR-X achieves 54.8% AP and 74 FPS, outperforming all YOLO detectors of the same scale in + both speed and accuracy. Furthermore, our RT-DETR-R50 achieves 53.1% AP and 108 FPS, outperforming + DINO-Deformable-DETR-R50 by 2.2% AP in accuracy and by about 21 times in FPS. Source code and pretrained models + will be available at PaddleDetection. + + **Tasks:** Detection + + **Datasets:** MS-COCO + + **Licence:** Apache 2.0 + + +++ + **Authors:** Wenyu Lv, Shangliang Xu, Yian Zhao, Guanzhong Wang, Jinman Wei, Cheng Cui, Yuning Du, Qingqing Dang, Yi Liu diff --git a/docs/source/models/segment_anything.rst b/docs/source/models/segment_anything.rst new file mode 100644 index 0000000..9624a56 --- /dev/null +++ b/docs/source/models/segment_anything.rst @@ -0,0 +1,309 @@ +Segment Anything (SAM) +====================== + +The Segment Anything Model (SAM) produces high quality object masks from input prompts such as points or boxes, and it +can be used to generate masks for all objects in an image. + +.. card:: + :link: https://segment-anything.com/ + + **Segment Anything** + ^^^ + **Abstract:** We introduce the Segment Anything (SAM) project: a new task, model, and dataset for image + segmentation. Using our efficient model in a data collection loop, we built the largest segmentation + dataset to date (by far), with over 1 billion masks on 11M licensed and privacy respecting images. The + model is designed and trained to be promptable, so it can transfer zero-shot to new image distributions + and tasks. We evaluate its capabilities on numerous tasks and find that its zero-shot performance is impressive + -- often competitive with or even superior to prior fully supervised results. We are releasing the Segment Anything + Model (SAM) and corresponding dataset (SA-1B) of 1B masks and 11M images at https://segment-anything.com to foster + research into foundation models for computer vision. + + **Tasks:** Segmentation + + **Datasets:** SA-1B + + **Licence:** Apache + + +++ + **Authors:** Alexander Kirillov and Eric Mintun and Nikhila Ravi and Hanzi Mao and Chloe Rolland and Laura + Gustafson and Alex Berg and Wan-Yen Lo and Piotr Dollar and Ross Girshick + + + +How to use SAM from Kornia +-------------------------- +The Kornia API for SAM try to provide a simple API to access initialize the model and load/download the weights. Also, +providing it to a high-level API called :code:`VisualPrompter`, which allow the users to set an image and run multiple +queries multiple times. + +The :code:`VisualPrompter` works querying on a single image, if you want to explore and query into a batch of images, +you can use the :code:`Sam` directly. But, for it you will need to write the boilerplate to preprocess and postprocess to +use it. This boilerplate, is already handle on the high-level API :code:`VisualPrompter`. + +Visual Prompter +^^^^^^^^^^^^^^^ +.. _anchor Prompter: + +The High level API :code:`VisualPrompter` handle with the image and prompt transformation, preprocessing and prediction for +a given SAM model. + +About the :code:`VisualPrompter`: + +#. From a `ModelConfig` loads the desired model with the desired checkpoint to be used as the model to receive the query + prompts. For know we just support Segment Anything model, where the *SAM-h* is the default option. + +#. Based on the model, the :code:`VisualPrompter` will handle with the necessary transformations to be done into the image + and prompts before apply it to the model. These transformations are done using PyTorch backed, by our API of + augmentations. Where we use the :class:`kornia.geometry.augmentation.AugmentationSequential` to handle with the different + data formats (keypoints, boxes, masks, image). + +#. When you use :code:`prompter.set_image(...)`, the prompter will preprocess this image, then pass it to the encoder, + and cache the embeddings to query it after. Note that the image should be scaled within the range [0,1]. + + * The preprocess steps are: 1) Resize the image to have its longer side the same size as :code:`image_encoder` image size + input. 2) Cache the information of this transformation to apply into the prompts. 3) normalize the image based on the + passed mean and standard deviation, or with the values of the SAM dataset. 4) pad on the bottom and right for the image + have the encoder expected resolution: :math:`(\text{image_encoder.img_size}, \text{image_encoder.img_size})`. + + * The best image to be used will always have the shape equals to + :math:`(\text{image_encoder.img_size}, \text{image_encoder.img_size})`. + +#. When you use :code:`prompter.predict(...)`, the prompter will apply the cached transformations on the coordinates of the + prompts, and then query this prompts into the cached embeddings. + + * If :code:`output_original_size=True`, the results structure will upsample the logits from it's resolution into the + image input original resolution. The output logits has the height and width equals to 256. + +#. You can benefit from using the :code:`torch.compile(...)` API (dynamo) for torch >= 2.0.0 version. To compile with dynamo + we provide the method :code:`prompter.compile(...)` which will optimize the right parts of the backend model and the + prompter itself. + +-------------- + +Example of using the :code:`VisualPrompter`: + +Exploring how to simple initialize the :code:`VisualPrompter`, automatically load the weights from a URL, +read the image and set it to be query, how to write the prompts, and the multiple ways we can use these prompts +to query the image masks from the SAM model. + + +.. code-block:: python + + import torch + + from kornia.models.sam import SamConfig + from kornia.contrib.visual_prompter import VisualPrompter + from kornia.io import load_image, ImageLoadType + from kornia.geometry.keypoints import Keypoints + from kornia.geometry.boxes import Boxes + from kornia.core.utils import get_cuda_or_mps_device_if_available + + model_type = 'vit_h' + checkpoint = './https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth' + device = get_cuda_or_mps_device_if_available + + # Load image + image = load_image('./example.jpg', ImageLoadType.RGB32, device) + + # Define the model config + config = SamConfig(model_type, checkpoint) + + # Load the prompter + prompter = VisualPrompter(config, device=device) + + # You can use torch dynamo/compile API with: + # prompter.compile() + + # set the image: This will preprocess the image and already generate the embeddings of it + prompter.set_image(image) + + # Generate the prompts + keypoints = Keypoints(torch.tensor([[[500, 375]]], device=device, dtype=torch.float32)) # BxNx2 + # For the keypoints label: 1 indicates a foreground point; 0 indicates a background point + keypoints_labels = torch.tensor([[1]], device=device) # BxN + boxes = Boxes( + torch.tensor([[[[425, 600], [425, 875], [700, 600], [700, 875]]]], device=device, dtype=torch.float32), mode='xyxy' + ) + + # Runs the prediction with all prompts + prediction = prompter.predict( + keypoints=keypoints, + keypoints_labels=keypoints_labels, + boxes=boxes, + multimask_output=True, + ) + + #---------------------------------------------- + # or run the prediction with just the keypoints + prediction = prompter.predict( + keypoints=keypoints, + keypoints_labels=keypoints_labels, + multimask_output=True, + ) + + #---------------------------------------------- + # or run the prediction with just the box + prediction = prompter.predict( + boxes=boxes, + multimask_output=True, + ) + + #---------------------------------------------- + # or run the prediction without prompts + prediction = prompter.predict( + multimask_output=True, + ) + + #------------------------------------------------ + # or run the prediction using the previous logits + prediction = prompter.predict( + masks=prediction.logits + multimask_output=True, + ) + + # The `prediction` is a SegmentationResults dataclass with the masks, scores and logits + print(prediction.masks.shape) + print(prediction.scores) + print(prediction.logits.shape) + + +Read more about the :code:`SegmentationResults` on :ref:`the official docs` + + + +Load from config +^^^^^^^^^^^^^^^^ +You can build a SAM model by specifying the encoder parameters on the :code:`SamConfig`, or from the model type. The +:code:`from_config` method will first try to build the model based on the model type, otherwise will try from the specified +parameters. If a checkpoint URL or path for a file is seted, the method will automatically load it. + +.. code-block:: python + + from kornia.models.sam import Sam, SamConfig + from kornia.core.utils import get_cuda_or_mps_device_if_available + + # model_type can be: + # 0, 'vit_h' or `kornia.models.sam.SamModelType.vit_h` + # 1, 'vit_l' or `kornia.models.sam.SamModelType.vit_l` + # 2, 'vit_b' or `kornia.models.sam.SamModelType.vit_b` + model_type = 'vit_b' + + # The checkpoint can be a filepath or a url + checkpoint = './path_for_the_model_checkpoint.pth' + device = get_cuda_or_mps_device_if_available() + + # Load config + config = SamConfig(model_type, checkpoint) + + # Load the model with checkpoint + sam_model = Sam.from_config(config) + + # Move to desired device + sam_model = sam_model.to(device) + + +Load checkpoint +^^^^^^^^^^^^^^^ +With the load checkpoint method you can load from a file or directly from a URL. The official (by meta) model weights are: + +#. `vit_h`: `ViT-H SAM model - https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth `_. +#. `vit_l`: `ViT-L SAM model - https://dl.fbaipublicfiles.com/segment_anything/sam_vit_l_0b3195.pth `_. +#. `vit_b`: `ViT-B SAM model - https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth `_. + +If a URL is passed the model will automatically download and cache the weights using +:code:`torch.hub.load_state_dict_from_url` + +.. code-block:: python + + from kornia.models.sam import Sam, SamConfig + from kornia.core.utils import get_cuda_or_mps_device_if_available + + model_type = 'vit_b' + + # The checkpoint can be a filepath or a url + checkpoint = './path_for_the_model_checkpoint.pth' + device = get_cuda_or_mps_device_if_available() + + # Load/build the model + sam_model = Sam.from_config(SamConfig(model_type)) + + # Load the checkpoint + sam_model.load_checkpoint(checkpoint, device) + + +.. Mask Generator +.. ^^^^^^^^^^^^^^ + + +Example of how to use the SAM model without API +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +This is a simple example, of how to directly use the SAM model loaded. We recommend the use of +:ref:`Prompter API` to handle/prepare the inputs. + +.. code-block:: python + + from kornia.models.sam import Sam + from kornia.models.structures import SegmentationResults + from kornia.io import load_image, ImageLoadType + from kornia.core.utils import get_cuda_or_mps_device_if_available + from kornia.geometry import resize + from kornia.enhance import normalize + + model_type = 'vit_b' # or can be a number `2` or the enum sam.SamModelType.vit_b + checkpoint_path = './path_for_the_model_checkpoint.pth' + device = get_cuda_or_mps_device_if_available() + + # Load the model + sam_model = Sam.from_pretrained(model_type, checkpoint_path, device) + + # Load image + image = load_image('./example.jpg', ImageLoadType.RGB32, device) + + # Transform the image (CxHxW) into a batched input (BxCxHxW) + image = image[None, ...] + + # Resize the image to have the maximum size 1024 on its largest side + data = resize(image, 1024, side='long') + + # Embed prompts -- ATTENTION: should match the coordinates after the resize of the image + sparse_embeddings, dense_embeddings = sam_model.prompt_encoder(points=None, boxes=None, masks=None) + + # define the info for normalize the input + pixel_mean = torch.tensor(...) + pixel_std = torch.tensor(...) + + # Preprocess input + data = normalize(data, pixel_mean, pixel_std) + padh = model_sam.image_encoder.img_size - h + padw = model_sam.image_encoder.img_size - w + data = pad(data, (0, padw, 0, padh)) + + #-------------------------------------------------------------------- + # Option A: Manually calling each API + #-------------------------------------------------------------------- + low_res_logits, iou_predictions = sam_model.mask_decoder( + image_embeddings=sam_model.image_encoder(data), + image_pe=sam_model.prompt_encoder.get_dense_pe(), + sparse_prompt_embeddings=sparse_embeddings, + dense_prompt_embeddings=dense_embeddings, + multimask_output=True, + ) + + prediction = SegmentationResults(low_res_logits, iou_predictions) + + #-------------------------------------------------------------------- + # Option B: Calling the model itself + #-------------------------------------------------------------------- + prediction = sam_model(data[None, ...], [{}], multimask_output=True) + + #-------------------------------------------------------------------- + # Post processing + #-------------------------------------------------------------------- + # Upscale the masks to the original image resolution + input_size = (data.shape[-2], data.shape[-1]) + original_size = (image.shape[-2], image.shape[-1]) + image_size_encoder = (model_sam.image_encoder.img_size, model_sam.image_encoder.img_size) + prediction.original_res_logits(input_size, original_size, image_size_encoder) + + # If wants to check the binary masks + masks = prediction.binary_masks diff --git a/docs/source/models/sold2.rst b/docs/source/models/sold2.rst new file mode 100644 index 0000000..b071fcb --- /dev/null +++ b/docs/source/models/sold2.rst @@ -0,0 +1,23 @@ +SOLD2 (Line detection and matching) +................................... + +.. card:: + :link: https://arxiv.org/abs/2104.03362 + + **SOLD²: Self-supervised Occlusion-aware Line Description and Detection** + ^^^ + **Abstract:** Compared to feature point detection and description, detecting and matching line segments offer additional challenges. Yet, line features represent a promising complement to points for multi-view tasks. Lines are indeed well-defined by the image gradient, frequently appear even in poorly textured areas and offer robust structural cues. We thus hereby introduce the first joint detection and description of line segments in a single deep network. Thanks to a self-supervised training, our method does not require any annotated line labels and can therefore generalize to any dataset. Our detector offers repeatable and accurate localization of line segments in images, departing from the wireframe parsing approach. Leveraging the recent progresses in descriptor learning, our proposed line descriptor is highly discriminative, while remaining robust to viewpoint changes and occlusions. We evaluate our approach against previous line detection and description methods on several multi-view datasets created with homographic warps as well as real-world viewpoint changes. Our full pipeline yields higher repeatability, localization accuracy and matching metrics, and thus represents a first step to bridge the gap with learned feature points methods. + + **Tasks:** Line detection, Line description, Line matching + + **Datasets:** Wireframe, YorkUrban, ETH3D + + **Conference:** CVPR 2021 + + **Licence:** MIT + + +++ + **Authors:** Rémi Pautrat*, Juan-Ting Lin*, Viktor Larsson, Martin R. Oswald, Marc Pollefeys + +.. image:: https://github.com/cvg/SOLD2/raw/main/assets/videos/demo_moving_camera.gif + :align: center diff --git a/docs/source/models/tiny_vit.rst b/docs/source/models/tiny_vit.rst new file mode 100644 index 0000000..e31e708 --- /dev/null +++ b/docs/source/models/tiny_vit.rst @@ -0,0 +1,36 @@ +.. _kornia_tiny_vit: + +TinyViT +......... + +.. card:: + :link: https://arxiv.org/abs/2110.02178 + + **TinyViT: Fast Pretraining Distillation for Small Vision Transformers** + ^^^ + **Abstract:** Vision transformer (ViT) recently has drawn great attention in computer vision due to its remarkable model capability. However, most prevailing ViT models suffer from huge number of parameters, restricting their applicability on devices with limited resources. To alleviate this issue, we propose TinyViT, a new family of tiny and efficient small vision transformers pretrained on large-scale datasets with our proposed fast distillation framework. The central idea is to transfer knowledge from large pretrained models to small ones, while enabling small models to get the dividends of massive pretraining data. More specifically, we apply distillation during pretraining for knowledge transfer. The logits of large teacher models are sparsified and stored in disk in advance to save the memory cost and computation overheads. The tiny student transformers are automatically scaled down from a large pretrained model with computation and parameter constraints. Comprehensive experiments demonstrate the efficacy of TinyViT. It achieves a top-1 accuracy of 84.8% on ImageNet-1k with only 21M parameters, being comparable to Swin-B pretrained on ImageNet-21k while using 4.2 times fewer parameters. Moreover, increasing image resolutions, TinyViT can reach 86.5% accuracy, being slightly better than Swin-L while using only 11% parameters. Last but not the least, we demonstrate a good transfer ability of TinyViT on various downstream tasks. Code and models are available at https://github.com/microsoft/Cream/tree/main/TinyViT. + + **Tasks:** Image Classification, Object Detection + + **Datasets:** ImageNet, MS-COCO + + +++ + **Authors:** Kan Wu, Jinnian Zhang, Houwen Peng, Mengchen Liu, Bin Xiao, Jianlong Fu, Lu Yuan + +.. image:: https://github.com/microsoft/Cream/blob/main/TinyViT/.figure/framework.png?raw=true + :align: center + +Usage +~~~~~ + +You can use TinyViT models as follows. + +.. code:: python + + import torch + from kornia.models.tiny_vit import TinyViT + + model = TinyViT.from_config("5m", pretrained=True) # ImageNet-1k pre-trained + + img = torch.rand(1, 3, 224, 224) + out = classifier(img) # 1x1000 diff --git a/docs/source/models/vit.rst b/docs/source/models/vit.rst new file mode 100644 index 0000000..359fa2c --- /dev/null +++ b/docs/source/models/vit.rst @@ -0,0 +1,84 @@ +.. _kornia_vit: + +Vision Transformer (ViT) +........................ + +.. card:: + :link: https://paperswithcode.com/paper/an-image-is-worth-16x16-words-transformers-1 + + **ViT: An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale** + ^^^ + **Abstract:** While the Transformer architecture has become the de-facto standard for natural language processing tasks, its applications to computer vision remain limited. In vision, attention is either applied in conjunction with convolutional networks, or used to replace certain components of convolutional networks while keeping their overall structure in place. We show that this reliance on CNNs is not necessary and a pure transformer applied directly to sequences of image patches can perform very well on image classification tasks. When pre-trained on large amounts of data and transferred to multiple mid-sized or small image recognition benchmarks (ImageNet, CIFAR-100, VTAB, etc. ), Vision Transformer (ViT) attains excellent results compared to state-of-the-art convolutional networks while requiring substantially fewer computational resources to train. + + **Tasks:** Image Classification, Fine-Grained Image Classification, Document Image Classification + + **Datasets:** CIFAR-10, ImageNet, CIFAR-100 + + **Conference:** ICLR 2021 + + **Licence:** Apache-2.0 + + +++ + **Authors:** Alexey Dosovitskiy, Lucas Beyer, Alexander Kolesnikov, Dirk Weissenborn, Xiaohua Zhai, Thomas Unterthiner, Mostafa Dehghani, Matthias Minderer, Georg Heigold, Sylvain Gelly, Jakob Uszkoreit, Neil Houlsby + +.. image:: https://github.com/google-research/vision_transformer/raw/main/vit_figure.png + :align: center + + +Kornia-ViT +---------- + +We provide the operator :py:class:`~kornia.models.vit.VisionTransformer` that is meant to be used across tasks. +One can use the *ViT* in Kornia as follows: + +.. code:: python + + img = torch.rand(1, 3, 224, 224) + vit = VisionTransformer(image_size=224, patch_size=16) + out = vit(img) + +Usage +~~~~~ + +``kornia-vit`` does not include any classification head. +You can easily add your own classification head using standard PyTorch modules. + +.. code:: python + + import torch.nn as nn + from kornia.models.vit import VisionTransformer + + classifier = nn.Sequential( + VisionTransformer(image_size=224, patch_size=16), + nn.Linear(768, 1000) # Example: 768 is the default hidden_dim, 1000 is num_classes + ) + + img = torch.rand(1, 3, 224, 224) + out = classifier(img) # BxN + scores = out.argmax(-1) # B + +In addition to create simple image classification, our API is flexible enough to design your pipelines e.g +to solve problems for multi-task, object detection, segmentation, etc. We show an example of a multi-task +class with two different classification heads: + +.. code:: python + + from kornia.models.vit import VisionTransformer + + class MultiTaskTransfornmer(nn.Module): + def __init__(self) -> None: + super().__init__() + self.transformer = VisionTransformer( + image_size=224, patch_size=16) + self.head1 = nn.Linear(768, 10) # Example: 768 is the default hidden_dim + self.head2 = nn.Linear(768, 50) + + def forward(self, x: torch.Tensor): + out = self.transformer(x) + return { + "head1": self.head1(out), + "head2": self.head2(out), + } + +.. tip:: + More heads, examples and a training API soon !! diff --git a/docs/source/models/vit_mobile.rst b/docs/source/models/vit_mobile.rst new file mode 100644 index 0000000..86987bb --- /dev/null +++ b/docs/source/models/vit_mobile.rst @@ -0,0 +1,55 @@ +.. _kornia_vit_mobile: + +MobileViT +......... + +.. card:: + :link: https://arxiv.org/abs/2110.02178 + + **MobileViT: Light-weight, General-purpose, and Mobile-friendly Vision Transformer** + ^^^ + **Abstract:** Light-weight convolutional neural networks (CNNs) are the de-facto for mobile vision tasks. Their spatial inductive biases allow them to learn representations with fewer parameters across different vision tasks. However, these networks are spatially local. To learn global representations, self-attention-based vision trans-formers (ViTs) have been adopted. Unlike CNNs, ViTs are heavy-weight. In this paper, we ask the following question: is it possible to combine the strengths of CNNs and ViTs to build a light-weight and low latency network for mobile vision tasks? Towards this end, we introduce MobileViT, a light-weight and general-purpose vision transformer for mobile devices. MobileViT presents a different perspective for the global processing of information with transformers, i.e., transformers as convolutions. Our results show that MobileViT significantly outperforms CNN- and ViT-based networks across different tasks and datasets. On the ImageNet-1k dataset, MobileViT achieves top-1 accuracy of 78.4% with about 6 million parameters, which is 3.2% and 6.2% more accurate than MobileNetv3 (CNN-based) and DeIT (ViT-based) for a similar number of parameters. On the MS-COCO object detection task, MobileViT is 5.7% more accurate than Mo-bileNetv3 for a similar number of parameters. + + **Tasks:** Image Classification, Object Detection, Semantic Segmentation + + **Datasets:** ImageNet, MS-COCO, PASCAL VOC + + +++ + **Authors:** Sachin Mehta, Mohammad Rastegari + +.. image:: https://user-images.githubusercontent.com/67839539/136470152-2573529e-1a24-4494-821d-70eb4647a51d.png + :align: center + + +Kornia-MobileViT +---------------- + +We provide :py:class:`~kornia.models.vit_mobile.MobileViT` which can be used for many downstream tasks, e.g., classification, object detection and semantic segmentation. +One can use the *MobileViT* in Kornia as follows: + +.. code:: python + + img = torch.rand(1, 3, 256, 256) + mvit = MobileViT(mode='xxs') + out = mvit(img) + + +Usage +~~~~~ + +Similar to ``Kornia-ViT``, ``Kornia-MobileViT`` does not include any classification head. But you can add it simply by doing: + +.. code:: python + + import torch.nn as nn + from kornia.models.vit_mobile import MobileViT + + classifier = nn.Sequential( + MobileViT(mode='xxs'), + nn.AvgPool2d(256 // 32, 1), + nn.Flatten(), + nn.Linear(320, 1000) + ) + + img = torch.rand(1, 3, 256, 256) + out = classifier(img) # 1x1000 diff --git a/docs/source/models/yunet.rst b/docs/source/models/yunet.rst new file mode 100644 index 0000000..c74b81c --- /dev/null +++ b/docs/source/models/yunet.rst @@ -0,0 +1,22 @@ +.. _yunet_model: + +YuNet +..... + +.. card:: + :link: https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=9429909 + + **A Systematic IoU-Related Method: Beyond Simplified Regression for Better Localization** + ^^^ + **Abstract:** Four-variable-independent-regression localization losses, such as Smooth- l 1 Loss, are used by default in modern detectors. Nevertheless, this kind of loss is oversimplified so that it is inconsistent with the final evaluation metric, intersection over union (IoU). Directly employing the standard IoU is also not infeasible, since the constant-zero plateau in the case of non-overlapping boxes and the non-zero gradient at the minimum may make it not trainable. Accordingly, we propose a systematic method to address these problems. Firstly, we propose a new metric, the extended IoU (EIoU), which is well-defined when two boxes are not overlapping and reduced to the standard IoU when overlapping. Secondly, we present the convexification technique (CT) to construct a loss on the basis of EIoU, which can guarantee the gradient at the minimum to be zero. Thirdly, we propose a steady optimization technique (SOT) to make the fractional EIoU loss approaching the minimum more steadily and smoothly. Fourthly, to fully exploit the capability of the EIoU based loss, we introduce an interrelated IoU-predicting head to further boost localization accuracy. With the proposed contributions, the new method incorporated into Faster R-CNN with ResNet50+FPN as the backbone yields 4.2 mAP gain on VOC2007 and 2.3 mAP gain on COCO2017 over the baseline Smooth- l 1 Loss, at almost no training and inferencing computational cost. Specifically, the stricter the metric is, the more notable the gain is, improving 8.2 mAP on VOC2007 and 5.4 mAP on COCO2017 at metric AP. + + **Tasks:** Face Detection + + **Datasets:** WIDER Face + + **Journal:** IEEE Transactions on Image Processing 2021 + + **Licence:** Apache-2.0 + + +++ + **Authors:** Hanyang Peng and Shiqi Yu diff --git a/docs/source/morphology.rst b/docs/source/morphology.rst new file mode 100644 index 0000000..682fe61 --- /dev/null +++ b/docs/source/morphology.rst @@ -0,0 +1,24 @@ +kornia.morphology +====================== + +.. meta:: + :name: description + :content: "The Kornia.morphology module offers a set of morphological image processing operations such as dilation, erosion, opening, closing, gradient, top hat, and bottom hat. It enables users to apply these transformations to images for advanced computer vision tasks. Try the interactive demo on Hugging Face Spaces." + +.. currentmodule:: kornia.morphology + +.. autofunction:: dilation +.. autofunction:: erosion +.. autofunction:: opening +.. autofunction:: closing +.. autofunction:: gradient +.. autofunction:: top_hat +.. autofunction:: bottom_hat + +Interactive Demo +~~~~~~~~~~~~~~~~ +.. raw:: html + + + +Visit the demo on `Hugging Face Spaces `_. diff --git a/docs/source/onnx.rst b/docs/source/onnx.rst new file mode 100644 index 0000000..b2e90ee --- /dev/null +++ b/docs/source/onnx.rst @@ -0,0 +1,187 @@ +ONNXSequential: Chain Multiple ONNX Models with Ease +==================================================== + +.. meta:: + :name: description + :content: "The `ONNXSequential` class enables users to effortlessly chain and execute multiple ONNX models in a sequence, simplifying the creation of complex pipelines. It offers flexibility in input/output mapping, optimized execution with ONNXRuntime's providers (CPU, CUDA, etc.), and allows for exporting combined models. Ideal for real-time inference and multi-model workflows, ONNXSequential provides a simple API for efficient model management and deployment." + +The `ONNXSequential` class is a powerful new feature that allows users to effortlessly combine and chain multiple ONNX models together. This is especially useful when you have several pre-trained models or custom ONNX operators that you want to execute sequentially as part of a larger pipeline. + +Whether you're working with models for inference, experimentation, or optimization, `ONNXSequential` makes it easier to manage, combine, and run ONNX models in a streamlined manner. It also supports flexibility in execution environments with ONNXRuntime's execution providers (CPU, CUDA, etc.). + +Key Features +------------ + +- **Seamless Model Chaining**: Combine multiple ONNX models into a single computational graph. +- **Flexible Input/Output Mapping**: Control how the outputs of one model are passed as inputs to the next. +- **Export to ONNX**: Save the combined model into a single ONNX file for easy deployment and sharing. +- **PyTorch-like Interface**: Use the `ONNXSequential` class like a PyTorch `nn.Sequential` model, including calling it directly for inference. + +Optimized Execution +------------------- +- **ONNXRuntime**: Automatically create optimized `ONNXRuntime` sessions to speed up inference. +- **Execution Providers Support**: Utilize ONNXRuntime's execution providers (e.g., `CUDAExecutionProvider`, `CPUExecutionProvider`, `TensorrtExecutionProvider`, `OpenVINOExecutionProvider`) for accelerated inference on different hardware. +- **Concurrent Sessions**: You can manage multiple inference sessions concurrently, allowing for parallel processing of multiple inputs. +- **Asynchronous API**: We offer asyncio-based execution along with the runtime's asynchronous functions to perform non-blocking inference. + +Quickstart Guide +---------------- + +Here's how you can quickly get started with `ONNXSequential`: + +1. **Install ONNX and ONNXRuntime** + + If you haven't already installed `onnx` and `onnxruntime`, you can install them using `pip`: + + .. code-block:: bash + + pip install onnx onnxruntime + +2. **Combining ONNX Models** + + You can initialize the `ONNXSequential` with a list of ONNX models or file paths. Models will be automatically chained together and optimized for inference. + + .. code-block:: python + + import numpy as np + from kornia.onnx import ONNXSequential + + # Initialize ONNXSequential with two models, loading from our only repo + onnx_seq = ONNXSequential( + "hf://operators/kornia.color.gray.RgbToGrayscale", + "hf://operators/kornia.geometry.transform.affwarp.Resize_512x512" + ) + + # Prepare some input data + input_data = np.random.randn(1, 3, 256, 512).astype(np.float32) + + # Perform inference + outputs = onnx_seq(input_data) + + # Print the model outputs + print(outputs) + + .. note:: + By default, we assume each ONNX model contains only one input node named "input" and one output node named "output". For complex models, you may need to pass an `io_maps` argument. + +3. **Input/Output Mapping Between Models** + + When combining models, you can specify how the outputs of one model are mapped to the inputs of the next. This allows you to chain models in custom ways. + + .. code-block:: python + + io_map = [("model1_output_0", "model2_input_0"), ("model1_output_1", "model2_input_1")] + onnx_seq = ONNXSequential("model1.onnx", "model2.onnx", io_map=io_map) + +4. **Exporting the Combined Model** + + You can easily export the combined model to an ONNX file: + + .. code-block:: python + + # Export the combined model to a file + onnx_seq.export("combined_model.onnx") + +5. **Optimizing with Execution Providers** + + Leverage ONNXRuntime's execution providers for optimized inference. For example, to run the model on a GPU: + + .. code-block:: python + + # Initialize with CUDA execution provider + onnx_seq = ONNXSequential( + "hf://operators/kornia.geometry.transform.flips.Hflip", + # Or you may use a local model with either a filepath "YOUR_OWN_MODEL.onnx" or a loaded ONNX model. + "hf://models/kornia.models.detection.rtdetr_r18vd_640x640", + providers=['CUDAExecutionProvider'] + ) + + # Run inference + outputs = onnx_seq(input_data) + +Frequently Asked Questions (FAQ) +-------------------------------- + +**1. Can I chain models from different sources?** + +Yes! You can chain models from different ONNX files or directly from `onnx.ModelProto` objects. `ONNXSequential` handles the integration and merging of their graphs. + +**2. What happens if the input/output sizes of models don't match?** + +You can use the `io_map` parameter to control how outputs of one model are mapped to the inputs of the next. This allows for greater flexibility when chaining models with different architectures. + +**3. Can I use custom ONNXRuntime session options?** + +Absolutely! You can pass your own session options to the `create_session` method to fine-tune performance, memory usage, or logging. + +**4. How to run with CUDA?** + +For using CUDA ONNXRuntime, you need to install `onnxruntime-gpu`. +For handling different CUDA version, you may refer to +https://github.com/microsoft/onnxruntime/issues/21769#issuecomment-2295342211. + +For example, to install `onnxruntime-gpu==1.19.2` under CUDA 11.X, you may install with: + +.. code-block:: console + + pip install onnxruntime-gpu==1.19.2 --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-11/pypi/simple/ + +You may then convert your sequence to CUDA, such as: + +.. code-block:: python + + import kornia + onnx_seq = ONNXSequential( + "hf://operators/kornia.geometry.transform.flips.Hflip", + "hf://models/kornia.models.detection.rtdetr_r18vd_640x640", # Or you may use "YOUR_OWN_MODEL.onnx" + ) + inp = kornia.utils.sample.get_sample_images()[0].numpy()[None] + import time + onnx_seq.as_cuda() + onnx_seq(inp) # GPU warm up + start_time = time.time() + onnx_seq(inp) + print("--- GPU %s seconds ---" % (time.time() - start_time)) + onnx_seq.as_cpu() + start_time = time.time() + onnx_seq(inp) + print("--- %s seconds ---" % (time.time() - start_time)) + +You may get a decent improvement: + +.. code-block:: console + + --- GPU 0.014804363250732422 seconds --- + --- CPU 0.17681646347045898 seconds --- + +Why Choose ONNXSequential? +-------------------------- + +With the increasing adoption of ONNX for model interoperability and deployment, `ONNXSequential` provides a simple yet powerful interface for combining models and operators. By leveraging ONNXRuntime's optimization and execution provider capabilities, it gives you the flexibility to: +- Deploy on different hardware (CPU, GPU, TensorRT, OpenVINO, etc.). +- Run complex pipelines in production environments. +- Combine and experiment with models effortlessly. + +Whether you're building an advanced deep learning pipeline or simply trying to chain pre-trained models, `ONNXSequential` makes it easy to manage, optimize, and execute ONNX models at scale. + +Get started today and streamline your ONNX workflows! + + +API Documentation +----------------- +.. autoclass:: kornia.onnx.module.ONNXModule + :members: + +.. autoclass:: kornia.onnx.sequential.ONNXSequential + :members: + +.. autoclass:: kornia.onnx.utils.ONNXLoader + + .. code-block:: python + + # Load a HuggingFace operator + ONNXLoader.load_model("hf://operators/kornia.color.gray.GrayscaleToRgb") # doctest: +SKIP + # Load a local converted/downloaded operator + ONNXLoader.load_model("operators/kornia.color.gray.GrayscaleToRgb") # doctest: +SKIP + + :members: diff --git a/docs/source/references.bib b/docs/source/references.bib new file mode 100644 index 0000000..2c083df --- /dev/null +++ b/docs/source/references.bib @@ -0,0 +1,431 @@ + +@article{zhao2023aliked, + author={Zhao, Xiaoming and Wu, Xingming and Chen, Weihai and Chen, Peter C. Y. and Xu, Qingsong and Li, Zhengguo}, + journal={IEEE Transactions on Instrumentation and Measurement}, + title={ALIKED: A Lighter Keypoint and Descriptor Extraction Network via Deformable Transformation}, + year={2023}, + volume={72}, + pages={1-16}, + doi={10.1109/TIM.2023.3271000}, +} + +@article{dosovitskiy2020vit, + title={An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale}, + author={Dosovitskiy, Alexey and Beyer, Lucas and Kolesnikov, Alexander and Weissenborn, Dirk and Zhai, Xiaohua and Unterthiner, Thomas and Dehghani, Mostafa and Minderer, Matthias and Heigold, Georg and Gelly, Sylvain and Uszkoreit, Jakob and Houlsby, Neil}, + journal={ICLR}, + year={2021} +} + +@inproceedings{DeFMO2021, + title={DeFMO: Deblurring and Shape Recovery of Fast Moving Objects}, + author={Rozumnyi, Denys and Oswald, Martin R. and Ferrari, Vittorio and Matas, Jiri and Pollefeys, Marc}, + booktitle={CVPR}, + year={2021} +} + +@inproceedings{zhang2019shiftinvar, + title={Making Convolutional Networks Shift-Invariant Again}, + author={Zhang, Richard}, + booktitle={ICML}, + year={2019} +} + +@inproceedings{baumberg2000, + title={Reliable feature matching across widely separated views}, + author={A. {Baumberg}}, + booktitle={CVPR}, + year={2000} +} + +@inproceedings{HardNet2017, + title={Working hard to know your neighbor's margins: Local descriptor learning loss}, + author={Anastasiya Mishchuk and Dmytro Mishkin and Filip Radenovic and Jiri Matas}, + booktitle={Proceedings of NeurIPS}, + year={2017} +} + +@article{HardNet2020, + title={Improving the HardNet Descriptor}, + author={Milan Pultar}, + journal={arXiv ePrint 2007.09699}, + year={2020} +} + +@inproceedings{FRN2019, + title={Filter Response Normalization Layer: Eliminating Batch Dependence in the Training of Deep Neural Networks}, + author={Saurabh Singh and Shankar Krishnan}, + year={2020}, + booktitle={CVPR} +} + +@article{LightGlue2023, + author={Philipp Lindenberger and + Paul-Edouard Sarlin and + Marc Pollefeys}, + title = {LightGlue: Local Feature Matching at Light Speed}, + journal={arXiv ePrint 2306.13643}, + year={2023} +} + +@inproceedings{hynet2020, + author = {Tian, Yurun and Barroso Laguna, Axel and Ng, Tony and Balntas, Vassileios and Mikolajczyk, Krystian}, + title = {HyNet: Learning Local Descriptor with Hybrid Similarity Measure and Triplet Loss}, + booktitle = {NeurIPS}, + year = {2020} +} + +@inproceedings{TFeat2016, + author = {Vassileios Balntas and Edgar Riba and Daniel Ponsa and Krystian Mikolajczyk}, + title = {Learning local feature descriptors with triplets and shallow convolutional neural networks}, + booktitle = {British Machine Vision Conference {(BMVC)}}, + year = {2016} +} + +@article{mukundan2019understanding, + title={Understanding and improving kernel local descriptors}, + author={Mukundan, Arun and Tolias, Giorgos and Bursuc, Andrei and J{\'e}gou, Herv{\'e} and Chum, Ond{\v{r}}ej}, + journal={International Journal of Computer Vision}, + year={2019} +} + + +@article{zhang2018mixup, + title={mixup: Beyond Empirical Risk Minimization}, + author={Hongyi Zhang and Moustapha Cisse nad Yann N. Dauphin and David Lopez-Paz}, + journal={International Conference on Learning Representations}, + year={2018}, + url={https://openreview.net/forum?id=r1Ddp1-Rb} +} + +@inproceedings{AffNet2018, + author = {D. Mishkin and F. Radenovic and J. Matas}, + title = {{Repeatability is Not Enough: Learning Affine Regions via Discriminability}}, + booktitle = {ECCV}, + year = {2018} +} + +@inproceedings{yun2019cutmix, + title={CutMix: Regularization Strategy to Train Strong Classifiers with Localizable Features}, + author={Yun, Sangdoo and Han, Dongyoon and Oh, Seong Joon and Chun, Sanghyuk and Choe, Junsuk and Yoo, Youngjoon}, + booktitle = {International Conference on Computer Vision (ICCV)}, + year={2019} +} + +@article{lin2018focal, + title={Focal Loss for Dense Object Detection}, + author={Tsung-Yi Lin and Priya Goyal and Ross Girshick and Kaiming He and Piotr Dollár}, + year={2018}, + journal={arXiv ePrint 1708.02002} +} + +@article{salehi2017tversky, + title={Tversky loss function for image segmentation using 3D fully convolutional deep networks}, + author={Seyed Sadegh Mohseni Salehi and Deniz Erdogmus and Ali Gholipour}, + year={2017}, + journal={arXiv ePrint 1706.05721} +} + +@article{Simard2003BestPF, + title={Best practices for convolutional neural networks applied to visual document analysis}, + author={P. Simard and David Steinkraus and John C. Platt}, + journal={Seventh International Conference on Document Analysis and Recognition, 2003. Proceedings.}, + year={2003}, + pages={958-963} +} + +@misc{lin2021patch, + title={Patch AutoAugment}, + author={Shiqi Lin and Tao Yu and Ruoyu Feng and Zhibo Chen}, + year={2021}, + eprint={2103.11099}, + archivePrefix={arXiv}, + primaryClass={cs.CV} +} + +@inproceedings{LoFTR2021, + title={{LoFTR}: Detector-Free Local Feature Matching with Transformers}, + author={Sun, Jiaming and Shen, Zehong and Wang, Yuang and Bao, Hujun and Zhou, Xiaowei}, + booktitle={CVPR}, + year={2021} +} + +@article{karimi2019reducing, + title={Reducing the hausdorff distance in medical image segmentation with convolutional neural networks}, + author={Karimi, Davood and Salcudean, Septimiu E}, + journal={IEEE Transactions on medical imaging}, + volume={39}, + number={2}, + pages={499--513}, + year={2019}, + publisher={IEEE} +} + +@article{Marquez-Neila2015, + author = {M\'{a}rquez-Neila, Pablo and L\'{o}pez-Alberca, Javier and Buenaposada, Jos\'{e} M. and Baumela, Luis}, + title = {Speeding-up Homography Estimation in Mobile Devices}, + year = {2016}, + publisher = {Springer-Verlag}, + address = {Berlin, Heidelberg}, + volume = {11}, + number = {1}, + issn = {1861-8200}, + url = {https://doi.org/10.1007/s11554-012-0314-1}, + doi = {10.1007/s11554-012-0314-1}, + journal = {J. Real-Time Image Process.}, + month = jan, + pages = {141–154}, + numpages = {14} +} + +@inproceedings{KeyNet2019, + author = {Barroso-Laguna, Axel and Riba, Edgar and Ponsa, Daniel and Mikolajczyk, Krystian}, + title = {{Key.Net: Keypoint Detection by Handcrafted and Learned CNN Filters}}, + booktitle = {ICCV}, + year = {2019} +} + +@article{facedetect-yu, + author={Yuantao Feng and Shiqi Yu and Hanyang Peng and Yan-ran Li and Jianguo Zhang}, + title={Detect Faces Efficiently: A Survey and Evaluations}, + journal={IEEE Transactions on Biometrics, Behavior, and Identity Science}, + year={2021} + } + + +@INPROCEEDINGS {xsoria2020dexined, +author = {X. Soria and E. Riba and A. Sappa}, +booktitle = {2020 IEEE Winter Conference on Applications of Computer Vision (WACV)}, +title = {Dense Extreme Inception Network: Towards a Robust CNN Model for Edge Detection}, +year = {2020}, +volume = {}, +issn = {}, +pages = {1912-1921}, +keywords = {image edge detection;convolution;training;feeds;machine learning;task analysis;kernel}, +doi = {10.1109/WACV45572.2020.9093290}, +url = {https://doi.ieeecomputersociety.org/10.1109/WACV45572.2020.9093290}, +publisher = {IEEE Computer Society}, +address = {Los Alamitos, CA, USA}, +month = {mar} +} + +@inproceedings{SOLD22021, + author = {Pautrat*, Rémi and Lin*, Juan-Ting and Larsson, Viktor and Oswald, Martin R. and Pollefeys, Marc}, + title = {SOLD2: Self-supervised Occlusion-aware Line Description and Detection}, + booktitle = {Computer Vision and Pattern Recognition (CVPR)}, + year = {2021}, +} + +@article{karam2019convdt, + author={Karam, Christina and Sugimoto, Kenjiro and Hirakawa, Keigo}, + journal={IEEE Signal Processing Letters}, + title={Fast Convolutional Distance Transform}, + year={2019}, + volume={26}, + number={6}, + pages={853-857}, + doi={10.1109/LSP.2019.2910466} +} + +@article{pham2021dtlayer, + title={A Differentiable Convolutional Distance Transform Layer for Improved Image Segmentation}, + author={Duc Duy Pham and Gurbandurdy Dovletov and Josef Pauli}, + journal={Pattern Recognition}, + year={2020}, + volume={12544}, + pages={432 - 444} +} + +@article{zini2022planckian, + title={Planckian jitter: enhancing the color quality of self-supervised visual representations}, + author={Zini, Simone and Buzzelli, Marco and Twardowski, Bart{\l}omiej and van de Weijer, Joost}, + journal={arXiv preprint arXiv:2202.07993}, + year={2022} +} + +@inproceedings{homolines2001, + author = {Guerrero J.J. and Sagues C.}, + title = {From Lines to Homographies between Uncalibrated Images}, + booktitle = {IX Spanish Symposium on Pattern Recognition and Image Analysis}, + year = {2001} +} + +@misc{tormentor, + doi = {10.48550/ARXIV.2204.03776}, + url = {https://arxiv.org/abs/2204.03776}, + author = {Nicolaou, Anguelos and Christlein, Vincent and Riba, Edgar and Shi, Jian and Vogeler, Georg and Seuret, Mathias}, + keywords = {Computer Vision and Pattern Recognition (cs.CV), Artificial Intelligence (cs.AI), FOS: Computer and information sciences, FOS: Computer and information sciences}, + title = {TorMentor: Deterministic dynamic-path, data augmentations with fractals}, + publisher = {arXiv}, + year = {2022}, + copyright = {Creative Commons Attribution 4.0 International} +} + +@article{MODS2015, + title = {MODS: Fast and robust method for two-view matching }, + author = {Dmytro Mishkin and Jiri Matas and Michal Perdoch}, + journal = {Computer Vision and Image Understanding }, + year = {2015}, + pages = {81 - 93}, + volume = {141} +} + +@incollection{burt1987laplacian, + title={The Laplacian pyramid as a compact image code}, + author={Burt, Peter J and Adelson, Edward H}, + booktitle={Readings in computer vision}, + pages={671--679}, + year={1987}, + publisher={Elsevier} +} + +@article{AdaLAM2020, + author = {Luca Cavalli and + Viktor Larsson and + Martin Ralf Oswald and + Torsten Sattler and + Marc Pollefeys}, + title = {AdaLAM: Revisiting Handcrafted Outlier Detection}, + journal = {CoRR}, + volume = {abs/2006.04250}, + year = {2020}, + url = {https://arxiv.org/abs/2006.04250}, + eprinttype = {arXiv}, + eprint = {2006.04250}, + timestamp = {Fri, 12 Jun 2020 14:02:57 +0200}, + biburl = {https://dblp.org/rec/journals/corr/abs-2006-04250.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} + +@article{rommel2022deep, + title={Deep invariant networks with differentiable augmentation layers}, + author={Rommel, C{\'e}dric and Moreau, Thomas and Gramfort, Alexandre}, + journal={arXiv preprint arXiv:2202.02142}, + year={2022} +} + +@article{cubuk2018autoaugment, + title={Autoaugment: Learning augmentation policies from data}, + author={Cubuk, Ekin D and Zoph, Barret and Mane, Dandelion and Vasudevan, Vijay and Le, Quoc V}, + journal={arXiv preprint arXiv:1805.09501}, + year={2018} +} + +@inproceedings{cubuk2020randaugment, + title={Randaugment: Practical automated data augmentation with a reduced search space}, + author={Cubuk, Ekin D and Zoph, Barret and Shlens, Jonathon and Le, Quoc V}, + booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition workshops}, + pages={702--703}, + year={2020} +} + +@inproceedings{muller2021trivialaugment, + title={Trivialaugment: Tuning-free yet state-of-the-art data augmentation}, + author={M{\"u}ller, Samuel G and Hutter, Frank}, + booktitle={Proceedings of the IEEE/CVF international conference on computer vision}, + pages={774--782}, + year={2021} +} + +@article{tyszkiewicz2020disk, + title={DISK: Learning local features with policy gradient}, + author={Tyszkiewicz, Micha{\l} and Fua, Pascal and Trulls, Eduard}, + journal={Advances in Neural Information Processing Systems}, + volume={33}, + pages={14254--14265}, + year={2020} +} + +@inproceedings{edstedt2024dedode, + title={{DeDoDe: Detect, Don't Describe --- Describe, Don't Detect for Local Feature Matching}}, + author = {Johan Edstedt and Georg Bökman and Mårten Wadenbäck and Michael Felsberg}, + booktitle={2024 International Conference on 3D Vision (3DV)}, + year={2024} +} + +@inproceedings{he2010guided, + title = {Guided Image Filtering}, + booktitle = {Proceedings of the 11th European Conference on Computer Vision: Part I}, + author = {He, Kaiming and Sun, Jian and Tang, Xiaoou}, + year = {2010}, + pages = {1-14}, +} + +@misc{he2015fast, + title={Fast Guided Filter}, + author={Kaiming He and Jian Sun}, + year={2015}, + eprint={1505.00996}, + archivePrefix={arXiv}, + primaryClass={cs.CV} +} + +@misc{sellner2023semantic, + title={Semantic segmentation of surgical hyperspectral images under geometric domain shifts}, + author={Jan Sellner and Silvia Seidlitz and Alexander Studier-Fischer and Alessandro Motta and Berkin Özdemir and Beat Peter Müller-Stich and Felix Nickel and Lena Maier-Hein}, + year={2023}, + eprint={2303.10972}, + archivePrefix={arXiv}, + primaryClass={eess.IV} +} + +@article{nister2004efficient, + author={Nist{\'e}r, David}, + journal={IEEE Transactions on Pattern Analysis and Machine Intelligence}, + title={An efficient solution to the five-point relative pose problem}, + year={2004}, + volume={26}, + number={6}, + pages={756-770} +} + +@inproceedings{barath2020magsac++, + title={MAGSAC++, a fast, reliable and accurate robust estimator}, + author={Barath, Daniel and Noskova, Jana and Ivashechkin, Maksym and Matas, Jiri}, + booktitle={CVPR}, + year={2020} +} + + +@inproceedings{shin2017, + title={JPEG-resistant Adversarial Images}, + author={Shin, Richard and Song, Dawn}, + booktitle={NIPS Workshop on Machine Learning and Computer Security}, + volume={1}, + pages={8}, + year={2017} +} + +@inproceedings{reich2024, + title={Differentiable JPEG: The Devil is in the Details}, + author={Reich, Christoph and Debnath, Biplob and Patel, Deep and Chakradhar, Srimat}, + booktitle={IEEE/CVF Winter Conference on Applications of Computer Vision (WACV)}, + year={2024} +} + +@inproceedings{wei2023generalized, + author = {Wei, Tong and Patel, Yash and Shekhovtsov, Alexander and Matas, Jiri and Barath, Daniel}, + title = {Generalized Differentiable RANSAC}, + booktitle = {ICCV}, + year = {2023} +} + +@article{wang2023vggsfm, + title={VGGSfM: Visual Geometry Grounded Deep Structure From Motion}, + author={Wang, Jianyuan and Karaev, Nikita and Rupprecht, Christian and Novotny, David}, + journal={arXiv preprint arXiv:2312.04563}, + year={2023} +} + +@misc{shi2024dissolving, + Author = {Jian Shi and Pengyi Zhang and Ni Zhang and Hakim Ghazzai and Peter Wonka}, + Title = {Dissolving Is Amplifying: Towards Fine-Grained Anomaly Detection}, + booktitle = {ECCV}, + Year = {2024}, +} + +@inproceedings{Karras2020ada, + title = {Training Generative Adversarial Networks with Limited Data}, + author = {Tero Karras and Miika Aittala and Janne Hellsten and Samuli Laine and Jaakko Lehtinen and Timo Aila}, + booktitle = {Proc. NeurIPS}, + year = {2020} +} diff --git a/docs/source/sensors.camera.rst b/docs/source/sensors.camera.rst new file mode 100644 index 0000000..8a462a6 --- /dev/null +++ b/docs/source/sensors.camera.rst @@ -0,0 +1,70 @@ +kornia.sensors.camera +====================== + +.. meta:: + :name: description + :content: "The `kornia.sensors.camera` module provides tools to define and manipulate various camera models, including the Pinhole model. It allows users to specify distortion and projection types in a differentiable way. While currently supporting only the Pinhole model, the module aims to extend its support to other models like Kannala Brandt and Orthographic. It also enables users to define custom camera models using distortion and projection types." + +.. currentmodule:: kornia.sensors.camera + +.. warning:: + :mod:`kornia.sensors.camera` is an experimental API and is subject to change. Once finished, it will subsume :mod:`kornia.geometry.camera` + +The objective of :mod:`kornia.sensors.camera` is to express well-known camera models such as Pinhole, Kannala Brandt, and others in terms of distortion and projection types while ensuring differentiability. +We also aim to equip the user with tools to define custom camera models. +As of now, only the Pinhole model is supported. + +Defining a `Pinhole` camera model is as simple as: + +.. code:: python + + from kornia.image import ImageSize + from kornia.sensors.camera import CameraModel, CameraModelType + + params = torch.tensor([328., 328., 320., 240.]) # fx, fy, cx, cy + cam = CameraModel(ImageSize(480, 640), CameraModelType.PINHOLE, params) + +To define a custom camera model based on distortion and projection types, one can use the :class:`CameraModelBase` api: + +.. code:: python + + from kornia.image import ImageSize + from kornia.sensors.camera import CameraModelBase + from kornia.sensors.camera.distortion_model import AffineTransform + from kornia.sensors.camera.projection_model import Z1Projection + + params = torch.tensor([328., 328., 320., 240.]) + cam = CameraModelBase(AffineTransform(), Z1Projection(), ImageSize(480, 640), params) + + +.. note:: + At the moment, the only supported model is Pinhole. However, we plan to add Kannala Brandt, Orthographic, and other models in the near future. + +.. autoclass:: CameraModelBase + :members: + +.. autoclass:: CameraModel + :members: + +.. autoclass:: CameraModelType + :members: + +.. autoclass:: PinholeModel + :members: + +Distortions +----------- + +.. currentmodule:: kornia.sensors.camera.distortion_model + +.. autoclass:: AffineTransform + :members: + + +Projections +----------- + +.. currentmodule:: kornia.sensors.camera.projection_model + +.. autoclass:: Z1Projection + :members: diff --git a/docs/source/sensors.rst b/docs/source/sensors.rst new file mode 100644 index 0000000..3b18840 --- /dev/null +++ b/docs/source/sensors.rst @@ -0,0 +1,19 @@ +kornia.sensors +=============== + +.. meta:: + :name: description + :content: "The Kornia.sensors module provides interfaces for sensor data handling, including Camera, IMU, and GNSS sensors. Note that this API is experimental and subject to future changes. Explore sensor-related functionalities within the module." + +Aims to provide interfaces to sensors like Camera, IMU, GNSS etc. + + +.. currentmodule:: kornia.sensors + +.. warning:: + :mod:`kornia.sensors` is an experimental API and is subject to change. + +.. toctree:: + :maxdepth: 3 + + sensors.camera diff --git a/docs/source/tracking.rst b/docs/source/tracking.rst new file mode 100644 index 0000000..66b66d0 --- /dev/null +++ b/docs/source/tracking.rst @@ -0,0 +1,11 @@ +kornia.tracking +=============== + +.. meta:: + :name: description + :content: "The `kornia.tracking` module provides tools for tracking objects across frames in computer vision tasks. It includes classes like `HomographyTracker` to estimate homography transformations and track objects over time." + +.. currentmodule:: kornia.tracking + +.. autoclass:: HomographyTracker + :members: diff --git a/kornia/__init__.py b/kornia/__init__.py new file mode 100644 index 0000000..9fa0189 --- /dev/null +++ b/kornia/__init__.py @@ -0,0 +1,248 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia — Differentiable computer vision and image processing for PyTorch. + +This package exposes core modules (filters, geometry, etc.) and provides +convenience imports at the top level. +""" + +from typing import Any + +# NOTE: kornia filters and geometry must go first since are the core of the library +# and by changing the import order you might get into a circular dependencies issue. +from . import filters +from . import geometry + +# import the other modules for convenience +from . import ( + augmentation, + color, + contrib, + core, + config, + enhance, + feature, + io, + losses, + metrics, + models, + morphology, + onnx, + tracking, +) + + +# Multi-framework support using ivy +from .transpiler import to_jax, to_numpy, to_tensorflow + +# NOTE: we are going to expose to top level very few things +from kornia.constants import pi + +# Deprecated top-level imports - use kornia.core.ops or kornia.core.utils instead +from kornia.core._compat import deprecated + +# Import the actual functions to wrap +from kornia.core.ops import eye_like as _eye_like, vec_like as _vec_like +from kornia.core.utils import ( + get_cuda_device_if_available as _get_cuda_device_if_available, + get_cuda_or_mps_device_if_available as _get_cuda_or_mps_device_if_available, + get_mps_device_if_available as _get_mps_device_if_available, + is_autocast_enabled as _is_autocast_enabled, + xla_is_available as _xla_is_available, +) +from kornia.geometry import ( + create_meshgrid as _create_meshgrid, + create_meshgrid3d as _create_meshgrid3d, + load_pointcloud_ply as _load_pointcloud_ply, + save_pointcloud_ply as _save_pointcloud_ply, +) +from kornia.image import ( + draw_convex_polygon as _draw_convex_polygon, + draw_line as _draw_line, + draw_point2d as _draw_point2d, + draw_rectangle as _draw_rectangle, + image_to_string as _image_to_string, + image_to_tensor as _image_to_tensor, + print_image as _print_image, + tensor_to_image as _tensor_to_image, +) +from kornia.losses.one_hot import one_hot as _one_hot + + +@deprecated(replace_with="kornia.core.ops.eye_like", version="0.8.3") +def eye_like(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.core.ops.eye_like` instead.""" + return _eye_like(*args, **kwargs) + + +@deprecated(replace_with="kornia.core.ops.vec_like", version="0.8.3") +def vec_like(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.core.ops.vec_like` instead.""" + return _vec_like(*args, **kwargs) + + +@deprecated(replace_with="kornia.core.utils.xla_is_available", version="0.8.3") +def xla_is_available(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.core.utils.xla_is_available` instead.""" + return _xla_is_available(*args, **kwargs) + + +@deprecated(replace_with="kornia.geometry.create_meshgrid", version="0.8.3") +def create_meshgrid(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.geometry.create_meshgrid` instead.""" + return _create_meshgrid(*args, **kwargs) + + +@deprecated(replace_with="kornia.image.image_to_tensor", version="0.8.3") +def image_to_tensor(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.image.image_to_tensor` instead.""" + return _image_to_tensor(*args, **kwargs) + + +@deprecated(replace_with="kornia.image.tensor_to_image", version="0.8.3") +def tensor_to_image(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.image.tensor_to_image` instead.""" + return _tensor_to_image(*args, **kwargs) + + +@deprecated(replace_with="kornia.core.utils.get_cuda_device_if_available", version="0.8.3") +def get_cuda_device_if_available(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.core.utils.get_cuda_device_if_available` instead.""" + return _get_cuda_device_if_available(*args, **kwargs) + + +@deprecated(replace_with="kornia.core.utils.get_mps_device_if_available", version="0.8.3") +def get_mps_device_if_available(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.core.utils.get_mps_device_if_available` instead.""" + return _get_mps_device_if_available(*args, **kwargs) + + +@deprecated(replace_with="kornia.core.utils.get_cuda_or_mps_device_if_available", version="0.8.3") +def get_cuda_or_mps_device_if_available(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.core.utils.get_cuda_or_mps_device_if_available` instead.""" + return _get_cuda_or_mps_device_if_available(*args, **kwargs) + + +@deprecated(replace_with="kornia.core.utils.is_autocast_enabled", version="0.8.3") +def is_autocast_enabled(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.core.utils.is_autocast_enabled` instead.""" + return _is_autocast_enabled(*args, **kwargs) + + +@deprecated( + replace_with="kornia.geometry.create_meshgrid3d", + version="0.8.3", + extra_reason=" Previously available as `kornia.utils.create_meshgrid3d`.", +) +def create_meshgrid3d(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.geometry.create_meshgrid3d` instead (previously `kornia.utils.create_meshgrid3d`).""" + return _create_meshgrid3d(*args, **kwargs) + + +@deprecated( + replace_with="kornia.geometry.load_pointcloud_ply", + version="0.8.3", + extra_reason=" Previously available as `kornia.utils.load_pointcloud_ply`.", +) +def load_pointcloud_ply(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.geometry.load_pointcloud_ply` instead (previously `kornia.utils.load_pointcloud_ply`).""" + return _load_pointcloud_ply(*args, **kwargs) + + +@deprecated( + replace_with="kornia.geometry.save_pointcloud_ply", + version="0.8.3", + extra_reason=" Previously available as `kornia.utils.save_pointcloud_ply`.", +) +def save_pointcloud_ply(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.geometry.save_pointcloud_ply` instead (previously `kornia.utils.save_pointcloud_ply`).""" + return _save_pointcloud_ply(*args, **kwargs) + + +@deprecated( + replace_with="kornia.image.draw_line", + version="0.8.3", + extra_reason=" Previously available as `kornia.utils.draw_line`.", +) +def draw_line(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.image.draw_line` instead (previously `kornia.utils.draw_line`).""" + return _draw_line(*args, **kwargs) + + +@deprecated( + replace_with="kornia.image.draw_rectangle", + version="0.8.3", + extra_reason=" Previously available as `kornia.utils.draw_rectangle`.", +) +def draw_rectangle(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.image.draw_rectangle` instead (previously `kornia.utils.draw_rectangle`).""" + return _draw_rectangle(*args, **kwargs) + + +@deprecated( + replace_with="kornia.image.draw_point2d", + version="0.8.3", + extra_reason=" Previously available as `kornia.utils.draw_point2d`.", +) +def draw_point2d(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.image.draw_point2d` instead (previously `kornia.utils.draw_point2d`).""" + return _draw_point2d(*args, **kwargs) + + +@deprecated( + replace_with="kornia.image.draw_convex_polygon", + version="0.8.3", + extra_reason=" Previously available as `kornia.utils.draw_convex_polygon`.", +) +def draw_convex_polygon(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.image.draw_convex_polygon` instead (previously `kornia.utils.draw_convex_polygon`).""" + return _draw_convex_polygon(*args, **kwargs) + + +@deprecated( + replace_with="kornia.image.image_to_string", + version="0.8.3", + extra_reason=" Previously available as `kornia.utils.image_to_string`.", +) +def image_to_string(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.image.image_to_string` instead (previously `kornia.utils.image_to_string`).""" + return _image_to_string(*args, **kwargs) + + +@deprecated( + replace_with="kornia.image.print_image", + version="0.8.3", + extra_reason=" Previously available as `kornia.utils.print_image`.", +) +def print_image(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.image.print_image` instead (previously `kornia.utils.print_image`).""" + return _print_image(*args, **kwargs) + + +@deprecated( + replace_with="kornia.losses.one_hot", + version="0.8.3", + extra_reason=" Previously available as `kornia.utils.one_hot`.", +) +def one_hot(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.losses.one_hot` instead (previously `kornia.utils.one_hot`).""" + return _one_hot(*args, **kwargs) + + +# Version variable +__version__ = "0.8.3" diff --git a/kornia/augmentation/README.md b/kornia/augmentation/README.md new file mode 100644 index 0000000..1a63365 --- /dev/null +++ b/kornia/augmentation/README.md @@ -0,0 +1,49 @@ +# Kornia Differentiable Data Augmentation + +## Supported Operations + + + + + + + + + + +
Geometric AugmentationsColor-space Augmentations
+ +| | 2D | 3D | +| ------------ | ----------- | ------------ | +| RandomHorizontalFlip | ✅ | ✅| +| RandomVerticalFlip | ✅ | ✅ | +| RandomDepthicalFlip | - | ✅ | +| RandomRotation | ✅ | ✅ | +| RandomAffine | ✅ | ✅ | +| RandomPerspective | ✅ | ✅ | +| RandomErasing | ✅ | ❌ | +| CenterCrop | ✅ | ✅ | +| RandomCrop | ✅ | ✅ | +| RandomResizedCrop | ✅ | - | +| RandomMotionBlur | ✅ | ✅ | + + + +| | 2D | 3D | +| ------------ | ----------- | ------------ | +| ColorJiggle | ✅ | ❌ | +| RandomGrayscale | ✅ | ❌ | +| RandomSolarize | ✅ | ❌ | +| RandomPosterize | ✅ | ❌ | +| RandomSharpness | ✅ | ❌ | +| RandomEqualize | ✅ | ✅ | + +
+ Mix Augmentations +
+ +| | 2D | 3D | +| ------------ | ----------- | ------------ | +| RandomMixUp | ✅ | ❌ | +| RandomCutMix       | ✅ | ❌ | +
diff --git a/kornia/augmentation/_2d/__init__.py b/kornia/augmentation/_2d/__init__.py new file mode 100644 index 0000000..b44c388 --- /dev/null +++ b/kornia/augmentation/_2d/__init__.py @@ -0,0 +1,20 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from kornia.augmentation._2d.geometric import * +from kornia.augmentation._2d.intensity import * +from kornia.augmentation._2d.mix import * diff --git a/kornia/augmentation/_2d/base.py b/kornia/augmentation/_2d/base.py new file mode 100644 index 0000000..9f395dc --- /dev/null +++ b/kornia/augmentation/_2d/base.py @@ -0,0 +1,180 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional + +import torch +from torch import float16, float32, float64 + +from kornia.augmentation.base import _AugmentationBase +from kornia.augmentation.utils import _transform_input, _transform_input_by_shape, _validate_input_dtype +from kornia.core.ops import eye_like +from kornia.core.utils import is_autocast_enabled +from kornia.geometry.boxes import Boxes +from kornia.geometry.keypoints import Keypoints + + +class AugmentationBase2D(_AugmentationBase): + r"""AugmentationBase2D base class for customized augmentation implementations. + + AugmentationBase2D aims at offering a generic base class for a greater level of customization. + If the subclass contains routined matrix-based transformations, `RigidAffineAugmentationBase2D` + might be a better fit. + + Args: + p: probability for applying an augmentation. This param controls the augmentation probabilities + element-wise for a batch. + p_batch: probability for applying an augmentation to a batch. This param controls the augmentation + probabilities batch-wise. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input ``True`` or broadcast it to the batch + form ``False``. + + """ + + def validate_tensor(self, input: torch.Tensor) -> None: + """Check if the input torch.Tensor is formatted as expected.""" + _validate_input_dtype(input, accepted_dtypes=[torch.bfloat16, float16, float32, float64]) + if len(input.shape) != 4: + raise RuntimeError(f"Expect (B, C, H, W). Got {input.shape}.") + + def transform_tensor( + self, input: torch.Tensor, *, shape: Optional[torch.Tensor] = None, match_channel: bool = True + ) -> torch.Tensor: + """Convert any incoming (H, W), (C, H, W) and (B, C, H, W) into (B, C, H, W).""" + _validate_input_dtype(input, accepted_dtypes=[torch.bfloat16, float16, float32, float64]) + + if shape is None: + return _transform_input(input) + else: + return _transform_input_by_shape(input, reference_shape=shape, match_channel=match_channel) + + +class RigidAffineAugmentationBase2D(AugmentationBase2D): + r"""AugmentationBase2D base class for rigid/affine augmentation implementations. + + RigidAffineAugmentationBase2D enables routined transformation with given transformation matrices + for different data types like masks, boxes, and keypoints. + + Args: + p: probability for applying an augmentation. This param controls the augmentation probabilities + element-wise for a batch. + p_batch: probability for applying an augmentation to a batch. This param controls the augmentation + probabilities batch-wise. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input ``True`` or broadcast it to the batch + form ``False``. + + """ + + _transform_matrix: Optional[torch.Tensor] + + @property + def transform_matrix(self) -> Optional[torch.Tensor]: + return self._transform_matrix + + def identity_matrix(self, input: torch.Tensor) -> torch.Tensor: + """Return 3x3 identity matrix.""" + return eye_like(3, input) + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + raise NotImplementedError + + def generate_transformation_matrix( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + """Generate transformation matrices with the given input and param settings.""" + batch_prob = params["batch_prob"] + to_apply = batch_prob > 0.5 + + in_tensor = self.transform_tensor(input) + + trans_matrix_applied = self.compute_transformation(in_tensor, params=params, flags=flags) + trans_matrix_identity = self.identity_matrix(in_tensor) + + if is_autocast_enabled(): + trans_matrix_applied = trans_matrix_applied.type(input.dtype) + trans_matrix_identity = trans_matrix_identity.type(input.dtype) + + # If batch sizes line up, do the where-blend. Otherwise (e.g. VideoSequential + # passes B-sized batch_prob into a B*T-sized input) fall back to all-or-nothing. + if trans_matrix_applied.shape[0] == to_apply.shape[0] == trans_matrix_identity.shape[0]: + to_apply_expanded = to_apply.view(-1, *([1] * (trans_matrix_applied.dim() - 1))) + trans_matrix = torch.where(to_apply_expanded, trans_matrix_applied, trans_matrix_identity) + else: + trans_matrix = trans_matrix_applied if bool(to_apply.any()) else trans_matrix_identity + + return trans_matrix + + def inverse_inputs( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + raise NotImplementedError + + def inverse_masks( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + raise NotImplementedError + + def inverse_boxes( + self, + input: Boxes, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> Boxes: + raise NotImplementedError + + def inverse_keypoints( + self, + input: Keypoints, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> Keypoints: + raise NotImplementedError + + def inverse_classes( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + raise NotImplementedError + + def apply_func( + self, in_tensor: torch.Tensor, params: Dict[str, torch.Tensor], flags: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + if flags is None: + flags = self.flags + + trans_matrix = self.generate_transformation_matrix(in_tensor, params, flags) + output = self.transform_inputs(in_tensor, params, flags, trans_matrix) + self._transform_matrix = trans_matrix + + return output diff --git a/kornia/augmentation/_2d/geometric/__init__.py b/kornia/augmentation/_2d/geometric/__init__.py new file mode 100644 index 0000000..9ef714e --- /dev/null +++ b/kornia/augmentation/_2d/geometric/__init__.py @@ -0,0 +1,32 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from kornia.augmentation._2d.geometric.affine import RandomAffine +from kornia.augmentation._2d.geometric.center_crop import CenterCrop +from kornia.augmentation._2d.geometric.crop import RandomCrop +from kornia.augmentation._2d.geometric.elastic_transform import RandomElasticTransform +from kornia.augmentation._2d.geometric.fisheye import RandomFisheye +from kornia.augmentation._2d.geometric.horizontal_flip import RandomHorizontalFlip +from kornia.augmentation._2d.geometric.pad import PadTo +from kornia.augmentation._2d.geometric.perspective import RandomPerspective +from kornia.augmentation._2d.geometric.resize import LongestMaxSize, Resize, SmallestMaxSize +from kornia.augmentation._2d.geometric.resized_crop import RandomResizedCrop +from kornia.augmentation._2d.geometric.rotation import RandomRotation, RandomRotation90 +from kornia.augmentation._2d.geometric.shear import RandomShear +from kornia.augmentation._2d.geometric.thin_plate_spline import RandomThinPlateSpline +from kornia.augmentation._2d.geometric.translate import RandomTranslate +from kornia.augmentation._2d.geometric.vertical_flip import RandomVerticalFlip diff --git a/kornia/augmentation/_2d/geometric/affine.py b/kornia/augmentation/_2d/geometric/affine.py new file mode 100644 index 0000000..6748602 --- /dev/null +++ b/kornia/augmentation/_2d/geometric/affine.py @@ -0,0 +1,178 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math +from typing import Any, Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.geometric.base import GeometricAugmentationBase2D +from kornia.constants import Resample, SamplePadding +from kornia.geometry.transform import get_affine_matrix2d, warp_affine + + +class RandomAffine(GeometricAugmentationBase2D): + r"""Apply a random 2D affine transformation to a torch.Tensor image. + + .. image:: _static/img/RandomAffine.png + + The transformation is computed so that the image center is kept invariant. + + Args: + degrees: Range of degrees to select from. + If degrees is a number instead of sequence like (min, max), the range of degrees + will be (-degrees, +degrees). Set to 0 to deactivate rotations. + translate: tuple of maximum absolute fraction for horizontal + and vertical translations. For example translate=(a, b), then horizontal shift + is randomly sampled in the range -img_width * a < dx < img_width * a and vertical shift is + randomly sampled in the range -img_height * b < dy < img_height * b. Will not translate by default. + scale: scaling factor interval. + If (a, b) represents isotropic scaling, the scale is randomly sampled from the range a <= scale <= b. + If (a, b, c, d), the scale is randomly sampled from the range a <= scale_x <= b, c <= scale_y <= d. + Will keep original scale by default. + shear: Range of degrees to select from. + If float, a shear parallel to the x axis in the range (-shear, +shear) will be applied. + If (a, b), a shear parallel to the x axis in the range (-shear, +shear) will be applied. + If (a, b, c, d), then x-axis shear in (shear[0], shear[1]) and y-axis shear in (shear[2], shear[3]) + will be applied. Will not apply shear by default. + resample: resample mode from "nearest" (0) or "bilinear" (1). + padding_mode: padding mode from "torch.zeros" (0), "border" (1), "reflection" (2) or "fill" (3). + fill_value: the value to be filled in the padding area when padding_mode="fill". + Can be a float, int, or a torch.Tensor of shape (C) or (1). + same_on_batch: apply the same transformation across the batch. + align_corners: interpolation flag. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.geometry.transform.warp_affine`. + + Examples: + >>> import torch + >>> rng = torch.manual_seed(0) + >>> input = torch.rand(1, 1, 3, 3) + >>> aug = RandomAffine((-15., 20.), p=1.) + >>> out = aug(input) + >>> out, aug.transform_matrix + (tensor([[[[0.3961, 0.7310, 0.1574], + [0.1781, 0.3074, 0.5648], + [0.4804, 0.8379, 0.4234]]]]), tensor([[[ 0.9923, -0.1241, 0.1319], + [ 0.1241, 0.9923, -0.1164], + [ 0.0000, 0.0000, 1.0000]]])) + >>> aug.inverse(out) + tensor([[[[0.3890, 0.6573, 0.1865], + [0.2063, 0.3074, 0.5459], + [0.3892, 0.7896, 0.4224]]]]) + >>> input + tensor([[[[0.4963, 0.7682, 0.0885], + [0.1320, 0.3074, 0.6341], + [0.4901, 0.8964, 0.4556]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomAffine((-15., 20.), p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + degrees: Union[torch.Tensor, float, Tuple[float, float]], + translate: Optional[Union[torch.Tensor, Tuple[float, float]]] = None, + scale: Optional[Union[torch.Tensor, Tuple[float, float], Tuple[float, float, float, float]]] = None, + shear: Optional[Union[torch.Tensor, float, Tuple[float, float]]] = None, + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + same_on_batch: bool = False, + align_corners: bool = False, + padding_mode: Union[str, int, SamplePadding] = SamplePadding.ZEROS.name, + fill_value: Optional[Union[float, int, torch.Tensor]] = None, # Updated type hint + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self._param_generator: rg.AffineGenerator = rg.AffineGenerator(degrees, translate, scale, shear) + + if fill_value is not None and not isinstance(fill_value, torch.Tensor): + fill_value = torch.as_tensor(fill_value) + + self.flags = { + "resample": Resample.get(resample), + "padding_mode": SamplePadding.get(padding_mode), + "align_corners": align_corners, + "fill_value": fill_value, + } + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + # ``torch.deg2rad`` lowers as ``aten::deg2rad`` which the legacy ONNX exporter + # does not support at opset 20. Multiply by ``pi/180`` explicitly same value, + # but only basic arithmetic ops in the trace. + deg2rad_factor: float = math.pi / 180.0 + shear_x = torch.as_tensor(params["shear_x"], device=input.device, dtype=input.dtype) * deg2rad_factor + shear_y = torch.as_tensor(params["shear_y"], device=input.device, dtype=input.dtype) * deg2rad_factor + return get_affine_matrix2d( + torch.as_tensor(params["translations"], device=input.device, dtype=input.dtype), + torch.as_tensor(params["center"], device=input.device, dtype=input.dtype), + torch.as_tensor(params["scale"], device=input.device, dtype=input.dtype), + torch.as_tensor(params["angle"], device=input.device, dtype=input.dtype), + shear_x, + shear_y, + ) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + _, _, height, width = input.shape + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + + return warp_affine( + input, + transform[:, :2, :], + (height, width), + flags["resample"].name.lower(), + align_corners=flags["align_corners"], + padding_mode=flags["padding_mode"].name.lower(), + fill_value=flags["fill_value"], # <--- PASS IT DOWN + ) + + def inverse_transform( + self, + input: torch.Tensor, + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + size: Optional[Tuple[int, int]] = None, + ) -> torch.Tensor: + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + return self.apply_transform( + input, + params=self._params, + transform=torch.as_tensor(transform, device=input.device, dtype=input.dtype), + flags=flags, + ) diff --git a/kornia/augmentation/_2d/geometric/base.py b/kornia/augmentation/_2d/geometric/base.py new file mode 100644 index 0000000..7837dfc --- /dev/null +++ b/kornia/augmentation/_2d/geometric/base.py @@ -0,0 +1,411 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple + +import torch + +from kornia.augmentation._2d.base import RigidAffineAugmentationBase2D +from kornia.constants import Resample +from kornia.core.utils import _torch_inverse_cast +from kornia.geometry.boxes import Boxes +from kornia.geometry.keypoints import Keypoints + + +class GeometricAugmentationBase2D(RigidAffineAugmentationBase2D): + r"""GeometricAugmentationBase2D base class for customized geometric augmentation implementations. + + Args: + p: probability for applying an augmentation. This param controls the augmentation probabilities + element-wise for a batch. + p_batch: probability for applying an augmentation to a batch. This param controls the augmentation + probabilities batch-wise. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input ``True`` or broadcast it + to the batch form ``False``. + + """ + + def inverse_transform( + self, + input: torch.Tensor, + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + size: Optional[Tuple[int, int]] = None, + ) -> torch.Tensor: + """By default, the exact transformation as ``apply_transform`` will be used.""" + raise NotImplementedError + + def compute_inverse_transformation(self, transform: torch.Tensor) -> torch.Tensor: + """Compute the inverse transform of given transformation matrices.""" + return _torch_inverse_cast(transform) + + def get_transformation_matrix( + self, + input: torch.Tensor, + params: Optional[Dict[str, torch.Tensor]] = None, + flags: Optional[Dict[str, Any]] = None, + ) -> torch.Tensor: + """Obtain transformation matrices. + + Return the current transformation matrix if existed. Generate a new one, otherwise. + """ + flags = self.flags if flags is None else flags + if params is not None: + transform = self.generate_transformation_matrix(input, params, flags) + elif self.transform_matrix is None: + params = self.forward_parameters(input.shape) + transform = self.generate_transformation_matrix(input, params, flags) + else: + transform = self.transform_matrix + return torch.as_tensor(transform, device=input.device, dtype=input.dtype) + + def apply_non_transform_mask( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Process masks corresponding to the inputs that are no transformation applied.""" + return input + + def apply_transform_mask( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Process masks corresponding to the inputs that are transformed. + + Note: + Convert "resample" arguments to "nearest" by default. + Normalize "align_corners" from None to False to match PyTorch's default behavior. + + """ + resample_method: Optional[Resample] = None + align_corners_was_none: bool = False + original_align_corners: Optional[bool] = None + + if "resample" in flags: + resample_method = flags["resample"] + flags["resample"] = Resample.get("nearest") + + # When align_corners=None is in flags (from extra_args), use the module's default + # This ensures masks use the same align_corners value as inputs for consistency + # However, for 'slice' cropping_mode with 'nearest' mode, align_corners must be None + # because crop_by_indices -> resize -> interpolate doesn't accept align_corners with nearest + # For 'resample' cropping_mode, warp_affine/grid_sample accepts align_corners with nearest + if "align_corners" in flags and flags["align_corners"] is None: + align_corners_was_none = True + original_align_corners = None + # Check if we're using 'slice' cropping_mode which uses interpolate + # interpolate doesn't accept align_corners with nearest mode + if flags.get("cropping_mode") == "slice": + # Keep align_corners=None for slice mode with nearest (interpolate requirement) + pass + else: + # Use the module's default align_corners value from self.flags + # This ensures masks use the same align_corners as inputs + # For 'resample' mode, warp_affine/grid_sample accepts align_corners with nearest + flags["align_corners"] = self.flags.get("align_corners", False) + + output = self.apply_transform(input, params, flags, transform) + + if resample_method is not None: + flags["resample"] = resample_method + + # Restore align_corners if it was modified + if align_corners_was_none: + flags["align_corners"] = original_align_corners + + return output + + def apply_non_transform_box( + self, + input: Boxes, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> Boxes: + """Process boxes corresponding to the inputs that are no transformation applied.""" + return input + + def apply_transform_box( + self, + input: Boxes, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> Boxes: + """Process boxes corresponding to the inputs that are transformed.""" + if transform is None: + if self.transform_matrix is None: + raise RuntimeError("No valid transformation matrix found. Please either pass one or forward one first.") + transform = self.transform_matrix + input = self.apply_non_transform_box(input, params, flags, transform) + return input.transform_boxes_(transform) + + def apply_non_transform_keypoint( + self, + input: Keypoints, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> Keypoints: + """Process keypoints corresponding to the inputs that are no transformation applied.""" + return input + + def apply_transform_keypoint( + self, + input: Keypoints, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> Keypoints: + """Process keypoints corresponding to the inputs that are transformed.""" + if transform is None: + if self.transform_matrix is None: + raise RuntimeError("No valid transformation matrix found. Please either pass one or forward one first.") + transform = self.transform_matrix + input = self.apply_non_transform_keypoint(input, params, flags, transform) + return input.transform_keypoints_(transform) + + def apply_non_transform_class( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Process class tags corresponding to the inputs that are no transformation applied.""" + return input + + def apply_transform_class( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Process class tags corresponding to the inputs that are transformed.""" + return input + + def inverse_inputs( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + **kwargs: Any, + ) -> torch.Tensor: + in_tensor = self.transform_tensor(input) + output = in_tensor.clone() + batch_prob = params["batch_prob"] + to_apply = batch_prob > 0.5 # NOTE: in case of Relaxed Distributions. + + params, flags = self._process_kwargs_to_params_and_flags( + self._params if params is None else params, flags, **kwargs + ) + + size: Optional[Tuple[int, int]] = None + if "forward_input_shape" in params: + # Majorly for cropping functions + _size = params["forward_input_shape"].tolist() + size = (_size[-2], _size[-1]) + + # if no augmentation needed + if not to_apply.any(): + output = in_tensor + # if all data needs to be augmented + elif to_apply.all(): + output = self.inverse_transform(in_tensor, flags=flags, transform=transform, size=size) + else: + output[to_apply] = self.inverse_transform( + in_tensor[to_apply], + transform=transform[to_apply] if transform is not None else transform, + size=size, + flags=flags, + ) + return output + + def inverse_masks( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + **kwargs: Any, + ) -> torch.Tensor: + resample_method: Optional[Resample] = None + align_corners_value: Optional[bool] = None + align_corners_was_none_in_kwargs: bool = False + if "resample" in flags: + resample_method = flags["resample"] + flags["resample"] = Resample.get("nearest") + # Preserve align_corners from extra_args (kwargs) if provided + # This ensures masks use the same align_corners setting in inverse as in forward + if "align_corners" in kwargs: + align_corners_value = flags.get("align_corners") + # When align_corners=None is in kwargs, use the module's default + # This ensures masks use the same align_corners value as inputs for consistency + # However, for 'slice' cropping_mode with 'nearest' mode, align_corners must be None + # because crop_by_indices -> resize -> interpolate doesn't accept align_corners with nearest + # For 'resample' cropping_mode, warp_affine/grid_sample accepts align_corners with nearest + # We need to normalize it in kwargs too, because inverse_inputs will call + # _process_kwargs_to_params_and_flags which merges kwargs into flags + if kwargs["align_corners"] is None: + align_corners_was_none_in_kwargs = True + # Check if we're using 'slice' cropping_mode which uses interpolate + # interpolate doesn't accept align_corners with nearest mode + if flags.get("cropping_mode") == "slice": + # Keep align_corners=None for slice mode with nearest (interpolate requirement) + # Don't modify flags or kwargs + pass + else: + # Use the module's default align_corners value + # This ensures masks use the same align_corners as inputs + # For 'resample' mode, warp_affine/grid_sample accepts align_corners with nearest + normalized_align_corners = self.flags.get("align_corners", False) + flags["align_corners"] = normalized_align_corners + # Also update kwargs to prevent _process_kwargs_to_params_and_flags from overwriting + kwargs["align_corners"] = normalized_align_corners + else: + flags["align_corners"] = kwargs["align_corners"] + output = self.inverse_inputs(input, params, flags, transform, **kwargs) + if resample_method is not None: + flags["resample"] = resample_method + # Restore align_corners if it was modified (mirror the modification condition) + # This ensures complete state restoration even if the original value was None + if "align_corners" in kwargs: + # Restore kwargs to original value if it was None + if align_corners_was_none_in_kwargs: + kwargs["align_corners"] = None + flags["align_corners"] = align_corners_value + return output + + def inverse_boxes( + self, + input: Boxes, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + **kwargs: Any, + ) -> Boxes: + output = input.clone() + batch_prob = params["batch_prob"] + to_apply = batch_prob > 0.5 # NOTE: in case of Relaxed Distributions. + + if transform is None: + raise RuntimeError("`transform` has to be a torch.Tensor. Got None.") + + params, flags = self._process_kwargs_to_params_and_flags( + self._params if params is None else params, flags, **kwargs + ) + + # if no augmentation needed + if not to_apply.any(): + output = input + # if all data needs to be augmented + elif to_apply.all(): + output = input.transform_boxes_(transform) + else: + output[to_apply] = input[to_apply].transform_boxes_(transform[to_apply]) + + return output + + def inverse_keypoints( + self, + input: Keypoints, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + **kwargs: Any, + ) -> Keypoints: + """Inverse the transformation on keypoints. + + Args: + input: input keypoints torch.Tensor or object. + params: the corresponding parameters for an operation. + flags: static parameters. + transform: the inverse transformation matrix + kwargs: additional arguments + + """ + output = input.clone() + batch_prob = params["batch_prob"] + to_apply = batch_prob > 0.5 # NOTE: in case of Relaxed Distributions. + + if transform is None: + raise RuntimeError("`transform` has to be a torch.Tensor. Got None.") + + params, flags = self._process_kwargs_to_params_and_flags( + self._params if params is None else params, flags, **kwargs + ) + + # if no augmentation needed + if not to_apply.any(): + output = input + # if all data needs to be augmented + elif to_apply.all(): + output = input.transform_keypoints_(transform) + else: + output[to_apply] = input[to_apply].transform_keypoints_(transform[to_apply]) + + return output + + def inverse_classes( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + **kwargs: Any, + ) -> torch.Tensor: + return input + + def inverse( + self, input: torch.Tensor, params: Optional[Dict[str, torch.Tensor]] = None, **kwargs: Any + ) -> torch.Tensor: + """Perform inverse operations. + + Args: + input: the input torch.Tensor. + params: the corresponding parameters for an operation. + If None, a new parameter suite will be generated. + **kwargs: key-value pairs to override the parameters and flags. + + """ + input_shape = input.shape + in_tensor = self.transform_tensor(input) + + params, flags = self._process_kwargs_to_params_and_flags( + self._params if params is None else params, self.flags, **kwargs + ) + + if params is None: + params = self._params + transform = self.get_transformation_matrix(in_tensor, params=params, flags=flags) + + transform = self.compute_inverse_transformation(transform) + output = self.inverse_inputs(in_tensor, params, flags, transform) + + if self.keepdim: + return self.transform_output_tensor(output, input_shape) + + return output diff --git a/kornia/augmentation/_2d/geometric/center_crop.py b/kornia/augmentation/_2d/geometric/center_crop.py new file mode 100644 index 0000000..30165fc --- /dev/null +++ b/kornia/augmentation/_2d/geometric/center_crop.py @@ -0,0 +1,166 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.geometric.base import GeometricAugmentationBase2D +from kornia.constants import Resample +from kornia.geometry.transform import crop_by_indices, crop_by_transform_mat, get_perspective_transform + + +class CenterCrop(GeometricAugmentationBase2D): + r"""Crop a given image torch.Tensor at the center. + + .. image:: _static/img/CenterCrop.png + + Args: + size: Desired output size (out_h, out_w) of the crop. + If integer, out_h = out_w = size. + If Tuple[int, int], out_h = size[0], out_w = size[1]. + align_corners: interpolation flag. + resample: The interpolation mode. + p: probability of applying the transformation for the whole batch. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + cropping_mode: The used algorithm to crop. ``slice`` will use advanced slicing to extract the torch.Tensor based + on the sampled indices. ``resample`` will use `warp_affine` using the affine transformation + to extract and resize at once. Use `slice` for efficiency, or `resample` for proper + differentiability. + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, out_h, out_w)` + + .. note:: + This function internally uses :func:`kornia.geometry.transform.crop_by_boxes`. + + Examples: + >>> import torch + >>> rng = torch.manual_seed(0) + >>> inputs = torch.randn(1, 1, 4, 4) + >>> inputs + tensor([[[[-1.1258, -1.1524, -0.2506, -0.4339], + [ 0.8487, 0.6920, -0.3160, -2.1152], + [ 0.3223, -1.2633, 0.3500, 0.3081], + [ 0.1198, 1.2377, 1.1168, -0.2473]]]]) + >>> aug = CenterCrop(2, p=1., cropping_mode="resample") + >>> out = aug(inputs) + >>> out + tensor([[[[ 0.6920, -0.3160], + [-1.2633, 0.3500]]]]) + >>> aug.inverse(out, padding_mode="border") + tensor([[[[ 0.6920, 0.6920, -0.3160, -0.3160], + [ 0.6920, 0.6920, -0.3160, -0.3160], + [-1.2633, -1.2633, 0.3500, 0.3500], + [-1.2633, -1.2633, 0.3500, 0.3500]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = CenterCrop(2, p=1., cropping_mode="resample") + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + size: Union[int, Tuple[int, int]], + align_corners: bool = True, + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + p: float = 1.0, + keepdim: bool = False, + cropping_mode: str = "slice", + ) -> None: + # same_on_batch is always True for CenterCrop + # Since PyTorch does not support ragged torch.Tensor. So cropping function happens batch-wisely. + super().__init__(p=1.0, same_on_batch=True, p_batch=p, keepdim=keepdim) + if isinstance(size, tuple): + self.size = (size[0], size[1]) + elif isinstance(size, int): + self.size = (size, size) + else: + raise Exception(f"Invalid size type. Expected (int, tuple(int, int). Got: {type(size)}.") + + self.flags = { + "resample": Resample.get(resample), + "cropping_mode": cropping_mode, + "align_corners": align_corners, + "size": self.size, + "padding_mode": "zeros", + } + + def generate_parameters(self, batch_shape: Tuple[int, ...]) -> Dict[str, torch.Tensor]: + return rg.center_crop_generator(batch_shape[0], batch_shape[-2], batch_shape[-1], self.size, self.device) + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + if flags["cropping_mode"] in ("resample", "slice"): + transform: torch.Tensor = get_perspective_transform(params["src"].to(input), params["dst"].to(input)) + transform = transform.expand(input.shape[0], -1, -1) + return transform + raise NotImplementedError(f"Not supported type: {flags['cropping_mode']}.") + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if flags["cropping_mode"] == "resample": # uses bilinear interpolation to crop + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + + return crop_by_transform_mat( + input, + transform[:, :2, :], + self.size, + flags["resample"].name.lower(), + "zeros", + flags["align_corners"], + ) + if flags["cropping_mode"] == "slice": # uses advanced slicing to crop + return crop_by_indices(input, params["src"], flags["size"]) + raise NotImplementedError(f"Not supported type: {flags['cropping_mode']}.") + + def inverse_transform( + self, + input: torch.Tensor, + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + size: Optional[Tuple[int, int]] = None, + ) -> torch.Tensor: + if flags["cropping_mode"] != "resample": + raise NotImplementedError( + f"`inverse` is only applicable for resample cropping mode. Got {flags['cropping_mode']}." + ) + if size is None: + size = self.size + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + return crop_by_transform_mat( + input, + transform[:, :2, :], + size, + flags["resample"].name.lower(), + flags["padding_mode"], + flags["align_corners"], + ) diff --git a/kornia/augmentation/_2d/geometric/crop.py b/kornia/augmentation/_2d/geometric/crop.py new file mode 100644 index 0000000..855f324 --- /dev/null +++ b/kornia/augmentation/_2d/geometric/crop.py @@ -0,0 +1,346 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.geometric.base import GeometricAugmentationBase2D +from kornia.constants import Resample +from kornia.geometry.boxes import Boxes +from kornia.geometry.keypoints import Keypoints +from kornia.geometry.transform import crop_by_indices, crop_by_transform_mat, get_perspective_transform + + +class RandomCrop(GeometricAugmentationBase2D): + r"""Crop random patches of a torch.Tensor image on a given size. + + .. image:: _static/img/RandomCrop.png + + Args: + size: Desired output size (out_h, out_w) of the crop. + Must be Tuple[int, int], then out_h = size[0], out_w = size[1]. + padding: Optional padding on each border + of the image. Default is None, i.e no padding. If a sequence of length + 4 is provided, it is used to F.pad left, top, right, bottom borders + respectively. If a sequence of length 2 is provided, it is used to + F.pad left/right, top/bottom borders, respectively. + pad_if_needed: It will F.pad the image if smaller than the + desired size to avoid raising an exception. Since cropping is done + after padding, the padding seems to be done at a random offset. + fill: Pixel fill value for constant fill. Default is 0. If a tuple of + length 3, it is used to fill R, G, B channels respectively. + This value is only used when the padding_mode is constant. + padding_mode: Type of padding. Should be: constant, reflect, replicate. + resample: the interpolation mode. + same_on_batch: apply the same transformation across the batch. + align_corners: interpolation flag. + p: probability of applying the transformation for the whole batch. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + cropping_mode: The used algorithm to crop. ``slice`` will use advanced slicing to extract the torch.Tensor based + on the sampled indices. ``resample`` will use `warp_affine` using the affine transformation + to extract and resize at once. Use `slice` for efficiency, or `resample` for proper + differentiability. + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, out_h, out_w)` + + Note: + Input torch.Tensor must be float and normalized into [0, 1] for the best differentiability support. + Additionally, this function accepts another transformation torch.Tensor (:math:`(B, 3, 3)`), then the + applied transformation will be merged int to the input transformation torch.Tensor and returned. + + Examples: + >>> import torch + >>> _ = torch.manual_seed(0) + >>> inputs = torch.arange(1*1*3*3.).view(1, 1, 3, 3) + >>> aug = RandomCrop((2, 2), p=1., cropping_mode="resample") + >>> out = aug(inputs) + >>> out + tensor([[[[3., 4.], + [6., 7.]]]]) + >>> aug.inverse(out, padding_mode="replicate") + tensor([[[[3., 4., 4.], + [3., 4., 4.], + [6., 7., 7.]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomCrop((2, 2), p=1., cropping_mode="resample") + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + size: Tuple[int, int], + padding: Optional[Union[int, Tuple[int, int], Tuple[int, int, int, int]]] = None, + pad_if_needed: Optional[bool] = False, + fill: int = 0, + padding_mode: str = "constant", + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + same_on_batch: bool = False, + align_corners: bool = True, + p: float = 1.0, + keepdim: bool = False, + cropping_mode: str = "slice", + ) -> None: + # Since PyTorch does not support ragged torch.Tensor. So cropping function happens batch-wisely. + super().__init__(p=1.0, same_on_batch=same_on_batch, p_batch=p, keepdim=keepdim) + self._param_generator = rg.CropGenerator(size) + self.flags = { + "size": size, + "padding": padding, + "pad_if_needed": pad_if_needed, + "fill": fill, + "padding_mode": padding_mode, + "resample": Resample.get(resample), + "align_corners": align_corners, + "cropping_mode": cropping_mode, + } + + def compute_padding(self, shape: Tuple[int, ...], flags: Optional[Dict[str, Any]] = None) -> List[int]: + flags = self.flags if flags is None else flags + if len(shape) != 4: + raise AssertionError(f"Expected BCHW. Got {shape}.") + padding = [0, 0, 0, 0] # left, right, top, bottom + if flags["padding"] is not None: + if isinstance(flags["padding"], int): + padding = [flags["padding"]] * 4 + elif isinstance(flags["padding"], tuple) and len(flags["padding"]) == 2: + padding = [flags["padding"][0], flags["padding"][0], flags["padding"][1], flags["padding"][1]] + elif isinstance(flags["padding"], tuple) and len(flags["padding"]) == 4: + padding = [flags["padding"][0], flags["padding"][2], flags["padding"][1], flags["padding"][3]] + else: + raise RuntimeError(f"Expect `padding` to be a scalar, or length 2/4 list. Got {flags['padding']}.") + + if flags["pad_if_needed"]: + needed_padding: Tuple[int, int] = (flags["size"][0] - shape[-2], flags["size"][1] - shape[-1]) # HW + # If crop width is larger than input width F.pad equally left and right + if needed_padding[1] > 0: + # Only use the extra padding if actually needed after possible fixed padding + padding[0] = max(needed_padding[1], padding[0]) + padding[1] = max(needed_padding[1], padding[1]) + # If crop height is larger than input height F.pad equally top and bottom + if needed_padding[0] > 0: + # Only use the extra padding if actually needed after possible fixed padding + padding[2] = max(needed_padding[0], padding[2]) + padding[3] = max(needed_padding[0], padding[3]) + return padding + + def precrop_padding( + self, input: torch.Tensor, padding: Optional[List[int]] = None, flags: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + flags = self.flags if flags is None else flags + if padding is None: + padding = self.compute_padding(input.shape) + + if any(padding): + input = F.pad(input, padding, value=flags["fill"], mode=flags["padding_mode"]) + + return input + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + if flags["cropping_mode"] in ("resample", "slice"): + src = params["src"].to(input) + dst = params["dst"].to(input) + transform: torch.Tensor = get_perspective_transform(src, dst) + + # Fast scaling correction when output exceeds input and padding disabled + if not flags.get("pad_if_needed", False): + h, w = input.shape[-2:] + h_out, w_out = flags["size"] + if h_out > h or w_out > w: + transform[:, 0, 0] *= w_out / w + transform[:, 1, 1] *= h_out / h + + return transform + raise NotImplementedError(f"Not supported type: {flags['cropping_mode']}.") + + def apply_transform_keypoint( + self, + input: Keypoints, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> Keypoints: + """Process keypoints corresponding to the inputs that are no transformation applied.""" + # For F.pad the keypoints properly. + padding_size = params["padding_size"].to(device=input.device) + input = input.pad(padding_size) + return super().apply_transform_keypoint(input=input, params=params, flags=flags, transform=transform) + + def apply_transform_box( + self, + input: Boxes, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> Boxes: + """Process keypoints corresponding to the inputs that are no transformation applied.""" + # For F.pad the boxes properly. + padding_size = params["padding_size"] + input = input.pad(padding_size) + return super().apply_transform_box(input=input, params=params, flags=flags, transform=transform) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + padding_size: Optional[List[int]] = None + if "padding_size" in params and isinstance(params["padding_size"], torch.Tensor): + padding_size = params["padding_size"].unique(dim=0).cpu().squeeze().tolist() + input = self.precrop_padding(input, padding_size, flags) + + flags = self.flags if flags is None else flags + if flags["cropping_mode"] == "resample": # uses bilinear interpolation to crop + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + # Fit the arg to F.F.pad + if flags["padding_mode"] == "constant": + padding_mode = "zeros" + elif flags["padding_mode"] == "replicate": + padding_mode = "border" + elif flags["padding_mode"] == "reflect": + padding_mode = "reflection" + else: + padding_mode = flags["padding_mode"] + + return crop_by_transform_mat( + input, + transform, + flags["size"], + mode=flags["resample"].name.lower(), + padding_mode=padding_mode, + align_corners=flags["align_corners"], + ) + if flags["cropping_mode"] == "slice": # uses advanced slicing to crop + return crop_by_indices(input, params["src"], flags["size"]) + raise NotImplementedError(f"Not supported type: {flags['cropping_mode']}.") + + def inverse_transform( + self, + input: torch.Tensor, + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + size: Optional[Tuple[int, int]] = None, + ) -> torch.Tensor: + if flags["cropping_mode"] != "resample": + raise NotImplementedError( + f"`inverse` is only applicable for resample cropping mode. Got {flags['cropping_mode']}." + ) + if size is None: + raise RuntimeError("`size` has to be a tuple. Got None.") + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + # Fit the arg to F.F.pad + if flags["padding_mode"] == "constant": + padding_mode = "zeros" + elif flags["padding_mode"] == "replicate": + padding_mode = "border" + elif flags["padding_mode"] == "reflect": + padding_mode = "reflection" + else: + padding_mode = flags["padding_mode"] + return crop_by_transform_mat( + input, + transform[:, :2, :], + size, + flags["resample"].name.lower(), + padding_mode=padding_mode, + align_corners=flags["align_corners"], + ) + + def inverse_inputs( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + **kwargs: Any, + ) -> torch.Tensor: + if flags["cropping_mode"] != "resample": + raise NotImplementedError( + f"`inverse` is only applicable for resample cropping mode. Got {flags['cropping_mode']}." + ) + out = super().inverse_inputs(input, params, flags, transform, **kwargs) + if not params["batch_prob"].all(): + return out + padding_size = params["padding_size"].unique(dim=0).cpu().squeeze().tolist() + padding_size = [-padding_size[0], -padding_size[1], -padding_size[2], -padding_size[3]] + return self.precrop_padding(out, padding_size) + + def inverse_boxes( + self, + input: Boxes, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + **kwargs: Any, + ) -> Boxes: + if flags["cropping_mode"] != "resample": + raise NotImplementedError( + f"`inverse` is only applicable for resample cropping mode. Got {flags['cropping_mode']}." + ) + output = super().inverse_boxes(input, params, flags, transform, **kwargs) + if not params["batch_prob"].all(): + return output + + return output.unpad(params["padding_size"]) + + def inverse_keypoints( + self, + input: Keypoints, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + **kwargs: Any, + ) -> Keypoints: + if flags["cropping_mode"] != "resample": + raise NotImplementedError( + f"`inverse` is only applicable for resample cropping mode. Got {flags['cropping_mode']}." + ) + output = super().inverse_keypoints(input, params, flags, transform, **kwargs) + if not params["batch_prob"].all(): + return output + + return output.unpad(params["padding_size"].to(device=input.device)) + + # Override parameters for precrop + def forward_parameters(self, batch_shape: Tuple[int, ...]) -> Dict[str, torch.Tensor]: + input_pad = self.compute_padding(batch_shape) + batch_shape_new = torch.Size( + ( + *batch_shape[:2], + batch_shape[2] + input_pad[2] + input_pad[3], # original height + top + bottom padding + batch_shape[3] + input_pad[0] + input_pad[1], # original width + left + right padding + ) + ) + padding_size = torch.tensor(tuple(input_pad), dtype=torch.long).expand(batch_shape[0], -1) + _params = super().forward_parameters(batch_shape_new) + _params.update({"padding_size": padding_size}) + return _params diff --git a/kornia/augmentation/_2d/geometric/elastic_transform.py b/kornia/augmentation/_2d/geometric/elastic_transform.py new file mode 100644 index 0000000..49ac10c --- /dev/null +++ b/kornia/augmentation/_2d/geometric/elastic_transform.py @@ -0,0 +1,152 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation._2d.base import AugmentationBase2D +from kornia.constants import Resample +from kornia.geometry.boxes import Boxes +from kornia.geometry.transform import elastic_transform2d + + +class RandomElasticTransform(AugmentationBase2D): + r"""Add random elastic transformation to a torch.Tensor image. + + .. image:: _static/img/RandomElasticTransform.png + + Args: + kernel_size: the size of the Gaussian kernel. + sigma: The standard deviation of the Gaussian in the y and x directions, + respectively. Larger sigma results in smaller pixel displacements. + alpha: The scaling factor that controls the intensity of the deformation + in the y and x directions, respectively. + align_corners: Interpolation flag used by `grid_sample`. + resample: Interpolation mode used by `grid_sample`. Either 'nearest' (0) or 'bilinear' (1). + padding_mode: The padding used by ```grid_sample```. Either 'torch.zeros', 'border' or 'refection'. + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + .. note:: + This function internally uses :func:`kornia.geometry.transform.elastic_transform2d`. + + Examples: + >>> import torch + >>> img = torch.ones(1, 1, 2, 2) + >>> out = RandomElasticTransform()(img) + >>> out.shape + torch.Size([1, 1, 2, 2]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomElasticTransform(p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + kernel_size: Tuple[int, int] = (63, 63), + sigma: Tuple[float, float] = (32.0, 32.0), + alpha: Tuple[float, float] = (1.0, 1.0), + align_corners: bool = False, + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + padding_mode: str = "zeros", + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + + self.flags = { + "kernel_size": kernel_size, + "sigma": sigma, + "alpha": alpha, + "align_corners": align_corners, + "resample": Resample.get(resample), + "padding_mode": padding_mode, + } + + def generate_parameters(self, shape: Tuple[int, ...]) -> Dict[str, torch.Tensor]: + B, _, H, W = shape + if self.same_on_batch: + noise = torch.rand(1, 2, H, W, device=self.device, dtype=self.dtype).expand(B, 2, H, W) + else: + noise = torch.rand(B, 2, H, W, device=self.device, dtype=self.dtype) + return {"noise": noise * 2 - 1} + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return elastic_transform2d( + input, + params["noise"].to(input), + flags["kernel_size"], + flags["sigma"], + flags["alpha"], + flags["align_corners"], + flags["resample"].name.lower(), + flags["padding_mode"], + ) + + def apply_non_transform_mask( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return input + + def apply_transform_mask( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Process masks corresponding to the inputs that are transformed.""" + return self.apply_transform(input, params=params, flags=flags, transform=transform) + + def apply_transform_box( + self, + input: Boxes, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> Boxes: + """Process masks corresponding to the inputs that are transformed.""" + # We assume that boxes may not be affected too much by the deformation. + return input + + def apply_transform_class( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Process class tags corresponding to the inputs that are transformed.""" + return input diff --git a/kornia/augmentation/_2d/geometric/fisheye.py b/kornia/augmentation/_2d/geometric/fisheye.py new file mode 100644 index 0000000..e57a14d --- /dev/null +++ b/kornia/augmentation/_2d/geometric/fisheye.py @@ -0,0 +1,106 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.base import AugmentationBase2D +from kornia.geometry.grid import create_meshgrid +from kornia.geometry.transform import remap + + +class RandomFisheye(AugmentationBase2D): + r"""Add random camera radial distortion. + + .. image:: _static/img/RandomFisheye.png + + Args: + center_x: Ranges to sample respect to x-coordinate center with shape (2,). + center_y: Ranges to sample respect to y-coordinate center with shape (2,). + gamma: Ranges to sample for the gamma values respect to optical center with shape (2,). + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Examples: + >>> import torch + >>> img = torch.ones(1, 1, 2, 2) + >>> center_x = torch.tensor([-.3, .3]) + >>> center_y = torch.tensor([-.3, .3]) + >>> gamma = torch.tensor([.9, 1.]) + >>> out = RandomFisheye(center_x, center_y, gamma)(img) + >>> out.shape + torch.Size([1, 1, 2, 2]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomFisheye(center_x, center_y, gamma, p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + center_x: torch.Tensor, + center_y: torch.Tensor, + gamma: torch.Tensor, + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + self._check_tensor(center_x) + self._check_tensor(center_y) + self._check_tensor(gamma) + self._param_generator = rg.PlainUniformGenerator( + (center_x[:, None], "center_x", None, None), + (center_y[:, None], "center_y", None, None), + (gamma[:, None], "gamma", None, None), + ) + + def _check_tensor(self, data: torch.Tensor) -> None: + if not isinstance(data, torch.Tensor): + raise TypeError(f"Invalid input type. Expected torch.Tensor - got: {type(data)}") + + if len(data.shape) != 1 and data.shape[0] != 2: + raise ValueError(f"torch.Tensor must be of shape (2,). Got: {data.shape}.") + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + # create the initial sampling fields + B, _, H, W = input.shape + grid = create_meshgrid(H, W, normalized_coordinates=True) + field_x = grid[..., 0].to(input) # 1xHxW + field_y = grid[..., 1].to(input) # 1xHxW + # vectorize the random parameters + center_x = params["center_x"].view(B, 1, 1).to(input) + center_y = params["center_y"].view(B, 1, 1).to(input) + gamma = params["gamma"].view(B, 1, 1).to(input) + # compute and apply the distances respect to the camera optical center + distance = ((center_x - field_x) ** 2 + (center_y - field_y) ** 2) ** 0.5 + field_x = field_x + field_x * distance**gamma # BxHxw + field_y = field_y + field_y * distance**gamma # BxHxW + return remap(input, field_x, field_y, normalized_coordinates=True, align_corners=True) diff --git a/kornia/augmentation/_2d/geometric/horizontal_flip.py b/kornia/augmentation/_2d/geometric/horizontal_flip.py new file mode 100644 index 0000000..bfaed3f --- /dev/null +++ b/kornia/augmentation/_2d/geometric/horizontal_flip.py @@ -0,0 +1,109 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple + +import torch + +# from torch import Tensor (use torch.Tensor instead) +from kornia.augmentation._2d.geometric.base import GeometricAugmentationBase2D +from kornia.geometry.transform import hflip + + +class RandomHorizontalFlip(GeometricAugmentationBase2D): + r"""Apply a random horizontal flip to a torch.Tensor image or a batch of torch.Tensor images. + + The flip is applied with a given probability. + + .. image:: _static/img/RandomHorizontalFlip.png + + Input should be a torch.Tensor of shape (C, H, W) or a batch of tensors :math:`(B, C, H, W)`. + If Input is a tuple it is assumed that the first element contains the aforementioned tensors and the second, + the corresponding transformation matrix that has been applied to them. In this case the module + will Horizontally flip the tensors and torch.cat the corresponding transformation matrix to the + previous one. This is especially useful when using this functionality as part of an ``nn.Sequential`` module. + + Args: + p: probability of the image being flipped. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.geometry.transform.hflip`. + + Examples: + >>> import torch + >>> input = torch.tensor([[[[0., 0., 0.], + ... [0., 0., 0.], + ... [0., 1., 1.]]]]) + >>> seq = RandomHorizontalFlip(p=1.0) + >>> seq(input), seq.transform_matrix + (tensor([[[[0., 0., 0.], + [0., 0., 0.], + [1., 1., 0.]]]]), tensor([[[-1., 0., 2.], + [ 0., 1., 0.], + [ 0., 0., 1.]]])) + >>> seq.inverse(seq(input)).equal(input) + True + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> seq = RandomHorizontalFlip(p=1.0) + >>> (seq(input) == seq(input, params=seq._params)).all() + tensor(True) + + """ + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + w: int = int(params["forward_input_shape"][-1]) + flip_mat: torch.Tensor = torch.tensor( + [[-1, 0, w - 1], [0, 1, 0], [0, 0, 1]], device=input.device, dtype=input.dtype + ) + + return flip_mat.expand(input.shape[0], 3, 3) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return hflip(input) + + def inverse_transform( + self, + input: torch.Tensor, + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + size: Optional[Tuple[int, int]] = None, + ) -> torch.Tensor: + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + return self.apply_transform( + input, + params=self._params, + transform=torch.as_tensor(transform, device=input.device, dtype=input.dtype), + flags=flags, + ) diff --git a/kornia/augmentation/_2d/geometric/pad.py b/kornia/augmentation/_2d/geometric/pad.py new file mode 100644 index 0000000..0172e46 --- /dev/null +++ b/kornia/augmentation/_2d/geometric/pad.py @@ -0,0 +1,96 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple + +import torch +from torch import Tensor + +from kornia.augmentation._2d.geometric.base import GeometricAugmentationBase2D + + +class PadTo(GeometricAugmentationBase2D): + r"""Pad the given sample to a specific size. Always occurs (p=1.0). + + .. image:: _static/img/PadTo.png + + Args: + size: a tuple of ints in the format (height, width) that give the spatial + dimensions to pad inputs to. + pad_mode: the type of padding to perform on the image (valid values + are those accepted by torch.nn.functional.pad) + pad_value: fill value for 'constant' padding applied to the image + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`torch.nn.functional.pad`. + + Examples: + >>> import torch + >>> img = torch.tensor([[[[0., 0., 0.], + ... [0., 0., 0.], + ... [0., 0., 0.]]]]) + >>> pad = PadTo((4, 5), pad_value=1.) + >>> out = pad(img) + >>> out + tensor([[[[0., 0., 0., 1., 1.], + [0., 0., 0., 1., 1.], + [0., 0., 0., 1., 1.], + [1., 1., 1., 1., 1.]]]]) + >>> pad.inverse(out) + tensor([[[[0., 0., 0.], + [0., 0., 0.], + [0., 0., 0.]]]]) + + """ + + def __init__( + self, size: Tuple[int, int], pad_mode: str = "constant", pad_value: float = 0, keepdim: bool = False + ) -> None: + super().__init__(p=1.0, same_on_batch=True, p_batch=1.0, keepdim=keepdim) + self.flags = {"size": size, "pad_mode": pad_mode, "pad_value": pad_value} + + # TODO: It is incorrect to return identity + # TODO: Having a resampled version with ``warp_affine`` + def compute_transformation(self, image: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any]) -> Tensor: + return self.identity_matrix(image) + + def apply_transform( + self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Tensor: + _, _, height, width = input.shape + height_pad: int = flags["size"][0] - height + width_pad: int = flags["size"][1] - width + return torch.nn.functional.pad( + input, [0, width_pad, 0, height_pad], mode=flags["pad_mode"], value=flags["pad_value"] + ) + + def inverse_transform( + self, + input: Tensor, + flags: Dict[str, Any], + transform: Optional[Tensor] = None, + size: Optional[Tuple[int, int]] = None, + ) -> Tensor: + if size is None: + raise RuntimeError("`size` has to be a tuple. Got None.") + return input[..., : size[0], : size[1]] diff --git a/kornia/augmentation/_2d/geometric/perspective.py b/kornia/augmentation/_2d/geometric/perspective.py new file mode 100644 index 0000000..0786788 --- /dev/null +++ b/kornia/augmentation/_2d/geometric/perspective.py @@ -0,0 +1,126 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.geometric.base import GeometricAugmentationBase2D +from kornia.constants import Resample +from kornia.geometry.transform import get_perspective_transform, warp_perspective + + +class RandomPerspective(GeometricAugmentationBase2D): + r"""Apply a random perspective transformation to an image torch.Tensor with a given probability. + + .. image:: _static/img/RandomPerspective.png + + Args: + distortion_scale: the degree of distortion, ranged from 0 to 1. + resample: the interpolation method to use. + same_on_batch: apply the same transformation across the batch. Default: False. + align_corners: interpolation flag. + p: probability of the image being perspectively transformed. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + sampling_method: ``'basic'`` | ``'area_preserving'``. Default: ``'basic'`` + If ``'basic'``, samples by translating the image corners randomly inwards. + If ``'area_preserving'``, samples by randomly translating the image corners in any direction. + Preserves area on average. See https://arxiv.org/abs/2104.03308 for further details. + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.geometry.transform.warp_pespective`. + + Examples: + >>> rng = torch.manual_seed(0) + >>> inputs= torch.tensor([[[[1., 0., 0.], + ... [0., 1., 0.], + ... [0., 0., 1.]]]]) + >>> aug = RandomPerspective(0.5, p=0.5) + >>> out = aug(inputs) + >>> out + tensor([[[[0.0000, 0.2289, 0.0000], + [0.0000, 0.4800, 0.0000], + [0.0000, 0.0000, 0.0000]]]]) + >>> aug.inverse(out) + tensor([[[[0.0500, 0.0961, 0.0000], + [0.2011, 0.3144, 0.0000], + [0.0031, 0.0130, 0.0053]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomPerspective(0.5, p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + distortion_scale: Union[torch.Tensor, float] = 0.5, + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + same_on_batch: bool = False, + align_corners: bool = False, + p: float = 0.5, + keepdim: bool = False, + sampling_method: str = "basic", + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self._param_generator = rg.PerspectiveGenerator(distortion_scale, sampling_method=sampling_method) + + self.flags: Dict[str, Any] = {"align_corners": align_corners, "resample": Resample.get(resample)} + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + return get_perspective_transform(params["start_points"].to(input), params["end_points"].to(input)) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + _, _, height, width = input.shape + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + + return warp_perspective( + input, transform, (height, width), mode=flags["resample"].name.lower(), align_corners=flags["align_corners"] + ) + + def inverse_transform( + self, + input: torch.Tensor, + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + size: Optional[Tuple[int, int]] = None, + ) -> torch.Tensor: + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + return self.apply_transform( + input, + params=self._params, + transform=torch.as_tensor(transform, device=input.device, dtype=input.dtype), + flags=flags, + ) diff --git a/kornia/augmentation/_2d/geometric/resize.py b/kornia/augmentation/_2d/geometric/resize.py new file mode 100644 index 0000000..6634a86 --- /dev/null +++ b/kornia/augmentation/_2d/geometric/resize.py @@ -0,0 +1,156 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.geometric.base import GeometricAugmentationBase2D +from kornia.constants import Resample +from kornia.core.ops import eye_like +from kornia.geometry.transform import crop_by_transform_mat, get_perspective_transform, resize + + +class Resize(GeometricAugmentationBase2D): + """Resize to size. + + Args: + size: Size (h, w) in pixels of the resized region or just one side. + side: Which side to resize, if size is only of type int. + resample: Resampling mode. + align_corners: interpolation flag. + antialias: if True, then image will be filtered with Gaussian before downscaling. No effect for upscaling. + p: probability of the augmentation been applied. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + """ + + def __init__( + self, + size: Union[int, Tuple[int, int]], + side: str = "short", + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + align_corners: bool = True, + antialias: bool = False, + p: float = 1.0, + keepdim: bool = False, + ) -> None: + super().__init__(p=1.0, same_on_batch=True, p_batch=p, keepdim=keepdim) + self._param_generator = rg.ResizeGenerator(resize_to=size, side=side) + self.flags = { + "size": size, + "side": side, + "resample": Resample.get(resample), + "align_corners": align_corners, + "antialias": antialias, + } + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + if params["output_size"] == input.shape[-2:]: + return eye_like(3, input) + + transform: torch.Tensor = torch.as_tensor( + get_perspective_transform(params["src"], params["dst"]), dtype=input.dtype, device=input.device + ) + transform = transform.expand(input.shape[0], -1, -1) + return transform + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + B, C, _, _ = input.shape + out_size = tuple(params["output_size"][0].tolist()) + out = torch.empty(B, C, *out_size, device=input.device, dtype=input.dtype) + + for i in range(B): + x1 = int(params["src"][i, 0, 0]) + x2 = int(params["src"][i, 1, 0]) + 1 + y1 = int(params["src"][i, 0, 1]) + y2 = int(params["src"][i, 3, 1]) + 1 + out[i] = resize( + input[i : i + 1, :, y1:y2, x1:x2], + out_size, + interpolation=flags["resample"].name.lower(), + align_corners=( + flags["align_corners"] if flags["resample"] in [Resample.BILINEAR, Resample.BICUBIC] else None + ), + antialias=flags["antialias"], + ) + return out + + def inverse_transform( + self, + input: torch.Tensor, + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + size: Optional[Tuple[int, int]] = None, + ) -> torch.Tensor: + if not isinstance(size, tuple): + raise TypeError(f"Expected the size be a tuple. Gotcha {type(size)}") + + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + + return crop_by_transform_mat( + input, transform[:, :2, :], size, flags["resample"].name.lower(), "zeros", flags["align_corners"] + ) + + +class LongestMaxSize(Resize): + """Rescale an image so that maximum side is equal to max_size, keeping the aspect ratio of the initial image. + + Args: + max_size: maximum size of the image after the transformation. + + """ + + def __init__( + self, + max_size: int, + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + align_corners: bool = True, + p: float = 1.0, + ) -> None: + # TODO: Support max_size list input to randomly select from + super().__init__(size=max_size, side="long", resample=resample, align_corners=align_corners, p=p) + + +class SmallestMaxSize(Resize): + """Rescale an image so that minimum side is equal to max_size, keeping the aspect ratio of the initial image. + + Args: + max_size: maximum size of the image after the transformation. + + """ + + def __init__( + self, + max_size: int, + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + align_corners: bool = True, + p: float = 1.0, + ) -> None: + # TODO: Support max_size list input to randomly select from + super().__init__(size=max_size, side="short", resample=resample, align_corners=align_corners, p=p) diff --git a/kornia/augmentation/_2d/geometric/resized_crop.py b/kornia/augmentation/_2d/geometric/resized_crop.py new file mode 100644 index 0000000..b24b337 --- /dev/null +++ b/kornia/augmentation/_2d/geometric/resized_crop.py @@ -0,0 +1,167 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.geometric.base import GeometricAugmentationBase2D +from kornia.constants import Resample +from kornia.geometry.transform import crop_by_indices, crop_by_transform_mat, get_perspective_transform + + +class RandomResizedCrop(GeometricAugmentationBase2D): + r"""Crop random patches in an image torch.Tensor and resizes to a given size. + + .. image:: _static/img/RandomResizedCrop.png + + Args: + size: Desired output size (out_h, out_w) of each edge. + Must be Tuple[int, int], then out_h = size[0], out_w = size[1]. + scale: range of size of the origin size cropped. + ratio: range of aspect ratio of the origin aspect ratio cropped. + resample: the interpolation mode. + same_on_batch: apply the same transformation across the batch. + align_corners: interpolation flag. + p: probability of the augmentation been applied. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + cropping_mode: The used algorithm to crop. ``slice`` will use advanced slicing to extract the torch.Tensor based + on the sampled indices. ``resample`` will use `warp_affine` using the affine transformation + to extract and resize at once. Use `slice` for efficiency, or `resample` for proper + differentiability. + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, out_h, out_w)` + + Note: + Input torch.Tensor must be float and normalized into [0, 1] for the best differentiability support. + Additionally, this function accepts another transformation torch.Tensor (:math:`(B, 3, 3)`), then the + applied transformation will be merged int to the input transformation torch.Tensor and returned. + + Example: + >>> rng = torch.manual_seed(0) + >>> inputs = torch.tensor([[[0., 1., 2.], + ... [3., 4., 5.], + ... [6., 7., 8.]]]) + >>> aug = RandomResizedCrop(size=(3, 3), scale=(3., 3.), ratio=(2., 2.), p=1., cropping_mode="resample") + >>> out = aug(inputs) + >>> out + tensor([[[[1.0000, 1.5000, 2.0000], + [4.0000, 4.5000, 5.0000], + [7.0000, 7.5000, 8.0000]]]]) + >>> aug.inverse(out, padding_mode="border") + tensor([[[[1., 1., 2.], + [4., 4., 5.], + [7., 7., 8.]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomResizedCrop(size=(3, 3), scale=(3., 3.), ratio=(2., 2.), p=1., cropping_mode="resample") + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + size: Tuple[int, int], + scale: Union[torch.Tensor, Tuple[float, float]] = (0.08, 1.0), + ratio: Union[torch.Tensor, Tuple[float, float]] = (3.0 / 4.0, 4.0 / 3.0), + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + same_on_batch: bool = False, + align_corners: bool = True, + p: float = 1.0, + keepdim: bool = False, + cropping_mode: str = "slice", + ) -> None: + # Since PyTorch does not support ragged torch.Tensor. So cropping function happens all the time. + super().__init__(p=1.0, same_on_batch=same_on_batch, p_batch=p, keepdim=keepdim) + self._param_generator = rg.ResizedCropGenerator(size, scale, ratio) + self.flags = { + "size": size, + "resample": Resample.get(resample), + "align_corners": align_corners, + "cropping_mode": cropping_mode, + "padding_mode": "zeros", + } + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + if flags["cropping_mode"] in ("resample", "slice"): + transform: torch.Tensor = get_perspective_transform(params["src"].to(input), params["dst"].to(input)) + transform = transform.expand(input.shape[0], -1, -1) + return transform + raise NotImplementedError(f"Not supported type: {flags['cropping_mode']}.") + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if flags["cropping_mode"] == "resample": # uses bilinear interpolation to crop + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + + return crop_by_transform_mat( + input, + transform, + flags["size"], + mode=flags["resample"].name.lower(), + padding_mode="zeros", + align_corners=flags["align_corners"], + ) + if flags["cropping_mode"] == "slice": # uses advanced slicing to crop + return crop_by_indices( + input, + params["src"], + flags["size"], + interpolation=flags["resample"].name.lower(), + align_corners=flags["align_corners"], + ) + raise NotImplementedError(f"Not supported type: {flags['cropping_mode']}.") + + def inverse_transform( + self, + input: torch.Tensor, + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + size: Optional[Tuple[int, int]] = None, + ) -> torch.Tensor: + if flags["cropping_mode"] != "resample": + raise NotImplementedError( + f"`inverse` is only applicable for resample cropping mode. Got {flags['cropping_mode']}." + ) + if not isinstance(size, tuple): + raise TypeError(f"Expected the size be a tuple. Gotcha {type(size)}") + + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + + return crop_by_transform_mat( + input, + transform[:, :2, :], + size, + flags["resample"].name.lower(), + flags["padding_mode"], + flags["align_corners"], + ) diff --git a/kornia/augmentation/_2d/geometric/rotation.py b/kornia/augmentation/_2d/geometric/rotation.py new file mode 100644 index 0000000..49845b0 --- /dev/null +++ b/kornia/augmentation/_2d/geometric/rotation.py @@ -0,0 +1,243 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.geometric.base import GeometricAugmentationBase2D +from kornia.constants import Resample +from kornia.core.ops import eye_like +from kornia.geometry.transform import affine +from kornia.geometry.transform.affwarp import _compute_rotation_matrix, _compute_tensor_center + + +class RandomRotation(GeometricAugmentationBase2D): + r"""Apply a random rotation to a torch.Tensor image or a batch of torch.Tensor images given an amount of degrees. + + .. image:: _static/img/RandomRotation.png + + Args: + degrees: range of degrees to select from. If degrees is a number the + range of degrees to select from will be (-degrees, +degrees). + resample: Default: the interpolation mode. + same_on_batch: apply the same transformation across the batch. + align_corners: interpolation flag. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.geometry.transform.affine`. + + Examples: + >>> rng = torch.manual_seed(0) + >>> input = torch.tensor([[1., 0., 0., 2.], + ... [0., 0., 0., 0.], + ... [0., 1., 2., 0.], + ... [0., 0., 1., 2.]]) + >>> aug = RandomRotation(degrees=45.0, p=1.) + >>> out = aug(input) + >>> out + tensor([[[[0.9824, 0.0088, 0.0000, 1.9649], + [0.0000, 0.0029, 0.0000, 0.0176], + [0.0029, 1.0000, 1.9883, 0.0000], + [0.0000, 0.0088, 1.0117, 1.9649]]]]) + >>> aug.transform_matrix + tensor([[[ 1.0000, -0.0059, 0.0088], + [ 0.0059, 1.0000, -0.0088], + [ 0.0000, 0.0000, 1.0000]]]) + >>> inv = aug.inverse(out) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomRotation(degrees=45.0, p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + # Note: Extra params, center=None, fill=0 in TorchVision + + def __init__( + self, + degrees: Union[torch.Tensor, float, Tuple[float, float], List[float]], + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + same_on_batch: bool = False, + align_corners: bool = True, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self._param_generator = rg.PlainUniformGenerator((degrees, "degrees", 0.0, (-360.0, 360.0))) + + self.flags = {"resample": Resample.get(resample), "align_corners": align_corners} + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + # TODO: Update to use `get_rotation_matrix2d` + angles: torch.Tensor = params["degrees"].to(input) + + center: torch.Tensor = _compute_tensor_center(input) + rotation_mat: torch.Tensor = _compute_rotation_matrix(angles, center.expand(angles.shape[0], -1)) + + # rotation_mat is B x 2 x 3 and we need a B x 3 x 3 matrix + trans_mat: torch.Tensor = eye_like(3, input, shared_memory=False) + trans_mat[:, 0] = rotation_mat[:, 0] + trans_mat[:, 1] = rotation_mat[:, 1] + + return trans_mat + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + + return affine(input, transform[..., :2, :3], flags["resample"].name.lower(), "zeros", flags["align_corners"]) + + def inverse_transform( + self, + input: torch.Tensor, + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + size: Optional[Tuple[int, int]] = None, + ) -> torch.Tensor: + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + return self.apply_transform( + input, + params=self._params, + transform=torch.as_tensor(transform, device=input.device, dtype=input.dtype), + flags=flags, + ) + + +class RandomRotation90(GeometricAugmentationBase2D): + r"""Apply a random 90 * n degree rotation to a torch.Tensor image or a batch of torch.Tensor images. + + Args: + times: the range of n times 90 degree rotation needs to be applied. + resample: Default: the interpolation mode. + same_on_batch: apply the same transformation across the batch. + align_corners: interpolation flag. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.geometry.transform.affine`. This version is relatively + slow as it operates based on affine transformations. + + Examples: + >>> rng = torch.manual_seed(1) + >>> input = torch.tensor([[1., 0., 0., 2.], + ... [0., 0., 0., 0.], + ... [0., 1., 2., 0.], + ... [0., 0., 1., 2.]]) + >>> aug = RandomRotation90(times=(1, 1), p=1.) + >>> out = aug(input) + >>> out + tensor([[[[2.0000e+00, 0.0000e+00, 0.0000e+00, 2.0000e+00], + [0.0000e+00, 0.0000e+00, 2.0000e+00, 1.0000e+00], + [5.9605e-08, 0.0000e+00, 1.0000e+00, 0.0000e+00], + [1.0000e+00, 5.9605e-08, 0.0000e+00, 0.0000e+00]]]]) + >>> aug.transform_matrix + tensor([[[-4.3711e-08, 1.0000e+00, 1.1921e-07], + [-1.0000e+00, -4.3711e-08, 3.0000e+00], + [ 0.0000e+00, 0.0000e+00, 1.0000e+00]]]) + >>> inv = aug.inverse(out) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomRotation90(times=(-1, 1), p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + times: tuple[int, int], + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + same_on_batch: bool = False, + align_corners: bool = True, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self._param_generator = rg.PlainUniformGenerator((times, "times", 0.0, (-3, 3))) + + self.flags = {"resample": Resample.get(resample), "align_corners": align_corners} + + def compute_transformation( + self, input: torch.Tensor, params: dict[str, torch.Tensor], flags: dict[str, Any] + ) -> torch.Tensor: + # TODO: Update to use `get_rotation_matrix2d` + angles: torch.Tensor = 90.0 * params["times"].round().to(input) + + center: torch.Tensor = _compute_tensor_center(input) + rotation_mat: torch.Tensor = _compute_rotation_matrix(angles, center.expand(angles.shape[0], -1)) + + # rotation_mat is B x 2 x 3 and we need a B x 3 x 3 matrix + trans_mat: torch.Tensor = eye_like(3, input, shared_memory=False) + trans_mat[:, 0] = rotation_mat[:, 0] + trans_mat[:, 1] = rotation_mat[:, 1] + + return trans_mat + + def apply_transform( + self, + input: torch.Tensor, + params: dict[str, torch.Tensor], + flags: dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + + return affine(input, transform[..., :2, :3], flags["resample"].name.lower(), "zeros", flags["align_corners"]) + + def inverse_transform( + self, + input: torch.Tensor, + flags: dict[str, Any], + transform: Optional[torch.Tensor] = None, + size: Optional[tuple[int, int]] = None, + ) -> torch.Tensor: + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + return self.apply_transform( + input, + params=self._params, + transform=torch.as_tensor(transform, device=input.device, dtype=input.dtype), + flags=flags, + ) diff --git a/kornia/augmentation/_2d/geometric/shear.py b/kornia/augmentation/_2d/geometric/shear.py new file mode 100644 index 0000000..65dc4ec --- /dev/null +++ b/kornia/augmentation/_2d/geometric/shear.py @@ -0,0 +1,147 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math +from typing import Any, Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.geometric.base import GeometricAugmentationBase2D +from kornia.constants import Resample, SamplePadding + +# from kornia.geometry.conversions import deg2rad (use torch.deg2rad instead) +from kornia.geometry.transform import get_shear_matrix2d, warp_affine + + +class RandomShear(GeometricAugmentationBase2D): + r"""Apply a random 2D shear transformation to a torch.Tensor image. + + The transformation is computed so that the image center is kept invariant. + + Args: + shear: Range of degrees to select from. + If float, a shear parallel to the x axis in the range (-shear, +shear) will be applied. + If (a, b), a shear parallel to the x axis in the range (-shear, +shear) will be applied. + If (a, b, c, d), then x-axis shear in (shear[0], shear[1]) and y-axis shear in (shear[2], shear[3]) + will be applied. Will not apply shear by default. + resample: resample mode from "nearest" (0) or "bilinear" (1). + padding_mode: padding mode from "torch.zeros" (0), "border" (1) or "reflection" (2). + same_on_batch: apply the same transformation across the batch. + align_corners: interpolation flag. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.geometry.transform.warp_affine`. + + Examples: + >>> import torch + >>> rng = torch.manual_seed(0) + >>> input = torch.rand(1, 1, 3, 3) + >>> aug = RandomShear((-5., 2., 5., 10.), p=1.) + >>> out = aug(input) + >>> out, aug.transform_matrix + (tensor([[[[0.4403, 0.7614, 0.1516], + [0.1753, 0.3074, 0.6127], + [0.4438, 0.8924, 0.4061]]]]), tensor([[[ 1.0000, 0.0100, -0.0100], + [-0.1183, 0.9988, 0.1194], + [ 0.0000, 0.0000, 1.0000]]])) + >>> aug.inverse(out) + tensor([[[[0.4045, 0.7577, 0.1393], + [0.2071, 0.3074, 0.5582], + [0.3958, 0.8868, 0.4265]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomShear((-15., 20.), p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + shear: Union[torch.Tensor, float, Tuple[float, float], Tuple[float, float, float, float]], + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + same_on_batch: bool = False, + align_corners: bool = False, + padding_mode: Union[str, int, SamplePadding] = SamplePadding.ZEROS.name, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self._param_generator: rg.ShearGenerator = rg.ShearGenerator(shear) + self.flags = { + "resample": Resample.get(resample), + "padding_mode": SamplePadding.get(padding_mode), + "align_corners": align_corners, + } + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + # Inline ``deg2rad`` so the trace lowers to plain multiply (legacy ONNX has + # no symbolic for ``aten::deg2rad`` at opset 20). + deg2rad_factor: float = math.pi / 180.0 + shear_x = torch.as_tensor(params["shear_x"], device=input.device, dtype=input.dtype) * deg2rad_factor + shear_y = torch.as_tensor(params["shear_y"], device=input.device, dtype=input.dtype) * deg2rad_factor + return get_shear_matrix2d( + torch.as_tensor(params["center"], device=input.device, dtype=input.dtype), + shear_x, + shear_y, + ) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + _, _, height, width = input.shape + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + + return warp_affine( + input, + transform[:, :2, :], + (height, width), + flags["resample"].name.lower(), + align_corners=flags["align_corners"], + padding_mode=flags["padding_mode"].name.lower(), + ) + + def inverse_transform( + self, + input: torch.Tensor, + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + size: Optional[Tuple[int, int]] = None, + ) -> torch.Tensor: + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + return self.apply_transform( + input, + params=self._params, + transform=torch.as_tensor(transform, device=input.device, dtype=input.dtype), + flags=flags, + ) diff --git a/kornia/augmentation/_2d/geometric/thin_plate_spline.py b/kornia/augmentation/_2d/geometric/thin_plate_spline.py new file mode 100644 index 0000000..d21ee53 --- /dev/null +++ b/kornia/augmentation/_2d/geometric/thin_plate_spline.py @@ -0,0 +1,108 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation._2d.base import AugmentationBase2D +from kornia.constants import SamplePadding +from kornia.geometry.transform import get_tps_transform, warp_image_tps + + +# NOTE: This NEEDS to be updated. It is out of the random generator controller. +class RandomThinPlateSpline(AugmentationBase2D): + r"""Add random noise to the Thin Plate Spline algorithm. + + .. image:: _static/img/RandomThinPlateSpline.png + + Args: + scale: the scale factor to apply to the destination points. + align_corners: Interpolation flag used by ``grid_sample``. + mode: Interpolation mode used by `grid_sample`. Either 'bilinear' or 'nearest'. + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + .. note:: + This function internally uses :func:`kornia.geometry.transform.warp_image_tps`. + + Examples: + >>> img = torch.ones(1, 1, 2, 2) + >>> out = RandomThinPlateSpline()(img) + >>> out.shape + torch.Size([1, 1, 2, 2]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomThinPlateSpline(p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + scale: float = 0.2, + align_corners: bool = False, + padding_mode: Union[str, int, SamplePadding] = SamplePadding.ZEROS.name, + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + self.flags = { + "align_corners": align_corners, + "padding_mode": SamplePadding.get(padding_mode), + } + self.dist = torch.distributions.Uniform(-scale, scale) + + def generate_parameters(self, shape: Tuple[int, ...]) -> Dict[str, torch.Tensor]: + B, _, _, _ = shape + + device = self.device + dtype = self.dtype + + # 5 TPS control points in normalized coordinates + src = torch.tensor( + [[[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0], [0.0, 0.0]]], + device=device, + dtype=dtype, + ).expand(B, 5, 2) + + if self.same_on_batch: + noise = self.dist.rsample((1, 5, 2)).to(device=device, dtype=dtype) + noise = noise.expand(B, 5, 2) + else: + noise = self.dist.rsample((B, 5, 2)).to(device=device, dtype=dtype) + + dst = src + noise + + return {"src": src, "dst": dst} + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + src = params["src"].to(input) + dst = params["dst"].to(input) + # NOTE: warp_image_tps need to use inverse parameters + kernel, affine = get_tps_transform(dst, src) + return warp_image_tps(input, src, kernel, affine, flags["align_corners"], flags["padding_mode"].name.lower()) diff --git a/kornia/augmentation/_2d/geometric/translate.py b/kornia/augmentation/_2d/geometric/translate.py new file mode 100644 index 0000000..967fcff --- /dev/null +++ b/kornia/augmentation/_2d/geometric/translate.py @@ -0,0 +1,136 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.geometric.base import GeometricAugmentationBase2D +from kornia.constants import Resample, SamplePadding +from kornia.geometry.transform import get_translation_matrix2d, warp_affine + + +class RandomTranslate(GeometricAugmentationBase2D): + r"""Apply a random 2D affine transformation to a torch.Tensor image. + + Args: + translate_x: tuple of maximum absolute fraction for horizontal translations. + For example translate_x=(a, b), then horizontal shift + is randomly sampled in the range img_width * a < dx < img_width * b + translate_y: tuple of maximum absolute fraction for vertical translations. + For example translate_y=(a, b), then vertical shift + is randomly sampled in the range img_height * a < dy < img_height * b. + resample: resample mode from "nearest" (0) or "bilinear" (1). + padding_mode: padding mode from "torch.zeros" (0), "border" (1) or "reflection" (2). + same_on_batch: apply the same transformation across the batch. + align_corners: interpolation flag. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.geometry.transform.warp_affine`. + + Examples: + >>> import torch + >>> rng = torch.manual_seed(0) + >>> input = torch.rand(1, 1, 3, 3) + >>> aug = RandomTranslate((-0.2, 0.2), (-0.1, 0.1), p=1.) + >>> out = aug(input) + >>> out, aug.transform_matrix + (tensor([[[[0.3403, 0.6439, 0.2920], + [0.1377, 0.3383, 0.5569], + [0.3226, 0.6909, 0.4844]]]]), tensor([[[ 1.0000, 0.0000, 0.1588], + [ 0.0000, 1.0000, -0.0907], + [ 0.0000, 0.0000, 1.0000]]])) + >>> aug.inverse(out) + tensor([[[[0.3565, 0.4839, 0.1922], + [0.2164, 0.4134, 0.3968], + [0.3797, 0.6075, 0.3765]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomTranslate((-0.2, 0.2), (-0.1, 0.1), p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + translate_x: Optional[Union[torch.Tensor, Tuple[float, float]]] = None, + translate_y: Optional[Union[torch.Tensor, Tuple[float, float]]] = None, + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + same_on_batch: bool = False, + align_corners: bool = False, + padding_mode: Union[str, int, SamplePadding] = SamplePadding.ZEROS.name, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self._param_generator: rg.TranslateGenerator = rg.TranslateGenerator(translate_x, translate_y) + self.flags = { + "resample": Resample.get(resample), + "padding_mode": SamplePadding.get(padding_mode), + "align_corners": align_corners, + } + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + translations = torch.stack([params["translate_x"], params["translate_y"]], dim=-1) + return get_translation_matrix2d(torch.as_tensor(translations, device=input.device, dtype=input.dtype)) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + _, _, height, width = input.shape + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + + return warp_affine( + input, + transform[:, :2, :], + (height, width), + flags["resample"].name.lower(), + align_corners=flags["align_corners"], + padding_mode=flags["padding_mode"].name.lower(), + ) + + def inverse_transform( + self, + input: torch.Tensor, + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + size: Optional[Tuple[int, int]] = None, + ) -> torch.Tensor: + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + return self.apply_transform( + input, + params=self._params, + transform=torch.as_tensor(transform, device=input.device, dtype=input.dtype), + flags=flags, + ) diff --git a/kornia/augmentation/_2d/geometric/vertical_flip.py b/kornia/augmentation/_2d/geometric/vertical_flip.py new file mode 100644 index 0000000..5fce10e --- /dev/null +++ b/kornia/augmentation/_2d/geometric/vertical_flip.py @@ -0,0 +1,100 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple + +import torch + +from kornia.augmentation._2d.geometric.base import GeometricAugmentationBase2D +from kornia.geometry.transform import vflip + + +class RandomVerticalFlip(GeometricAugmentationBase2D): + r"""Apply a random vertical flip to a torch.Tensor image or a batch of torch.Tensor images with a given probability. + + .. image:: _static/img/RandomVerticalFlip.png + + Args: + p: probability of the image being flipped. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.geometry.transform.vflip`. + + Examples: + >>> import torch + >>> input = torch.tensor([[[[0., 0., 0.], + ... [0., 0., 0.], + ... [0., 1., 1.]]]]) + >>> seq = RandomVerticalFlip(p=1.0) + >>> seq(input), seq.transform_matrix + (tensor([[[[0., 1., 1.], + [0., 0., 0.], + [0., 0., 0.]]]]), tensor([[[ 1., 0., 0.], + [ 0., -1., 2.], + [ 0., 0., 1.]]])) + >>> seq.inverse(seq(input)).equal(input) + True + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> seq = RandomVerticalFlip(p=1.0) + >>> (seq(input) == seq(input, params=seq._params)).all() + tensor(True) + + """ + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + h: int = int(params["forward_input_shape"][-2]) + flip_mat: torch.Tensor = torch.tensor( + [[1, 0, 0], [0, -1, h - 1], [0, 0, 1]], device=input.device, dtype=input.dtype + ) + + return flip_mat.expand(input.shape[0], 3, 3) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return vflip(input) + + def inverse_transform( + self, + input: torch.Tensor, + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + size: Optional[Tuple[int, int]] = None, + ) -> torch.Tensor: + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the `transform` be a torch.Tensor. Got {type(transform)}.") + return self.apply_transform( + input, + params=self._params, + transform=torch.as_tensor(transform, device=input.device, dtype=input.dtype), + flags=flags, + ) diff --git a/kornia/augmentation/_2d/intensity/__init__.py b/kornia/augmentation/_2d/intensity/__init__.py new file mode 100644 index 0000000..6f0eaae --- /dev/null +++ b/kornia/augmentation/_2d/intensity/__init__.py @@ -0,0 +1,55 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from kornia.augmentation._2d.intensity.auto_contrast import RandomAutoContrast +from kornia.augmentation._2d.intensity.box_blur import RandomBoxBlur +from kornia.augmentation._2d.intensity.brightness import RandomBrightness +from kornia.augmentation._2d.intensity.channel_dropout import RandomChannelDropout +from kornia.augmentation._2d.intensity.channel_shuffle import RandomChannelShuffle +from kornia.augmentation._2d.intensity.clahe import RandomClahe +from kornia.augmentation._2d.intensity.color_jiggle import ColorJiggle +from kornia.augmentation._2d.intensity.color_jitter import ColorJitter +from kornia.augmentation._2d.intensity.contrast import RandomContrast +from kornia.augmentation._2d.intensity.denormalize import Denormalize +from kornia.augmentation._2d.intensity.dissolving import RandomDissolving +from kornia.augmentation._2d.intensity.equalize import RandomEqualize +from kornia.augmentation._2d.intensity.erasing import RandomErasing +from kornia.augmentation._2d.intensity.gamma import RandomGamma +from kornia.augmentation._2d.intensity.gaussian_blur import RandomGaussianBlur +from kornia.augmentation._2d.intensity.gaussian_illumination import RandomGaussianIllumination +from kornia.augmentation._2d.intensity.gaussian_noise import RandomGaussianNoise +from kornia.augmentation._2d.intensity.grayscale import RandomGrayscale +from kornia.augmentation._2d.intensity.hue import RandomHue +from kornia.augmentation._2d.intensity.invert import RandomInvert +from kornia.augmentation._2d.intensity.jpeg import RandomJPEG +from kornia.augmentation._2d.intensity.linear_illumination import ( + RandomLinearCornerIllumination, + RandomLinearIllumination, +) +from kornia.augmentation._2d.intensity.median_blur import RandomMedianBlur +from kornia.augmentation._2d.intensity.motion_blur import RandomMotionBlur +from kornia.augmentation._2d.intensity.normalize import Normalize +from kornia.augmentation._2d.intensity.planckian_jitter import RandomPlanckianJitter +from kornia.augmentation._2d.intensity.plasma import RandomPlasmaBrightness, RandomPlasmaContrast, RandomPlasmaShadow +from kornia.augmentation._2d.intensity.posterize import RandomPosterize +from kornia.augmentation._2d.intensity.random_rain import RandomRain +from kornia.augmentation._2d.intensity.random_rgb_shift import RandomRGBShift +from kornia.augmentation._2d.intensity.random_snow import RandomSnow +from kornia.augmentation._2d.intensity.salt_pepper_noise import RandomSaltAndPepperNoise +from kornia.augmentation._2d.intensity.saturation import RandomSaturation +from kornia.augmentation._2d.intensity.sharpness import RandomSharpness +from kornia.augmentation._2d.intensity.solarize import RandomSolarize diff --git a/kornia/augmentation/_2d/intensity/auto_contrast.py b/kornia/augmentation/_2d/intensity/auto_contrast.py new file mode 100644 index 0000000..0b03cb9 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/auto_contrast.py @@ -0,0 +1,63 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional + +import torch + +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.enhance import normalize_min_max + + +class RandomAutoContrast(IntensityAugmentationBase2D): + r"""Apply a random auto-contrast of a torch.Tensor image. + + Args: + p: probability of applying the transformation. + clip_output: if true clip output + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.enhance.normalize_min_max` + + """ + + def __init__( + self, clip_output: bool = True, same_on_batch: bool = False, p: float = 1.0, keepdim: bool = False + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + + self.clip_output = clip_output + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + out = normalize_min_max(input) + + if self.clip_output: + return out.clamp(0.0, 1.0) + + return out diff --git a/kornia/augmentation/_2d/intensity/base.py b/kornia/augmentation/_2d/intensity/base.py new file mode 100644 index 0000000..ec6ff2c --- /dev/null +++ b/kornia/augmentation/_2d/intensity/base.py @@ -0,0 +1,88 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional + +from torch import Tensor + +from kornia.augmentation._2d.base import RigidAffineAugmentationBase2D +from kornia.geometry.boxes import Boxes +from kornia.geometry.keypoints import Keypoints + + +class IntensityAugmentationBase2D(RigidAffineAugmentationBase2D): + r"""IntensityAugmentationBase2D base class for customized intensity augmentation implementations. + + Args: + p: probability for applying an augmentation. This param controls the augmentation probabilities + element-wise for a batch. + p_batch: probability for applying an augmentation to a batch. This param controls the augmentation + probabilities batch-wise. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input ``True`` or broadcast it + to the batch form ``False``. + + """ + + def compute_transformation(self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any]) -> Tensor: + return self.identity_matrix(input) + + def apply_non_transform( + self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Tensor: + # For the images where batch_prob == False. + return input + + def apply_non_transform_mask( + self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Tensor: + return input + + def apply_transform_mask( + self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Tensor: + return input + + def apply_non_transform_boxes( + self, input: Boxes, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Boxes: + return input + + def apply_transform_boxes( + self, input: Boxes, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Boxes: + return input + + def apply_non_transform_keypoint( + self, input: Keypoints, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Keypoints: + return input + + def apply_transform_keypoint( + self, input: Keypoints, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Keypoints: + return input + + def apply_non_transform_class( + self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Tensor: + return input + + def apply_transform_class( + self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Tensor: + return input diff --git a/kornia/augmentation/_2d/intensity/box_blur.py b/kornia/augmentation/_2d/intensity/box_blur.py new file mode 100644 index 0000000..4365612 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/box_blur.py @@ -0,0 +1,72 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple + +from torch import Tensor + +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.filters import box_blur + + +class RandomBoxBlur(IntensityAugmentationBase2D): + """Add random blur with a box filter to an image tensor. + + .. image:: _static/img/RandomBoxBlur.png + + Args: + kernel_size: the blurring kernel size. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``constant``, ``reflect``, ``replicate`` or ``circular``. + normalized: if True, L1 norm of the kernel is set to 1. + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + .. note:: + This function internally uses :func:`kornia.filters.box_blur`. + + Examples: + >>> img = torch.ones(1, 1, 24, 24) + >>> out = RandomBoxBlur((7, 7))(img) + >>> out.shape + torch.Size([1, 1, 24, 24]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomBoxBlur((7, 7), p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + kernel_size: Tuple[int, int] = (3, 3), + border_type: str = "reflect", + normalized: bool = True, + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + self.flags = {"kernel_size": kernel_size, "border_type": border_type, "normalized": normalized} + + def apply_transform( + self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Tensor: + return box_blur(input, flags["kernel_size"], flags["border_type"], flags["normalized"]) diff --git a/kornia/augmentation/_2d/intensity/brightness.py b/kornia/augmentation/_2d/intensity/brightness.py new file mode 100644 index 0000000..d13c58a --- /dev/null +++ b/kornia/augmentation/_2d/intensity/brightness.py @@ -0,0 +1,98 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.augmentation.utils import _range_bound +from kornia.enhance.adjust import adjust_brightness + + +class RandomBrightness(IntensityAugmentationBase2D): + r"""Apply a random transformation to the brightness of a torch.Tensor image. + + This implementation aligns PIL. Hence, the output is close to TorchVision. + + .. image:: _static/img/RandomBrightness.png + + Args: + brightness: the brightness factor to apply + clip_output: if true clip output + silence_instantiation_warning: if True, silence the warning at instantiation. + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.enhance.adjust_brightness` + + Examples: + >>> rng = torch.manual_seed(0) + >>> inputs = torch.rand(1, 3, 3, 3) + >>> aug = RandomBrightness(brightness = (0.5,2.),p=1.) + >>> aug(inputs) + tensor([[[[0.0505, 0.3225, 0.0000], + [0.0000, 0.0000, 0.1883], + [0.0443, 0.4507, 0.0099]], + + [[0.1866, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000], + [0.0728, 0.2519, 0.3543]], + + [[0.0000, 0.0000, 0.2359], + [0.4694, 0.0000, 0.4284], + [0.0000, 0.1072, 0.5070]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + + >>> input = torch.rand(1, 3, 32, 32) + >>> aug = RandomBrightness((0.8,1.2), p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + brightness: Tuple[float, float] = (1.0, 1.0), + clip_output: bool = True, + same_on_batch: bool = False, + p: float = 1.0, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self.brightness: torch.Tensor = _range_bound(brightness, "brightness", center=1.0, bounds=(0.0, 2.0)) + self._param_generator = rg.PlainUniformGenerator((self.brightness, "brightness_factor", None, None)) + + self.clip_output = clip_output + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + brightness_factor = params["brightness_factor"].to(input) + return adjust_brightness(input, brightness_factor - 1, self.clip_output) diff --git a/kornia/augmentation/_2d/intensity/channel_dropout.py b/kornia/augmentation/_2d/intensity/channel_dropout.py new file mode 100644 index 0000000..5a83a5a --- /dev/null +++ b/kornia/augmentation/_2d/intensity/channel_dropout.py @@ -0,0 +1,121 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional + +import torch + +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.augmentation.random_generator._2d import ChannelDropoutGenerator +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SHAPE, KORNIA_CHECK_TYPE + + +class RandomChannelDropout(IntensityAugmentationBase2D): + r"""Apply random channel dropout to a batch of images. + + .. image:: _static/img/RandomChannelDropout.png + + Args: + num_drop_channels: Number of channels to drop randomly. Default is 1. + fill_value: Value to fill the dropped channels with. Default is 0.0. + same_on_batch: Apply the same transformation across the batch. Defaults to False. + p: Probability of applying the transformation. Defaults to 0.5. + keepdim: Whether to keep the output shape the same as input ``True`` or broadcast it + to the batch form ``False``. Defaults to False. + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)` + - Output: :math:`(C, H, W)` or :math:`(B, C, H, W)` + + .. note:: + If `num_drop_channels` is set to 1, it means that for each image in the batch, + we will randomly choose one channel to drop. + If `num_drop_channels` is set to 2, it means that for each image in the batch, + we will randomly choose two channels to drop. + If num_drop_channels is set to 3, it means that for each image in the batch, + we will randomly choose three channels to drop (all image). + + Examples: + >>> rng = torch.manual_seed(1) + >>> img = torch.ones(1, 3, 3, 3) + >>> aug = RandomChannelDropout(num_drop_channels=1, fill_value=0.0, p=1.0) + >>> aug(img) + tensor([[[[1., 1., 1.], + [1., 1., 1.], + [1., 1., 1.]], + + [[0., 0., 0.], + [0., 0., 0.], + [0., 0., 0.]], + + [[1., 1., 1.], + [1., 1., 1.], + [1., 1., 1.]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.rand(1, 3, 32, 32) + >>> aug = RandomChannelDropout(num_drop_channels=1, fill_value=0.0, p=1.0) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + num_drop_channels: int = 1, + fill_value: float = 0.0, + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + + KORNIA_CHECK_TYPE(fill_value, float, f"`fill_value` must be a float. Got: {type(fill_value)}") + KORNIA_CHECK( + 0.0 <= fill_value <= 1.0, + f"Invalid value in `fill_value`. Must be a float between 0 and 1. Got: {fill_value}", + ) + self.fill_value = torch.tensor(fill_value) + + KORNIA_CHECK_TYPE(num_drop_channels, int, f"`num_drop_channels` must be an int. Got: {type(num_drop_channels)}") + KORNIA_CHECK( + num_drop_channels >= 1, + f"Invalid value in `num_drop_channels`. Must be an int greater than 1. Got: {num_drop_channels}", + ) + self.num_drop_channels = num_drop_channels + # Generator of random parameters. + self._param_generator = ChannelDropoutGenerator(self.num_drop_channels) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + KORNIA_CHECK( + self.num_drop_channels <= input.shape[1], + "Invalid value in `num_drop_channels`. Cannot be greater than the number of channels of `input`.", + ) + + out = input.clone() + out[params["batch_idx"], params["channel_idx"], ...] = self.fill_value.to( + device=input.device, dtype=input.dtype + ) + + return out diff --git a/kornia/augmentation/_2d/intensity/channel_shuffle.py b/kornia/augmentation/_2d/intensity/channel_shuffle.py new file mode 100644 index 0000000..31a4194 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/channel_shuffle.py @@ -0,0 +1,69 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D + + +class RandomChannelShuffle(IntensityAugmentationBase2D): + r"""Shuffle the channels of a batch of multi-dimensional images. + + .. image:: _static/img/RandomChannelShuffle.png + + Args: + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input ``True`` or broadcast it + to the batch form ``False``. + + Examples: + >>> rng = torch.manual_seed(0) + >>> img = torch.arange(1*2*2*2.).view(1,2,2,2) + >>> RandomChannelShuffle()(img) + tensor([[[[4., 5.], + [6., 7.]], + + [[0., 1.], + [2., 3.]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomChannelShuffle(p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__(self, same_on_batch: bool = False, p: float = 0.5, keepdim: bool = False) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + self._param_generator = rg.ChannelShuffleGenerator() + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + out = torch.empty_like(input) + for i in range(out.shape[0]): + out[i] = input[i, params["channels"][i]] + return out diff --git a/kornia/augmentation/_2d/intensity/clahe.py b/kornia/augmentation/_2d/intensity/clahe.py new file mode 100644 index 0000000..69efe2c --- /dev/null +++ b/kornia/augmentation/_2d/intensity/clahe.py @@ -0,0 +1,88 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Any, Optional + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.enhance import equalize_clahe + + +class RandomClahe(IntensityAugmentationBase2D): + r"""Apply CLAHE equalization on the input torch.Tensor randomly. + + .. image:: _static/img/equalize_clahe.png + + Args: + clip_limit: threshold value for contrast limiting. If 0 clipping is disabled. + grid_size: number of tiles to be cropped in each direction (GH, GW). + slow_and_differentiable: flag to select implementation + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + .. note:: + This function internally uses :func:`kornia.enhance.equalize_clahe`. + + Examples: + >>> img = torch.rand(1, 10, 20) + >>> aug = RandomClahe() + >>> res = aug(img) + >>> res.shape + torch.Size([1, 1, 10, 20]) + + >>> img = torch.rand(2, 3, 10, 20) + >>> aug = RandomClahe() + >>> res = aug(img) + >>> res.shape + torch.Size([2, 3, 10, 20]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.rand(1, 3, 32, 32) + >>> aug = RandomClahe(p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + clip_limit: tuple[float, float] = (40.0, 40.0), + grid_size: tuple[int, int] = (8, 8), + slow_and_differentiable: bool = False, + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + self.clip_limit = clip_limit + self._param_generator = rg.PlainUniformGenerator((self.clip_limit, "clip_limit_factor", None, None)) + self.flags = {"grid_size": grid_size, "slow_and_differentiable": slow_and_differentiable} + + def apply_transform( + self, + input: torch.Tensor, + params: dict[str, torch.Tensor], + flags: dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + clip_limit = float(params["clip_limit_factor"][0]) + return equalize_clahe(input, clip_limit, flags["grid_size"], flags["slow_and_differentiable"]) diff --git a/kornia/augmentation/_2d/intensity/color_jiggle.py b/kornia/augmentation/_2d/intensity/color_jiggle.py new file mode 100644 index 0000000..816b322 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/color_jiggle.py @@ -0,0 +1,120 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.constants import pi +from kornia.enhance import adjust_brightness, adjust_contrast, adjust_hue, adjust_saturation + + +class ColorJiggle(IntensityAugmentationBase2D): + r"""Apply a random transformation to the brightness, contrast, saturation and hue of a torch.Tensor image. + + .. image:: _static/img/ColorJiggle.png + + Args: + p: probability of applying the transformation. + brightness: The brightness factor to apply. + contrast: The contrast factor to apply. + saturation: The saturation factor to apply. + hue: The hue factor to apply. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.enhance.adjust_brightness`, + :func:`kornia.enhance.adjust_contrast`. :func:`kornia.enhance.adjust_saturation`, + :func:`kornia.enhance.adjust_hue`. + + Examples: + >>> rng = torch.manual_seed(0) + >>> inputs = torch.ones(1, 3, 3, 3) + >>> aug = ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.) + >>> aug(inputs) + tensor([[[[0.9993, 0.9993, 0.9993], + [0.9993, 0.9993, 0.9993], + [0.9993, 0.9993, 0.9993]], + + [[0.9993, 0.9993, 0.9993], + [0.9993, 0.9993, 0.9993], + [0.9993, 0.9993, 0.9993]], + + [[0.9993, 0.9993, 0.9993], + [0.9993, 0.9993, 0.9993], + [0.9993, 0.9993, 0.9993]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + brightness: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 0.0, + contrast: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 0.0, + saturation: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 0.0, + hue: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 0.0, + same_on_batch: bool = False, + p: float = 1.0, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self.brightness = brightness + self.contrast = contrast + self.saturation = saturation + self.hue = hue + self._param_generator = rg.ColorJiggleGenerator(brightness, contrast, saturation, hue) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + transforms = [ + lambda img: ( + adjust_brightness(img, params["brightness_factor"] - 1) + if (params["brightness_factor"] - 1 != 0).any() + else img + ), + lambda img: ( + adjust_contrast(img, params["contrast_factor"]) if (params["contrast_factor"] != 1).any() else img + ), + lambda img: ( + adjust_saturation(img, params["saturation_factor"]) if (params["saturation_factor"] != 1).any() else img + ), + lambda img: adjust_hue(img, params["hue_factor"] * 2 * pi) if (params["hue_factor"] != 0).any() else img, + ] + + jittered = input + for idx in params["order"].tolist(): + t = transforms[idx] + jittered = t(jittered) + + return jittered diff --git a/kornia/augmentation/_2d/intensity/color_jitter.py b/kornia/augmentation/_2d/intensity/color_jitter.py new file mode 100644 index 0000000..bae5b22 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/color_jitter.py @@ -0,0 +1,189 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.constants import pi +from kornia.enhance import ( + adjust_brightness_accumulative, + adjust_contrast_with_mean_subtraction, + adjust_hue, + adjust_saturation_with_gray_subtraction, +) + + +class ColorJitter(IntensityAugmentationBase2D): + r"""Apply a random transformation to the brightness, contrast, saturation and hue of a torch.Tensor image. + + This implementation aligns PIL. Hence, the output is close to TorchVision. However, it does not + follow the color theory and is not be actively maintained. Prefer using + :func:`kornia.augmentation.ColorJiggle` + + .. image:: _static/img/ColorJitter.png + + Args: + brightness: The brightness factor to apply. + contrast: The contrast factor to apply. + saturation: The saturation factor to apply. + hue: The hue factor to apply. + silence_instantiation_warning: if True, silence the warning at instantiation. + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.enhance.adjust_brightness_accumulative`, + :func:`kornia.enhance.adjust_contrast_with_mean_subtraction`, + :func:`kornia.enhance.adjust_saturation_with_gray_subtraction`, + :func:`kornia.enhance.adjust_hue`. + + Examples: + >>> rng = torch.manual_seed(0) + >>> inputs = torch.ones(1, 3, 3, 3) + >>> aug = ColorJitter(0.1, 0.1, 0.1, 0.1, p=1.) + >>> aug(inputs) + tensor([[[[0.9993, 0.9993, 0.9993], + [0.9993, 0.9993, 0.9993], + [0.9993, 0.9993, 0.9993]], + + [[0.9993, 0.9993, 0.9993], + [0.9993, 0.9993, 0.9993], + [0.9993, 0.9993, 0.9993]], + + [[0.9993, 0.9993, 0.9993], + [0.9993, 0.9993, 0.9993], + [0.9993, 0.9993, 0.9993]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = ColorJitter(0.1, 0.1, 0.1, 0.1, p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + brightness: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 0.0, + contrast: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 0.0, + saturation: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 0.0, + hue: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 0.0, + same_on_batch: bool = False, + p: float = 1.0, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + + self.brightness = brightness + self.contrast = contrast + self.saturation = saturation + self.hue = hue + self._param_generator = rg.ColorJitterGenerator(brightness, contrast, saturation, hue) + + # native functions + self._brightness_fn = adjust_brightness_accumulative + self._contrast_fn = adjust_contrast_with_mean_subtraction + self._saturation_fn = adjust_saturation_with_gray_subtraction + self._hue_fn = adjust_hue + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + transforms = [ + lambda img: ( + self._brightness_fn(img, params["brightness_factor"]) + if (params["brightness_factor"] != 0).any() + else img + ), + lambda img: ( + self._contrast_fn(img, params["contrast_factor"]) if (params["contrast_factor"] != 1).any() else img + ), + lambda img: ( + self._saturation_fn(img, params["saturation_factor"]) + if (params["saturation_factor"] != 1).any() + else img + ), + lambda img: self._hue_fn(img, params["hue_factor"] * 2 * pi) if (params["hue_factor"] != 0).any() else img, + ] + + jittered = input + for idx in params["order"]: + t = transforms[idx] + jittered = t(jittered) + + return jittered + + def compile( + self, + *, + fullgraph: bool = False, + dynamic: bool = False, + backend: str = "inductor", + mode: Optional[str] = None, + options: Optional[Dict[Any, Any]] = None, + disable: bool = False, + ) -> "ColorJitter": + self.brightness_fn = torch.compile( + self._brightness_fn, + fullgraph=fullgraph, + dynamic=dynamic, + backend=backend, + mode=mode, + options=options, + disable=disable, + ) + self.contrast_fn = torch.compile( + self._contrast_fn, + fullgraph=fullgraph, + dynamic=dynamic, + backend=backend, + mode=mode, + options=options, + disable=disable, + ) + self.saturation_fn = torch.compile( + self._saturation_fn, + fullgraph=fullgraph, + dynamic=dynamic, + backend=backend, + mode=mode, + options=options, + disable=disable, + ) + self.hue_fn = torch.compile( + self._hue_fn, + fullgraph=fullgraph, + dynamic=dynamic, + backend=backend, + mode=mode, + options=options, + disable=disable, + ) + return self diff --git a/kornia/augmentation/_2d/intensity/contrast.py b/kornia/augmentation/_2d/intensity/contrast.py new file mode 100644 index 0000000..6020f33 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/contrast.py @@ -0,0 +1,97 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.augmentation.utils import _range_bound +from kornia.enhance.adjust import adjust_contrast + + +class RandomContrast(IntensityAugmentationBase2D): + r"""Apply a random transformation to the contrast of a torch.Tensor image. + + This implementation aligns PIL. Hence, the output is close to TorchVision. + + .. image:: _static/img/RandomContrast.png + + Args: + contrast: the contrast factor to apply. + clip_output: if true clip output. + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.enhance.adjust_contrast` + + Examples: + >>> rng = torch.manual_seed(0) + >>> inputs = torch.rand(1, 3, 3, 3) + >>> aug = RandomContrast(contrast = (0.5, 2.), p = 1.) + >>> aug(inputs) + tensor([[[[0.2750, 0.4258, 0.0490], + [0.0732, 0.1704, 0.3514], + [0.2716, 0.4969, 0.2525]], + + [[0.3505, 0.1934, 0.2227], + [0.0124, 0.0936, 0.1629], + [0.2874, 0.3867, 0.4434]], + + [[0.0893, 0.1564, 0.3778], + [0.5072, 0.2201, 0.4845], + [0.2325, 0.3064, 0.5281]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + + >>> input = torch.rand(1, 3, 32, 32) + >>> aug = RandomContrast((0.8,1.2), p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + contrast: Tuple[float, float] = (1.0, 1.0), + clip_output: bool = True, + same_on_batch: bool = False, + p: float = 1.0, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self.contrast: torch.Tensor = _range_bound(contrast, "contrast", center=1.0) + self._param_generator = rg.PlainUniformGenerator((self.contrast, "contrast_factor", None, None)) + + self.clip_output = clip_output + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + contrast_factor = params["contrast_factor"].to(input) + return adjust_contrast(input, contrast_factor, self.clip_output) diff --git a/kornia/augmentation/_2d/intensity/denormalize.py b/kornia/augmentation/_2d/intensity/denormalize.py new file mode 100644 index 0000000..ffc03d3 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/denormalize.py @@ -0,0 +1,83 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +from torch import Tensor + +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.enhance import denormalize + + +class Denormalize(IntensityAugmentationBase2D): + r"""Denormalize tensor images with mean and standard deviation. + + .. math:: + \text{input[channel] = (input[channel] * std[channel]) + mean[channel]} + + Where `mean` is :math:`(M_1, ..., M_n)` and `std` :math:`(S_1, ..., S_n)` for `n` channels, + + Args: + mean: Mean for each channel. + std: Standard deviations for each channel. + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Return: + Denormalised tensor with same size as input :math:`(*, C, H, W)`. + + .. note:: + This function internally uses :func:`kornia.enhance.denormalize`. + + Examples: + >>> norm = Denormalize(mean=torch.zeros(1, 4), std=torch.ones(1, 4)) + >>> x = torch.rand(1, 4, 3, 3) + >>> out = norm(x) + >>> out.shape + torch.Size([1, 4, 3, 3]) + + """ + + def __init__( + self, + mean: Union[Tensor, Tuple[float], List[float], float], + std: Union[Tensor, Tuple[float], List[float], float], + p: float = 1.0, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=True, keepdim=keepdim) + if isinstance(mean, float): + mean = torch.tensor([mean]) + + if isinstance(std, float): + std = torch.tensor([std]) + + if isinstance(mean, (tuple, list)): + mean = torch.tensor(mean) + + if isinstance(std, (tuple, list)): + std = torch.tensor(std) + + self.flags = {"mean": mean, "std": std} + + def apply_transform( + self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Tensor: + return denormalize(input, flags["mean"], flags["std"]) diff --git a/kornia/augmentation/_2d/intensity/dissolving.py b/kornia/augmentation/_2d/intensity/dissolving.py new file mode 100644 index 0000000..67104b7 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/dissolving.py @@ -0,0 +1,80 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.filters import StableDiffusionDissolving + + +class RandomDissolving(IntensityAugmentationBase2D): + r"""Perform dissolving transformation using StableDiffusion models. + + Based on :cite:`shi2024dissolving`, the dissolving transformation is essentially applying one-step + reverse diffusion. Our implementation currently supports HuggingFace implementations of SD 1.4, 1.5 + and 2.1. SD 1.X tends to remove more details than SD2.1. + + .. list-table:: Title + :widths: 32 32 32 + :header-rows: 1 + + * - SD 1.4 + - SD 1.5 + - SD xl + * - figure:: https://raw.githubusercontent.com/kornia/data/main/dslv-sd-1.4.png + - figure:: https://raw.githubusercontent.com/kornia/data/main/dslv-sd-1.5.png + - figure:: https://raw.githubusercontent.com/kornia/data/main/dslv-sd-2.1.png + + Args: + p: probability of applying the transformation. + version: the version of the stable diffusion model. + step_range: the step range of the diffusion model steps. Higher the step, stronger + the dissolving effects. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + **kwargs: additional arguments for `.from_pretrained` for HF StableDiffusionPipeline. + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`. + - Output: :math:`(B, C, H, W)` + + """ + + def __init__( + self, + step_range: Tuple[float, float] = (100, 500), + version: str = "1.5", + p: float = 0.5, + keepdim: bool = False, + **kwargs: Any, + ) -> None: + super().__init__(p=p, same_on_batch=True, keepdim=keepdim) + self.step_range = step_range + self._dslv = StableDiffusionDissolving(version, **kwargs) + self._param_generator = rg.PlainUniformGenerator((self.step_range, "step_range_factor", None, None)) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return self._dslv(input, params["step_range_factor"][0].long().item()) diff --git a/kornia/augmentation/_2d/intensity/equalize.py b/kornia/augmentation/_2d/intensity/equalize.py new file mode 100644 index 0000000..5c22af4 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/equalize.py @@ -0,0 +1,69 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional + +from torch import Tensor + +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.enhance import equalize + + +class RandomEqualize(IntensityAugmentationBase2D): + r"""Equalize given tensor image or a batch of tensor images randomly. + + .. image:: _static/img/RandomEqualize.png + + Args: + p: Probability to equalize an image. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.enhance.equalize`. + + Examples: + >>> rng = torch.manual_seed(0) + >>> input = torch.rand(1, 1, 5, 5) + >>> equalize = RandomEqualize(p=1.) + >>> equalize(input) + tensor([[[[0.4963, 0.7682, 0.0885, 0.1320, 0.3074], + [0.6341, 0.4901, 0.8964, 0.4556, 0.6323], + [0.3489, 0.4017, 0.0223, 0.1689, 0.2939], + [0.5185, 0.6977, 0.8000, 0.1610, 0.2823], + [0.6816, 0.9152, 0.3971, 0.8742, 0.4194]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.rand(1, 3, 32, 32) + >>> aug = RandomEqualize(p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__(self, same_on_batch: bool = False, p: float = 0.5, keepdim: bool = False) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + + def apply_transform( + self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Tensor: + return equalize(input) diff --git a/kornia/augmentation/_2d/intensity/erasing.py b/kornia/augmentation/_2d/intensity/erasing.py new file mode 100644 index 0000000..f55be6c --- /dev/null +++ b/kornia/augmentation/_2d/intensity/erasing.py @@ -0,0 +1,122 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.geometry.bbox import bbox_generator, bbox_to_mask + + +class RandomErasing(IntensityAugmentationBase2D): + r"""Erase a random rectangle of a torch.Tensor image according to a probability p value. + + .. image:: _static/img/RandomErasing.png + + The operator removes image parts and fills them with zero values at a selected rectangle + for each of the images in the batch. + + The rectangle will have an area equal to the original image area multiplied by a value uniformly + sampled between the range [scale[0], scale[1]) and an aspect ratio sampled + between [ratio[0], ratio[1]) + + Args: + scale: range of proportion of erased area against input image. + ratio: range of aspect ratio of erased area. + value: the value to fill the erased area. + same_on_batch: apply the same transformation across the batch. + p: probability that the random erasing operation will be performed. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + Note: + Input torch.Tensor must be float and normalized into [0, 1] for the best differentiability support. + Additionally, this function accepts another transformation torch.Tensor (:math:`(B, 3, 3)`), then the + applied transformation will be merged int to the input transformation torch.Tensor and returned. + + Examples: + >>> rng = torch.manual_seed(0) + >>> inputs = torch.ones(1, 1, 3, 3) + >>> aug = RandomErasing((.4, .8), (.3, 1/.3), p=0.5) + >>> aug(inputs) + tensor([[[[1., 0., 0.], + [1., 0., 0.], + [1., 0., 0.]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomErasing((.4, .8), (.3, 1/.3), p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + scale: Union[torch.Tensor, Tuple[float, float]] = (0.02, 0.33), + ratio: Union[torch.Tensor, Tuple[float, float]] = (0.3, 3.3), + value: float = 0.0, + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self.scale = scale + self.ratio = ratio + self.value = value + self._param_generator = rg.RectangleEraseGenerator(scale, ratio, value) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + _, c, h, w = input.size() + values = params["values"].unsqueeze(-1).unsqueeze(-1).unsqueeze(-1).repeat(1, *input.shape[1:]).to(input) + + bboxes = bbox_generator(params["xs"], params["ys"], params["widths"], params["heights"]) + mask = bbox_to_mask(bboxes, w, h) # Returns B, H, W + mask = mask.unsqueeze(1).repeat(1, c, 1, 1).to(input) # Transform to B, c, H, W + transformed = torch.where(mask == 1.0, values, input) + return transformed + + def apply_transform_mask( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + _, c, h, w = input.size() + + values = params["values"][..., None, None, None].repeat(1, *input.shape[1:]).to(input) + # Erase the corresponding areas on masks. + values = values.zero_() + + bboxes = bbox_generator(params["xs"], params["ys"], params["widths"], params["heights"]) + mask = bbox_to_mask(bboxes, w, h) # Returns B, H, W + mask = mask.unsqueeze(1).repeat(1, c, 1, 1).to(input) # Transform to B, c, H, W + transformed = torch.where(mask == 1.0, values, input) + return transformed diff --git a/kornia/augmentation/_2d/intensity/gamma.py b/kornia/augmentation/_2d/intensity/gamma.py new file mode 100644 index 0000000..46b9299 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/gamma.py @@ -0,0 +1,95 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.enhance.adjust import adjust_gamma + + +class RandomGamma(IntensityAugmentationBase2D): + r"""Apply a random transformation to the gamma of a torch.Tensor image. + + This implementation aligns PIL. Hence, the output is close to TorchVision. + + .. image:: _static/img/RandomGamma.png + + Args: + p: probability of applying the transformation. + gamma: the gamma factor to apply. + gain: the gain factor to apply. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.enhance.adjust_gamma` + + Examples: + >>> rng = torch.manual_seed(0) + >>> inputs = torch.rand(1, 3, 3, 3) + >>> aug = RandomGamma((0.5,2.),(1.5,1.5),p=1.) + >>> aug(inputs) + tensor([[[[1.0000, 1.0000, 0.3912], + [0.4883, 0.7801, 1.0000], + [1.0000, 1.0000, 0.9702]], + + [[1.0000, 0.8368, 0.9048], + [0.1824, 0.5597, 0.7609], + [1.0000, 1.0000, 1.0000]], + + [[0.5452, 0.7441, 1.0000], + [1.0000, 0.8990, 1.0000], + [0.9267, 1.0000, 1.0000]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.rand(1, 3, 32, 32) + >>> aug = RandomGamma((0.8,1.2), p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + gamma: Tuple[float, float] = (1.0, 1.0), + gain: Tuple[float, float] = (1.0, 1.0), + same_on_batch: bool = False, + p: float = 1.0, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self._param_generator = rg.PlainUniformGenerator( + (gamma, "gamma_factor", None, None), (gain, "gain_factor", None, None) + ) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + gamma_factor = params["gamma_factor"].to(input) + gain_factor = params["gain_factor"].to(input) + return adjust_gamma(input, gamma_factor, gain_factor) diff --git a/kornia/augmentation/_2d/intensity/gaussian_blur.py b/kornia/augmentation/_2d/intensity/gaussian_blur.py new file mode 100644 index 0000000..3bc6eaf --- /dev/null +++ b/kornia/augmentation/_2d/intensity/gaussian_blur.py @@ -0,0 +1,130 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, Union + +import torch +from torch import Tensor + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.constants import BorderType +from kornia.filters import gaussian_blur2d + + +class RandomGaussianBlur(IntensityAugmentationBase2D): + r"""Apply gaussian blur given tensor image or a batch of tensor images randomly. + + The standard deviation is sampled for each instance. + + .. image:: _static/img/RandomGaussianBlur.png + + Args: + kernel_size: the size of the kernel. + sigma: the range for the standard deviation of the kernel. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``constant``, ``reflect``, ``replicate`` or ``circular``. + separable: run as composition of two 1d-convolutions. + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + silence_instantiation_warning: if True, silence the warning at instantiation. + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.filters.gaussian_blur2d`. + + Examples: + >>> rng = torch.manual_seed(0) + >>> input = torch.rand(1, 1, 5, 5) + >>> blur = RandomGaussianBlur((3, 3), (0.1, 2.0), p=1.) + >>> blur(input) + tensor([[[[0.5941, 0.5833, 0.5022, 0.4384, 0.3934], + [0.5310, 0.4964, 0.4113, 0.3637, 0.3472], + [0.4991, 0.4997, 0.4312, 0.3620, 0.3081], + [0.6082, 0.5667, 0.4954, 0.3825, 0.3508], + [0.7042, 0.6849, 0.6275, 0.4753, 0.4105]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomGaussianBlur((3, 3), (0.1, 2.0), p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + kernel_size: Union[Tuple[int, int], int], + sigma: Union[Tuple[float, float], Tensor], + border_type: str = "reflect", + separable: bool = True, + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + + self.flags = { + "kernel_size": kernel_size, + "separable": separable, + "border_type": BorderType.get(border_type), + } + self._param_generator = rg.RandomGaussianBlurGenerator(sigma) + + self._gaussian_blur2d_fn = gaussian_blur2d + + def apply_transform( + self, + input: Tensor, + params: Dict[str, Tensor], + flags: Dict[str, Any], + transform: Optional[Tensor] = None, + ) -> Tensor: + sigma = params["sigma"].unsqueeze(-1).expand(-1, 2) + return self._gaussian_blur2d_fn( + input, + kernel_size=self.flags["kernel_size"], + sigma=sigma, + border_type=self.flags["border_type"].name.lower(), + separable=self.flags["separable"], + ) + + def compile( + self, + *, + fullgraph: bool = False, + dynamic: bool = False, + backend: str = "inductor", + mode: Optional[str] = None, + options: Optional[Dict[Any, Any]] = None, + disable: bool = False, + ) -> "RandomGaussianBlur": + self._gaussian_blur2d_fn = torch.compile( + self._gaussian_blur2d_fn, + fullgraph=fullgraph, + dynamic=dynamic, + backend=backend, + mode=mode, + options=options, + disable=disable, + ) + return self diff --git a/kornia/augmentation/_2d/intensity/gaussian_illumination.py b/kornia/augmentation/_2d/intensity/gaussian_illumination.py new file mode 100644 index 0000000..84a28c9 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/gaussian_illumination.py @@ -0,0 +1,207 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.augmentation.random_generator._2d import GaussianIlluminationGenerator +from kornia.core.check import KORNIA_CHECK + + +class RandomGaussianIllumination(IntensityAugmentationBase2D): + r"""Applies random 2D Gaussian illumination patterns to a batch of images. + + .. image:: _static/img/RandomGaussianIllumination.png + + Args: + gain: Range for the gain factor (intensity) applied to the generated illumination. + center: The center coordinates of the Gaussian distribution are expressed as a + percentage of the spatial dimensions :math:(H, W). + sigma: The sigma values (standard deviation) of the Gaussian distribution are expressed as a + percentage of the spatial dimensions :math:(H, W). + sign: Range for the sign of the Gaussian distribution. If only one sign is needed, + insert only as a tuple or float. + p: Probability of applying the transformation. + same_on_batch: If True, apply the same transformation across the entire batch. Default is False. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + .. note:: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + Examples: + >>> rng = torch.manual_seed(1) + >>> input = torch.ones(1, 3, 3, 3) * 0.5 + >>> aug = RandomGaussianIllumination(gain=0.5, p=1.) + >>> aug(input) + tensor([[[[0.7266, 1.0000, 0.7266], + [0.6621, 0.9121, 0.6621], + [0.5000, 0.6911, 0.5000]], + + [[0.7266, 1.0000, 0.7266], + [0.6621, 0.9121, 0.6621], + [0.5000, 0.6911, 0.5000]], + + [[0.7266, 1.0000, 0.7266], + [0.6621, 0.9121, 0.6621], + [0.5000, 0.6911, 0.5000]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.rand(1, 3, 32, 32) + >>> aug = RandomGaussianIllumination(p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + gain: Optional[Union[float, Tuple[float, float]]] = (0.01, 0.15), + center: Optional[Union[float, Tuple[float, float]]] = (0.1, 0.9), + sigma: Optional[Union[float, Tuple[float, float]]] = (0.2, 1.0), + sign: Optional[Union[float, Tuple[float, float]]] = (-1.0, 1.0), + p: float = 0.5, + same_on_batch: bool = False, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + + # Validation and initialization of amount parameter. + if isinstance(gain, (tuple, float)): + if isinstance(gain, float): + gain = (gain, gain) + elif len(gain) == 1: + gain = (gain[0], gain[0]) + elif len(gain) > 2 or len(gain) <= 0: + raise ValueError( + "The length of gain must be greater than 0 \ + and less than or equal to 2, and it should be a tuple or a float." + ) + else: + raise ValueError("gain must be a tuple or a float") + KORNIA_CHECK( + all(0 <= el <= 1 for el in gain), + "gain values must be between 0 and 1. Recommended values less than 0.2.", + ) + + if isinstance(center, (tuple, float)): + if isinstance(center, float): + center = (center, center) + elif len(center) == 1: + center = (center[0], center[0]) + elif len(center) > 2 or len(center) <= 0: + raise ValueError( + "The length of center must be greater than 0 \ + and less than or equal to 2, and it should be a tuple or a float." + ) + else: + raise ValueError("center must be a tuple or a float") + KORNIA_CHECK( + all(0 <= el <= 1 for el in center), + "center of gaussian value must be between 0 and 1.", + ) + + if isinstance(sigma, (tuple, float)): + if isinstance(sigma, float): + sigma = (sigma, sigma) + elif len(sigma) == 1: + sigma = (sigma[0], sigma[0]) + elif len(sigma) > 2 or len(sigma) <= 0: + raise ValueError( + "The length of sigma must be greater than 0 \ + and less than or equal to 2, and it should be a tuple or a float." + ) + else: + raise ValueError("sigma must be a tuple or a float") + KORNIA_CHECK( + all(0 <= el <= 1 for el in sigma), + "sigma of gaussian value must be between 0 and 1.", + ) + + if isinstance(sign, (tuple, float)): + if isinstance(sign, float): + sign = (sign, sign) + elif len(sign) == 1: + sign = (sign[0], sign[0]) + elif len(sign) > 2 or len(sign) <= 0: + raise ValueError( + "The length of sign must be greater than 0 \ + and less than or equal to 2, and it should be a tuple or a float." + ) + else: + raise ValueError("sign must be a tuple or a float") + KORNIA_CHECK( + all(-1 <= el <= 1 for el in sign), + "sign of gaussian value must be between -1 and 1.", + ) + + # Generator of random parameters and masks. + self._param_generator = GaussianIlluminationGenerator(gain, center, sigma, sign) + + def _apply_transform( + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + # The gradient comes from the generator on CPU. + # We must explicitly move it to the input's device before adding. + gradient = params["gradient"].to(input.device, input.dtype) + + return input.add_(gradient).clamp_(0, 1) + + self._fn = _apply_transform + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + r"""Apply random gaussian gradient illumination to the input image.""" + return self._fn(input=input, params=params, flags=flags, transform=transform) + + def compile( + self, + *, + fullgraph: bool = False, + dynamic: bool = False, + backend: str = "inductor", + mode: Optional[str] = None, + options: Optional[Dict[Any, Any]] = None, + disable: bool = False, + ) -> "RandomGaussianIllumination": + self._fn = torch.compile( + self._fn, + fullgraph=fullgraph, + dynamic=dynamic, + backend=backend, + mode=mode, + options=options, + disable=disable, + ) + + return self diff --git a/kornia/augmentation/_2d/intensity/gaussian_noise.py b/kornia/augmentation/_2d/intensity/gaussian_noise.py new file mode 100644 index 0000000..a05bde5 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/gaussian_noise.py @@ -0,0 +1,71 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D + + +class RandomGaussianNoise(IntensityAugmentationBase2D): + r"""Add gaussian noise to a batch of multi-dimensional images. + + .. image:: _static/img/RandomGaussianNoise.png + + Args: + mean: The mean of the gaussian distribution. + std: The standard deviation of the gaussian distribution. + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Examples: + >>> rng = torch.manual_seed(0) + >>> img = torch.ones(1, 1, 2, 2) + >>> RandomGaussianNoise(mean=0., std=1., p=1.)(img) + tensor([[[[ 2.5410, 0.7066], + [-1.1788, 1.5684]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomGaussianNoise(mean=0., std=1., p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, mean: float = 0.0, std: float = 1.0, same_on_batch: bool = False, p: float = 0.5, keepdim: bool = False + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + self._param_generator = rg.GaussianNoiseGenerator(mean, std) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + gaussian_noise = params["gaussian_noise"].to(input) + # When same_on_batch=True the generator produces shape (1, C, H, W); expand to full batch. + if gaussian_noise.shape[0] != input.shape[0]: + gaussian_noise = gaussian_noise.expand_as(input) + return input + gaussian_noise diff --git a/kornia/augmentation/_2d/intensity/grayscale.py b/kornia/augmentation/_2d/intensity/grayscale.py new file mode 100644 index 0000000..8ce2b30 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/grayscale.py @@ -0,0 +1,84 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional + +import torch +from torch import Tensor + +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.color import rgb_to_grayscale + + +class RandomGrayscale(IntensityAugmentationBase2D): + r"""Apply random transformation to Grayscale according to a probability p value. + + .. image:: _static/img/RandomGrayscale.png + + Args: + rgb_weights: Weights that will be applied on each channel (RGB). + The sum of the weights should add up to one. + p: probability of the image to be transformed to grayscale. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.color.rgb_to_grayscale`. + + Examples: + >>> rng = torch.manual_seed(0) + >>> inputs = torch.randn((1, 3, 3, 3)) + >>> aug = RandomGrayscale(p=1.0) + >>> aug(inputs) + tensor([[[[-1.1344, -0.1330, 0.1517], + [-0.0791, 0.6711, -0.1413], + [-0.1717, -0.9023, 0.0819]], + + [[-1.1344, -0.1330, 0.1517], + [-0.0791, 0.6711, -0.1413], + [-0.1717, -0.9023, 0.0819]], + + [[-1.1344, -0.1330, 0.1517], + [-0.0791, 0.6711, -0.1413], + [-0.1717, -0.9023, 0.0819]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomGrayscale(p=1.0) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, rgb_weights: Optional[Tensor] = None, same_on_batch: bool = False, p: float = 0.1, keepdim: bool = False + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self.rgb_weights = rgb_weights + + def apply_transform( + self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Tensor: + # Make sure it returns (*, 3, H, W) + grayscale = torch.ones_like(input) + grayscale[:] = rgb_to_grayscale(input, rgb_weights=self.rgb_weights) + return grayscale diff --git a/kornia/augmentation/_2d/intensity/hue.py b/kornia/augmentation/_2d/intensity/hue.py new file mode 100644 index 0000000..a6262cc --- /dev/null +++ b/kornia/augmentation/_2d/intensity/hue.py @@ -0,0 +1,90 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.augmentation.utils import _range_bound +from kornia.constants import pi +from kornia.enhance.adjust import adjust_hue + + +class RandomHue(IntensityAugmentationBase2D): + r"""Apply a random transformation to the hue of a torch.Tensor image. + + This implementation aligns PIL. Hence, the output is close to TorchVision. + + .. image:: _static/img/RandomHue.png + + Args: + hue: the saturation factor to apply. + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.enhance.adjust_hue` + + Examples: + >>> rng = torch.manual_seed(0) + >>> inputs = torch.rand(1, 3, 3, 3) + >>> aug = RandomHue(hue = (-0.5,0.5),p=1.) + >>> aug(inputs) + tensor([[[[0.3993, 0.2823, 0.6816], + [0.6117, 0.2090, 0.4081], + [0.4693, 0.5529, 0.9527]], + + [[0.1610, 0.5962, 0.4971], + [0.9152, 0.3971, 0.8742], + [0.4194, 0.6771, 0.7162]], + + [[0.6323, 0.7682, 0.0885], + [0.0223, 0.1689, 0.2939], + [0.5185, 0.8964, 0.4556]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + + >>> input = torch.rand(1, 3, 32, 32) + >>> aug = RandomHue((-0.2,0.2), p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, hue: Tuple[float, float] = (0.0, 0.0), same_on_batch: bool = False, p: float = 1.0, keepdim: bool = False + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self.hue: torch.Tensor = _range_bound(hue, "hue", bounds=(-0.5, 0.5)) + self._param_generator = rg.PlainUniformGenerator((self.hue, "hue_factor", None, None)) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + hue_factor = params["hue_factor"].to(input) + return adjust_hue(input, hue_factor * 2 * pi) diff --git a/kornia/augmentation/_2d/intensity/invert.py b/kornia/augmentation/_2d/intensity/invert.py new file mode 100644 index 0000000..26d23d5 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/invert.py @@ -0,0 +1,74 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Union + +import torch +from torch import Tensor + +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.enhance import invert + + +class RandomInvert(IntensityAugmentationBase2D): + r"""Invert the tensor images values randomly. + + .. image:: _static/img/RandomInvert.png + + Args: + max_val: The expected maximum value in the input tensor. The shape has to + according to the input tensor shape, or at least has to work with broadcasting. + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + .. note:: + This function internally uses :func:`kornia.enhance.invert`. + + Examples: + >>> rng = torch.manual_seed(0) + >>> img = torch.rand(1, 1, 5, 5) + >>> inv = RandomInvert() + >>> inv(img) + tensor([[[[0.4963, 0.7682, 0.0885, 0.1320, 0.3074], + [0.6341, 0.4901, 0.8964, 0.4556, 0.6323], + [0.3489, 0.4017, 0.0223, 0.1689, 0.2939], + [0.5185, 0.6977, 0.8000, 0.1610, 0.2823], + [0.6816, 0.9152, 0.3971, 0.8742, 0.4194]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomInvert(p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + max_val: Union[float, Tensor] = 1.0, + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + self.flags = {"max_val": max_val} + + def apply_transform( + self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Tensor: + return invert(input, torch.as_tensor(flags["max_val"], device=input.device, dtype=input.dtype)) diff --git a/kornia/augmentation/_2d/intensity/jpeg.py b/kornia/augmentation/_2d/intensity/jpeg.py new file mode 100644 index 0000000..7581f4c --- /dev/null +++ b/kornia/augmentation/_2d/intensity/jpeg.py @@ -0,0 +1,80 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.enhance import jpeg_codec_differentiable + + +class RandomJPEG(IntensityAugmentationBase2D): + r"""Applies random (differentiable) JPEG coding to a torch.Tensor image. + + .. image:: _static/img/RandomJPEG.png + + Args: + jpeg_quality: The range of compression rates to be applied. + p: probability of applying the transformation. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.enhance.jpeg_codec_differentiable`. + + Examples: + >>> import torch + >>> rng = torch.manual_seed(0) + >>> images = 0.1904 * torch.ones(2, 3, 32, 32) + >>> aug = RandomJPEG(jpeg_quality=(1.0, 50.0), p=1.) + >>> images_jpeg = aug(images) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> images = 0.1904 * torch.ones(2, 3, 32, 32) + >>> aug = RandomJPEG(jpeg_quality=20.0, p=1.) # Samples a JPEG quality from the range [30.0, 70.0] + >>> (aug(images) == aug(images, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + jpeg_quality: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 50.0, + same_on_batch: bool = False, + p: float = 1.0, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self.jpeg_quality = jpeg_quality + self._param_generator = rg.JPEGGenerator(jpeg_quality) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + jpeg_output: torch.Tensor = jpeg_codec_differentiable(input, params["jpeg_quality"]) + return jpeg_output diff --git a/kornia/augmentation/_2d/intensity/linear_illumination.py b/kornia/augmentation/_2d/intensity/linear_illumination.py new file mode 100644 index 0000000..07443c7 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/linear_illumination.py @@ -0,0 +1,238 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.augmentation.random_generator._2d import LinearCornerIlluminationGenerator, LinearIlluminationGenerator +from kornia.core.check import KORNIA_CHECK + + +class RandomLinearIllumination(IntensityAugmentationBase2D): + r"""Applies random 2D Linear illumination patterns to a batch of images. + + .. image:: _static/img/RandomLinearIllumination.png + + Args: + gain: Range for the gain factor (intensity) applied to the generated illumination. + sign: Range for the sign of the distribution. If only one sign is needed, + insert only as a tuple or float. + p: Probability of applying the transformation. + same_on_batch: If True, apply the same transformation across the entire batch. Default is False. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + .. note:: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + Examples: + >>> rng = torch.manual_seed(1) + >>> input = torch.ones(1, 3, 3, 3) * 0.5 + >>> aug = RandomLinearIllumination(gain=0.25, p=1.) + >>> aug(input) + tensor([[[[0.2500, 0.2500, 0.2500], + [0.3750, 0.3750, 0.3750], + [0.5000, 0.5000, 0.5000]], + + [[0.2500, 0.2500, 0.2500], + [0.3750, 0.3750, 0.3750], + [0.5000, 0.5000, 0.5000]], + + [[0.2500, 0.2500, 0.2500], + [0.3750, 0.3750, 0.3750], + [0.5000, 0.5000, 0.5000]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.rand(1, 3, 32, 32) + >>> aug = RandomLinearIllumination(p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + gain: Optional[Union[float, Tuple[float, float]]] = (0.01, 0.2), + sign: Optional[Union[float, Tuple[float, float]]] = (-1.0, 1.0), + p: float = 0.5, + same_on_batch: bool = False, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + + # Validation and initialization of amount parameter. + if isinstance(gain, (tuple, float)): + if isinstance(gain, float): + gain = (gain, gain) + elif len(gain) == 1: + gain = (gain[0], gain[0]) + elif len(gain) > 2 or len(gain) <= 0: + raise ValueError( + "The length of gain must be greater than 0 \ + and less than or equal to 2, and it should be a tuple or a float." + ) + else: + raise ValueError("gain must be a tuple or a float") + KORNIA_CHECK( + all(0 <= el <= 1 for el in gain), + "gain values must be between 0 and 1. Recommended values less than 0.2.", + ) + + if isinstance(sign, (tuple, float)): + if isinstance(sign, float): + sign = (sign, sign) + elif len(sign) == 1: + sign = (sign[0], sign[0]) + elif len(sign) > 2 or len(sign) <= 0: + raise ValueError( + "The length of sign must be greater than 0 \ + and less than or equal to 2, and it should be a tuple or a float." + ) + else: + raise ValueError("sign must be a tuple or a float") + KORNIA_CHECK( + all(-1 <= el <= 1 for el in sign), + "sign of linear value must be between -1 and 1.", + ) + + # Generator of random parameters and masks. + self._param_generator = LinearIlluminationGenerator(gain, sign) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + r"""Apply random gaussian gradient illumination to the input image.""" + return input.add(params["gradient"].to(input)).clamp(0, 1) + + +class RandomLinearCornerIllumination(IntensityAugmentationBase2D): + r"""Applies random 2D Linear from corner illumination patterns to a batch of images. + + .. image:: _static/img/RandomLinearCornerIllumination.png + + Args: + gain: Range for the gain factor (intensity) applied to the generated illumination. + sign: Range for the sign of the distribution. If only one sign is needed, + insert only as a tuple or float. + p: Probability of applying the transformation. + same_on_batch: If True, apply the same transformation across the entire batch. Default is False. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + .. note:: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + Examples: + >>> rng = torch.manual_seed(1) + >>> input = torch.ones(1, 3, 3, 3) * 0.5 + >>> aug = RandomLinearCornerIllumination(gain=0.25, p=1.) + >>> aug(input) + tensor([[[[0.3750, 0.4375, 0.5000], + [0.3125, 0.3750, 0.4375], + [0.2500, 0.3125, 0.3750]], + + [[0.3750, 0.4375, 0.5000], + [0.3125, 0.3750, 0.4375], + [0.2500, 0.3125, 0.3750]], + + [[0.3750, 0.4375, 0.5000], + [0.3125, 0.3750, 0.4375], + [0.2500, 0.3125, 0.3750]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.rand(1, 3, 32, 32) + >>> aug = RandomLinearCornerIllumination(p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + gain: Optional[Union[float, Tuple[float, float]]] = (0.01, 0.2), + sign: Optional[Union[float, Tuple[float, float]]] = (-1.0, 1.0), + p: float = 0.5, + same_on_batch: bool = False, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + + # Validation and initialization of amount parameter. + if isinstance(gain, (tuple, float)): + if isinstance(gain, float): + gain = (gain, gain) + elif len(gain) == 1: + gain = (gain[0], gain[0]) + elif len(gain) > 2 or len(gain) <= 0: + raise ValueError( + "The length of gain must be greater than 0 \ + and less than or equal to 2, and it should be a tuple or a float." + ) + else: + raise ValueError("gain must be a tuple or a float") + KORNIA_CHECK( + all(0 <= el <= 1 for el in gain), + "gain values must be between 0 and 1. Recommended values less than 0.2.", + ) + + if isinstance(sign, (tuple, float)): + if isinstance(sign, float): + sign = (sign, sign) + elif len(sign) == 1: + sign = (sign[0], sign[0]) + elif len(sign) > 2 or len(sign) <= 0: + raise ValueError( + "The length of sign must be greater than 0 \ + and less than or equal to 2, and it should be a tuple or a float." + ) + else: + raise ValueError("sign must be a tuple or a float") + KORNIA_CHECK( + all(-1 <= el <= 1 for el in sign), + "sign of linear value must be between -1 and 1.", + ) + + # Generator of random parameters and masks. + self._param_generator = LinearCornerIlluminationGenerator(gain, sign) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + r"""Apply random gaussian gradient illumination to the input image.""" + return input.add(params["gradient"].to(input)).clamp(0, 1) diff --git a/kornia/augmentation/_2d/intensity/median_blur.py b/kornia/augmentation/_2d/intensity/median_blur.py new file mode 100644 index 0000000..5795da3 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/median_blur.py @@ -0,0 +1,68 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple + +from torch import Tensor + +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.filters import median_blur + + +class RandomMedianBlur(IntensityAugmentationBase2D): + """Add random blur with a median filter to an image tensor. + + .. image:: _static/img/RandomMedianBlur.png + + Args: + kernel_size: the blurring kernel size. + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + .. note:: + This function internally uses :func:`kornia.filters.median_blur`. + + Examples: + >>> img = torch.ones(1, 1, 4, 4) + >>> out = RandomMedianBlur((3, 3), p = 1)(img) + >>> out.shape + torch.Size([1, 1, 4, 4]) + >>> out + tensor([[[[0., 1., 1., 0.], + [1., 1., 1., 1.], + [1., 1., 1., 1.], + [0., 1., 1., 0.]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomMedianBlur((7, 7), p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, kernel_size: Tuple[int, int] = (3, 3), same_on_batch: bool = False, p: float = 0.5, keepdim: bool = False + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + self.flags = {"kernel_size": kernel_size} + + def apply_transform( + self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Tensor: + return median_blur(input, flags["kernel_size"]) diff --git a/kornia/augmentation/_2d/intensity/motion_blur.py b/kornia/augmentation/_2d/intensity/motion_blur.py new file mode 100644 index 0000000..6befbcd --- /dev/null +++ b/kornia/augmentation/_2d/intensity/motion_blur.py @@ -0,0 +1,128 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, List, Optional, Tuple, Union, cast + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.constants import BorderType, Resample +from kornia.filters import motion_blur + + +class RandomMotionBlur(IntensityAugmentationBase2D): + r"""Perform motion blur on 2D images (4D torch.Tensor). + + .. image:: _static/img/RandomMotionBlur.png + + Args: + p: probability of applying the transformation. + kernel_size: motion kernel size (odd and positive). + If int, the kernel will have a fixed size. + If Tuple[int, int], it will randomly generate the value from the range batch-wisely. + angle: angle of the motion blur in degrees (anti-clockwise rotation). + If float, it will generate the value from (-angle, angle). + direction: forward/backward direction of the motion blur. + Lower values towards -1.0 will point the motion blur towards the back (with angle provided via angle), + while higher values towards 1.0 will point the motion blur forward. A value of 0.0 leads to a + uniformly (but still angled) motion blur. + If float, it will generate the value from (-direction, direction). + If Tuple[int, int], it will randomly generate the value from the range. + border_type: the padding mode to be applied before convolving. + CONSTANT = 0, REFLECT = 1, REPLICATE = 2, CIRCULAR = 3. + resample: the interpolation mode. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + Note: + Input torch.Tensor must be float and normalized into [0, 1] for the best differentiability support. + Additionally, this function accepts another transformation torch.Tensor (:math:`(B, 3, 3)`), then the + applied transformation will be merged int to the input transformation torch.Tensor and returned. + + Please set ``resample`` to ``'bilinear'`` if more meaningful gradients wanted. + + .. note:: + This function internally uses :func:`kornia.filters.motion_blur`. + + Examples: + >>> rng = torch.manual_seed(0) + >>> input = torch.ones(1, 1, 5, 5) + >>> motion_blur = RandomMotionBlur(3, 35., 0.5, p=1.) + >>> motion_blur(input) + tensor([[[[0.5773, 1.0000, 1.0000, 1.0000, 0.7561], + [0.5773, 1.0000, 1.0000, 1.0000, 0.7561], + [0.5773, 1.0000, 1.0000, 1.0000, 0.7561], + [0.5773, 1.0000, 1.0000, 1.0000, 0.7561], + [0.5773, 1.0000, 1.0000, 1.0000, 0.7561]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomMotionBlur(3, 35., 0.5, p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + kernel_size: Union[int, Tuple[int, int]], + angle: Union[torch.Tensor, float, Tuple[float, float]], + direction: Union[torch.Tensor, float, Tuple[float, float]], + border_type: Union[int, str, BorderType] = BorderType.CONSTANT.name, + resample: Union[str, int, Resample] = Resample.NEAREST.name, + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self._param_generator = rg.MotionBlurGenerator(kernel_size, angle, direction) + self.flags = {"border_type": BorderType.get(border_type), "resample": Resample.get(resample)} + + def generate_parameters(self, batch_shape: Tuple[int, ...]) -> Dict[str, torch.Tensor]: + params = super().generate_parameters(batch_shape) + params["idx"] = torch.tensor([0]) if batch_shape[0] == 0 else torch.randint(batch_shape[0], (1,)) + return params + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + # sample a kernel size + kernel_size_list: List[int] = params["ksize_factor"].tolist() + + # 1. We have to apply the same kernel size to all samples in the batch, thus we take the previously + # selected random index --- `params["idx"][0]` --- to determine the applied kernel size. + # 2. The `VideoSequential` flattens the first two dimensions, effectively creating a larger batch. + # Its method `VideoSequential.__repeat_param_across_channels__` repeats the previously selected index, + # creating a torch.Tensor with equal values. Hence, taking the first one (`params["idx"][0]`) is legit. + idx: int = cast(int, params["idx"][0]) + return motion_blur( + input, + kernel_size=kernel_size_list[idx], + angle=params["angle_factor"], + direction=params["direction_factor"], + border_type=flags["border_type"].name.lower(), + mode=flags["resample"].name.lower(), + ) diff --git a/kornia/augmentation/_2d/intensity/normalize.py b/kornia/augmentation/_2d/intensity/normalize.py new file mode 100644 index 0000000..74c41b1 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/normalize.py @@ -0,0 +1,91 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Any, Optional + +import torch +from torch import Tensor + +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.enhance import normalize + + +class Normalize(IntensityAugmentationBase2D): + r"""Normalize tensor images with mean and standard deviation. + + .. math:: + \text{input[channel] = (input[channel] - mean[channel]) / std[channel]} + + Where `mean` is :math:`(M_1, ..., M_n)` and `std` :math:`(S_1, ..., S_n)` for `n` channels, + + Args: + mean: Mean for each channel. + std: Standard deviations for each channel. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Return: + Normalised tensor with same size as input :math:`(*, C, H, W)`. + + .. note:: + This function internally uses :func:`kornia.enhance.normalize`. + + Examples: + >>> norm = Normalize(mean=torch.zeros(4), std=torch.ones(4)) + >>> x = torch.rand(1, 4, 3, 3) + >>> out = norm(x) + >>> out.shape + torch.Size([1, 4, 3, 3]) + + """ + + def __init__( + self, + mean: Tensor | tuple[float, ...] | list[float] | float, + std: Tensor | tuple[float, ...] | list[float] | float, + p: float = 1.0, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=True, keepdim=keepdim) + if isinstance(mean, (int, float)): + mean = torch.tensor([mean]) + + if isinstance(std, (int, float)): + std = torch.tensor([std]) + + if isinstance(mean, (tuple, list)): + mean = torch.tensor(mean) + + if isinstance(std, (tuple, list)): + std = torch.tensor(std) + + self.flags = {"mean": mean, "std": std} + + def apply_transform( + self, input: Tensor, params: dict[str, Tensor], flags: dict[str, Any], transform: Optional[Tensor] = None + ) -> Tensor: + mean: Tensor = flags["mean"] + std: Tensor = flags["std"] + if torch.onnx.is_in_onnx_export(): + if mean.dim() == 1: + mean = mean.view(1, -1) + if std.dim() == 1: + std = std.view(1, -1) + return normalize(input, mean, std) diff --git a/kornia/augmentation/_2d/intensity/planckian_jitter.py b/kornia/augmentation/_2d/intensity/planckian_jitter.py new file mode 100644 index 0000000..d04c3f6 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/planckian_jitter.py @@ -0,0 +1,208 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, List, Optional, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.core.check import KORNIA_CHECK_SHAPE + + +def get_planckian_coeffs(mode: str) -> torch.Tensor: + if mode.lower() == "blackbody": + coefs = torch.tensor( + [ + [0.6743, 0.4029, 0.0013], + [0.6281, 0.4241, 0.1665], + [0.5919, 0.4372, 0.2513], + [0.5623, 0.4457, 0.3154], + [0.5376, 0.4515, 0.3672], + [0.5163, 0.4555, 0.4103], + [0.4979, 0.4584, 0.4468], + [0.4816, 0.4604, 0.4782], + [0.4672, 0.4619, 0.5053], + [0.4542, 0.4630, 0.5289], + [0.4426, 0.4638, 0.5497], + [0.4320, 0.4644, 0.5681], + [0.4223, 0.4648, 0.5844], + [0.4135, 0.4651, 0.5990], + [0.4054, 0.4653, 0.6121], + [0.3980, 0.4654, 0.6239], + [0.3911, 0.4655, 0.6346], + [0.3847, 0.4656, 0.6444], + [0.3787, 0.4656, 0.6532], + [0.3732, 0.4656, 0.6613], + [0.3680, 0.4655, 0.6688], + [0.3632, 0.4655, 0.6756], + [0.3586, 0.4655, 0.6820], + [0.3544, 0.4654, 0.6878], + [0.3503, 0.4653, 0.6933], + ] + ) + + elif mode.upper() == "CIED": + coefs = torch.tensor( + [ + [0.5829, 0.4421, 0.2288], + [0.5510, 0.4514, 0.2948], + [0.5246, 0.4576, 0.3488], + [0.5021, 0.4618, 0.3941], + [0.4826, 0.4646, 0.4325], + [0.4654, 0.4667, 0.4654], + [0.4502, 0.4681, 0.4938], + [0.4364, 0.4692, 0.5186], + [0.4240, 0.4700, 0.5403], + [0.4127, 0.4705, 0.5594], + [0.4023, 0.4709, 0.5763], + [0.3928, 0.4713, 0.5914], + [0.3839, 0.4715, 0.6049], + [0.3757, 0.4716, 0.6171], + [0.3681, 0.4717, 0.6281], + [0.3609, 0.4718, 0.6380], + [0.3543, 0.4719, 0.6472], + [0.3480, 0.4719, 0.6555], + [0.3421, 0.4719, 0.6631], + [0.3365, 0.4719, 0.6702], + [0.3313, 0.4719, 0.6766], + [0.3263, 0.4719, 0.6826], + [0.3217, 0.4719, 0.6882], + ] + ) + else: + raise RuntimeError(f"Unexpected mode. Gotcha {mode}") + + return torch.stack((coefs[:, 0] / coefs[:, 1], coefs[:, 2] / coefs[:, 1]), 1) + + +class RandomPlanckianJitter(IntensityAugmentationBase2D): + r"""Apply planckian jitter transformation to input torch.Tensor. + + .. image:: _static/img/RandomPlanckianJitter.png + + This is physics based color augmentation, that creates realistic + variations in chromaticity, this can simulate the illumination + changes in the scene. + + See :cite:`zini2022planckian` for more details. + + Args: + mode: 'blackbody' or 'CIED'. + select_from: choose a list of jitters to apply from. `blackbody` range [0-24], `CIED` range [0-22] + same_on_batch: apply the same transformation across the batch. + p: probability that the random erasing operation will be performed. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + .. note:: + Input torch.Tensor must be float and normalized into [0, 1]. + + Examples: + To apply planckian jitter based on mode + + >>> rng = torch.manual_seed(0) + >>> input = torch.randn(1, 3, 2, 2) + >>> aug = RandomPlanckianJitter(mode='CIED') + >>> aug(input) + tensor([[[[ 1.0000, -0.2389], + [-1.7740, 0.4628]], + + [[-1.0845, -1.3986], + [ 0.4033, 0.8380]], + + [[-0.9228, -0.5175], + [-0.7654, 0.2335]]]]) + + To apply planckian jitter on image(s) from list of interested jitters + + >>> rng = torch.manual_seed(0) + >>> input = torch.randn(2, 3, 2, 2) + >>> aug = RandomPlanckianJitter(mode='blackbody', select_from=[23, 24, 1, 2]) + >>> aug(input) + tensor([[[[-1.1258, -1.1524], + [-0.2506, -0.4339]], + + [[ 0.8487, 0.6920], + [-0.3160, -2.1152]], + + [[ 0.4681, -0.1577], + [ 1.4437, 0.2660]]], + + + [[[ 0.2465, 1.0000], + [-0.2125, -0.1653]], + + [[ 0.9318, 1.0000], + [ 1.0000, 0.0537]], + + [[ 0.2426, -0.1621], + [-0.3302, -0.9093]]]]) + + """ + + def __init__( + self, + mode: str = "blackbody", + select_from: Optional[Union[int, List[int]]] = None, + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + + if isinstance(select_from, int): + select_from = [select_from] + + _pl = get_planckian_coeffs(mode) + if select_from is not None: + _pl = _pl[select_from] + self.register_buffer("pl", _pl) + self.pl: torch.Tensor + + # the range of the sampling parameters + _param_min: float = 0.0 + _param_max: float = float(self.pl.shape[0]) + + self._param_generator = rg.PlanckianJitterGenerator([_param_min, _param_max]) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + list_idx = params["idx"].tolist() + KORNIA_CHECK_SHAPE(input, ["*", "3", "H", "W"]) + self.pl = self.pl.to(device=input.device) + coeffs = self.pl[list_idx] + + r_w = coeffs[:, 0][..., None, None] + b_w = coeffs[:, 1][..., None, None] + + r = input[..., 0, :, :] * r_w + g = input[..., 1, :, :] + b = input[..., 2, :, :] * b_w + + output = torch.stack([r, g, b], -3) + + return output.clamp(max=1.0) diff --git a/kornia/augmentation/_2d/intensity/plasma.py b/kornia/augmentation/_2d/intensity/plasma.py new file mode 100644 index 0000000..c48fe81 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/plasma.py @@ -0,0 +1,194 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.contrib import diamond_square + + +class RandomPlasmaBrightness(IntensityAugmentationBase2D): + r"""Adds brightness to the image based on a fractal map generated by the diamond square algorithm. + + .. image:: _static/img/RandomPlasmaBrightness.png + + This is based on the original paper: TorMentor: Deterministic dynamic-path, data augmentations with fractals. + See: :cite:`tormentor` for more details. + + .. note:: + This function internally uses :func:`kornia.contrib.diamond_square`. + + Args: + roughness: value to scale during the recursion in the generation of the fractal map. + intensity: value that scales the intensity values of the generated maps. + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Examples: + >>> rng = torch.manual_seed(0) + >>> img = torch.ones(1, 1, 3, 4) + >>> RandomPlasmaBrightness(roughness=(0.1, 0.7), p=1.)(img) + tensor([[[[0.6415, 1.0000, 0.3142, 0.6836], + [1.0000, 0.5593, 0.5556, 0.4566], + [0.5809, 1.0000, 0.7005, 1.0000]]]]) + + """ + + def __init__( + self, + roughness: Tuple[float, float] = (0.1, 0.7), + intensity: Tuple[float, float] = (0.0, 1.0), + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + self._param_generator = rg.PlainUniformGenerator( + (roughness, "roughness", None, None), (intensity, "intensity", None, None) + ) + + def apply_transform( + self, + image: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + B, C, H, W = image.shape + roughness = params["roughness"].to(image) + intensity = params["intensity"].to(image).view(-1, 1, 1, 1) + brightness_map = 2 * diamond_square((B, C, H, W), roughness, device=image.device, dtype=image.dtype) - 1 + brightness_map = brightness_map * intensity + return (image + brightness_map).clamp_(0, 1) + + +class RandomPlasmaContrast(IntensityAugmentationBase2D): + r"""Adds contrast to the image based on a fractal map generated by the diamond square algorithm. + + .. image:: _static/img/RandomPlasmaContrast.png + + This is based on the original paper: TorMentor: Deterministic dynamic-path, data augmentations with fractals. + See: :cite:`tormentor` for more details. + + .. note:: + This function internally uses :func:`kornia.contrib.diamond_square`. + + Args: + roughness: value to scale during the recursion in the generation of the fractal map. + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Examples: + >>> rng = torch.manual_seed(0) + >>> img = torch.ones(1, 1, 3, 4) + >>> RandomPlasmaContrast(roughness=(0.1, 0.7), p=1.)(img) + tensor([[[[0.9651, 1.0000, 1.0000, 1.0000], + [1.0000, 0.9103, 0.8038, 0.9263], + [0.6882, 1.0000, 0.9544, 1.0000]]]]) + + """ + + def __init__( + self, + roughness: Tuple[float, float] = (0.1, 0.7), + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + self._param_generator = rg.PlainUniformGenerator((roughness, "roughness", None, None)) + + def apply_transform( + self, + image: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + B, C, H, W = image.shape + roughness = params["roughness"].to(image) + contrast_map = 4 * diamond_square((B, C, H, W), roughness, device=image.device, dtype=image.dtype) + return ((image - 0.5) * contrast_map + 0.5).clamp_(0, 1) + + +class RandomPlasmaShadow(IntensityAugmentationBase2D): + r"""Add gaussian noise to a batch of multi-dimensional images. + + .. image:: _static/img/RandomPlasmaShadow.png + + This is based on the original paper: TorMentor: Deterministic dynamic-path, data augmentations with fractals. + See: :cite:`tormentor` for more details. + + .. note:: + This function internally uses :func:`kornia.contrib.diamond_square`. + + Args: + roughness: value to scale during the recursion in the generation of the fractal map. + shade_intensity: value that scales the intensity values of the generated maps. + shade_quantity: value to select the pixels to mask. + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Examples: + >>> rng = torch.manual_seed(0) + >>> img = torch.ones(1, 1, 3, 4) + >>> RandomPlasmaShadow(roughness=(0.1, 0.7), p=1.)(img) + tensor([[[[0.7682, 1.0000, 1.0000, 1.0000], + [1.0000, 1.0000, 1.0000, 1.0000], + [1.0000, 1.0000, 1.0000, 1.0000]]]]) + + """ + + def __init__( + self, + roughness: Tuple[float, float] = (0.1, 0.7), + shade_intensity: Tuple[float, float] = (-1.0, 0.0), + shade_quantity: Tuple[float, float] = (0.0, 1.0), + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + self._param_generator = rg.PlainUniformGenerator( + (roughness, "roughness", None, None), + (shade_intensity, "shade_intensity", None, None), + (shade_quantity, "shade_quantity", None, None), + ) + + def apply_transform( + self, + image: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + B, _, H, W = image.shape + roughness = params["roughness"].to(image) + shade_intensity = params["shade_intensity"].to(image).view(-1, 1, 1, 1) + shade_quantity = params["shade_quantity"].to(image).view(-1, 1, 1, 1) + shade_map = diamond_square((B, 1, H, W), roughness, device=image.device, dtype=image.dtype) + shade_map = (shade_map < shade_quantity).to(image.dtype) * shade_intensity + return (image + shade_map).clamp_(0, 1) diff --git a/kornia/augmentation/_2d/intensity/posterize.py b/kornia/augmentation/_2d/intensity/posterize.py new file mode 100644 index 0000000..b35fe63 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/posterize.py @@ -0,0 +1,85 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.enhance import posterize + + +class RandomPosterize(IntensityAugmentationBase2D): + r"""Posterize given torch.Tensor image or a batch of torch.Tensor images randomly. + + .. image:: _static/img/RandomPosterize.png + + Args: + p: probability of applying the transformation. + bits: Integer that ranged from (0, 8], in which 0 gives black image and 8 gives the original. + If int x, bits will be generated from (x, 8) then convert to int. + If tuple (x, y), bits will be generated from (x, y) then convert to int. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.enhance.posterize`. + + Examples: + >>> rng = torch.manual_seed(0) + >>> input = torch.rand(1, 1, 5, 5) + >>> posterize = RandomPosterize(3., p=1.) + >>> posterize(input) + tensor([[[[0.4863, 0.7529, 0.0784, 0.1255, 0.2980], + [0.6275, 0.4863, 0.8941, 0.4549, 0.6275], + [0.3451, 0.3922, 0.0157, 0.1569, 0.2824], + [0.5176, 0.6902, 0.8000, 0.1569, 0.2667], + [0.6745, 0.9098, 0.3922, 0.8627, 0.4078]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomPosterize(3., p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + bits: Union[float, Tuple[float, float], torch.Tensor] = 3, + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + # TODO: the generator should receive the device + self._param_generator = rg.PosterizeGenerator(bits) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return posterize(input, params["bits_factor"].to(input.device)) diff --git a/kornia/augmentation/_2d/intensity/random_rain.py b/kornia/augmentation/_2d/intensity/random_rain.py new file mode 100644 index 0000000..2223d35 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/random_rain.py @@ -0,0 +1,112 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Any, Optional + +import torch + +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.augmentation.random_generator._2d import RainGenerator +from kornia.core.check import KORNIA_CHECK + + +class RandomRain(IntensityAugmentationBase2D): + r"""Add Random Rain to the image. + + Args: + p: probability of applying the transformation. + number_of_drops: number of drops per image + drop_height: height of the drop in image(same for each drops in one image) + drop_width: width of the drop in image(same for each drops in one image) + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + Examples: + >>> rng = torch.manual_seed(0) + >>> input = torch.rand(1, 1, 5, 5) + >>> rain = RandomRain(p=1,drop_height=(1,2),drop_width=(1,2),number_of_drops=(1,1)) + >>> rain(input) + tensor([[[[0.4963, 0.7843, 0.0885, 0.1320, 0.3074], + [0.6341, 0.4901, 0.8964, 0.4556, 0.6323], + [0.3489, 0.4017, 0.0223, 0.1689, 0.2939], + [0.5185, 0.6977, 0.8000, 0.1610, 0.2823], + [0.6816, 0.9152, 0.3971, 0.8742, 0.4194]]]]) + + """ + + def __init__( + self, + number_of_drops: tuple[int, int] = (1000, 2000), + drop_height: tuple[int, int] = (5, 20), + drop_width: tuple[int, int] = (-5, 5), + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + self._param_generator = RainGenerator(number_of_drops, drop_height, drop_width) + + def apply_transform( + self, + image: torch.Tensor, + params: dict[str, torch.Tensor], + flags: dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + # Check array and drops size + KORNIA_CHECK(image.shape[1] in {3, 1}, "Number of color channels should be 1 or 3.") + KORNIA_CHECK( + bool( + torch.all(params["drop_height_factor"] <= image.shape[2]) + and torch.all(params["drop_height_factor"] > 0) + ), + "Height of drop should be greater than zero and less than image height.", + ) + + KORNIA_CHECK( + bool(torch.all(torch.abs(params["drop_width_factor"]) <= image.shape[3])), + "Width of drop should be less than image width.", + ) + modeified_img = image.clone() + for i in range(image.shape[0]): + number_of_drops: int = int(params["number_of_drops_factor"][i]) + # We generate torch.Tensor with maximum number of drops, and then remove unnecessary drops. + + coordinates_of_drops: torch.Tensor = params["coordinates_factor"][i][:number_of_drops] + height_of_drop: int = int(params["drop_height_factor"][i]) + width_of_drop: int = int(params["drop_width_factor"][i]) + + # Generate start coordinates for each drop + random_y_coords = coordinates_of_drops[:, 0] * (image.shape[2] - height_of_drop - 1) + if width_of_drop > 0: + random_x_coords = coordinates_of_drops[:, 1] * (image.shape[3] - width_of_drop - 1) + else: + random_x_coords = coordinates_of_drops[:, 1] * (image.shape[3] + width_of_drop - 1) - width_of_drop + + coords = torch.cat([random_y_coords[None], random_x_coords[None]], dim=0).to(image.device, dtype=torch.long) + + # Generate how our drop will look like into the image + size_of_line: int = max(height_of_drop, abs(width_of_drop)) + x = torch.linspace(start=0, end=height_of_drop, steps=size_of_line, dtype=torch.long).to(image.device) + y = torch.linspace(start=0, end=width_of_drop, steps=size_of_line, dtype=torch.long).to(image.device) + # Draw lines + for k in range(x.shape[0]): + modeified_img[i, :, coords[0] + x[k], coords[1] + y[k]] = 200 / 255 + return modeified_img diff --git a/kornia/augmentation/_2d/intensity/random_rgb_shift.py b/kornia/augmentation/_2d/intensity/random_rgb_shift.py new file mode 100644 index 0000000..571c828 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/random_rgb_shift.py @@ -0,0 +1,118 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.enhance import shift_rgb + + +class RandomRGBShift(IntensityAugmentationBase2D): + """Randomly shift each channel of an image. + + Args: + r_shift_limit: maximum value up to which the shift value can be generated for red channel; + recommended interval - [0, 1], should always be positive + g_shift_limit: maximum value up to which the shift value can be generated for green channel; + recommended interval - [0, 1], should always be positive + b_shift_limit: maximum value up to which the shift value can be generated for blue channel; + recommended interval - [0, 1], should always be positive + same_on_batch: apply the same transformation across the batch. + p: probability of applying the transformation. + keepdim: whether to keep the output shape the same as input ``True`` or broadcast it + to the batch form ``False``. + + Note: + Input torch.Tensor must be float and normalized into [0, 1]. + + Examples: + >>> import torch + >>> rng = torch.manual_seed(0) + >>> inp = torch.rand(1, 3, 5, 5) + >>> aug = RandomRGBShift(0, 0, 0) + >>> ((inp == aug(inp)).double()).all() + tensor(True) + + >>> rng = torch.manual_seed(0) + >>> inp = torch.rand(1, 3, 5, 5) + >>> inp + tensor([[[[0.4963, 0.7682, 0.0885, 0.1320, 0.3074], + [0.6341, 0.4901, 0.8964, 0.4556, 0.6323], + [0.3489, 0.4017, 0.0223, 0.1689, 0.2939], + [0.5185, 0.6977, 0.8000, 0.1610, 0.2823], + [0.6816, 0.9152, 0.3971, 0.8742, 0.4194]], + + [[0.5529, 0.9527, 0.0362, 0.1852, 0.3734], + [0.3051, 0.9320, 0.1759, 0.2698, 0.1507], + [0.0317, 0.2081, 0.9298, 0.7231, 0.7423], + [0.5263, 0.2437, 0.5846, 0.0332, 0.1387], + [0.2422, 0.8155, 0.7932, 0.2783, 0.4820]], + + [[0.8198, 0.9971, 0.6984, 0.5675, 0.8352], + [0.2056, 0.5932, 0.1123, 0.1535, 0.2417], + [0.7262, 0.7011, 0.2038, 0.6511, 0.7745], + [0.4369, 0.5191, 0.6159, 0.8102, 0.9801], + [0.1147, 0.3168, 0.6965, 0.9143, 0.9351]]]]) + >>> aug = RandomRGBShift(p=1.) + >>> aug(inp) + tensor([[[[0.9374, 1.0000, 0.5297, 0.5732, 0.7486], + [1.0000, 0.9313, 1.0000, 0.8968, 1.0000], + [0.7901, 0.8429, 0.4635, 0.6100, 0.7351], + [0.9597, 1.0000, 1.0000, 0.6022, 0.7234], + [1.0000, 1.0000, 0.8383, 1.0000, 0.8606]], + + [[0.6524, 1.0000, 0.1357, 0.2847, 0.4729], + [0.4046, 1.0000, 0.2754, 0.3693, 0.2502], + [0.1312, 0.3076, 1.0000, 0.8226, 0.8418], + [0.6258, 0.3432, 0.6841, 0.1327, 0.2382], + [0.3417, 0.9150, 0.8927, 0.3778, 0.5815]], + + [[0.3850, 0.5623, 0.2636, 0.1328, 0.4005], + [0.0000, 0.1584, 0.0000, 0.0000, 0.0000], + [0.2914, 0.2663, 0.0000, 0.2163, 0.3397], + [0.0021, 0.0843, 0.1811, 0.3754, 0.5453], + [0.0000, 0.0000, 0.2617, 0.4795, 0.5003]]]]) + + """ + + def __init__( + self, + r_shift_limit: float = 0.5, + g_shift_limit: float = 0.5, + b_shift_limit: float = 0.5, + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self._param_generator = rg.PlainUniformGenerator( + (r_shift_limit, "r_shift", 0, (-r_shift_limit, r_shift_limit)), + (g_shift_limit, "g_shift", 0, (-g_shift_limit, g_shift_limit)), + (b_shift_limit, "b_shift", 0, (-b_shift_limit, b_shift_limit)), + ) + + def apply_transform( + self, + inp: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return shift_rgb(inp, params["r_shift"], params["g_shift"], params["b_shift"]) diff --git a/kornia/augmentation/_2d/intensity/random_snow.py b/kornia/augmentation/_2d/intensity/random_snow.py new file mode 100644 index 0000000..22892e3 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/random_snow.py @@ -0,0 +1,97 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.color import hls_to_rgb, rgb_to_hls +from kornia.core.check import KORNIA_CHECK + + +class RandomSnow(IntensityAugmentationBase2D): + r"""Generates snow effect on given torch.Tensor image or a batch torch.Tensor images. + + Args: + snow_coefficient: A tuple of floats (lower and upper bound) between 0 and 1 that control + the amount of snow to add to the image, the larger value corresponds to the more snow. + brightness: A tuple of floats (lower and upper bound) greater than 1 that controls the + brightness of the snow. + same_on_batch: If True, apply the same transformation to each image in a batch. Default: False. + p: Probability of applying the transformation. Default: 0.5. + keepdim: Keep the output torch.Tensor with the same shape as input. Default: False. + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + Examples: + >>> inputs = torch.rand(2, 3, 4, 4) + >>> snow = kornia.augmentation.RandomSnow(p=1.0, snow_coefficient=(0.1, 0.6), brightness=(1.0, 5.0)) + >>> output = snow(inputs) + >>> output.shape + torch.Size([2, 3, 4, 4]) + + """ + + def __init__( + self, + snow_coefficient: Tuple[float, float] = (0.5, 0.5), + brightness: Tuple[float, float] = (2, 2), + same_on_batch: bool = False, + p: float = 1.0, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + KORNIA_CHECK(all(0 <= el <= 1 for el in snow_coefficient), "Snow coefficient values must be between 0 and 1.") + KORNIA_CHECK(all(1 <= el for el in brightness), "Brightness values must be greater than 1.") + + self._param_generator = rg.PlainUniformGenerator( + (snow_coefficient, "snow_coefficient", 0.5, (0.0, 1.0)), (brightness, "brightness", None, None) + ) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + KORNIA_CHECK(input.shape[1] == 3, "Number of color channels should be 3.") + KORNIA_CHECK(len(input.shape) in (3, 4), "Wrong input dimension.") + + if len(input.shape) == 3: + input = input[None, :, :, :] + input_HLS = rgb_to_hls(input) + + mask = torch.zeros_like(input_HLS) + # Retrieve generated parameters + snow_coefficient = params["snow_coefficient"].to(input) + brightness = params["brightness"].to(input) + snow_coefficient = snow_coefficient[:, None, None, None] + brightness = brightness[:, None, None, None] + + mask[:, 1, :, :] = torch.where(input_HLS[:, 1, :, :] < snow_coefficient[:, 0, :, :], 1, 0) + + # Increase Light channel of the image by given brightness for areas based on snow coefficient. + new_light = (input_HLS * mask * brightness).clamp(min=0.0, max=1.0) + input_HLS = input_HLS * (1 - mask) + new_light + + output = hls_to_rgb(input_HLS) + return output diff --git a/kornia/augmentation/_2d/intensity/salt_pepper_noise.py b/kornia/augmentation/_2d/intensity/salt_pepper_noise.py new file mode 100644 index 0000000..11ddd2f --- /dev/null +++ b/kornia/augmentation/_2d/intensity/salt_pepper_noise.py @@ -0,0 +1,149 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.augmentation.random_generator._2d import SaltAndPepperGenerator +from kornia.core.check import KORNIA_CHECK + + +class RandomSaltAndPepperNoise(IntensityAugmentationBase2D): + r"""Apply random Salt and Pepper noise to input images. + + .. image:: _static/img/RandomSaltAndPepperNoise.png + + Args: + amount: A float or a tuple representing the range for the amount of noise to apply. + salt_vs_pepper: A float or a tuple representing the range for the ratio of Salt to Pepper noise. + p: The probability of applying the transformation. Default is 0.5. + same_on_batch: If True, apply the same transformation across the entire batch. Default is False. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + .. note:: + The `amount` parameter controls the intensity of the noise, while `salt_vs_pepper` controls the ratio + of Salt to Pepper noise. + + The values for `amount` and `salt_vs_pepper` should be between 0 and 1. The recommended value for + `salt_vs_pepper` is 0.5, and for `amount`, values less than 0.2 are recommended. + + If `amount` and `salt_vs_pepper` are floats (unique values), the transformation is applied with these + exact values, rather than randomly sampling from the specified range. However, the masks are still + generated randomly using these exact parameters. + + Examples: + >>> rng = torch.manual_seed(5) + >>> inputs = torch.rand(1, 3, 3, 3) + >>> aug = RandomSaltAndPepperNoise(amount=0.5, salt_vs_pepper=0.5, p=1.) + >>> aug(inputs) + tensor([[[[1.0000, 0.0000, 0.0000], + [1.0000, 1.0000, 0.1166], + [0.1644, 0.7379, 0.0000]], + + [[1.0000, 0.0000, 0.0000], + [1.0000, 1.0000, 0.7150], + [0.5793, 0.9809, 0.0000]], + + [[1.0000, 0.0000, 0.0000], + [1.0000, 1.0000, 0.7850], + [0.9752, 0.0903, 0.0000]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.rand(1, 3, 32, 32) + >>> aug = RandomSaltAndPepperNoise(amount=0.05, salt_vs_pepper=0.5, p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + amount: Optional[Union[float, Tuple[float, float]]] = (0.01, 0.06), + salt_vs_pepper: Optional[Union[float, Tuple[float, float]]] = (0.4, 0.6), + p: float = 0.5, + same_on_batch: bool = False, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + + # Validation and initialization of amount and salt_vs_pepper parameters. + if isinstance(salt_vs_pepper, (tuple, float)): + if isinstance(salt_vs_pepper, float): + salt_vs_pepper = (salt_vs_pepper, salt_vs_pepper) + elif len(salt_vs_pepper) == 1: + salt_vs_pepper = (salt_vs_pepper[0], salt_vs_pepper[0]) + elif len(salt_vs_pepper) > 2 or len(salt_vs_pepper) <= 0: + raise ValueError( + "The length of salt_vs_pepper must be greater than 0 \ + and less than or equal to 2, and it should be a tuple." + ) + else: + raise ValueError("salt_vs_pepper must be a tuple or a float") + KORNIA_CHECK( + all(0 <= el <= 1 for el in salt_vs_pepper), + "Salt_vs_pepper values must be between 0 and 1. \ + Recommended value 0.5.", + ) + + if isinstance(amount, (tuple, float)): + if isinstance(amount, float): + amount = (amount, amount) + elif len(amount) == 1: + amount = (amount[0], amount[0]) + elif len(amount) > 2 or len(amount) <= 0: + raise ValueError( + "The length of amount must be greater than 0 \ + and less than or equal to 2, and it should be a tuple." + ) + else: + raise ValueError("amount must be a tuple or a float") + KORNIA_CHECK( + all(0 <= el <= 1 for el in amount), + "amount of noise values must be between 0 and 1. \ + Recommended values less than 0.2.", + ) + + # Generator of random parameters and masks. + self._param_generator = SaltAndPepperGenerator(amount, salt_vs_pepper) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + r"""Apply random Salt and Pepper noise transformation to the input image.""" + KORNIA_CHECK(len(input.shape) in (3, 4), "Wrong input dimension.") + if len(input.shape) == 3: + input = input[None, :, :, :] + KORNIA_CHECK(input.shape[1] in {3, 1}, "Number of color channels should be 1 or 3.") + + noisy_image = input.clone() + + # Apply noise masks using indexing. + noisy_image[params["mask_salt"].to(input.device)] = 1.0 + noisy_image[params["mask_pepper"].to(input.device)] = 0.0 + + return noisy_image diff --git a/kornia/augmentation/_2d/intensity/saturation.py b/kornia/augmentation/_2d/intensity/saturation.py new file mode 100644 index 0000000..37f1b00 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/saturation.py @@ -0,0 +1,93 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.augmentation.utils import _range_bound +from kornia.enhance.adjust import adjust_saturation + + +class RandomSaturation(IntensityAugmentationBase2D): + r"""Apply a random transformation to the saturation of a torch.Tensor image. + + This implementation aligns PIL. Hence, the output is close to TorchVision. + + .. image:: _static/img/RandomSaturation.png + + Args: + p: probability of applying the transformation. + saturation: the saturation factor to apply. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.enhance.adjust_saturation` + + Examples: + >>> rng = torch.manual_seed(0) + >>> inputs = torch.rand(1, 3, 3, 3) + >>> aug = RandomSaturation(saturation = (0.5,2.),p=1.) + >>> aug(inputs) + tensor([[[[0.5569, 0.7682, 0.3529], + [0.4811, 0.3474, 0.7411], + [0.5028, 0.8964, 0.6772]], + + [[0.6323, 0.5358, 0.5265], + [0.4203, 0.2706, 0.5525], + [0.5185, 0.7863, 0.8681]], + + [[0.3711, 0.4989, 0.6816], + [0.9152, 0.3971, 0.8742], + [0.4636, 0.7060, 0.9527]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + + >>> input = torch.rand(1, 3, 32, 32) + >>> aug = RandomSaturation((0.8,1.2), p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + saturation: Tuple[float, float] = (1.0, 1.0), + same_on_batch: bool = False, + p: float = 1.0, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self.saturation: torch.Tensor = _range_bound(saturation, "saturation", center=1.0) + self._param_generator = rg.PlainUniformGenerator((self.saturation, "saturation_factor", None, None)) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + saturation_factor = params["saturation_factor"].to(input) + return adjust_saturation(input, saturation_factor) diff --git a/kornia/augmentation/_2d/intensity/sharpness.py b/kornia/augmentation/_2d/intensity/sharpness.py new file mode 100644 index 0000000..6a095c1 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/sharpness.py @@ -0,0 +1,83 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.enhance import sharpness + + +class RandomSharpness(IntensityAugmentationBase2D): + r"""Sharpen given torch.Tensor image or a batch of torch.Tensor images randomly. + + .. image:: _static/img/RandomSharpness.png + + Args: + p: probability of applying the transformation. + sharpness: factor of sharpness strength. Must be above 0. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.enhance.sharpness`. + + Examples: + >>> rng = torch.manual_seed(0) + >>> input = torch.rand(1, 1, 5, 5) + >>> sharpness = RandomSharpness(1., p=1.) + >>> sharpness(input) + tensor([[[[0.4963, 0.7682, 0.0885, 0.1320, 0.3074], + [0.6341, 0.4810, 0.7367, 0.4177, 0.6323], + [0.3489, 0.4428, 0.1562, 0.2443, 0.2939], + [0.5185, 0.6462, 0.7050, 0.2288, 0.2823], + [0.6816, 0.9152, 0.3971, 0.8742, 0.4194]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomSharpness(1., p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + sharpness: Union[torch.Tensor, float, Tuple[float, float]] = 0.5, + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self._param_generator = rg.PlainUniformGenerator((sharpness, "sharpness", 0.0, (0, float("inf")))) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + factor = params["sharpness"] + return sharpness(input, factor) diff --git a/kornia/augmentation/_2d/intensity/solarize.py b/kornia/augmentation/_2d/intensity/solarize.py new file mode 100644 index 0000000..d195f31 --- /dev/null +++ b/kornia/augmentation/_2d/intensity/solarize.py @@ -0,0 +1,96 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.enhance import solarize + + +class RandomSolarize(IntensityAugmentationBase2D): + r"""Solarize given torch.Tensor image or a batch of torch.Tensor images randomly. + + .. image:: _static/img/RandomSolarize.png + + Args: + p: probability of applying the transformation. + thresholds: + If float x, threshold will be generated from (0.5 - x, 0.5 + x). + If tuple (x, y), threshold will be generated from (x, y). + additions: + If float x, addition will be generated from (-x, x). + If tuple (x, y), addition will be generated from (x, y). + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, H, W)` or :math:`(B, C, H, W)`, Optional: :math:`(B, 3, 3)` + - Output: :math:`(B, C, H, W)` + + .. note:: + This function internally uses :func:`kornia.enhance.solarize`. + + Examples: + >>> rng = torch.manual_seed(0) + >>> input = torch.rand(1, 1, 5, 5) + >>> solarize = RandomSolarize(0.1, 0.1, p=1.) + >>> solarize(input) + tensor([[[[0.4132, 0.1412, 0.1790, 0.2226, 0.3980], + [0.2754, 0.4194, 0.0130, 0.4538, 0.2771], + [0.4394, 0.4923, 0.1129, 0.2594, 0.3844], + [0.3909, 0.2118, 0.1094, 0.2516, 0.3728], + [0.2278, 0.0000, 0.4876, 0.0353, 0.5100]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.randn(1, 3, 32, 32) + >>> aug = RandomSolarize(0.1, 0.1, p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + thresholds: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 0.1, + additions: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 0.1, + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self._param_generator = rg.PlainUniformGenerator( + (thresholds, "thresholds", 0.5, (0.0, 1.0)), (additions, "additions", 0.0, (-0.5, 0.5)) + ) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + thresholds = params["thresholds"] + additions: Optional[torch.Tensor] + if "additions" in params: + additions = params["additions"] + else: + additions = None + return solarize(input, thresholds, additions) diff --git a/kornia/augmentation/_2d/mix/__init__.py b/kornia/augmentation/_2d/mix/__init__.py new file mode 100644 index 0000000..6379154 --- /dev/null +++ b/kornia/augmentation/_2d/mix/__init__.py @@ -0,0 +1,23 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from kornia.augmentation._2d.mix.cutmix import RandomCutMixV2 +from kornia.augmentation._2d.mix.jigsaw import RandomJigsaw +from kornia.augmentation._2d.mix.mixup import RandomMixUpV2 +from kornia.augmentation._2d.mix.mosaic import RandomMosaic +from kornia.augmentation._2d.mix.patchmix import PatchMix +from kornia.augmentation._2d.mix.transplantation import RandomTransplantation diff --git a/kornia/augmentation/_2d/mix/base.py b/kornia/augmentation/_2d/mix/base.py new file mode 100644 index 0000000..7c629dd --- /dev/null +++ b/kornia/augmentation/_2d/mix/base.py @@ -0,0 +1,269 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, List, Optional, Union + +import torch + +from kornia.augmentation.base import _BasicAugmentationBase +from kornia.augmentation.utils import ( + _transform_input, + _transform_input_by_shape, + _transform_output_shape, + _validate_input_dtype, +) +from kornia.constants import DataKey, DType +from kornia.core.check import KORNIA_UNWRAP +from kornia.geometry.boxes import Boxes + + +class MixAugmentationBaseV2(_BasicAugmentationBase): + r"""MixAugmentationBase base class for customized mix augmentation implementations. + + For any augmentation, the implementation of "generate_parameters" and "apply_transform" are required. + "apply_transform" will need to handle the probabilities internally. + + Args: + p: probability for applying an augmentation. This param controls if to apply the augmentation for the batch. + p_batch: probability for applying an augmentation to a batch. This param controls the augmentation + probabilities batch-wise. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input ``True`` or broadcast it + to the batch form ``False``. + data_keys: the input type sequential for applying augmentations. + Accepts "input", "image", "mask", "bbox", "bbox_xyxy", "bbox_xywh", "keypoints", "class", "label". + + """ + + def __init__( + self, + p: float, + p_batch: float, + same_on_batch: bool = False, + keepdim: bool = False, + data_keys: Optional[List[Union[str, int, DataKey]]] = None, + ) -> None: + super().__init__(p, p_batch=p_batch, same_on_batch=same_on_batch, keepdim=keepdim) + self.data_keys = [DataKey.INPUT] + if data_keys is not None: + self.data_keys = [DataKey.get(inp) for inp in data_keys] + + def transform_tensor( + self, input: torch.Tensor, *, shape: Optional[torch.Tensor] = None, match_channel: bool = True + ) -> torch.Tensor: + """Convert any incoming (H, W), (C, H, W) and (B, C, H, W) into (B, C, H, W).""" + _validate_input_dtype(input, accepted_dtypes=[torch.bfloat16, torch.float16, torch.float32, torch.float64]) + + if shape is None: + return _transform_input(input) + else: + return _transform_input_by_shape(input, reference_shape=shape, match_channel=match_channel) + + def apply_transform( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + # NOTE: apply_transform receives the whole torch.Tensor, but returns only altered elements. + raise NotImplementedError + + def apply_non_transform( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + # For the images where batch_prob == False. + return input + + def transform_input( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + batch_prob = params["batch_prob"] + to_apply = batch_prob > 0.5 + ori_shape = input.shape + in_tensor = self.transform_tensor(input) + + # Compute the non-transform branch first; if no element is to be transformed, + # short-circuit (mix transforms like RandomJigsaw subset their input internally + # and can't operate on an empty subset). + non_applied = self.apply_non_transform(in_tensor, params, flags) + if not bool(to_apply.any()): + output = non_applied + return _transform_output_shape(output, ori_shape) if self.keepdim else output + + # Run the transform branch on the full batch (the random-generator contract is + # full-batch sized now), then pass it through ``apply_non_transform`` so any + # subclass post-processing (e.g. resize to ``output_size`` in ``RandomMosaic``) + # is applied uniformly to both branches. + applied = self.apply_transform(in_tensor, params, flags) + applied_post = self.apply_non_transform(applied, params, flags) + + if applied_post.shape == non_applied.shape and applied_post.shape[0] == to_apply.shape[0]: + to_apply_expanded = to_apply.view(-1, *([1] * (applied_post.dim() - 1))) + output = torch.where(to_apply_expanded, applied_post, non_applied) + else: + # Shape-changing mix augmentations (e.g. RandomMosaic when ``output_size`` + # differs from the input size) cannot be where-blended. Fall back to the + # all-applied branch. + output = applied_post + + output = _transform_output_shape(output, ori_shape) if self.keepdim else output + return output + + def transform_mask( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + batch_prob = params["batch_prob"] + to_apply = batch_prob > 0.5 # NOTE: in case of Relaxed Distributions. + output = input + if sum(to_apply) != len(to_apply): + output = self.apply_non_transform_mask(input, params, flags) + if sum(to_apply) != 0: + output = self.apply_transform_mask(input, params, flags) + return output + + def transform_boxes( + self, input: Union[torch.Tensor, Boxes], params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> Boxes: + # input is BxNx4x2 or Boxes. + if isinstance(input, torch.Tensor): + if not (len(input.shape) == 4 and input.shape[2:] == torch.Size([4, 2])): + raise RuntimeError(f"Only BxNx4x2 torch.Tensor is supported. Got {input.shape}.") + input = Boxes(input, False, mode="vertices_plus") + batch_prob = params["batch_prob"] + to_apply = batch_prob > 0.5 # NOTE: in case of Relaxed Distributions. + output = input + if sum(to_apply) != len(to_apply): + output = self.apply_non_transform_boxes(input, params, flags) + if sum(to_apply) != 0: + output = self.apply_transform_boxes(output, params, flags) + return output + + def transform_keypoint( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + batch_prob = params["batch_prob"] + to_apply = batch_prob > 0.5 # NOTE: in case of Relaxed Distributions. + output = input + if sum(to_apply) != len(to_apply): + output = self.apply_non_transform_keypoint(input, params, flags) + if sum(to_apply) != 0: + output = self.apply_transform_keypoint(input, params, flags) + return output + + def transform_class( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + batch_prob = params["batch_prob"] + to_apply = batch_prob > 0.5 # NOTE: in case of Relaxed Distributions. + output = input + if sum(to_apply) != len(to_apply): + output = self.apply_non_transform_class(input, params, flags) + if sum(to_apply) != 0: + output = self.apply_transform_class(input, params, flags) + return output + + def apply_non_transform_mask( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + raise NotImplementedError + + def apply_transform_mask( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + raise NotImplementedError + + def apply_non_transform_boxes(self, input: Boxes, params: Dict[str, torch.Tensor], flags: Dict[str, Any]) -> Boxes: + return input + + def apply_transform_boxes(self, input: Boxes, params: Dict[str, torch.Tensor], flags: Dict[str, Any]) -> Boxes: + raise NotImplementedError + + def apply_non_transform_keypoint( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + return input + + def apply_transform_keypoint( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + raise NotImplementedError + + def apply_non_transform_class( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + return input + + def apply_transform_class( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + raise NotImplementedError + + def forward( # type: ignore[override] + self, + *input: torch.Tensor, + params: Optional[Dict[str, torch.Tensor]] = None, + data_keys: Optional[List[Union[str, int, DataKey]]] = None, + ) -> Union[torch.Tensor, List[torch.Tensor]]: + keys: List[DataKey] + if data_keys is None: + keys = self.data_keys + else: + keys = [DataKey.get(inp) for inp in data_keys] + + if params is None: + in_tensor_idx: int = keys.index(DataKey.INPUT) + in_tensor: torch.Tensor = input[in_tensor_idx] + in_tensor = self.transform_tensor(in_tensor) + self._params = self.forward_parameters(in_tensor.shape) + self._params.update({"dtype": torch.tensor(DType.get(in_tensor.dtype).value)}) + else: + self._params = params + + outputs: List[torch.Tensor] = [] + for dcate, _input in zip(keys, input): + output: torch.Tensor + if dcate == DataKey.INPUT: + output = self.transform_input(_input, self._params, self.flags) + elif dcate == DataKey.MASK: + output = self.transform_mask(_input, self._params, self.flags) + elif dcate == DataKey.BBOX: + box = Boxes.from_tensor(_input, mode="vertices", validate_boxes=False) + box = self.transform_boxes(box, self._params, self.flags) + output = KORNIA_UNWRAP(box.to_tensor("vertices"), torch.Tensor) + elif dcate == DataKey.BBOX_XYXY: + box = Boxes.from_tensor(_input, mode="xyxy", validate_boxes=False) + box = self.transform_boxes(box, self._params, self.flags) + output = KORNIA_UNWRAP(box.to_tensor("xyxy"), torch.Tensor) + elif dcate == DataKey.BBOX_XYWH: + box = Boxes.from_tensor(_input, mode="xywh", validate_boxes=False) + box = self.transform_boxes(box, self._params, self.flags) + output = KORNIA_UNWRAP(box.to_tensor("xywh"), torch.Tensor) + elif dcate == DataKey.KEYPOINTS: + output = self.transform_keypoint(_input, self._params, self.flags) + elif dcate == DataKey.CLASS: + output = self.transform_class(_input, self._params, self.flags) + else: + raise NotImplementedError + outputs.append(output) + if len(outputs) == 1: + return outputs[0] + return outputs + + @torch.jit.ignore + def inverse(self, **kwargs: Any) -> Optional[torch.Tensor]: + raise RuntimeError(f"Inverse for {self.__class__.__name__} is not supported.") + + @property + def transform_matrix(self) -> Optional[torch.Tensor]: + raise RuntimeError(f"Transformation matrices for {self.__class__.__name__} is not supported.") diff --git a/kornia/augmentation/_2d/mix/cutmix.py b/kornia/augmentation/_2d/mix/cutmix.py new file mode 100644 index 0000000..298c788 --- /dev/null +++ b/kornia/augmentation/_2d/mix/cutmix.py @@ -0,0 +1,173 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import warnings +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.mix.base import MixAugmentationBaseV2 +from kornia.constants import DataKey, DType +from kornia.geometry.bbox import bbox_to_mask, infer_bbox_shape + + +class RandomCutMixV2(MixAugmentationBaseV2): + r"""Apply CutMix augmentation to a batch of torch.Tensor images. + + .. image:: _static/img/RandomCutMixV2.png + + Implementation for `CutMix: Regularization Strategy to Train Strong Classifiers with + Localizable Features` :cite:`yun2019cutmix`. + + The function returns (inputs, labels), in which the inputs is the torch.Tensor that contains the mixup images + while the labels is a :math:`(\text{num_mixes}, B, 3)` torch.Tensor that contains (label_permuted_batch, lambda) + for each cutmix. + + The implementation referred to the following repository: `https://github.com/clovaai/CutMix-PyTorch + `_. + + Args: + height: the width of the input image. + width: the width of the input image. + p: probability for applying an augmentation to a batch. This param controls the augmentation + probabilities batch-wisely. + num_mix: cut mix times. + beta: hyperparameter for generating cut size from beta distribution. + Beta cannot be set to 0 after torch 1.8.0. If None, it will be set to 1. + cut_size: controlling the minimum and maximum cut ratio from [0, 1]. + If None, it will be set to [0, 1], which means no restriction. + same_on_batch: apply the same transformation across the batch. + This flag will not maintain permutation order. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + use_correct_lambda: if True, compute lambda according to the CutMix paper + (`lam = 1 - area_ratio`). Defaults to False (`lam = area_ratio`) for backward compatibility, + but will raise a deprecation warning when False. + + Inputs: + - Input image tensors, shape of :math:`(B, C, H, W)`. + - Raw labels, shape of :math:`(B)`. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: + - Adjusted image, shape of :math:`(B, C, H, W)`. + - Raw labels, permuted labels and lambdas for each mix, shape of :math:`(B, num_mix, 3)`. + + Note: + This implementation would randomly cutmix images in a batch. Ideally, the larger batch size would be preferred. + + Examples: + >>> rng = torch.manual_seed(3) + >>> input = torch.rand(2, 1, 3, 3) + >>> input[0] = torch.ones((1, 3, 3)) + >>> label = torch.tensor([0, 1]) + >>> cutmix = RandomCutMixV2(data_keys=["input", "class"], use_correct_lambda=True) + >>> cutmix(input, label) + [tensor([[[[0.8879, 0.4510, 1.0000], + [0.1498, 0.4015, 1.0000], + [1.0000, 1.0000, 1.0000]]], + + + [[[1.0000, 1.0000, 0.7995], + [1.0000, 1.0000, 0.0542], + [0.4594, 0.1756, 0.9492]]]]), tensor([[[0.0000, 1.0000, 0.5556], + [1.0000, 0.0000, 0.5556]]])] + + """ + + def __init__( + self, + num_mix: int = 1, + cut_size: Optional[Union[torch.Tensor, Tuple[float, float]]] = None, + beta: Optional[Union[torch.Tensor, float]] = None, + same_on_batch: bool = False, + p: float = 1.0, + keepdim: bool = False, + data_keys: Optional[List[Union[str, int, DataKey]]] = None, + use_correct_lambda: bool = False, + ) -> None: + super().__init__(p=1.0, p_batch=p, same_on_batch=same_on_batch, keepdim=keepdim, data_keys=data_keys) + self._param_generator: rg.CutmixGenerator = rg.CutmixGenerator(cut_size, beta, num_mix, p=p) + + self.use_correct_lambda = use_correct_lambda + if not self.use_correct_lambda: + warnings.warn( + "RandomCutMixV2 currently uses the old (inconsistent) lambda computation. " + "Set `use_correct_lambda=True` to align with the original CutMix paper. " + "This default will change in a future release.", + DeprecationWarning, + stacklevel=2, + ) + + def apply_transform_class( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + height, width = params["image_shape"] + + out_labels = [] + for pair, crop in zip(params["mix_pairs"], params["crop_src"]): + labels_permute = input.index_select(dim=0, index=pair.to(input.device)) + w, h = infer_bbox_shape(crop) + + lam_val = w.to(input.dtype) * h.to(input.dtype) / (width * height) + lam = 1 - lam_val if self.use_correct_lambda else lam_val + + out_labels.append( + torch.stack( + [ + input.to(device=input.device, dtype=DType.to_torch(int(params["dtype"].item()))), + labels_permute.to(device=input.device, dtype=DType.to_torch(int(params["dtype"].item()))), + lam.to(device=input.device, dtype=DType.to_torch(int(params["dtype"].item()))), + ], + 1, + ) + ) + + return torch.stack(out_labels, 0) + + def apply_non_transform_class( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + out_labels = [] + lam = torch.zeros((len(input)), device=input.device, dtype=DType.to_torch(int(params["dtype"].item()))) + for _ in range(self._param_generator.num_mix): + out_labels.append( + torch.stack( + [ + input.to(device=input.device, dtype=DType.to_torch(int(params["dtype"].item()))), + input.to(device=input.device, dtype=DType.to_torch(int(params["dtype"].item()))), + lam, + ], + 1, + ) + ) + + return torch.stack(out_labels, 0) + + def apply_transform( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], maybe_flags: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + height, width = input.size(2), input.size(3) + + out_inputs = input.clone() + for pair, crop in zip(params["mix_pairs"], params["crop_src"]): + input_permute = input.index_select(dim=0, index=pair.to(input.device)) + # compute mask to match input shape + mask = bbox_to_mask(crop, width, height).bool().unsqueeze(dim=1).repeat(1, input.size(1), 1, 1) + out_inputs[mask] = input_permute[mask] + + return out_inputs diff --git a/kornia/augmentation/_2d/mix/jigsaw.py b/kornia/augmentation/_2d/mix/jigsaw.py new file mode 100644 index 0000000..b56feb8 --- /dev/null +++ b/kornia/augmentation/_2d/mix/jigsaw.py @@ -0,0 +1,104 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.mix.base import MixAugmentationBaseV2 +from kornia.constants import DataKey + +__all__ = ["RandomJigsaw"] + + +class RandomJigsaw(MixAugmentationBaseV2): + r"""RandomJigsaw augmentation. + + .. image:: _static/img/RandomJigsaw.png + + Make Jigsaw puzzles for each image individually. To mix with different images in a + batch, referring to :class:`kornia.augmentation.RandomMosic`. + + Args: + grid: the Jigsaw puzzle grid. e.g. (2, 2) means + each output will mix image patches in a 2x2 grid. + ensure_perm: to ensure the nonidentical patch permutation generation against + the original one. + data_keys: the input type sequential for applying augmentations. + Accepts "input", "image", "mask", "bbox", "bbox_xyxy", "bbox_xywh", "keypoints", + "class", "label". + p: probability of applying the transformation for the whole batch. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input ``True`` or broadcast it + to the batch form ``False``. + + Examples: + >>> jigsaw = RandomJigsaw((4, 4)) + >>> input = torch.randn(8, 3, 256, 256) + >>> out = jigsaw(input) + >>> out.shape + torch.Size([8, 3, 256, 256]) + + """ + + def __init__( + self, + grid: Tuple[int, int] = (4, 4), + data_keys: Optional[List[Union[str, int, DataKey]]] = None, + p: float = 0.5, + same_on_batch: bool = False, + keepdim: bool = False, + ensure_perm: bool = True, + ) -> None: + super().__init__(p=p, p_batch=1.0, same_on_batch=same_on_batch, keepdim=keepdim, data_keys=data_keys) + self._param_generator = rg.JigsawGenerator(grid, ensure_perm) + self.flags = {"grid": grid} + + def apply_transform( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], maybe_flags: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + # different from the Base class routine. This function will not refer to any non-transformation images. + batch_prob = params["batch_prob"] + to_apply = batch_prob > 0.5 # NOTE: in case of Relaxed Distributions. + input = input[to_apply].clone() + + b, c, h, w = input.shape + # Params are generated for the full batch; slice to subset. + perm = params["permutation"][to_apply] + # Note: with a 100x100 image and a grid size of 3x3, it could work if + # we make h = piece_size_h * self.flags["grid"][0] with one pixel loss, then resize to 100 x 100. + # Probably worth to check if we should tolerate such "errorness" or to raise it as an error. + piece_size_h, piece_size_w = input.shape[-2] // self.flags["grid"][0], input.shape[-1] // self.flags["grid"][1] + # Convert to C BxN H' W' + input = ( + input.unfold(2, piece_size_h, piece_size_h) + .unfold(3, piece_size_w, piece_size_w) + .reshape(b, c, -1, piece_size_h, piece_size_w) + .permute(1, 0, 2, 3, 4) + .reshape(c, -1, piece_size_h, piece_size_w) + ) + perm = (perm + torch.arange(0, b, device=perm.device)[:, None] * perm.shape[1]).view(-1) + input = input[:, perm, :, :] + input = ( + input.reshape(-1, b, self.flags["grid"][1], h, piece_size_w) + .permute(0, 1, 2, 4, 3) + .reshape(-1, b, w, h) + .permute(0, 1, 3, 2) + .permute(1, 0, 2, 3) + ) + return input diff --git a/kornia/augmentation/_2d/mix/mixup.py b/kornia/augmentation/_2d/mix/mixup.py new file mode 100644 index 0000000..914890d --- /dev/null +++ b/kornia/augmentation/_2d/mix/mixup.py @@ -0,0 +1,143 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.mix.base import MixAugmentationBaseV2 +from kornia.constants import DataKey, DType + + +class RandomMixUpV2(MixAugmentationBaseV2): + r"""Apply MixUp augmentation to a batch of torch.Tensor images. + + .. image:: _static/img/RandomMixUpV2.png + + Implementation for `mixup: BEYOND EMPIRICAL RISK MINIMIZATION` :cite:`zhang2018mixup`. + + The function returns (inputs, labels), in which the inputs is the torch.Tensor that contains the mixup images + while the labels is a :math:`(B, 3)` torch.Tensor that contains (label_batch, label_permuted_batch, lambda) for + each image. + + The implementation is on top of the following repository: + `https://github.com/hongyi-zhang/mixup/blob/master/cifar/utils.py + `_. + + The loss and accuracy are computed as: + + .. code-block:: python + + def loss_mixup(y, logits): + criterion = F.cross_entropy + loss_a = criterion(logits, y[:, 0].long(), reduction='none') + loss_b = criterion(logits, y[:, 1].long(), reduction='none') + return ((1 - y[:, 2]) * loss_a + y[:, 2] * loss_b).mean() + + .. code-block:: python + + def acc_mixup(y, logits): + pred = torch.argmax(logits, dim=1).to(y.device) + return (1 - y[:, 2]) * pred.eq(y[:, 0]).float() + y[:, 2] * pred.eq(y[:, 1]).float() + + Args: + p: probability for applying an augmentation to a batch. This param controls the augmentation + probabilities batch-wisely. + lambda_val: min-max value of mixup strength. Default is 0-1. + same_on_batch: apply the same transformation across the batch. + This flag will not maintain permutation order. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Inputs: + - Input image tensors, shape of :math:`(B, C, H, W)`. + - Label: raw labels, shape of :math:`(B)`. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: + - Adjusted image, shape of :math:`(B, C, H, W)`. + - Raw labels, permuted labels and lambdas for each mix, shape of :math:`(B, 3)`. + + Note: + This implementation would randomly mixup images in a batch. Ideally, the larger batch size would be preferred. + + Examples: + >>> rng = torch.manual_seed(1) + >>> input = torch.rand(2, 1, 3, 3) + >>> label = torch.tensor([0, 1]) + >>> mixup = RandomMixUpV2(data_keys=["input", "class"]) + >>> mixup(input, label) + [tensor([[[[0.7576, 0.2793, 0.4031], + [0.7347, 0.0293, 0.7999], + [0.3971, 0.7544, 0.5695]]], + + + [[[0.4388, 0.6387, 0.5247], + [0.6826, 0.3051, 0.4635], + [0.4550, 0.5725, 0.4980]]]]), tensor([[0.0000, 0.0000, 0.1980], + [1.0000, 1.0000, 0.4162]])] + + """ + + def __init__( + self, + lambda_val: Optional[Union[torch.Tensor, Tuple[float, float]]] = None, + same_on_batch: bool = False, + p: float = 1.0, + keepdim: bool = False, + data_keys: Optional[List[Union[str, int, DataKey]]] = None, + ) -> None: + super().__init__(p=1.0, p_batch=p, same_on_batch=same_on_batch, keepdim=keepdim, data_keys=data_keys) + self._param_generator = rg.MixupGenerator(lambda_val, p=p) + + def apply_transform( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], maybe_flags: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + input_permute = input.index_select(dim=0, index=params["mixup_pairs"].to(input.device)) + + lam = params["mixup_lambdas"].view(-1, 1, 1, 1).expand_as(input).to(input.device) + inputs = input * (1 - lam) + input_permute * lam + return inputs + + def apply_non_transform_class( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], maybe_flags: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + out_labels = torch.stack( + [ + input.to(device=input.device, dtype=DType.to_torch(int(params["dtype"].item()))), + input.to(device=input.device, dtype=DType.to_torch(int(params["dtype"].item()))), + torch.zeros((len(input),), device=input.device, dtype=DType.to_torch(int(params["dtype"].item()))), + ], + -1, + ) + return out_labels + + def apply_transform_class( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], maybe_flags: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + labels_permute = input.index_select(dim=0, index=params["mixup_pairs"].to(input.device)) + + out_labels = torch.stack( + [ + input.to(device=input.device, dtype=DType.to_torch(int(params["dtype"].item()))), + labels_permute.to(device=input.device, dtype=DType.to_torch(int(params["dtype"].item()))), + params["mixup_lambdas"].to(device=input.device, dtype=DType.to_torch(int(params["dtype"].item()))), + ], + -1, + ) + return out_labels diff --git a/kornia/augmentation/_2d/mix/mosaic.py b/kornia/augmentation/_2d/mix/mosaic.py new file mode 100644 index 0000000..320ef59 --- /dev/null +++ b/kornia/augmentation/_2d/mix/mosaic.py @@ -0,0 +1,242 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.mix.base import MixAugmentationBaseV2 +from kornia.constants import DataKey, Resample +from kornia.core.check import KORNIA_UNWRAP +from kornia.core.ops import eye_like +from kornia.geometry.boxes import Boxes +from kornia.geometry.transform import crop_by_indices, crop_by_transform_mat, get_perspective_transform + +__all__ = ["RandomMosaic"] + + +class RandomMosaic(MixAugmentationBaseV2): + r"""Mosaic augmentation. + + .. image:: https://raw.githubusercontent.com/kornia/data/main/random_mosaic.png + + Given a certain number of images, mosaic transform combines them into one output image. + The output image is composed of the parts from each sub-image. To mess up each image individually, + referring to :class:`kornia.augmentation.RandomJigsaw`. + + The mosaic transform steps are as follows: + + 1. Concate selected images into a super-image. + 2. Crop out the outcome image according to the top-left corner and crop size. + + Args: + output_size: the output torch.Tensor width and height after mosaicing. + start_ratio_range: top-left (x, y) position for cropping the mosaic images. + mosaic_grid: the number of images and image arrangement. e.g. (2, 2) means + each output will mix 4 images in a 2x2 grid. + min_bbox_size: minimum area of bounding boxes. Default to 0. + data_keys: the input type sequential for applying augmentations. + Accepts "input", "image", "mask", "bbox", "bbox_xyxy", "bbox_xywh", "keypoints", + "class", "label". + p: probability of applying the transformation for the whole batch. + keepdim: whether to keep the output shape the same as input ``True`` or broadcast it + to the batch form ``False``. + padding_mode: Type of padding. Should be: constant, reflect, replicate. + resample: the interpolation mode. + align_corners: interpolation flag. + cropping_mode: The used algorithm to crop. ``slice`` will use advanced slicing to extract the torch.Tensor based + on the sampled indices. ``resample`` will use `warp_affine` using the affine transformation + to extract and resize at once. Use `slice` for efficiency, or `resample` for proper + differentiability. + + Examples: + >>> mosaic = RandomMosaic((300, 300), data_keys=["input", "bbox_xyxy"]) + >>> boxes = torch.tensor([[ + ... [70, 5, 150, 100], + ... [60, 180, 175, 220], + ... ]]).repeat(8, 1, 1) + >>> input = torch.randn(8, 3, 224, 224) + >>> out = mosaic(input, boxes) + >>> out[0].shape, out[1].shape + (torch.Size([8, 3, 300, 300]), torch.Size([8, 8, 4])) + + """ + + def __init__( + self, + output_size: Optional[Tuple[int, int]] = None, + mosaic_grid: Tuple[int, int] = (2, 2), + start_ratio_range: Tuple[float, float] = (0.3, 0.7), + min_bbox_size: float = 0.0, + data_keys: Optional[List[Union[str, int, DataKey]]] = None, + p: float = 0.7, + keepdim: bool = False, + padding_mode: str = "constant", + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + align_corners: bool = True, + cropping_mode: str = "slice", + ) -> None: + super().__init__(p=p, p_batch=1.0, same_on_batch=False, keepdim=keepdim, data_keys=data_keys) + self.start_ratio_range = start_ratio_range + self._param_generator = rg.MosaicGenerator(output_size, mosaic_grid, start_ratio_range) + + self.flags = { + "mosaic_grid": mosaic_grid, + "output_size": output_size, + "min_bbox_size": min_bbox_size, + "padding_mode": padding_mode, + "resample": Resample.get(resample), + "align_corners": align_corners, + "cropping_mode": cropping_mode, + } + + def apply_transform_mask( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + raise NotImplementedError + + @torch.no_grad() + def apply_transform_boxes(self, input: Boxes, params: Dict[str, torch.Tensor], flags: Dict[str, Any]) -> Boxes: + to_apply = params["batch_prob"] > 0.5 + src_box = torch.as_tensor(params["src"], device=input.device, dtype=input.dtype) + dst_box = torch.as_tensor(params["dst"], device=input.device, dtype=input.dtype) + # Boxes is BxNx4x2 only. + batch_shapes = torch.as_tensor(params["batch_shapes"], device=input.device, dtype=input.dtype) + offset = torch.zeros((len(to_apply), 2), device=input.device, dtype=input.dtype) # Bx2 + # NOTE: not a pretty good line I think. + offset_end = dst_box[0, 2].repeat(input.data.shape[0], 1) + idx = torch.arange(0, input.data.shape[0], device=input.device, dtype=torch.long)[to_apply] + + maybe_out_boxes: Optional[Boxes] = None + # ``batch_shapes`` and ``src_box`` are full-batch sized. + # Subset them to match ``idx`` (the to_apply indices). + batch_shapes_idx = batch_shapes[to_apply] + src_box_idx = src_box[to_apply] + for i in range(flags["mosaic_grid"][0]): + for j in range(flags["mosaic_grid"][1]): + _offset = offset.clone() + _offset[idx, 0] = batch_shapes_idx[:, -2] * i - src_box_idx[:, 0, 0] + _offset[idx, 1] = batch_shapes_idx[:, -1] * j - src_box_idx[:, 0, 1] + _box = input.clone() + _idx = i * flags["mosaic_grid"][1] + j + _box._data[params["permutation"][:, 0]] = _box._data[params["permutation"][:, _idx]] + _box.translate(_offset, inplace=True) + # zero-out unrelated batch elements. + _box._data[~to_apply] = 0 + if maybe_out_boxes is None: + _box._data[~to_apply] = input._data[~to_apply] + maybe_out_boxes = _box + else: + KORNIA_UNWRAP(maybe_out_boxes, Boxes).merge(_box, inplace=True) + out_boxes: Boxes = KORNIA_UNWRAP(maybe_out_boxes, Boxes) + out_boxes.clamp(offset, offset_end, inplace=True) + out_boxes.filter_boxes_by_area(flags["min_bbox_size"], inplace=True) + return out_boxes + + def apply_transform_keypoint( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + raise NotImplementedError + + def apply_transform_class( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + raise RuntimeError(f"{self.__class__.__name__} does not support `TAG` types.") + + @torch.no_grad() + def _compose_images( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + out = [] + for i in range(flags["mosaic_grid"][0]): + out_row = [] + for j in range(flags["mosaic_grid"][1]): + img_idx = flags["mosaic_grid"][1] * i + j + image = input[params["permutation"][:, img_idx]] + out_row.append(image) + out.append(torch.cat(out_row, -2)) + return torch.cat(out, -1) + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + if flags["cropping_mode"] == "resample": + transform: torch.Tensor = get_perspective_transform(params["src"].to(input), params["dst"].to(input)) + return transform + if flags["cropping_mode"] == "slice": # Skip the computation for slicing. + return eye_like(3, input) + raise NotImplementedError(f"Not supported type: {flags['cropping_mode']}.") + + def _crop_images( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + flags = self.flags if flags is None else flags + if flags["cropping_mode"] == "resample": # uses bilinear interpolation to crop + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the transform to be a torch.Tensor. Gotcha {type(transform)}") + + # Fit the arg to F.F.pad + if flags["padding_mode"] == "constant": + padding_mode = "zeros" + elif flags["padding_mode"] == "replicate": + padding_mode = "border" + elif flags["padding_mode"] == "reflect": + padding_mode = "reflection" + else: + padding_mode = flags["padding_mode"] + + return crop_by_transform_mat( + input, + transform, + flags["output_size"], + mode=flags["resample"].name.lower(), + padding_mode=padding_mode, + align_corners=flags["align_corners"], + ) + if flags["cropping_mode"] == "slice": # uses advanced slicing to crop + return crop_by_indices(input, params["src"], flags["output_size"], shape_compensation="F.pad") + raise NotImplementedError(f"Not supported type: {flags['cropping_mode']}.") + + def apply_non_transform( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + if flags is not None and flags["output_size"] is not None: + output_size = KORNIA_UNWRAP(flags["output_size"], Tuple[int, int]) + return F.pad(input, [0, output_size[1] - input.shape[-1], 0, output_size[0] - input.shape[-2]]) + # NOTE: resize is not suitable for being consistent with bounding boxes. + # return resize( + # input, + # size=flags["output_size"], + # interpolation=flags["resample"].name.lower(), + # align_corners=flags["align_corners"] + # ) + return input + + def apply_transform( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], maybe_flags: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + flags = KORNIA_UNWRAP(maybe_flags, Dict[str, Any]) + output = self._compose_images(input, params, flags=flags) + transform = self.compute_transformation(output, params, flags=flags) + output = self._crop_images(output, params, flags=flags, transform=transform) + return output diff --git a/kornia/augmentation/_2d/mix/patchmix.py b/kornia/augmentation/_2d/mix/patchmix.py new file mode 100644 index 0000000..b919cb3 --- /dev/null +++ b/kornia/augmentation/_2d/mix/patchmix.py @@ -0,0 +1,87 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, List, Optional, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._2d.mix.base import MixAugmentationBaseV2 +from kornia.constants import DataKey + + +class PatchMix(MixAugmentationBaseV2): + r"""PatchMix augmentation. + + .. image:: _static/img/PatchMix.png + + Replaces a random patch in each image of a batch with the corresponding + region from a randomly chosen different image in the batch. + + Implementation for `CutMix: Regularization Strategy to Train Strong + Classifiers with Localizable Features` :cite:`yun2019cutmix`. + + Args: + alpha: Hyperparameter for the Beta distribution used to generate the + mixing parameter lambda. + patch_size: The size of the square patch to mix. + p: Probability for applying the augmentation. + same_on_batch: Apply the same transformation across the batch. + keepdim: Whether to keep the output shape the same as input ``True`` + or broadcast it to the batch form ``False``. + + Examples: + >>> aug = PatchMix(alpha=1.0, patch_size=4) + >>> x = torch.rand(2, 3, 32, 32) + >>> out = aug(x) + >>> out.shape + torch.Size([2, 3, 32, 32]) + """ + + def __init__( + self, + alpha: float = 1.0, + patch_size: int = 16, + p: float = 1.0, + same_on_batch: bool = False, + keepdim: bool = False, + data_keys: Optional[List[Union[str, int, DataKey]]] = None, + ) -> None: + super().__init__(p=1.0, p_batch=p, same_on_batch=same_on_batch, keepdim=keepdim, data_keys=data_keys) + self.alpha = alpha + self.patch_size = patch_size + self._param_generator = rg.PatchMixGenerator(alpha, patch_size, p) + + def generate_parameters(self, batch_shape: torch.Size) -> Dict[str, torch.Tensor]: + return self._param_generator(batch_shape, self.same_on_batch) + + def apply_transform( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], extra_args: Dict[str, Any] + ) -> torch.Tensor: + B, _, _, _ = input.shape + idx = params["mix_pairs"].to(input.device) + patch_coords = params["patch_coords"].to(input.device) + + out = input.clone() + for i in range(B): + x, y = patch_coords[i] + x, y = int(x.item()), int(y.item()) + out[i, :, y : y + self.patch_size, x : x + self.patch_size] = input[ + idx[i], :, y : y + self.patch_size, x : x + self.patch_size + ] + + return out diff --git a/kornia/augmentation/_2d/mix/transplantation.py b/kornia/augmentation/_2d/mix/transplantation.py new file mode 100644 index 0000000..204c6f5 --- /dev/null +++ b/kornia/augmentation/_2d/mix/transplantation.py @@ -0,0 +1,353 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Any, Optional, Sequence, Union + +import torch + +from kornia.augmentation._2d.mix.base import MixAugmentationBaseV2 +from kornia.augmentation.utils import _validate_input_dtype +from kornia.constants import DataKey +from kornia.core.check import KORNIA_CHECK + +__all__ = ["RandomTransplantation"] + + +class RandomTransplantation(MixAugmentationBaseV2): + r"""RandomTransplantation augmentation. + + .. image:: _static/img/RandomTransplantation.png + + Randomly transplant (copy and paste) image features and corresponding segmentation masks between images in a batch. + The transplantation transform works as follows: + + 1. Based on the parameter `p`, a certain number of images in the batch are selected as acceptor of a + transplantation. + 2. For each acceptor, the image below in the batch is selected as donor (via circling: :math:`i - 1 \mod B`). + 3. From the donor, a random label is selected and the corresponding image features and segmentation mask are + transplanted to the acceptor. + + The augmentation is described in `Semantic segmentation of surgical hyperspectral images under geometric domain + shifts` :cite:`sellner2023semantic`. + + Args: + excluded_labels: sequence of labels which should not be transplanted from a donor. This can be useful if only + parts of the image are annotated and the non-annotated regions (with a specific label index) should be + excluded from the augmentation. If no label is left in the donor image, nothing is transplanted. + p: probability for applying an augmentation to an image. This parameter controls how many images in a batch + receive a transplant. + p_batch: probability for applying an augmentation to a batch. This param controls the augmentation + probabilities batch-wise. + data_keys: the input type sequential for applying augmentations. There must be at least one "mask" torch.Tensor. + If no data keys are given, the first torch.Tensor is assumed to be `DataKey.INPUT` and the second + torch.Tensor `DataKey.MASK`. + Accepts "input", "mask". + + Note: + - This augmentation requires that segmentation masks are available for all images in the batch and that at + least some objects in the image are annotated. + - When using this class directly (`RandomTransplantation()(...)`), it works for arbitrary spatial dimensions + including 2D and 3D images. When wrapping in :class:`kornia.augmentation.AugmentationSequential`, use + :class:`kornia.augmentation.RandomTransplantation` for 2D and + :class:`kornia.augmentation.RandomTransplantation3D` for 3D images. + + Inputs: + - Segmentation mask torch.Tensor which is used to determine the objects for transplantation: :math:`(B, *)`. + - (optional) Additional image or mask tensors where the features are transplanted based on the first + segmentation mask: :math:`(B, C, *)` (`DataKey.INPUT`) or :math:`(B, *)` (`DataKey.MASK`). + + Returns: + torch.Tensor | list[torch.Tensor]: + + torch.Tensor: + - Augmented mask tensors: :math:`(B, *)`. + list[torch.Tensor]: + - Augmented mask tensors: :math:`(B, *)`. + - Additional augmented image or mask tensors: :math:`(B, C, *)` (`DataKey.INPUT`) or :math:`(B, *)` + (`DataKey.MASK`). + + Examples: + >>> import torch + >>> rng = torch.manual_seed(0) + >>> aug = RandomTransplantation(p=1.) + >>> image = torch.randn(2, 3, 5, 5) + >>> mask = torch.randint(0, 3, (2, 5, 5)) + >>> mask + tensor([[[0, 0, 1, 1, 0], + [1, 2, 0, 0, 0], + [1, 2, 1, 1, 0], + [0, 0, 0, 0, 2], + [2, 2, 2, 0, 2]], + + [[2, 0, 0, 2, 1], + [2, 1, 0, 2, 1], + [2, 0, 1, 0, 2], + [2, 2, 2, 0, 2], + [2, 1, 0, 0, 0]]]) + >>> image_out, mask_out = aug(image, mask) + >>> image_out.shape + torch.Size([2, 3, 5, 5]) + >>> mask_out.shape + torch.Size([2, 5, 5]) + >>> mask_out + tensor([[[2, 0, 1, 2, 0], + [2, 2, 0, 2, 0], + [2, 2, 1, 1, 2], + [2, 2, 2, 0, 2], + [2, 2, 2, 0, 2]], + + [[0, 0, 0, 2, 0], + [2, 1, 0, 0, 0], + [2, 0, 1, 0, 0], + [0, 0, 0, 0, 2], + [2, 1, 0, 0, 0]]]) + >>> aug._params["selected_labels"] # Image 0 received label 2 from image 1 and image 1 label 0 from image 0 + tensor([2, 0]) + + You can apply the same augmentation again in which case the same objects get transplanted between the images: + + >>> aug._params["selection"] # The pixels (objects) which get transplanted + tensor([[[ True, False, False, True, False], + [ True, False, False, True, False], + [ True, False, False, False, True], + [ True, True, True, False, True], + [ True, False, False, False, False]], + + [[ True, True, False, False, True], + [False, False, True, True, True], + [False, False, False, False, True], + [ True, True, True, True, False], + [False, False, False, True, False]]]) + >>> image2 = torch.zeros(2, 3, 5, 5) + >>> image2[1] = 1 + >>> image2[:, 0] + tensor([[[0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.]], + + [[1., 1., 1., 1., 1.], + [1., 1., 1., 1., 1.], + [1., 1., 1., 1., 1.], + [1., 1., 1., 1., 1.], + [1., 1., 1., 1., 1.]]]) + >>> image_out2, mask_out2 = aug(image2, mask, params=aug._params) + >>> image_out2[:, 0] + tensor([[[1., 0., 0., 1., 0.], + [1., 0., 0., 1., 0.], + [1., 0., 0., 0., 1.], + [1., 1., 1., 0., 1.], + [1., 0., 0., 0., 0.]], + + [[0., 0., 1., 1., 0.], + [1., 1., 0., 0., 0.], + [1., 1., 1., 1., 0.], + [0., 0., 0., 0., 1.], + [1., 1., 1., 0., 1.]]]) + + """ + + def __init__( + self, + excluded_labels: Optional[Union[Sequence[int], torch.Tensor]] = None, + p: float = 0.5, + p_batch: float = 1.0, + data_keys: Optional[list[str | int | DataKey]] = None, + ) -> None: + super().__init__(p=p, p_batch=p_batch) + + if excluded_labels is None: + excluded_labels = [] + if not isinstance(excluded_labels, torch.Tensor): + excluded_labels = torch.tensor(excluded_labels) + self.excluded_labels: torch.Tensor = excluded_labels + KORNIA_CHECK( + self.excluded_labels.ndim == 1, + f"excluded_labels must be a 1-dimensional sequence, but got {self.excluded_labels.ndim} dimensions.", + ) + + if data_keys is None: + data_keys = [DataKey.INPUT, DataKey.MASK] + self.data_keys = [DataKey.get(inp) for inp in data_keys] + self._channel_dim = 1 + + def apply_non_transform_mask( + self, input: torch.Tensor, params: dict[str, torch.Tensor], flags: dict[str, Any] + ) -> torch.Tensor: + return input + + def transform_input(self, acceptor: torch.Tensor, donor: torch.Tensor, selection: torch.Tensor) -> torch.Tensor: # type: ignore[override] + # Expand selection to the channel dimension + selection = selection.unsqueeze(dim=self._channel_dim).expand_as(donor) + acceptor[selection] = donor[selection] + return acceptor + + def transform_mask(self, acceptor: torch.Tensor, donor: torch.Tensor, selection: torch.Tensor) -> torch.Tensor: # type: ignore[override] + acceptor[selection] = donor[selection] + return acceptor + + def params_from_input( + self, + *input: torch.Tensor, + data_keys: list[DataKey], + params: dict[str, torch.Tensor], + extra_args: Optional[dict[DataKey, dict[str, Any]]] = None, + ) -> dict[str, torch.Tensor]: + """Compute parameters for the transformation which are based on one or more input tensors. + + This function is, for example, called by :class:`kornia.augmentation.container.ops.AugmentationSequentialOps` + before the augmentation is applied on the individual input tensors. + + Args: + *input: All input tensors passed to the augmentation pipeline. + data_keys: Associated data key for every input torch.Tensor. + params: Dictionary of parameters computed so far by the augmentation pipeline (e.g. including the + `batch_prob`). + extra_args: Optional dictionary of extra arguments with specific options for different input types. + + Returns: + Updated dictionary of parameters with the necessary information to apply the augmentation on all input + tensors separately. + + """ + KORNIA_CHECK( + len(data_keys) == len(input), + f"Length of keys ({len(data_keys)}) does not match number of inputs ({len(input)}).", + ) + + # The first mask key will be used for the transplantation + mask: torch.Tensor = input[data_keys.index(DataKey.MASK)] + for _input, key in zip(input, data_keys): + if key == DataKey.INPUT: + KORNIA_CHECK( + _input.ndim == mask.ndim + 1, + "Every image input must have one additional dimension (channel dimension) than the segmentation " + f"mask, but got {_input.ndim} for the input image and {mask.ndim} for the segmentation mask.", + ) + KORNIA_CHECK( + mask.size() == torch.Size([s for i, s in enumerate(_input.size()) if i != self._channel_dim]), + "The dimensions of the input image and segmentation mask must match except for the channel " + f"dimension, but got {_input.size()} for the input image and {mask.size()} for the segmentation " + "mask.", + ) + + if "acceptor_indices" not in params: + params["acceptor_indices"] = torch.where(params["batch_prob"] > 0.5)[0] + if "donor_indices" not in params: + params["donor_indices"] = (params["acceptor_indices"] - 1) % len(params["batch_prob"]) + + if "selected_labels" not in params: + if self.excluded_labels.device != mask.device: + self.excluded_labels = self.excluded_labels.to(mask.device) + + donor_labels: list[torch.Tensor] = [] + for d in range(len(params["donor_indices"])): + # Select a random label from the donor image + current_mask = mask[params["donor_indices"][d]] + labels = current_mask.unique() + + # Remove any label which is part of the excluded labels + labels = labels[(labels.view(1, -1) != self.excluded_labels.view(-1, 1)).all(dim=0)] + + if len(labels) > 0: + selected_label = labels[torch.randperm(len(labels))[0]] + donor_labels.append(selected_label) + + params["selected_labels"] = torch.stack(donor_labels) if len(donor_labels) > 0 else torch.empty(0) + + if "selection" not in params: + selection = torch.zeros( + (len(params["acceptor_indices"]), *mask.shape[1:]), dtype=torch.bool, device=mask.device + ) + selected_labels: torch.Tensor = params["selected_labels"] + KORNIA_CHECK( + selected_labels.ndim == 1, + f"selected_labels must be a 1-dimensional torch.tensor, but got {selected_labels.ndim} dimensions.", + ) + KORNIA_CHECK( + len(selected_labels) <= len(params["acceptor_indices"]), + f"There cannot be more selected labels ({len(selected_labels)}) than images " + f"torch.where this augmentation " + f"should be applied ({len(params['acceptor_indices'])}).", + ) + + for d, selected_label in zip(range(len(params["donor_indices"])), selected_labels): + current_mask = mask[params["donor_indices"][d]] + selection[d].masked_fill_(current_mask == selected_label, True) + + params["selection"] = selection + + return params + + def forward( # type: ignore[override] + self, + *input: torch.Tensor, + params: Optional[dict[str, torch.Tensor]] = None, + data_keys: Optional[list[str | int | DataKey]] = None, + **kwargs: dict[str, Any], + ) -> torch.Tensor | list[torch.Tensor]: + keys: list[DataKey] + if data_keys is None: + keys = self.data_keys + else: + keys = [DataKey.get(inp) for inp in data_keys] + + if params is None: + mask: torch.Tensor = input[keys.index(DataKey.MASK)] + self._params = self.forward_parameters(mask.shape) + else: + self._params = params + + if any(k not in self._params for k in ["acceptor_indices", "donor_indices", "selection"]): + self._params.update(self.params_from_input(*input, data_keys=keys, params=self._params)) + + outputs: list[torch.Tensor] = [] + for dcate, _input in zip(keys, input): + acceptor = _input[self._params["acceptor_indices"]].clone() + donor = _input[self._params["donor_indices"]] + + output: torch.Tensor + if dcate == DataKey.INPUT: + _validate_input_dtype( + _input, accepted_dtypes=[torch.bfloat16, torch.float16, torch.float32, torch.float64] + ) + + applied = self.transform_input(acceptor, donor, self._params["selection"]) + output = self.apply_non_transform(_input, self._params, self.flags) + output = output.index_put( + (self._params["acceptor_indices"],), + self.apply_non_transform_mask(applied, self._params, self.flags), + ) + elif dcate == DataKey.MASK: + applied = self.transform_mask(acceptor, donor, self._params["selection"]) + output = self.apply_non_transform_mask(_input, self._params, self.flags) + output = output.index_put( + (self._params["acceptor_indices"],), + self.apply_non_transform_mask(applied, self._params, self.flags), + ) + else: + raise NotImplementedError + + outputs.append(output) + + if len(outputs) == 1: + return outputs[0] + else: + return outputs diff --git a/kornia/augmentation/_3d/__init__.py b/kornia/augmentation/_3d/__init__.py new file mode 100644 index 0000000..570e7e3 --- /dev/null +++ b/kornia/augmentation/_3d/__init__.py @@ -0,0 +1,20 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from kornia.augmentation._3d.geometric import * +from kornia.augmentation._3d.intensity import * +from kornia.augmentation._3d.mix import * diff --git a/kornia/augmentation/_3d/base.py b/kornia/augmentation/_3d/base.py new file mode 100644 index 0000000..b8d208b --- /dev/null +++ b/kornia/augmentation/_3d/base.py @@ -0,0 +1,166 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional + +import torch +from torch import float16, float32, float64 + +import kornia +from kornia.augmentation.base import _AugmentationBase +from kornia.augmentation.utils import _transform_input3d, _transform_input3d_by_shape, _validate_input_dtype +from kornia.geometry.boxes import Boxes3D +from kornia.geometry.keypoints import Keypoints3D + + +class AugmentationBase3D(_AugmentationBase): + r"""AugmentationBase3D base class for customized augmentation implementations. + + Args: + p: probability for applying an augmentation. This param controls the augmentation probabilities + element-wise for a batch. + p_batch: probability for applying an augmentation to a batch. This param controls the augmentation + probabilities batch-wise. + same_on_batch: apply the same transformation across the batch. + + """ + + def validate_tensor(self, input: torch.Tensor) -> None: + """Check if the input torch.Tensor is formatted as expected.""" + _validate_input_dtype(input, accepted_dtypes=[torch.bfloat16, float16, float32, float64]) + if len(input.shape) != 5: + raise RuntimeError(f"Expect (B, C, D, H, W). Got {input.shape}.") + + def transform_tensor( + self, input: torch.Tensor, *, shape: Optional[torch.Tensor] = None, match_channel: bool = True + ) -> torch.Tensor: + """Convert any incoming (D, H, W), (C, D, H, W) and (B, C, D, H, W) into (B, C, D, H, W).""" + _validate_input_dtype(input, accepted_dtypes=[torch.bfloat16, float16, float32, float64]) + if shape is None: + return _transform_input3d(input) + else: + return _transform_input3d_by_shape(input, reference_shape=shape, match_channel=match_channel) + + def identity_matrix(self, input: torch.Tensor) -> torch.Tensor: + """Return 4x4 identity matrix.""" + return kornia.core.ops.eye_like(4, input) + + +class RigidAffineAugmentationBase3D(AugmentationBase3D): + r"""AugmentationBase2D base class for rigid/affine augmentation implementations. + + RigidAffineAugmentationBase2D enables routined transformation with given transformation matrices + for different data types like masks, boxes, and keypoints. + + Args: + p: probability for applying an augmentation. This param controls the augmentation probabilities + element-wise for a batch. + p_batch: probability for applying an augmentation to a batch. This param controls the augmentation + probabilities batch-wise. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input ``True`` or broadcast it to the batch + form ``False``. + + """ + + _transform_matrix: Optional[torch.Tensor] + + @property + def transform_matrix(self) -> Optional[torch.Tensor]: + return self._transform_matrix + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + raise NotImplementedError + + def generate_transformation_matrix( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + """Generate transformation matrices with the given input and param settings.""" + batch_prob = params["batch_prob"] + to_apply = batch_prob > 0.5 + + in_tensor = self.transform_tensor(input) + + trans_matrix_applied = self.compute_transformation(in_tensor, params=params, flags=flags) + trans_matrix_identity = self.identity_matrix(in_tensor) + + if trans_matrix_applied.shape[0] == to_apply.shape[0] == trans_matrix_identity.shape[0]: + to_apply_expanded = to_apply.view(-1, *([1] * (trans_matrix_applied.dim() - 1))) + trans_matrix = torch.where(to_apply_expanded, trans_matrix_applied, trans_matrix_identity) + else: + trans_matrix = trans_matrix_applied if bool(to_apply.any()) else trans_matrix_identity + + return trans_matrix + + def inverse_inputs( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + raise NotImplementedError + + def inverse_masks( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + raise NotImplementedError + + def inverse_boxes( + self, + input: Boxes3D, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> Boxes3D: + raise NotImplementedError + + def inverse_keypoints( + self, + input: Keypoints3D, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> Keypoints3D: + raise NotImplementedError + + def inverse_classes( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + raise NotImplementedError + + def apply_func( + self, in_tensor: torch.Tensor, params: Dict[str, torch.Tensor], flags: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + if flags is None: + flags = self.flags + + trans_matrix = self.generate_transformation_matrix(in_tensor, params, flags) + output = self.transform_inputs(in_tensor, params, flags, trans_matrix) + self._transform_matrix = trans_matrix + + return output diff --git a/kornia/augmentation/_3d/geometric/__init__.py b/kornia/augmentation/_3d/geometric/__init__.py new file mode 100644 index 0000000..1be9b50 --- /dev/null +++ b/kornia/augmentation/_3d/geometric/__init__.py @@ -0,0 +1,25 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from kornia.augmentation._3d.geometric.affine import RandomAffine3D +from kornia.augmentation._3d.geometric.center_crop import CenterCrop3D +from kornia.augmentation._3d.geometric.crop import RandomCrop3D +from kornia.augmentation._3d.geometric.depthical_flip import RandomDepthicalFlip3D +from kornia.augmentation._3d.geometric.horizontal_flip import RandomHorizontalFlip3D +from kornia.augmentation._3d.geometric.perspective import RandomPerspective3D +from kornia.augmentation._3d.geometric.rotation import RandomRotation3D +from kornia.augmentation._3d.geometric.vertical_flip import RandomVerticalFlip3D diff --git a/kornia/augmentation/_3d/geometric/affine.py b/kornia/augmentation/_3d/geometric/affine.py new file mode 100644 index 0000000..f01b30b --- /dev/null +++ b/kornia/augmentation/_3d/geometric/affine.py @@ -0,0 +1,185 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math +from typing import Any, Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._3d.geometric.base import GeometricAugmentationBase3D +from kornia.constants import Resample +from kornia.geometry.transform.imgwarp import get_affine_matrix3d, warp_affine3d + + +class RandomAffine3D(GeometricAugmentationBase3D): + r"""Apply affine transformation 3D volumes (5D torch.Tensor). + + The transformation is computed so that the center is kept invariant. + + Args: + degrees: Range of yaw (x-axis), pitch (y-axis), roll (z-axis) to select from. + If degrees is a number, then yaw, pitch, roll will be generated from the range of (-degrees, +degrees). + If degrees is a tuple of (min, max), then yaw, pitch, roll will be generated from the range of (min, max). + If degrees is a list of floats [a, b, c], then yaw, pitch, roll will be generated from (-a, a), (-b, b) + and (-c, c). + If degrees is a list of tuple ((a, b), (m, n), (x, y)), then yaw, pitch, roll will be generated from + (a, b), (m, n) and (x, y). + Set to 0 to deactivate rotations. + translate: tuple of maximum absolute fraction for horizontal, vertical and + depthical translations (dx,dy,dz). For example translate=(a, b, c), then + horizontal shift will be randomly sampled in the range -img_width * a < dx < img_width * a + vertical shift will be randomly sampled in the range -img_height * b < dy < img_height * b. + depthical shift will be randomly sampled in the range -img_depth * c < dz < img_depth * c. + Will not translate by default. + scale: scaling factor interval. + If (a, b) represents isotropic scaling, the scale is randomly sampled from the range a <= scale <= b. + If ((a, b), (c, d), (e, f)), the scale is randomly sampled from the range a <= scale_x <= b, + c <= scale_y <= d, e <= scale_z <= f. Will keep original scale by default. + shears: Range of degrees to select from. + If shear is a number, a shear to the 6 facets in the range (-shear, +shear) will be applied. + If shear is a tuple of 2 values, a shear to the 6 facets in the range (shear[0], shear[1]) will be applied. + If shear is a tuple of 6 values, a shear to the i-th facet in the range (-shear[i], shear[i]) + will be applied. + If shear is a tuple of 6 tuples, a shear to the i-th facet in the range (-shear[i, 0], shear[i, 1]) + will be applied. + resample: resample mode from "nearest" (0) or "bilinear" (1). + same_on_batch: apply the same transformation across the batch. + align_corners: interpolation flag. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). Default: False. + + Shape: + - Input: :math:`(C, D, H, W)` or :math:`(B, C, D, H, W)`, Optional: :math:`(B, 4, 4)` + - Output: :math:`(B, C, D, H, W)` + + Note: + Input torch.Tensor must be float and normalized into [0, 1] for the best differentiability support. + Additionally, this function accepts another transformation torch.Tensor (:math:`(B, 4, 4)`), then the + applied transformation will be merged int to the input transformation torch.Tensor and returned. + + Examples: + >>> import torch + >>> rng = torch.manual_seed(0) + >>> input = torch.rand(1, 1, 3, 3, 3) + >>> aug = RandomAffine3D((15., 20., 20.), p=1.) + >>> aug(input), aug.transform_matrix + (tensor([[[[[0.4503, 0.4763, 0.1680], + [0.2029, 0.4267, 0.3515], + [0.3195, 0.5436, 0.3706]], + + [[0.5255, 0.3508, 0.4858], + [0.0795, 0.1689, 0.4220], + [0.5306, 0.7234, 0.6879]], + + [[0.2971, 0.2746, 0.3471], + [0.4924, 0.4960, 0.6460], + [0.3187, 0.4556, 0.7596]]]]]), tensor([[[ 0.9722, -0.0603, 0.2262, -0.1381], + [ 0.1131, 0.9669, -0.2286, 0.1486], + [-0.2049, 0.2478, 0.9469, 0.0102], + [ 0.0000, 0.0000, 0.0000, 1.0000]]])) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.rand(1, 3, 32, 32, 32) + >>> aug = RandomAffine3D((15., 20., 20.), p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + degrees: Union[ + torch.Tensor, + float, + Tuple[float, float], + Tuple[float, float, float], + Tuple[Tuple[float, float], Tuple[float, float], Tuple[float, float]], + ], + translate: Optional[Union[torch.Tensor, Tuple[float, float, float]]] = None, + scale: Optional[ + Union[ + torch.Tensor, Tuple[float, float], Tuple[Tuple[float, float], Tuple[float, float], Tuple[float, float]] + ] + ] = None, + shears: Union[ + None, + torch.Tensor, + float, + Tuple[float, float], + Tuple[float, float, float, float, float, float], + Tuple[ + Tuple[float, float], + Tuple[float, float], + Tuple[float, float], + Tuple[float, float], + Tuple[float, float], + Tuple[float, float], + ], + ] = None, + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + same_on_batch: bool = False, + align_corners: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self.degrees = degrees + self.shears = shears + self.translate = translate + self.scale = scale + + self.flags = {"resample": Resample.get(resample), "align_corners": align_corners} + self._param_generator = rg.AffineGenerator3D(degrees, translate, scale, shears) + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + # Inline ``deg2rad`` so the trace lowers to plain multiply (legacy ONNX has + # no symbolic for ``aten::deg2rad`` at opset 20). + deg2rad_factor: float = math.pi / 180.0 + transform: torch.Tensor = get_affine_matrix3d( + params["translations"], + params["center"], + params["scale"], + params["angles"], + params["sxy"] * deg2rad_factor, + params["sxz"] * deg2rad_factor, + params["syx"] * deg2rad_factor, + params["syz"] * deg2rad_factor, + params["szx"] * deg2rad_factor, + params["szy"] * deg2rad_factor, + ).to(input) + return transform + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the transform to be a torch.Tensor. Gotcha {type(transform)}") + + return warp_affine3d( + input, + transform[:, :3, :], + (input.shape[-3], input.shape[-2], input.shape[-1]), + flags["resample"].name.lower(), + align_corners=flags["align_corners"], + ) diff --git a/kornia/augmentation/_3d/geometric/base.py b/kornia/augmentation/_3d/geometric/base.py new file mode 100644 index 0000000..c6a25e3 --- /dev/null +++ b/kornia/augmentation/_3d/geometric/base.py @@ -0,0 +1,22 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from kornia.augmentation._3d.base import RigidAffineAugmentationBase3D + + +class GeometricAugmentationBase3D(RigidAffineAugmentationBase3D): + pass diff --git a/kornia/augmentation/_3d/geometric/center_crop.py b/kornia/augmentation/_3d/geometric/center_crop.py new file mode 100644 index 0000000..0b03358 --- /dev/null +++ b/kornia/augmentation/_3d/geometric/center_crop.py @@ -0,0 +1,121 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, Union, cast + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._3d.geometric.base import GeometricAugmentationBase3D +from kornia.constants import Resample +from kornia.geometry import crop_by_transform_mat3d, get_perspective_transform3d + + +class CenterCrop3D(GeometricAugmentationBase3D): + r"""Apply center crop on 3D volumes (5D torch.Tensor). + + Args: + p: probability of applying the transformation for the whole batch. + size (Tuple[int, int, int] or int): Desired output size (out_d, out_h, out_w) of the crop. + If integer, out_d = out_h = out_w = size. + If Tuple[int, int, int], out_d = size[0], out_h = size[1], out_w = size[2]. + resample: resample mode from "nearest" (0) or "bilinear" (1). + align_corners: interpolation flag. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, D, H, W)` or :math:`(B, C, D, H, W)`, Optional: :math:`(B, 4, 4)` + - Output: :math:`(B, C, out_d, out_h, out_w)` + + Note: + Input torch.Tensor must be float and normalized into [0, 1] for the best differentiability support. + Additionally, this function accepts another transformation torch.Tensor (:math:`(B, 4, 4)`), then the + applied transformation will be merged int to the input transformation torch.Tensor and returned. + + Examples: + >>> import torch + >>> rng = torch.manual_seed(0) + >>> inputs = torch.randn(1, 1, 2, 4, 6) + >>> inputs + tensor([[[[[-1.1258, -1.1524, -0.2506, -0.4339, 0.8487, 0.6920], + [-0.3160, -2.1152, 0.3223, -1.2633, 0.3500, 0.3081], + [ 0.1198, 1.2377, 1.1168, -0.2473, -1.3527, -1.6959], + [ 0.5667, 0.7935, 0.5988, -1.5551, -0.3414, 1.8530]], + + [[ 0.7502, -0.5855, -0.1734, 0.1835, 1.3894, 1.5863], + [ 0.9463, -0.8437, -0.6136, 0.0316, -0.4927, 0.2484], + [ 0.4397, 0.1124, 0.6408, 0.4412, -0.1023, 0.7924], + [-0.2897, 0.0525, 0.5229, 2.3022, -1.4689, -1.5867]]]]]) + >>> aug = CenterCrop3D(2, p=1.) + >>> aug(inputs) + tensor([[[[[ 0.3223, -1.2633], + [ 1.1168, -0.2473]], + + [[-0.6136, 0.0316], + [ 0.6408, 0.4412]]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.rand(1, 3, 32, 32, 32) + >>> aug = CenterCrop3D(24, p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + size: Union[int, Tuple[int, int, int]], + align_corners: bool = True, + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + p: float = 1.0, + keepdim: bool = False, + ) -> None: + # same_on_batch is always True for CenterCrop + # Since PyTorch does not support ragged torch.Tensor. So cropping function happens batch-wisely. + super().__init__(p=1.0, same_on_batch=True, p_batch=p, keepdim=keepdim) + if isinstance(size, tuple): + self.size = (size[0], size[1], size[2]) + elif isinstance(size, int): + self.size = (size, size, size) + else: + raise Exception(f"Invalid size type. Expected (int, tuple(int, int int). Got: {size}.") + self.flags = {"align_corners": align_corners, "resample": Resample.get(resample)} + + def generate_parameters(self, batch_shape: Tuple[int, ...]) -> Dict[str, torch.Tensor]: + return rg.center_crop_generator3d( + batch_shape[0], batch_shape[-3], batch_shape[-2], batch_shape[-1], self.size, device=self.device + ) + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + transform: torch.Tensor = get_perspective_transform3d(params["src"].to(input), params["dst"].to(input)) + transform = transform.expand(input.shape[0], -1, -1) + return transform + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + transform = cast(torch.Tensor, transform) + return crop_by_transform_mat3d( + input, transform, self.size, mode=flags["resample"].name.lower(), align_corners=flags["align_corners"] + ) diff --git a/kornia/augmentation/_3d/geometric/crop.py b/kornia/augmentation/_3d/geometric/crop.py new file mode 100644 index 0000000..e40ac22 --- /dev/null +++ b/kornia/augmentation/_3d/geometric/crop.py @@ -0,0 +1,165 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, Union + +import torch +import torch.nn.functional as F + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._3d.geometric.base import GeometricAugmentationBase3D +from kornia.constants import Resample +from kornia.geometry import crop_by_transform_mat3d, get_perspective_transform3d + + +class RandomCrop3D(GeometricAugmentationBase3D): + r"""Apply random crop on 3D volumes (5D torch.Tensor). + + Crops random sub-volumes on a given size. + + Args: + p: probability of applying the transformation for the whole batch. + size: Desired output size (out_d, out_h, out_w) of the crop. + Must be Tuple[int, int, int], then out_d = size[0], out_h = size[1], out_w = size[2]. + padding: Optional padding on each border of the image. + Default is None, i.e no padding. If a sequence of length 6 is provided, it is used to F.pad + left, top, right, bottom, front, back borders respectively. + If a sequence of length 3 is provided, it is used to F.pad left/right, + top/bottom, front/back borders, respectively. + pad_if_needed: It will F.pad the image if smaller than the + desired size to avoid raising an exception. Since cropping is done + after padding, the padding seems to be done at a random offset. + fill: Pixel fill value for constant fill. Default is 0. If a tuple of + length 3, it is used to fill R, G, B channels respectively. + This value is only used when the padding_mode is constant. + padding_mode: Type of padding. Should be: constant, edge, reflect or symmetric. Default is constant. + resample: resample mode from "nearest" (0) or "bilinear" (1). + same_on_batch: apply the same transformation across the batch. + align_corners: interpolation flag. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, D, H, W)` or :math:`(B, C, D, H, W)`, Optional: :math:`(B, 4, 4)` + - Output: :math:`(B, C, , out_d, out_h, out_w)` + + Note: + Input torch.Tensor must be float and normalized into [0, 1] for the best differentiability support. + Additionally, this function accepts another transformation torch.Tensor (:math:`(B, 4, 4)`), then the + applied transformation will be merged int to the input transformation torch.Tensor and returned. + + Examples: + >>> import torch + >>> rng = torch.manual_seed(0) + >>> inputs = torch.randn(1, 1, 3, 3, 3) + >>> aug = RandomCrop3D((2, 2, 2), p=1.) + >>> aug(inputs) + tensor([[[[[-1.1258, -1.1524], + [-0.4339, 0.8487]], + + [[-1.2633, 0.3500], + [ 0.1665, 0.8744]]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.rand(1, 3, 32, 32, 32) + >>> aug = RandomCrop3D((24, 24, 24), p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + size: Tuple[int, int, int], + padding: Optional[Union[int, Tuple[int, int, int], Tuple[int, int, int, int, int, int]]] = None, + pad_if_needed: Optional[bool] = False, + fill: int = 0, + padding_mode: str = "constant", + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + same_on_batch: bool = False, + align_corners: bool = True, + p: float = 1.0, + keepdim: bool = False, + ) -> None: + # Since PyTorch does not support ragged torch.Tensor. So cropping function happens batch-wisely. + super().__init__(p=1.0, same_on_batch=same_on_batch, p_batch=p, keepdim=keepdim) + self.flags = { + "size": size, + "padding": padding, + "pad_if_needed": pad_if_needed, + "padding_mode": padding_mode, + "fill": fill, + "resample": Resample.get(resample), + "align_corners": align_corners, + } + self._param_generator = rg.CropGenerator3D(size, None) + + def precrop_padding(self, input: torch.Tensor, flags: Optional[Dict[str, Any]] = None) -> torch.Tensor: + flags = self.flags if flags is None else flags + padding = flags["padding"] + if padding is not None: + if isinstance(padding, int): + padding = [padding, padding, padding, padding, padding, padding] + elif isinstance(padding, (tuple, list)) and len(padding) == 3: + padding = [padding[0], padding[0], padding[1], padding[1], padding[2], padding[2]] + elif isinstance(padding, (tuple, list)) and len(padding) == 6: + padding = [padding[0], padding[1], padding[2], padding[3], padding[4], padding[5]] + else: + raise ValueError(f"`padding` must be an integer, 3-element-list or 6-element-list. Got {padding}.") + input = F.pad(input, padding, value=flags["fill"], mode=flags["padding_mode"]) + + if flags["pad_if_needed"] and input.shape[-3] < flags["size"][0]: + padding = [0, 0, 0, 0, flags["size"][0] - input.shape[-3], flags["size"][0] - input.shape[-3]] + input = F.pad(input, padding, value=flags["fill"], mode=flags["padding_mode"]) + + if flags["pad_if_needed"] and input.shape[-2] < flags["size"][1]: + padding = [0, 0, (flags["size"][1] - input.shape[-2]), flags["size"][1] - input.shape[-2], 0, 0] + input = F.pad(input, padding, value=flags["fill"], mode=flags["padding_mode"]) + + if flags["pad_if_needed"] and input.shape[-1] < flags["size"][2]: + padding = [flags["size"][2] - input.shape[-1], flags["size"][2] - input.shape[-1], 0, 0, 0, 0] + input = F.pad(input, padding, value=flags["fill"], mode=flags["padding_mode"]) + + return input + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + transform: torch.Tensor = get_perspective_transform3d(params["src"].to(input), params["dst"].to(input)) + transform = transform.expand(input.shape[0], -1, -1) + return transform + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the transform to be a torch.Tensor. Gotcha {type(transform)}") + + return crop_by_transform_mat3d( + input, transform, flags["size"], mode=flags["resample"].name.lower(), align_corners=flags["align_corners"] + ) + + def forward( + self, input: torch.Tensor, params: Optional[Dict[str, torch.Tensor]] = None, **kwargs: Any + ) -> torch.Tensor: + # TODO: need to align 2D implementations + input = self.precrop_padding(input) + return super().forward(input, params) diff --git a/kornia/augmentation/_3d/geometric/depthical_flip.py b/kornia/augmentation/_3d/geometric/depthical_flip.py new file mode 100644 index 0000000..8b6ed4f --- /dev/null +++ b/kornia/augmentation/_3d/geometric/depthical_flip.py @@ -0,0 +1,91 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional + +import torch +from torch import Tensor + +from kornia.augmentation._3d.geometric.base import GeometricAugmentationBase3D + + +class RandomDepthicalFlip3D(GeometricAugmentationBase3D): + r"""Apply random flip along the depth axis of 3D volumes (5D tensor). + + Input should be a tensor of shape :math:`(C, D, H, W)` or a batch of tensors :math:`(*, C, D, H, W)`. + If Input is a tuple it is assumed that the first element contains the aforementioned tensors and the second, + the corresponding transformation matrix that has been applied to them. In this case the module + will Depthically flip the tensors and concatenate the corresponding transformation matrix to the + previous one. This is especially useful when using this functionality as part of an ``nn.Sequential`` module. + + Args: + p: probability of the image being flipped. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input ``True`` or broadcast it + to the batch form ``False``. + + Shape: + - Input: :math:`(C, D, H, W)` or :math:`(B, C, D, H, W)`, Optional: :math:`(B, 4, 4)` + - Output: :math:`(B, C, D, H, W)` + + Note: + Input tensor must be float and normalized into [0, 1] for the best differentiability support. + Additionally, this function accepts another transformation torch.tensor(:math:`(B, 4, 4)`), then the + applied transformation will be merged int to the input transformation tensor and returned. + + Examples: + >>> import torch + >>> x = torch.eye(3).repeat(3, 1, 1) + >>> seq = RandomDepthicalFlip3D(p=1.0) + >>> seq(x), seq.transform_matrix + (tensor([[[[[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]], + + [[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]], + + [[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]]]]), tensor([[[ 1., 0., 0., 0.], + [ 0., 1., 0., 0.], + [ 0., 0., -1., 2.], + [ 0., 0., 0., 1.]]])) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.rand(1, 3, 32, 32, 32) + >>> aug = RandomDepthicalFlip3D(p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__(self, same_on_batch: bool = False, p: float = 0.5, keepdim: bool = False) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + + def compute_transformation(self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any]) -> Tensor: + d: int = input.shape[-3] + flip_mat: Tensor = torch.tensor( + [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, -1, d - 1], [0, 0, 0, 1]], device=input.device, dtype=input.dtype + ) + return flip_mat.expand(input.shape[0], 4, 4) + + def apply_transform( + self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Tensor: + return torch.flip(input, [-3]) diff --git a/kornia/augmentation/_3d/geometric/horizontal_flip.py b/kornia/augmentation/_3d/geometric/horizontal_flip.py new file mode 100644 index 0000000..849fcf8 --- /dev/null +++ b/kornia/augmentation/_3d/geometric/horizontal_flip.py @@ -0,0 +1,85 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional + +import torch +from torch import Tensor + +from kornia.augmentation._3d.geometric.base import GeometricAugmentationBase3D + + +class RandomHorizontalFlip3D(GeometricAugmentationBase3D): + r"""Apply random horizontal flip to 3D volumes (5D tensor). + + Args: + p: probability of the image being flipped. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input ``True`` or broadcast it + to the batch form ``False``. + + Shape: + - Input: :math:`(C, D, H, W)` or :math:`(B, C, D, H, W)`, Optional: :math:`(B, 4, 4)` + - Output: :math:`(B, C, D, H, W)` + + Note: + Input tensor must be float and normalized into [0, 1] for the best differentiability support. + Additionally, this function accepts another transformation torch.tensor(:math:`(B, 4, 4)`), then the + applied transformation will be merged int to the input transformation tensor and returned. + + Examples: + >>> import torch + >>> x = torch.eye(3).repeat(3, 1, 1) + >>> seq = RandomHorizontalFlip3D(p=1.0) + >>> seq(x), seq.transform_matrix + (tensor([[[[[0., 0., 1.], + [0., 1., 0.], + [1., 0., 0.]], + + [[0., 0., 1.], + [0., 1., 0.], + [1., 0., 0.]], + + [[0., 0., 1.], + [0., 1., 0.], + [1., 0., 0.]]]]]), tensor([[[-1., 0., 0., 2.], + [ 0., 1., 0., 0.], + [ 0., 0., 1., 0.], + [ 0., 0., 0., 1.]]])) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.rand(1, 3, 32, 32, 32) + >>> aug = RandomHorizontalFlip3D(p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__(self, same_on_batch: bool = False, p: float = 0.5, keepdim: bool = False) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + + def compute_transformation(self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any]) -> Tensor: + w: int = input.shape[-1] + flip_mat: Tensor = torch.tensor( + [[-1, 0, 0, w - 1], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], device=input.device, dtype=input.dtype + ) + return flip_mat.expand(input.shape[0], 4, 4) + + def apply_transform( + self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Tensor: + return torch.flip(input, [-1]) diff --git a/kornia/augmentation/_3d/geometric/perspective.py b/kornia/augmentation/_3d/geometric/perspective.py new file mode 100644 index 0000000..98165b4 --- /dev/null +++ b/kornia/augmentation/_3d/geometric/perspective.py @@ -0,0 +1,119 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._3d.geometric.base import GeometricAugmentationBase3D +from kornia.constants import Resample +from kornia.geometry import get_perspective_transform3d, warp_perspective3d + + +class RandomPerspective3D(GeometricAugmentationBase3D): + r"""Apply andom perspective transformation to 3D volumes (5D torch.Tensor). + + Args: + p: probability of the image being perspectively transformed. + distortion_scale: it controls the degree of distortion and ranges from 0 to 1. + resample: resample mode from "nearest" (0) or "bilinear" (1). + same_on_batch: apply the same transformation across the batch. + align_corners: interpolation flag. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, D, H, W)` or :math:`(B, C, D, H, W)`, Optional: :math:`(B, 4, 4)` + - Output: :math:`(B, C, D, H, W)` + + Note: + Input torch.Tensor must be float and normalized into [0, 1] for the best differentiability support. + Additionally, this function accepts another transformation torch.Tensor (:math:`(B, 4, 4)`), then the + applied transformation will be merged int to the input transformation torch.Tensor and returned. + + Examples: + >>> import torch + >>> rng = torch.manual_seed(0) + >>> inputs= torch.tensor([[[ + ... [[1., 0., 0.], + ... [0., 1., 0.], + ... [0., 0., 1.]], + ... [[1., 0., 0.], + ... [0., 1., 0.], + ... [0., 0., 1.]], + ... [[1., 0., 0.], + ... [0., 1., 0.], + ... [0., 0., 1.]] + ... ]]]) + >>> aug = RandomPerspective3D(0.5, p=1., align_corners=True) + >>> aug(inputs) + tensor([[[[[0.3976, 0.5507, 0.0000], + [0.0901, 0.3668, 0.0000], + [0.0000, 0.0000, 0.0000]], + + [[0.2651, 0.4657, 0.0000], + [0.1390, 0.5174, 0.0000], + [0.0000, 0.0000, 0.0000]], + + [[0.0000, 0.1153, 0.0000], + [0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000]]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.rand(1, 3, 32, 32, 32) + >>> aug = RandomPerspective3D(0.5, p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + distortion_scale: Union[torch.Tensor, float] = 0.5, + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + same_on_batch: bool = False, + align_corners: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self.flags = {"resample": Resample.get(resample), "align_corners": align_corners} + self._param_generator = rg.PerspectiveGenerator3D(distortion_scale) + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + return get_perspective_transform3d(params["start_points"], params["end_points"]).to(input) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the transform to be a torch.Tensor. Gotcha {type(transform)}") + + return warp_perspective3d( + input, + transform, + (input.shape[-3], input.shape[-2], input.shape[-1]), + flags=flags["resample"].name.lower(), + align_corners=flags["align_corners"], + ) diff --git a/kornia/augmentation/_3d/geometric/rotation.py b/kornia/augmentation/_3d/geometric/rotation.py new file mode 100644 index 0000000..cf933cc --- /dev/null +++ b/kornia/augmentation/_3d/geometric/rotation.py @@ -0,0 +1,138 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, Union + +import torch + +import kornia +from kornia.augmentation import random_generator as rg +from kornia.augmentation._3d.geometric.base import GeometricAugmentationBase3D +from kornia.constants import Resample +from kornia.geometry import affine3d +from kornia.geometry.transform.affwarp import _compute_rotation_matrix3d, _compute_tensor_center3d + + +class RandomRotation3D(GeometricAugmentationBase3D): + r"""Apply random rotations to 3D volumes (5D torch.Tensor). + + Input should be a torch.Tensor of shape (C, D, H, W) or a batch of tensors :math:`(B, C, D, H, W)`. + If Input is a tuple it is assumed that the first element contains the aforementioned tensors and the second, + the corresponding transformation matrix that has been applied to them. In this case the module + will rotate the tensors and torch.cat the corresponding transformation matrix to the + previous one. This is especially useful when using this functionality as part of an ``nn.Sequential`` module. + + Args: + degrees: Range of degrees to select from. + If degrees is a number, then yaw, pitch, roll will be generated from the range of (-degrees, +degrees). + If degrees is a tuple of (min, max), then yaw, pitch, roll will be generated from the range of (min, max). + If degrees is a list of floats [a, b, c], then yaw, pitch, roll will be generated from (-a, a), (-b, b) + and (-c, c). + If degrees is a list of tuple ((a, b), (m, n), (x, y)), then yaw, pitch, roll will be generated from + (a, b), (m, n) and (x, y). + Set to 0 to deactivate rotations. + resample: resample mode from "nearest" (0) or "bilinear" (1). + same_on_batch: apply the same transformation across the batch. + align_corners: interpolation flag. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, D, H, W)` or :math:`(B, C, D, H, W)`, Optional: :math:`(B, 4, 4)` + - Output: :math:`(B, C, D, H, W)` + + Note: + Input torch.Tensor must be float and normalized into [0, 1] for the best differentiability support. + Additionally, this function accepts another transformation torch.Tensor (:math:`(B, 4, 4)`), then the + applied transformation will be merged int to the input transformation torch.Tensor and returned. + + Examples: + >>> import torch + >>> rng = torch.manual_seed(0) + >>> input = torch.rand(1, 1, 3, 3, 3) + >>> aug = RandomRotation3D((15., 20., 20.), p=1.0) + >>> aug(input), aug.transform_matrix + (tensor([[[[[0.3819, 0.4886, 0.2111], + [0.1196, 0.3833, 0.4722], + [0.3432, 0.5951, 0.4223]], + + [[0.5553, 0.4374, 0.2780], + [0.2423, 0.1689, 0.4009], + [0.4516, 0.6376, 0.7327]], + + [[0.1605, 0.3112, 0.3673], + [0.4931, 0.4620, 0.5700], + [0.3505, 0.4685, 0.8092]]]]]), tensor([[[ 0.9722, 0.1131, -0.2049, 0.1196], + [-0.0603, 0.9669, 0.2478, -0.1545], + [ 0.2262, -0.2286, 0.9469, 0.0556], + [ 0.0000, 0.0000, 0.0000, 1.0000]]])) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.rand(1, 3, 32, 32, 32) + >>> aug = RandomRotation3D((15., 20., 20.), p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + degrees: Union[ + torch.Tensor, + float, + Tuple[float, float, float], + Tuple[Tuple[float, float], Tuple[float, float], Tuple[float, float]], + ], + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + same_on_batch: bool = False, + align_corners: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + self.flags = {"resample": Resample.get(resample), "align_corners": align_corners} + self._param_generator = rg.RotationGenerator3D(degrees) + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + yaw: torch.Tensor = params["yaw"].to(input) + pitch: torch.Tensor = params["pitch"].to(input) + roll: torch.Tensor = params["roll"].to(input) + + center: torch.Tensor = _compute_tensor_center3d(input) + rotation_mat: torch.Tensor = _compute_rotation_matrix3d(yaw, pitch, roll, center.expand(yaw.shape[0], -1)) + + # rotation_mat is B x 3 x 4 and we need a B x 4 x 4 matrix + trans_mat: torch.Tensor = kornia.core.ops.eye_like(4, input) + trans_mat[:, 0] = rotation_mat[:, 0] + trans_mat[:, 1] = rotation_mat[:, 1] + trans_mat[:, 2] = rotation_mat[:, 2] + + return trans_mat + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if not isinstance(transform, torch.Tensor): + raise TypeError(f"Expected the transform to be a torch.Tensor. Gotcha {type(transform)}") + + return affine3d(input, transform[..., :3, :4], flags["resample"].name.lower(), "zeros", flags["align_corners"]) diff --git a/kornia/augmentation/_3d/geometric/vertical_flip.py b/kornia/augmentation/_3d/geometric/vertical_flip.py new file mode 100644 index 0000000..0aaa8aa --- /dev/null +++ b/kornia/augmentation/_3d/geometric/vertical_flip.py @@ -0,0 +1,91 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional + +import torch +from torch import Tensor + +from kornia.augmentation._3d.geometric.base import GeometricAugmentationBase3D + + +class RandomVerticalFlip3D(GeometricAugmentationBase3D): + r"""Apply random vertical flip to 3D volumes (5D tensor). + + Input should be a tensor of shape :math:`(C, D, H, W)` or a batch of tensors :math:`(*, C, D, H, W)`. + If Input is a tuple it is assumed that the first element contains the aforementioned tensors and the second, + the corresponding transformation matrix that has been applied to them. In this case the module + will Vertically flip the tensors and concatenate the corresponding transformation matrix to the + previous one. This is especially useful when using this functionality as part of an ``nn.Sequential`` module. + + Args: + p: probability of the image being flipped. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input ``True`` or broadcast it + to the batch form ``False``. + + Shape: + - Input: :math:`(C, D, H, W)` or :math:`(B, C, D, H, W)`, Optional: :math:`(B, 4, 4)` + - Output: :math:`(B, C, D, H, W)` + + Note: + Input tensor must be float and normalized into [0, 1] for the best differentiability support. + Additionally, this function accepts another transformation torch.tensor(:math:`(B, 4, 4)`), then the + applied transformation will be merged int to the input transformation tensor and returned. + + Examples: + >>> import torch + >>> x = torch.eye(3).repeat(3, 1, 1) + >>> seq = RandomVerticalFlip3D(p=1.0) + >>> seq(x), seq.transform_matrix + (tensor([[[[[0., 0., 1.], + [0., 1., 0.], + [1., 0., 0.]], + + [[0., 0., 1.], + [0., 1., 0.], + [1., 0., 0.]], + + [[0., 0., 1.], + [0., 1., 0.], + [1., 0., 0.]]]]]), tensor([[[ 1., 0., 0., 0.], + [ 0., -1., 0., 2.], + [ 0., 0., 1., 0.], + [ 0., 0., 0., 1.]]])) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.rand(1, 3, 32, 32, 32) + >>> aug = RandomVerticalFlip3D(p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__(self, same_on_batch: bool = False, p: float = 0.5, keepdim: bool = False) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + + def compute_transformation(self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any]) -> Tensor: + h: int = input.shape[-2] + flip_mat: Tensor = torch.tensor( + [[1, 0, 0, 0], [0, -1, 0, h - 1], [0, 0, 1, 0], [0, 0, 0, 1]], device=input.device, dtype=input.dtype + ) + return flip_mat.expand(input.shape[0], 4, 4) + + def apply_transform( + self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Tensor: + return torch.flip(input, [-2]) diff --git a/kornia/augmentation/_3d/intensity/__init__.py b/kornia/augmentation/_3d/intensity/__init__.py new file mode 100644 index 0000000..11edb47 --- /dev/null +++ b/kornia/augmentation/_3d/intensity/__init__.py @@ -0,0 +1,19 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from kornia.augmentation._3d.intensity.equalize import RandomEqualize3D +from kornia.augmentation._3d.intensity.motion_blur import RandomMotionBlur3D diff --git a/kornia/augmentation/_3d/intensity/base.py b/kornia/augmentation/_3d/intensity/base.py new file mode 100644 index 0000000..fe26ec1 --- /dev/null +++ b/kornia/augmentation/_3d/intensity/base.py @@ -0,0 +1,22 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from kornia.augmentation._3d.base import RigidAffineAugmentationBase3D + + +class IntensityAugmentationBase3D(RigidAffineAugmentationBase3D): + pass diff --git a/kornia/augmentation/_3d/intensity/equalize.py b/kornia/augmentation/_3d/intensity/equalize.py new file mode 100644 index 0000000..c82747c --- /dev/null +++ b/kornia/augmentation/_3d/intensity/equalize.py @@ -0,0 +1,79 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional + +from torch import Tensor + +from kornia.augmentation._3d.intensity.base import IntensityAugmentationBase3D +from kornia.enhance import equalize3d + + +class RandomEqualize3D(IntensityAugmentationBase3D): + r"""Apply random equalization to 3D volumes (5D tensor). + + Args: + p: probability of the image being equalized. + same_on_batch): apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). + + Shape: + - Input: :math:`(C, D, H, W)` or :math:`(B, C, D, H, W)`, Optional: :math:`(B, 4, 4)` + - Output: :math:`(B, C, D, H, W)` + + Note: + Input tensor must be float and normalized into [0, 1] for the best differentiability support. + Additionally, this function accepts another transformation tensor (:math:`(B, 4, 4)`), then the + applied transformation will be merged int to the input transformation tensor and returned. + + Examples: + >>> import torch + >>> rng = torch.manual_seed(0) + >>> input = torch.rand(1, 1, 3, 3, 3) + >>> aug = RandomEqualize3D(p=1.0) + >>> aug(input) + tensor([[[[[0.4963, 0.7682, 0.0885], + [0.1320, 0.3074, 0.6341], + [0.4901, 0.8964, 0.4556]], + + [[0.6323, 0.3489, 0.4017], + [0.0223, 0.1689, 0.2939], + [0.5185, 0.6977, 0.8000]], + + [[0.1610, 0.2823, 0.6816], + [0.9152, 0.3971, 0.8742], + [0.4194, 0.5529, 0.9527]]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.rand(1, 3, 32, 32, 32) + >>> aug = RandomEqualize3D(p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__(self, p: float = 0.5, same_on_batch: bool = False, keepdim: bool = False) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, keepdim=keepdim) + + def compute_transformation(self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any]) -> Tensor: + return self.identity_matrix(input) + + def apply_transform( + self, input: Tensor, params: Dict[str, Tensor], flags: Dict[str, Any], transform: Optional[Tensor] = None + ) -> Tensor: + return equalize3d(input) diff --git a/kornia/augmentation/_3d/intensity/motion_blur.py b/kornia/augmentation/_3d/intensity/motion_blur.py new file mode 100644 index 0000000..a34c160 --- /dev/null +++ b/kornia/augmentation/_3d/intensity/motion_blur.py @@ -0,0 +1,138 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation import random_generator as rg +from kornia.augmentation._3d.intensity.base import IntensityAugmentationBase3D +from kornia.constants import BorderType, Resample +from kornia.filters import motion_blur3d + + +class RandomMotionBlur3D(IntensityAugmentationBase3D): + r"""Apply random motion blur on 3D volumes (5D torch.Tensor). + + Args: + p: probability of applying the transformation. + kernel_size: motion kernel size (odd and positive). + If int, the kernel will have a fixed size. + If Tuple[int, int], it will randomly generate the value from the range batch-wisely. + angle: Range of degrees to select from. + If angle is a number, then yaw, pitch, roll will be generated from the range of (-angle, +angle). + If angle is a tuple of (min, max), then yaw, pitch, roll will be generated from the range of (min, max). + If angle is a list of floats [a, b, c], then yaw, pitch, roll will be generated from (-a, a), (-b, b) + and (-c, c). + If angle is a list of tuple ((a, b), (m, n), (x, y)), then yaw, pitch, roll will be generated from + (a, b), (m, n) and (x, y). + Set to 0 to deactivate rotations. + direction: forward/backward direction of the motion blur. + Lower values towards -1.0 will point the motion blur towards the back (with angle provided via angle), + while higher values towards 1.0 will point the motion blur forward. A value of 0.0 leads to a + uniformly (but still angled) motion blur. + If float, it will generate the value from (-direction, direction). + If Tuple[int, int], it will randomly generate the value from the range. + border_type: the padding mode to be applied before convolving. + CONSTANT = 0, REFLECT = 1, REPLICATE = 2, CIRCULAR = 3. Default: BorderType.CONSTANT. + resample: resample mode from "nearest" (0) or "bilinear" (1). + keepdim: whether to keep the output shape the same as input (True) or broadcast it to the batch form (False). + + Shape: + - Input: :math:`(C, D, H, W)` or :math:`(B, C, D, H, W)`, Optional: :math:`(B, 4, 4)` + - Output: :math:`(B, C, D, H, W)` + + Note: + Input torch.Tensor must be float and normalized into [0, 1] for the best differentiability support. + Additionally, this function accepts another transformation torch.Tensor (:math:`(B, 4, 4)`), then the + applied transformation will be merged int to the input transformation torch.Tensor and returned. + + Examples: + >>> import torch + >>> rng = torch.manual_seed(0) + >>> input = torch.rand(1, 1, 3, 5, 5) + >>> motion_blur = RandomMotionBlur3D(3, 35., 0.5, p=1.) + >>> motion_blur(input) + tensor([[[[[0.1654, 0.4772, 0.2004, 0.3566, 0.2613], + [0.4557, 0.3131, 0.4809, 0.2574, 0.2696], + [0.2721, 0.5998, 0.3956, 0.5363, 0.1541], + [0.3006, 0.4773, 0.6395, 0.2856, 0.3989], + [0.4491, 0.5595, 0.1836, 0.3811, 0.1398]], + + [[0.1843, 0.4240, 0.3370, 0.1231, 0.2186], + [0.4047, 0.3332, 0.1901, 0.5329, 0.3023], + [0.3070, 0.3088, 0.4807, 0.4928, 0.2590], + [0.2416, 0.4614, 0.7091, 0.5237, 0.1433], + [0.1582, 0.4577, 0.2749, 0.1369, 0.1607]], + + [[0.2733, 0.4040, 0.4396, 0.2284, 0.3319], + [0.3856, 0.6730, 0.4624, 0.3878, 0.3076], + [0.4307, 0.4217, 0.2977, 0.5086, 0.5406], + [0.3686, 0.2778, 0.5228, 0.7592, 0.6455], + [0.2033, 0.3014, 0.4898, 0.6164, 0.3117]]]]]) + + To apply the exact augmenation again, you may take the advantage of the previous parameter state: + >>> input = torch.rand(1, 3, 32, 32, 32) + >>> aug = RandomMotionBlur3D(3, 35., 0.5, p=1.) + >>> (aug(input) == aug(input, params=aug._params)).all() + tensor(True) + + """ + + def __init__( + self, + kernel_size: Union[int, Tuple[int, int]], + angle: Union[ + torch.Tensor, + float, + Tuple[float, float, float], + Tuple[Tuple[float, float], Tuple[float, float], Tuple[float, float]], + ], + direction: Union[torch.Tensor, float, Tuple[float, float]], + border_type: Union[int, str, BorderType] = BorderType.CONSTANT.name, + resample: Union[str, int, Resample] = Resample.NEAREST.name, + same_on_batch: bool = False, + p: float = 0.5, + keepdim: bool = False, + ) -> None: + super().__init__(p=p, same_on_batch=same_on_batch, p_batch=1.0, keepdim=keepdim) + self.flags = {"border_type": BorderType.get(border_type), "resample": Resample.get(resample)} + self._param_generator = rg.MotionBlurGenerator3D(kernel_size, angle, direction) + + def compute_transformation( + self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any] + ) -> torch.Tensor: + return self.identity_matrix(input) + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + kernel_size = int(params["ksize_factor"].unique().item()) + angle = params["angle_factor"] + direction = params["direction_factor"] + return motion_blur3d( + input, + kernel_size, + angle, + direction, + self.flags["border_type"].name.lower(), + self.flags["resample"].name.lower(), + ) diff --git a/kornia/augmentation/_3d/mix/__init__.py b/kornia/augmentation/_3d/mix/__init__.py new file mode 100644 index 0000000..2c668c4 --- /dev/null +++ b/kornia/augmentation/_3d/mix/__init__.py @@ -0,0 +1,18 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from kornia.augmentation._3d.mix.transplantation import RandomTransplantation3D diff --git a/kornia/augmentation/_3d/mix/transplantation.py b/kornia/augmentation/_3d/mix/transplantation.py new file mode 100644 index 0000000..3f343cd --- /dev/null +++ b/kornia/augmentation/_3d/mix/transplantation.py @@ -0,0 +1,31 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from kornia.augmentation._2d.mix.transplantation import RandomTransplantation +from kornia.augmentation._3d.base import AugmentationBase3D + +__all__ = ["RandomTransplantation3D"] + + +class RandomTransplantation3D(RandomTransplantation, AugmentationBase3D): # type: ignore + """RandomTransplantation3D augmentation. + + 3D version of the :class:`kornia.augmentation.RandomTransplantation` augmentation intended to be used with + :class:`kornia.augmentation.AugmentationSequential`. The interface is identical to the 2D version. + """ + + pass diff --git a/kornia/augmentation/__init__.py b/kornia/augmentation/__init__.py new file mode 100644 index 0000000..fea7e17 --- /dev/null +++ b/kornia/augmentation/__init__.py @@ -0,0 +1,206 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Augmentation — Differentiable image, video, and 3D data augmentation for PyTorch. + +This subpackage provides a wide range of augmentation modules for computer vision tasks. +""" + +# Lazy loading auto module +from kornia.augmentation import auto, container +from kornia.augmentation._2d import ( + CenterCrop, + ColorJiggle, + ColorJitter, + Denormalize, + LongestMaxSize, + Normalize, + PadTo, + PatchMix, + RandomAffine, + RandomAutoContrast, + RandomBoxBlur, + RandomBrightness, + RandomChannelDropout, + RandomChannelShuffle, + RandomClahe, + RandomContrast, + RandomCrop, + RandomCutMixV2, + RandomDissolving, + RandomElasticTransform, + RandomEqualize, + RandomErasing, + RandomFisheye, + RandomGamma, + RandomGaussianBlur, + RandomGaussianIllumination, + RandomGaussianNoise, + RandomGrayscale, + RandomHorizontalFlip, + RandomHue, + RandomInvert, + RandomJigsaw, + RandomJPEG, + RandomLinearCornerIllumination, + RandomLinearIllumination, + RandomMedianBlur, + RandomMixUpV2, + RandomMosaic, + RandomMotionBlur, + RandomPerspective, + RandomPlanckianJitter, + RandomPlasmaBrightness, + RandomPlasmaContrast, + RandomPlasmaShadow, + RandomPosterize, + RandomRain, + RandomResizedCrop, + RandomRGBShift, + RandomRotation, + RandomRotation90, + RandomSaltAndPepperNoise, + RandomSaturation, + RandomSharpness, + RandomShear, + RandomSnow, + RandomSolarize, + RandomThinPlateSpline, + RandomTranslate, + RandomTransplantation, + RandomVerticalFlip, + Resize, + SmallestMaxSize, +) +from kornia.augmentation._2d.base import AugmentationBase2D, RigidAffineAugmentationBase2D +from kornia.augmentation._2d.geometric.base import GeometricAugmentationBase2D +from kornia.augmentation._2d.intensity.base import IntensityAugmentationBase2D +from kornia.augmentation._2d.mix.base import MixAugmentationBaseV2 +from kornia.augmentation._3d import ( + CenterCrop3D, + RandomAffine3D, + RandomCrop3D, + RandomDepthicalFlip3D, + RandomEqualize3D, + RandomHorizontalFlip3D, + RandomMotionBlur3D, + RandomPerspective3D, + RandomRotation3D, + RandomTransplantation3D, + RandomVerticalFlip3D, +) +from kornia.augmentation._3d.base import AugmentationBase3D, RigidAffineAugmentationBase3D +from kornia.augmentation._3d.geometric.base import GeometricAugmentationBase3D +from kornia.augmentation._3d.intensity.base import IntensityAugmentationBase3D +from kornia.augmentation.container import ( + AugmentationSequential, + ImageSequential, + ManyToManyAugmentationDispather, + ManyToOneAugmentationDispather, + PatchSequential, + VideoSequential, +) + +__all__ = [ + "AugmentationBase2D", + "AugmentationBase3D", + "AugmentationSequential", + "CenterCrop", + "CenterCrop3D", + "ColorJiggle", + "ColorJitter", + "Denormalize", + "GeometricAugmentationBase2D", + "GeometricAugmentationBase3D", + "ImageSequential", + "IntensityAugmentationBase2D", + "IntensityAugmentationBase3D", + "LongestMaxSize", + "ManyToManyAugmentationDispather", + "ManyToOneAugmentationDispather", + "MixAugmentationBaseV2", + "Normalize", + "PadTo", + "PatchMix", + "PatchSequential", + "RandomAffine", + "RandomAffine3D", + "RandomAutoContrast", + "RandomBoxBlur", + "RandomBrightness", + "RandomChannelDropout", + "RandomChannelShuffle", + "RandomContrast", + "RandomCrop", + "RandomCrop3D", + "RandomCutMixV2", + "RandomDepthicalFlip3D", + "RandomDissolving", + "RandomElasticTransform", + "RandomEqualize", + "RandomEqualize3D", + "RandomErasing", + "RandomFisheye", + "RandomGamma", + "RandomGaussianBlur", + "RandomGaussianIllumination", + "RandomGaussianNoise", + "RandomGrayscale", + "RandomHorizontalFlip", + "RandomHorizontalFlip3D", + "RandomHue", + "RandomInvert", + "RandomJigsaw", + "RandomLinearCornerIllumination", + "RandomLinearIllumination", + "RandomMedianBlur", + "RandomMixUpV2", + "RandomMosaic", + "RandomMotionBlur", + "RandomMotionBlur3D", + "RandomPerspective", + "RandomPerspective3D", + "RandomPlanckianJitter", + "RandomPlasmaBrightness", + "RandomPlasmaContrast", + "RandomPlasmaShadow", + "RandomPosterize", + "RandomRGBShift", + "RandomRGBShift", + "RandomRain", + "RandomResizedCrop", + "RandomRotation", + "RandomRotation3D", + "RandomRotation90", + "RandomSaltAndPepperNoise", + "RandomSaturation", + "RandomSharpness", + "RandomShear", + "RandomSnow", + "RandomSolarize", + "RandomThinPlateSpline", + "RandomTranslate", + "RandomVerticalFlip", + "RandomVerticalFlip3D", + "Resize", + "RigidAffineAugmentationBase2D", + "RigidAffineAugmentationBase3D", + "SmallestMaxSize", + "VideoSequential", + "auto", + "container", +] diff --git a/kornia/augmentation/auto/__init__.py b/kornia/augmentation/auto/__init__.py new file mode 100644 index 0000000..c82cf27 --- /dev/null +++ b/kornia/augmentation/auto/__init__.py @@ -0,0 +1,29 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Augmentation Auto — Automated augmentation policies for computer vision. + +This subpackage provides AutoAugment and related policy-based augmentation utilities. +""" + +from .autoaugment import AutoAugment +from .base import PolicyAugmentBase +from .operations import PolicySequential +from .rand_augment import RandAugment +from .trivial_augment import TrivialAugment + +__all__ = ["AutoAugment", "PolicyAugmentBase", "PolicySequential", "RandAugment", "TrivialAugment"] diff --git a/kornia/augmentation/auto/autoaugment/__init__.py b/kornia/augmentation/auto/autoaugment/__init__.py new file mode 100644 index 0000000..0d6758b --- /dev/null +++ b/kornia/augmentation/auto/autoaugment/__init__.py @@ -0,0 +1,23 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia AutoAugment — AutoAugment policy implementations for Kornia augmentation. + +This subpackage provides the AutoAugment class and related utilities. +""" + +from .autoaugment import AutoAugment diff --git a/kornia/augmentation/auto/autoaugment/autoaugment.py b/kornia/augmentation/auto/autoaugment/autoaugment.py new file mode 100644 index 0000000..74f739c --- /dev/null +++ b/kornia/augmentation/auto/autoaugment/autoaugment.py @@ -0,0 +1,183 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Iterator, List, Optional, Tuple, Union + +import torch +from torch import nn +from torch.distributions import Categorical + +from kornia.augmentation.auto.base import SUBPOLICY_CONFIG, PolicyAugmentBase +from kornia.augmentation.auto.operations.policy import PolicySequential +from kornia.augmentation.container.params import ParamItem + +from . import ops + +imagenet_policy: List[SUBPOLICY_CONFIG] = [ + [("posterize", 0.4, 8), ("rotate", 0.6, 9)], + [("solarize", 0.6, 5), ("auto_contrast", 0.6, None)], + [("equalize", 0.8, None), ("equalize", 0.6, None)], + [("posterize", 0.6, 7), ("posterize", 0.6, 6)], + [("equalize", 0.4, None), ("solarize", 0.2, 4)], + [("equalize", 0.4, None), ("rotate", 0.8, 8)], + [("solarize", 0.6, 3), ("equalize", 0.6, None)], + [("posterize", 0.8, 5), ("equalize", 1.0, None)], + [("rotate", 0.2, 3), ("solarize", 0.6, 8)], + [("equalize", 0.6, None), ("posterize", 0.4, 6)], + [("rotate", 0.8, 8), ("color", 0.4, 0)], + [("rotate", 0.4, 9), ("equalize", 0.6, None)], + [("equalize", 0.0, None), ("equalize", 0.8, None)], + [("invert", 0.6, None), ("equalize", 1.0, None)], + [("color", 0.6, 4), ("contrast", 1.0, 8)], + [("rotate", 0.8, 8), ("color", 1.0, 2)], + [("color", 0.8, 8), ("solarize", 0.8, 7)], + [("sharpness", 0.4, 7), ("invert", 0.6, None)], + [("shear_x", 0.6, 5), ("equalize", 1.0, None)], + [("color", 0.4, 0), ("equalize", 0.6, None)], + [("equalize", 0.4, None), ("solarize", 0.2, 4)], + [("solarize", 0.6, 5), ("auto_contrast", 0.6, None)], + [("invert", 0.6, None), ("equalize", 1.0, None)], + [("color", 0.6, 4), ("contrast", 1.0, 8)], + [("equalize", 0.8, None), ("equalize", 0.6, None)], +] + + +cifar10_policy: List[SUBPOLICY_CONFIG] = [ + [("invert", 0.1, None), ("contrast", 0.2, 6)], + [("rotate", 0.7, 2), ("translate_x", 0.3, 9)], + [("sharpness", 0.8, 1), ("sharpness", 0.9, 3)], + [("shear_y", 0.5, 8), ("translate_y", 0.7, 9)], + [("auto_contrast", 0.5, None), ("equalize", 0.9, None)], + [("shear_y", 0.2, 7), ("posterize", 0.3, 7)], + [("color", 0.4, 3), ("brightness", 0.6, 7)], + [("sharpness", 0.3, 9), ("brightness", 0.7, 9)], + [("equalize", 0.6, None), ("equalize", 0.5, None)], + [("contrast", 0.6, 7), ("sharpness", 0.6, 5)], + [("color", 0.7, 7), ("translate_x", 0.5, 8)], + [("equalize", 0.3, None), ("auto_contrast", 0.4, None)], + [("translate_y", 0.4, 3), ("sharpness", 0.2, 6)], + [("brightness", 0.9, 6), ("color", 0.2, 8)], + [("solarize", 0.5, 2), ("invert", 0.0, None)], + [("equalize", 0.2, None), ("auto_contrast", 0.6, None)], + [("equalize", 0.2, None), ("equalize", 0.6, None)], + [("color", 0.9, 9), ("equalize", 0.6, None)], + [("auto_contrast", 0.8, None), ("solarize", 0.2, 8)], + [("brightness", 0.1, 3), ("color", 0.7, 0)], + [("solarize", 0.4, 5), ("auto_contrast", 0.9, None)], + [("translate_y", 0.9, 9), ("translate_y", 0.7, 9)], + [("auto_contrast", 0.9, None), ("solarize", 0.8, 3)], + [("equalize", 0.8, None), ("invert", 0.1, None)], + [("translate_y", 0.7, 9), ("auto_contrast", 0.9, None)], +] + + +svhn_policy: List[SUBPOLICY_CONFIG] = [ + [("shear_x", 0.9, 4), ("invert", 0.2, None)], + [("shear_y", 0.9, 8), ("invert", 0.7, None)], + [("equalize", 0.6, None), ("solarize", 0.6, 6)], + [("invert", 0.9, None), ("equalize", 0.6, None)], + [("equalize", 0.6, None), ("rotate", 0.9, 3)], + [("shear_x", 0.9, 4), ("auto_contrast", 0.8, None)], + [("shear_y", 0.9, 8), ("invert", 0.4, None)], + [("shear_y", 0.9, 5), ("solarize", 0.2, 6)], + [("invert", 0.9, None), ("auto_contrast", 0.8, None)], + [("equalize", 0.6, None), ("rotate", 0.9, 3)], + [("shear_x", 0.9, 4), ("solarize", 0.3, 3)], + [("shear_y", 0.8, 8), ("invert", 0.7, None)], + [("equalize", 0.9, None), ("translate_y", 0.6, 6)], + [("invert", 0.9, None), ("equalize", 0.6, None)], + [("contrast", 0.3, 3), ("rotate", 0.8, 4)], + [("invert", 0.8, None), ("translate_y", 0.0, 2)], + [("shear_y", 0.7, 6), ("solarize", 0.4, 8)], + [("invert", 0.6, None), ("rotate", 0.8, 4)], + [("shear_y", 0.3, 7), ("translate_x", 0.9, 3)], + [("shear_x", 0.1, 6), ("invert", 0.6, None)], + [("solarize", 0.7, 2), ("translate_y", 0.6, 7)], + [("shear_y", 0.8, 4), ("invert", 0.8, None)], + [("shear_x", 0.7, 9), ("translate_y", 0.8, 3)], + [("shear_y", 0.8, 5), ("auto_contrast", 0.7, None)], + [("shear_x", 0.7, 2), ("invert", 0.1, None)], +] + + +class AutoAugment(PolicyAugmentBase): + """Apply AutoAugment :cite:`cubuk2018autoaugment` searched strategies. + + Args: + policy: a customized policy config or presets of "imagenet", "cifar10", and "svhn". + transformation_matrix_mode: computation mode for the chained transformation matrix, via `.transform_matrix` + attribute. + If `silent`, transformation matrix will be computed silently and the non-rigid + modules will be ignored as identity transformations. + If `rigid`, transformation matrix will be computed silently and the non-rigid + modules will trigger errors. + If `skip`, transformation matrix will be totally ignored. + + Examples: + >>> import torch + >>> import kornia.augmentation as K + >>> in_tensor = torch.rand(5, 3, 30, 30) + >>> aug = K.AugmentationSequential(AutoAugment()) + >>> aug(in_tensor).shape + torch.Size([5, 3, 30, 30]) + + """ + + def __init__( + self, policy: Union[str, List[SUBPOLICY_CONFIG]] = "imagenet", transformation_matrix_mode: str = "silent" + ) -> None: + if policy == "imagenet": + _policy = imagenet_policy + elif policy == "cifar10": + _policy = cifar10_policy + elif policy == "svhn": + _policy = svhn_policy + elif isinstance(policy, (list, tuple)): + _policy = policy + else: + raise NotImplementedError(f"Invalid policy `{policy}`.") + + super().__init__(_policy, transformation_matrix_mode=transformation_matrix_mode) + selection_weights = torch.tensor([1.0 / len(self)] * len(self)) + self.rand_selector = Categorical(selection_weights) + + def compose_subpolicy_sequential(self, subpolicy: SUBPOLICY_CONFIG) -> PolicySequential: + """Build a :class:`PolicySequential` from one AutoAugment sub-policy. + + Args: + subpolicy: Sequence of ``(name, probability, magnitude)`` operation tuples. + + Returns: + A sequential container that applies the configured operations in order. + """ + return PolicySequential(*[getattr(ops, name)(prob, mag) for name, prob, mag in subpolicy]) + + def get_forward_sequence(self, params: Optional[List[ParamItem]] = None) -> Iterator[Tuple[str, nn.Module]]: + """Return the operations to run for the current forward call. + + Args: + params: Optional recorded parameters. When provided, the sequence is + reconstructed from them. + + Returns: + Iterator of ``(name, module)`` pairs. + """ + if params is None: + idx = self.rand_selector.sample((1,)) + return self.get_children_by_indices(idx) + + return self.get_children_by_params(params) diff --git a/kornia/augmentation/auto/autoaugment/ops.py b/kornia/augmentation/auto/autoaugment/ops.py new file mode 100644 index 0000000..24ee74b --- /dev/null +++ b/kornia/augmentation/auto/autoaugment/ops.py @@ -0,0 +1,150 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""AutoAugment operation wrapper.""" + +import torch + +from kornia.augmentation.auto.operations import ( + AutoContrast, + Brightness, + Contrast, + Equalize, + Invert, + OperationBase, + Posterize, + Rotate, + Saturate, + Sharpness, + ShearX, + ShearY, + Solarize, + TranslateX, + TranslateY, +) + + +def shear_x(probability: float, magnitude: int) -> OperationBase: + """Return ShearX op.""" + magnitudes = torch.linspace(-0.3, 0.3, 11) * 180.0 + return ShearX( + None, + probability, + magnitude_range=(magnitudes[magnitude].item(), magnitudes[magnitude + 1].item()), + symmetric_megnitude=False, + ) + + +def shear_y(probability: float, magnitude: int) -> OperationBase: + """Return ShearY op.""" + magnitudes = torch.linspace(-0.3, 0.3, 11) * 180.0 + return ShearY( + None, + probability, + magnitude_range=(magnitudes[magnitude].item(), magnitudes[magnitude + 1].item()), + symmetric_megnitude=False, + ) + + +def translate_x(probability: float, magnitude: int) -> OperationBase: + """Return TranslateX op.""" + magnitudes = torch.linspace(-0.5, 0.5, 11) + return TranslateX( + None, + probability, + magnitude_range=(magnitudes[magnitude].item(), magnitudes[magnitude + 1].item()), + symmetric_megnitude=False, + ) + + +def translate_y(probability: float, magnitude: int) -> OperationBase: + """Return TranslateY op.""" + magnitudes = torch.linspace(-0.5, 0.5, 11) + return TranslateY( + None, + probability, + magnitude_range=(magnitudes[magnitude].item(), magnitudes[magnitude + 1].item()), + symmetric_megnitude=False, + ) + + +def rotate(probability: float, magnitude: int) -> OperationBase: + """Return rotate op.""" + magnitudes = torch.linspace(-30, 30, 11) + return Rotate( + None, + probability, + magnitude_range=(magnitudes[magnitude].item(), magnitudes[magnitude + 1].item()), + symmetric_megnitude=False, + ) + + +def auto_contrast(probability: float, _: int) -> OperationBase: + """Return AutoConstrast op.""" + return AutoContrast(probability) + + +def invert(probability: float, _: int) -> OperationBase: + """Return invert op.""" + return Invert(probability) + + +def equalize(probability: float, _: int) -> OperationBase: + """Return equalize op.""" + return Equalize(probability) + + +def solarize(probability: float, magnitude: int) -> OperationBase: + """Return solarize op.""" + magnitudes = torch.linspace(0, 255, 11) / 255.0 + return Solarize(None, probability, magnitude_range=(magnitudes[magnitude].item(), magnitudes[magnitude + 1].item())) + + +def posterize(probability: float, magnitude: int) -> OperationBase: + """Return posterize op.""" + magnitudes = torch.linspace(4, 8, 11) + return Posterize( + None, probability, magnitude_range=(magnitudes[magnitude].item(), magnitudes[magnitude + 1].item()) + ) + + +def contrast(probability: float, magnitude: int) -> OperationBase: + """Return contrast op.""" + magnitudes = torch.linspace(0.1, 1.9, 11) + return Contrast(None, probability, magnitude_range=(magnitudes[magnitude].item(), magnitudes[magnitude + 1].item())) + + +def brightness(probability: float, magnitude: int) -> OperationBase: + """Return brightness op.""" + magnitudes = torch.linspace(0.1, 1.9, 11) + return Brightness( + None, probability, magnitude_range=(magnitudes[magnitude].item(), magnitudes[magnitude + 1].item()) + ) + + +def sharpness(probability: float, magnitude: int) -> OperationBase: + """Return sharpness op.""" + magnitudes = torch.linspace(0.1, 1.9, 11) + return Sharpness( + None, probability, magnitude_range=(magnitudes[magnitude].item(), magnitudes[magnitude + 1].item()) + ) + + +def color(probability: float, magnitude: int) -> OperationBase: + """Return color op.""" + magnitudes = torch.linspace(0.1, 1.9, 11) + return Saturate(None, probability, magnitude_range=(magnitudes[magnitude].item(), magnitudes[magnitude + 1].item())) diff --git a/kornia/augmentation/auto/base.py b/kornia/augmentation/auto/base.py new file mode 100644 index 0000000..3487dac --- /dev/null +++ b/kornia/augmentation/auto/base.py @@ -0,0 +1,183 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Iterator, List, Optional, Tuple, Union, cast + +import torch +from torch import nn + +from kornia.augmentation.auto.operations.base import OperationBase +from kornia.augmentation.auto.operations.policy import PolicySequential +from kornia.augmentation.container.base import ImageSequentialBase, TransformMatrixMinIn +from kornia.augmentation.container.ops import InputSequentialOps +from kornia.augmentation.container.params import ParamItem +from kornia.core.ops import eye_like + +NUMBER = Union[float, int] +OP_CONFIG = Tuple[str, NUMBER, Optional[NUMBER]] +SUBPOLICY_CONFIG = List[OP_CONFIG] + + +class PolicyAugmentBase(ImageSequentialBase, TransformMatrixMinIn): + """Policy-based image augmentation.""" + + def __init__(self, policy: List[SUBPOLICY_CONFIG], transformation_matrix_mode: str = "silence") -> None: + policies = self.compose_policy(policy) + super().__init__(*policies) + self._parse_transformation_matrix_mode(transformation_matrix_mode) + self._valid_ops_for_transform_computation: Tuple[Any, ...] = (PolicySequential,) + + def _update_transform_matrix_for_valid_op(self, module: PolicySequential) -> None: # type: ignore + self._transform_matrices.append(module.transform_matrix) + + def clear_state(self) -> None: + """Reset cached params and transformation-matrix state.""" + self._reset_transform_matrix_state() + return super().clear_state() + + def compose_policy(self, policy: List[SUBPOLICY_CONFIG]) -> List[PolicySequential]: + """Compose policy by the provided policy config.""" + return [self.compose_subpolicy_sequential(subpolicy) for subpolicy in policy] + + def compose_subpolicy_sequential(self, subpolicy: SUBPOLICY_CONFIG) -> PolicySequential: + """Build a :class:`PolicySequential` from a single sub-policy spec. + + Args: + subpolicy: Sub-policy definition used by the concrete augmentation. + + Returns: + The sequential module representing that sub-policy. + """ + raise NotImplementedError + + def identity_matrix(self, input: torch.Tensor) -> torch.Tensor: + """Return identity matrix.""" + return eye_like(3, input) + + def get_transformation_matrix( + self, + input: torch.Tensor, + params: Optional[List[ParamItem]] = None, + recompute: bool = False, + extra_args: Optional[Dict[str, Any]] = None, + ) -> Optional[torch.Tensor]: + """Compute the transformation matrix according to the provided parameters. + + Args: + input: the input torch.Tensor. + params: params for the sequence. + recompute: if to recompute the transformation matrix according to the params. + default: False. + extra_args: Optional dictionary of extra arguments with specific options for different input types. + + """ + if params is None: + raise NotImplementedError("requires params to be provided.") + named_modules: Iterator[Tuple[str, nn.Module]] = self.get_forward_sequence(params) + + # Define as 1 for broadcasting + res_mat: Optional[torch.Tensor] = None + for (_, module), param in zip(named_modules, params if params is not None else []): + module = cast(PolicySequential, module) + mat = module.get_transformation_matrix( + input, params=cast(Optional[List[ParamItem]], param.data), recompute=recompute, extra_args=extra_args + ) + res_mat = mat if res_mat is None else mat @ res_mat + return res_mat + + def is_intensity_only(self, params: Optional[List[ParamItem]] = None) -> bool: + """Check whether all selected sub-policies are intensity-only. + + Args: + params: Optional parameters that define which sub-policies to inspect. + + Returns: + ``True`` if no geometric transform is selected, ``False`` otherwise. + """ + named_modules: Iterator[Tuple[str, nn.Module]] = self.get_forward_sequence(params) + for _, module in named_modules: + module = cast(PolicySequential, module) + if not module.is_intensity_only(): + return False + return True + + def forward_parameters(self, batch_shape: torch.Size) -> List[ParamItem]: + """Generate per-module parameters for one policy forward pass. + + Args: + batch_shape: Input shape used to sample operation parameters. + + Returns: + Parameters for each selected child module, in execution order. + """ + named_modules: Iterator[Tuple[str, nn.Module]] = self.get_forward_sequence() + + params: List[ParamItem] = [] + mod_param: Union[Dict[str, torch.Tensor], List[ParamItem]] + for name, module in named_modules: + module = cast(OperationBase, module) + mod_param = module.forward_parameters(batch_shape) + param = ParamItem(name, mod_param) + params.append(param) + return params + + def transform_inputs( + self, input: torch.Tensor, params: List[ParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Apply a prepared parameter list to the input tensor. + + Args: + input: Input tensor. + params: Parameters produced by :meth:`forward_parameters`. + extra_args: Optional overrides forwarded to child transforms. + + Returns: + Transformed tensor. + """ + for param in params: + module = self.get_submodule(param.name) + input = InputSequentialOps.transform(input, module=module, param=param, extra_args=extra_args) + return input + + def forward( + self, input: torch.Tensor, params: Optional[List[ParamItem]] = None, extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Apply the policy to ``input`` and cache the parameters used. + + Args: + input: Input tensor. + params: Optional precomputed parameters. If omitted, parameters are + sampled on-the-fly. + extra_args: Optional overrides forwarded to child transforms. + + Returns: + Augmented tensor. + """ + self.clear_state() + + if params is None: + inp = input + _, out_shape = self.autofill_dim(inp, dim_range=(2, 4)) + params = self.forward_parameters(out_shape) + + for param in params: + module = self.get_submodule(param.name) + input = InputSequentialOps.transform(input, module=module, param=param, extra_args=extra_args) + self._update_transform_matrix_by_module(module) + + self._params = params + return input diff --git a/kornia/augmentation/auto/operations/__init__.py b/kornia/augmentation/auto/operations/__init__.py new file mode 100644 index 0000000..82e55d6 --- /dev/null +++ b/kornia/augmentation/auto/operations/__init__.py @@ -0,0 +1,25 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia AutoAugment Operations — Building blocks for policy-based augmentation. + +This subpackage provides operation classes and utilities for AutoAugment policies. +""" + +from .base import OperationBase +from .ops import * +from .policy import PolicySequential diff --git a/kornia/augmentation/auto/operations/base.py b/kornia/augmentation/auto/operations/base.py new file mode 100644 index 0000000..3edcb9b --- /dev/null +++ b/kornia/augmentation/auto/operations/base.py @@ -0,0 +1,234 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Callable, Dict, List, Optional, Tuple, TypeVar + +import torch +from torch import nn +from torch.distributions import Bernoulli, RelaxedBernoulli +from typing_extensions import Self + +from kornia.augmentation.base import _AugmentationBase + +T = TypeVar("T", bound="OperationBase") + + +class OperationBase(nn.Module): + """Base class of differentiable augmentation operations. + + Args: + operation: Kornia augmentation module. + initial_magnitude: targeted magnitude parameter name and its initial magnitude value. + The magnitude parameter name shall align with the attribute inside the random_generator + in each augmentation. If None, the augmentation will be randomly applied according to + the augmentation sampling range. + temperature: temperature for RelaxedBernoulli distribution used during training. + is_batch_operation: determine if to obtain the probability from `p` or `p_batch`. + Set to True for most non-shape-persistent operations (e.g. cropping). + + """ + + def __init__( + self, + operation: _AugmentationBase, + initial_magnitude: Optional[List[Tuple[str, Optional[float]]]] = None, + temperature: float = 0.1, + is_batch_operation: bool = False, + magnitude_fn: Optional[Callable[[torch.Tensor], torch.Tensor]] = None, + symmetric_megnitude: bool = False, + ) -> None: + super().__init__() + if not isinstance(operation, _AugmentationBase): + raise ValueError(f"Only Kornia augmentations supported. Got {operation}.") + + self.op = operation + + self._init_magnitude(initial_magnitude) + + # Avoid skipping the sampling in `__batch_prob_generator__` + self.probability_range = (1e-7, 1 - 1e-7) + self._is_batch_operation = is_batch_operation + if is_batch_operation: + self._probability = nn.Parameter(torch.empty(1).fill_(self.op.p_batch)) + else: + self._probability = nn.Parameter(torch.empty(1).fill_(self.op.p)) + + if temperature < 0: + raise ValueError(f"Expect temperature value greater than 0. Got {temperature}.") + self.register_buffer("temperature", torch.empty(1).fill_(temperature)) + + self.symmetric_megnitude = symmetric_megnitude + self._magnitude_fn = self._init_magnitude_fn(magnitude_fn) + + def _init_magnitude_fn( + self, magnitude_fn: Optional[Callable[[torch.Tensor], torch.Tensor]] + ) -> Callable[[torch.Tensor], torch.Tensor]: + def _identity(x: torch.Tensor) -> torch.Tensor: + return x + + def _random_flip(fn: Callable[[torch.Tensor], torch.Tensor]) -> Callable[[torch.Tensor], torch.Tensor]: + def f(x: torch.Tensor) -> torch.Tensor: + flip = torch.rand((x.shape[0],), device=x.device) > 0.5 + return fn(x) * flip + + return f + + if magnitude_fn is None: + magnitude_fn = _identity + + if self.symmetric_megnitude: + return _random_flip(magnitude_fn) + + return magnitude_fn + + def _init_magnitude(self, initial_magnitude: Optional[List[Tuple[str, Optional[float]]]]) -> None: + if isinstance(initial_magnitude, (list, tuple)): + if not all(isinstance(ini_mag, (list, tuple)) and len(ini_mag) == 2 for ini_mag in initial_magnitude): + raise ValueError(f"`initial_magnitude` shall be a list of 2-element tuples. Got {initial_magnitude}") + if len(initial_magnitude) != 1: + raise NotImplementedError("Multi magnitudes operations are not yet supported.") + + if initial_magnitude is None: + self._factor_name = None + self._magnitude = None + self.magnitude_range = None + else: + self._factor_name = initial_magnitude[0][0] + if self.op._param_generator is not None: + self.magnitude_range = getattr(self.op._param_generator, self._factor_name) + else: + raise ValueError(f"No valid magnitude `{self._factor_name}` found in `{self.op._param_generator}`.") + + self._magnitude = None + if initial_magnitude[0][1] is not None: + self._magnitude = nn.Parameter(torch.empty(1).fill_(initial_magnitude[0][1])) + + def _update_probability_gen(self, relaxation: bool) -> None: + if relaxation: + if self._is_batch_operation: + self.op._p_batch_gen = RelaxedBernoulli(self.temperature, self.probability) + else: + self.op._p_gen = RelaxedBernoulli(self.temperature, self.probability) + elif self._is_batch_operation: + self.op._p_batch_gen = Bernoulli(self.probability) + else: + self.op._p_gen = Bernoulli(self.probability) + + def train(self, mode: bool = True) -> Self: + """Switch training mode and refresh probability samplers. + + Args: + mode: ``True`` for training mode, ``False`` for evaluation mode. + + Returns: + This module. + """ + self._update_probability_gen(relaxation=mode) + return super().train(mode=mode) + + def eval(self) -> Self: + """Switch to evaluation mode. + + Returns: + This module. + """ + return self.train(False) + + def forward_parameters( + self, batch_shape: torch.Size, mag: Optional[torch.Tensor] = None + ) -> Dict[str, torch.Tensor]: + """Sample parameters for the wrapped augmentation op. + + Args: + batch_shape: Input batch shape used for sampling. + mag: Optional magnitude override. When set, it replaces the sampled + factor before the magnitude mapping function is applied. + + Returns: + Parameter dictionary consumed by :meth:`forward`. + """ + if mag is None: + mag = self.magnitude + + self._update_probability_gen(relaxation=True) + params = self.op.forward_parameters(batch_shape) + + if mag is not None: + if self._factor_name is None: + raise RuntimeError("No factor found in the params while `mag` is provided.") + params[self._factor_name] = params[self._factor_name].zero_() + mag + + if self._factor_name is not None: + params[self._factor_name] = self._magnitude_fn(params[self._factor_name]) + + return params + + def forward(self, input: torch.Tensor, params: Optional[Dict[str, torch.Tensor]] = None) -> torch.Tensor: + """Apply the operation with probabilistic gating. + + Args: + input: Input tensor. + params: Optional precomputed parameters. If omitted, parameters are + sampled from ``input.shape``. + + Returns: + Tensor where each sample is either transformed or left unchanged + according to ``batch_prob``. + """ + if params is None: + params = self.forward_parameters(input.shape) + + batch_prob = params["batch_prob"][(...,) + ((None,) * (len(input.shape) - 1))].to(device=input.device) + + return batch_prob * self.op(input, params=params) + (1 - batch_prob) * input + + @property + def transform_matrix(self) -> Optional[torch.Tensor]: + """Return the latest transform matrix from the wrapped op, if available. + + Returns: + Transform matrix tensor or ``None`` for non-geometric operations. + """ + if hasattr(self.op, "transform_matrix"): + return self.op.transform_matrix + return None + + @property + def magnitude(self) -> Optional[torch.Tensor]: + """Return the learned magnitude value for the operation. + + Returns: + Magnitude tensor, clamped to ``magnitude_range`` when defined; otherwise + the raw learnable value. Returns ``None`` when this operation has no + magnitude parameter. + """ + if self._magnitude is None: + return None + mag = self._magnitude + if self.magnitude_range is not None: + return mag.clamp(*self.magnitude_range) + return mag + + @property + def probability(self) -> torch.Tensor: + """Return the operation probability after applying configured bounds. + + Returns: + Probability tensor clamped to ``probability_range``. + """ + p = self._probability.clamp(*self.probability_range) + return p diff --git a/kornia/augmentation/auto/operations/ops.py b/kornia/augmentation/auto/operations/ops.py new file mode 100644 index 0000000..ddcd16f --- /dev/null +++ b/kornia/augmentation/auto/operations/ops.py @@ -0,0 +1,589 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Optional, Tuple + +import torch + +import kornia.augmentation as K +from kornia.augmentation.auto.operations.base import OperationBase + +__all__ = [ + "AutoContrast", + "Brightness", + "Contrast", + "Equalize", + "Gray", + "HorizontalFlip", + "Hue", + "Invert", + "Posterize", + "Rotate", + "Saturate", + "Sharpness", + "ShearX", + "ShearY", + "Solarize", + "SolarizeAdd", + "TranslateX", + "TranslateY", + "VerticalFlip", +] + + +class AutoContrast(OperationBase): + """Apply auto_contrast operation. + + Args: + initial_probability: the initial probability. If None, the augmentation will be randomly + applied according to he augmentation sampling range. + temperature: temperature for RelaxedBernoulli distribution used during training. + symmetric_megnitude: if to randomly assign the magnitude as negative or not. + + """ + + def __init__( + self, initial_probability: float = 0.5, temperature: float = 0.1, symmetric_megnitude: bool = False + ) -> None: + super().__init__( + K.RandomAutoContrast(same_on_batch=False, p=initial_probability), + initial_magnitude=None, + temperature=temperature, + symmetric_megnitude=symmetric_megnitude, + ) + + +class Brightness(OperationBase): + """Apply brightness operation. + + Args: + initial_probability: the initial probability. If None, the augmentation will be randomly + applied according to he augmentation sampling range. + initial_magnitude: the initial magnitude. + magnitude_range: the sampling range for random sampling and clamping the optimized magnitude. + temperature: temperature for RelaxedBernoulli distribution used during training. + symmetric_megnitude: if to randomly assign the magnitude as negative or not. + + """ + + def __init__( + self, + initial_magnitude: Optional[float] = 0.5, + initial_probability: float = 0.5, + magnitude_range: Tuple[float, float] = (0.2, 1.8), + temperature: float = 0.1, + symmetric_megnitude: bool = False, + ) -> None: + super().__init__( + K.RandomBrightness(magnitude_range, same_on_batch=False, p=initial_probability), + initial_magnitude=[("brightness_factor", initial_magnitude)], + temperature=temperature, + symmetric_megnitude=symmetric_megnitude, + ) + + +class Contrast(OperationBase): + """Apply contrast operation. + + Args: + initial_probability: the initial probability. If None, the augmentation will be randomly + applied according to he augmentation sampling range. + initial_magnitude: the initial magnitude. + magnitude_range: the sampling range for random sampling and clamping the optimized magnitude. + temperature: temperature for RelaxedBernoulli distribution used during training. + symmetric_megnitude: if to randomly assign the magnitude as negative or not. + + """ + + def __init__( + self, + initial_magnitude: Optional[float] = 0.5, + initial_probability: float = 0.5, + magnitude_range: Tuple[float, float] = (0.2, 1.8), + temperature: float = 0.1, + symmetric_megnitude: bool = False, + ) -> None: + super().__init__( + K.RandomContrast(magnitude_range, same_on_batch=False, p=initial_probability), + initial_magnitude=[("contrast_factor", initial_magnitude)], + temperature=temperature, + symmetric_megnitude=symmetric_megnitude, + ) + + +class Hue(OperationBase): + """Apply hue operation. + + Args: + initial_probability: the initial probability. If None, the augmentation will be randomly + applied according to he augmentation sampling range. + initial_magnitude: the initial magnitude. + magnitude_range: the sampling range for random sampling and clamping the optimized magnitude. + temperature: temperature for RelaxedBernoulli distribution used during training. + symmetric_megnitude: if to randomly assign the magnitude as negative or not. + + """ + + def __init__( + self, + initial_magnitude: Optional[float] = 0.0, + initial_probability: float = 0.5, + magnitude_range: Tuple[float, float] = (-0.5, 0.5), + temperature: float = 0.1, + symmetric_megnitude: bool = False, + ) -> None: + super().__init__( + K.RandomHue(magnitude_range, same_on_batch=False, p=initial_probability), + initial_magnitude=[("hue_factor", initial_magnitude)], + temperature=temperature, + symmetric_megnitude=symmetric_megnitude, + ) + + +class Saturate(OperationBase): + """Apply saturation operation. + + Args: + initial_probability: the initial probability. If None, the augmentation will be randomly + applied according to he augmentation sampling range. + initial_magnitude: the initial magnitude. + magnitude_range: the sampling range for random sampling and clamping the optimized magnitude. + temperature: temperature for RelaxedBernoulli distribution used during training. + symmetric_megnitude: if to randomly assign the magnitude as negative or not. + + """ + + def __init__( + self, + initial_magnitude: Optional[float] = 0.5, + initial_probability: float = 0.5, + magnitude_range: Tuple[float, float] = (0.2, 1.8), + temperature: float = 0.1, + symmetric_megnitude: bool = False, + ) -> None: + super().__init__( + K.RandomSaturation(magnitude_range, same_on_batch=False, p=initial_probability), + initial_magnitude=[("saturation_factor", initial_magnitude)], + temperature=temperature, + symmetric_megnitude=symmetric_megnitude, + ) + + +# TODO: Equalize cannot update probabilities yet. +class Equalize(OperationBase): + """Apply equalize operation. + + Args: + initial_probability: the initial probability. If None, the augmentation will be randomly + applied according to he augmentation sampling range. + temperature: temperature for RelaxedBernoulli distribution used during training. + + Note: + Equalize cannot update probabilities yet. + + """ + + def __init__(self, initial_probability: float = 0.5, temperature: float = 0.1) -> None: + super().__init__( + K.RandomEqualize(same_on_batch=False, p=initial_probability), + initial_magnitude=None, + temperature=temperature, + symmetric_megnitude=False, + ) + + +class Gray(OperationBase): + """Apply grayscale operation. + + Args: + initial_probability: the initial probability. If None, the augmentation will be randomly + applied according to he augmentation sampling range. + temperature: temperature for RelaxedBernoulli distribution used during training. + + """ + + def __init__(self, initial_probability: float = 0.5, temperature: float = 0.1) -> None: + super().__init__( + K.RandomGrayscale(same_on_batch=False, p=initial_probability), + initial_magnitude=None, + temperature=temperature, + symmetric_megnitude=False, + ) + + +class Invert(OperationBase): + """Apply invert operation. + + Args: + initial_probability: the initial probability. If None, the augmentation will be randomly + applied according to he augmentation sampling range. + temperature: temperature for RelaxedBernoulli distribution used during training. + + """ + + def __init__(self, initial_probability: float = 0.5, temperature: float = 0.1) -> None: + super().__init__( + K.RandomInvert(same_on_batch=False, p=initial_probability), + initial_magnitude=None, + temperature=temperature, + symmetric_megnitude=False, + ) + + +class Posterize(OperationBase): + """Apply posterize operation. + + Args: + initial_magnitude: the initial magnitude. + initial_probability: the initial probability. If None, the augmentation will be randomly + applied according to he augmentation sampling range. + magnitude_range: the sampling range for random sampling and clamping the optimized magnitude. + temperature: temperature for RelaxedBernoulli distribution used during training. + symmetric_megnitude: if to randomly assign the magnitude as negative or not. + + """ + + @staticmethod + def _process_magnitude(magnitude: torch.Tensor) -> torch.Tensor: + return magnitude.long() + + def __init__( + self, + initial_magnitude: Optional[float] = 4.0, + initial_probability: float = 0.5, + magnitude_range: Tuple[float, float] = (1.0, 8.0), + temperature: float = 0.1, + symmetric_megnitude: bool = False, + ) -> None: + super().__init__( + K.RandomPosterize(magnitude_range, same_on_batch=False, p=initial_probability), + initial_magnitude=[("bits_factor", initial_magnitude)], + temperature=temperature, + symmetric_megnitude=symmetric_megnitude, + magnitude_fn=Posterize._process_magnitude, + ) + + +class Solarize(OperationBase): + """Apply solarize operation. + + Args: + initial_magnitude: the initial magnitude. + initial_probability: the initial probability. If None, the augmentation will be randomly + applied according to he augmentation sampling range. + magnitude_range: the sampling range for random sampling and clamping the optimized + symmetric_megnitude: if to randomly assign the magnitude as negative or not.magnitude. + temperature: temperature for RelaxedBernoulli distribution used during training. + + """ + + def __init__( + self, + initial_magnitude: Optional[float] = 0.5, + initial_probability: float = 0.5, + magnitude_range: Tuple[float, float] = (0.0, 1.0), + temperature: float = 0.1, + symmetric_megnitude: bool = False, + ) -> None: + super().__init__( + K.RandomSolarize(magnitude_range, additions=0.0, same_on_batch=False, p=initial_probability), + initial_magnitude=[("thresholds", initial_magnitude)], + temperature=temperature, + symmetric_megnitude=symmetric_megnitude, + ) + + +class SolarizeAdd(OperationBase): + """Apply solarize-addition operation with a fixed thresholds of 0.5. + + Args: + initial_magnitude: the initial magnitude. + initial_probability: the initial probability. If None, the augmentation will be randomly + applied according to he augmentation sampling range. + magnitude_range: the sampling range for random sampling and clamping the optimized magnitude. + temperature: temperature for RelaxedBernoulli distribution used during training. + symmetric_megnitude: if to randomly assign the magnitude as negative or not. + + """ + + def __init__( + self, + initial_magnitude: Optional[float] = 0.0, + initial_probability: float = 0.5, + magnitude_range: Tuple[float, float] = (-0.3, 0.3), + temperature: float = 0.1, + symmetric_megnitude: bool = False, + ) -> None: + super().__init__( + K.RandomSolarize(thresholds=0.5, additions=magnitude_range, same_on_batch=False, p=initial_probability), + initial_magnitude=[("additions", initial_magnitude)], + temperature=temperature, + symmetric_megnitude=symmetric_megnitude, + ) + + +class Sharpness(OperationBase): + """Apply sharpness operation. + + Args: + initial_magnitude: the initial magnitude. + initial_probability: the initial probability. If None, the augmentation will be randomly + applied according to he augmentation sampling range. + magnitude_range: the sampling range for random sampling and clamping the optimized magnitude. + temperature: temperature for RelaxedBernoulli distribution used during training. + symmetric_megnitude: if to randomly assign the magnitude as negative or not. + + """ + + def __init__( + self, + initial_magnitude: Optional[float] = 0.5, + initial_probability: float = 0.5, + magnitude_range: Tuple[float, float] = (0.1, 1.9), + temperature: float = 0.1, + symmetric_megnitude: bool = False, + ) -> None: + super().__init__( + K.RandomSharpness(magnitude_range, same_on_batch=False, p=initial_probability), + initial_magnitude=[("sharpness", initial_magnitude)], + temperature=temperature, + symmetric_megnitude=symmetric_megnitude, + ) + + +class HorizontalFlip(OperationBase): + """Apply horizontal flip operation. + + Args: + initial_probability: the initial probability. If None, the augmentation will be randomly + applied according to he augmentation sampling range. + temperature: temperature for RelaxedBernoulli distribution used during training. + + """ + + def __init__(self, initial_probability: float = 0.5, temperature: float = 0.1) -> None: + super().__init__( + K.RandomHorizontalFlip(same_on_batch=False, p=initial_probability), + initial_magnitude=None, + temperature=temperature, + symmetric_megnitude=False, + ) + + +class VerticalFlip(OperationBase): + """Apply vertical flip operation. + + Args: + initial_magnitude: the initial magnitude. + temperature: temperature for RelaxedBernoulli distribution used during training. + + """ + + def __init__(self, initial_probability: float = 0.5, temperature: float = 0.1) -> None: + super().__init__( + K.RandomVerticalFlip(same_on_batch=False, p=initial_probability), + initial_magnitude=None, + temperature=temperature, + symmetric_megnitude=False, + ) + + +class Rotate(OperationBase): + """Apply rotate operation. + + Args: + initial_magnitude: the initial magnitude. + initial_probability: the initial probability. If None, the augmentation will be randomly + applied according to he augmentation sampling range. + magnitude_range: the sampling range for random sampling and clamping the optimized magnitude. + temperature: temperature for RelaxedBernoulli distribution used during training. + symmetric_megnitude: if to randomly assign the magnitude as negative or not. + + """ + + def __init__( + self, + initial_magnitude: Optional[float] = 15.0, + initial_probability: float = 0.5, + magnitude_range: Tuple[float, float] = (0.0, 30.0), + temperature: float = 0.1, + symmetric_megnitude: bool = True, + ) -> None: + if symmetric_megnitude and magnitude_range[0] < 0: + raise ValueError( + f"Lower bound of {self.__class__.__name__} is a symmetric operation. " + f"The lower bound must above 0. Got {magnitude_range[0]}." + ) + super().__init__( + K.RandomRotation(magnitude_range, same_on_batch=False, p=initial_probability), + initial_magnitude=[("degrees", initial_magnitude)], + temperature=temperature, + symmetric_megnitude=symmetric_megnitude, + ) + + +class ShearX(OperationBase): + """Apply shear operation along x-axis. + + Args: + initial_magnitude: the initial magnitude. + initial_probability: the initial probability. If None, the augmentation will be randomly + applied according to he augmentation sampling range. + magnitude_range: the sampling range for random sampling and clamping the optimized magnitude. + temperature: temperature for RelaxedBernoulli distribution used during training. + symmetric_megnitude: if to randomly assign the magnitude as negative or not. + + """ + + @staticmethod + def _process_magnitude(magnitude: torch.Tensor) -> torch.Tensor: + # make it sign-agnostic + return magnitude * 180 + + def __init__( + self, + initial_magnitude: Optional[float] = 0.1, + initial_probability: float = 0.5, + magnitude_range: Tuple[float, float] = (0.0, 0.3), + temperature: float = 0.1, + symmetric_megnitude: bool = True, + ) -> None: + if symmetric_megnitude and magnitude_range[0] < 0: + raise ValueError( + f"Lower bound of {self.__class__.__name__} is a symmetric operation. " + f"The lower bound must above 0. Got {magnitude_range[0]}." + ) + super().__init__( + K.RandomShear(magnitude_range, same_on_batch=False, p=initial_probability, align_corners=True), + initial_magnitude=[("shear_x", initial_magnitude)], + temperature=temperature, + symmetric_megnitude=symmetric_megnitude, + magnitude_fn=ShearX._process_magnitude, + ) + + +class ShearY(OperationBase): + """Apply shear operation along y-axis. + + Args: + initial_magnitude: the initial magnitude. + initial_probability: the initial probability. If None, the augmentation will be randomly + applied according to he augmentation sampling range. + magnitude_range: the sampling range for random sampling and clamping the optimized magnitude. + temperature: temperature for RelaxedBernoulli distribution used during training. + symmetric_megnitude: if to randomly assign the magnitude as negative or not. + + """ + + @staticmethod + def _process_magnitude(magnitude: torch.Tensor) -> torch.Tensor: + # make it sign-agnostic + return magnitude * 180 + + def __init__( + self, + initial_magnitude: Optional[float] = 0.1, + initial_probability: float = 0.5, + magnitude_range: Tuple[float, float] = (0.0, 0.3), + temperature: float = 0.1, + symmetric_megnitude: bool = True, + ) -> None: + if symmetric_megnitude and magnitude_range[0] < 0: + raise ValueError( + f"Lower bound of {self.__class__.__name__} is a symmetric operation. " + f"The lower bound must above 0. Got {magnitude_range[0]}." + ) + super().__init__( + K.RandomShear( + (0.0, 0.0, magnitude_range[0], magnitude_range[1]), + same_on_batch=False, + p=initial_probability, + align_corners=True, + ), + initial_magnitude=[("shear_y", initial_magnitude)], + temperature=temperature, + symmetric_megnitude=symmetric_megnitude, + magnitude_fn=ShearY._process_magnitude, + ) + + +class TranslateX(OperationBase): + """Apply translate operation along x-axis. + + Args: + initial_magnitude: the initial magnitude. + initial_probability: the initial probability. If None, the augmentation will be randomly + applied according to he augmentation sampling range. + magnitude_range: the sampling range for random sampling and clamping the optimized magnitude. + temperature: temperature for RelaxedBernoulli distribution used during training. + symmetric_megnitude: if to randomly assign the magnitude as negative or not. + + """ + + def __init__( + self, + initial_magnitude: Optional[float] = 0.2, + initial_probability: float = 0.5, + magnitude_range: Tuple[float, float] = (0.0, 0.5), + temperature: float = 0.1, + symmetric_megnitude: bool = True, + ) -> None: + if symmetric_megnitude and magnitude_range[0] < 0: + raise ValueError( + f"Lower bound of {self.__class__.__name__} is a symmetric operation. " + f"The lower bound must above 0. Got {magnitude_range[0]}." + ) + super().__init__( + K.RandomTranslate(magnitude_range, same_on_batch=False, p=initial_probability, align_corners=True), + initial_magnitude=[("translate_x", initial_magnitude)], + temperature=temperature, + symmetric_megnitude=symmetric_megnitude, + ) + + +class TranslateY(OperationBase): + """Apply translate operation along y-axis. + + Args: + initial_magnitude: the initial magnitude. + initial_probability: the initial probability. If None, the augmentation will be randomly + applied according to he augmentation sampling range. + magnitude_range: the sampling range for random sampling and clamping the optimized magnitude. + temperature: temperature for RelaxedBernoulli distribution used during training. + symmetric_megnitude: if to randomly assign the magnitude as negative or not. + + """ + + def __init__( + self, + initial_magnitude: Optional[float] = 0.2, + initial_probability: float = 0.5, + magnitude_range: Tuple[float, float] = (0.0, 0.5), + temperature: float = 0.1, + symmetric_megnitude: bool = True, + ) -> None: + if symmetric_megnitude and magnitude_range[0] < 0: + raise ValueError( + f"Lower bound of {self.__class__.__name__} is a symmetric operation. " + f"The lower bound must above 0. Got {magnitude_range[0]}." + ) + super().__init__( + K.RandomTranslate(None, magnitude_range, same_on_batch=False, p=initial_probability, align_corners=True), + initial_magnitude=[("translate_y", initial_magnitude)], + temperature=temperature, + symmetric_megnitude=symmetric_megnitude, + ) diff --git a/kornia/augmentation/auto/operations/policy.py b/kornia/augmentation/auto/operations/policy.py new file mode 100644 index 0000000..5c1566d --- /dev/null +++ b/kornia/augmentation/auto/operations/policy.py @@ -0,0 +1,178 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Iterator, List, Optional, Tuple, Union, cast + +import torch +from torch import Size, nn + +import kornia.augmentation as K +from kornia.augmentation.auto.operations import OperationBase +from kornia.augmentation.container.base import ImageSequentialBase, TransformMatrixMinIn +from kornia.augmentation.container.ops import InputSequentialOps +from kornia.augmentation.container.params import ParamItem +from kornia.augmentation.utils import _transform_input, override_parameters +from kornia.core.ops import eye_like + + +class PolicySequential(TransformMatrixMinIn, ImageSequentialBase): + """Policy tuple for applying multiple operations. + + Args: + operations: a list of operations to perform. + + """ + + def __init__(self, *operations: OperationBase) -> None: + self.validate_operations(*operations) + super().__init__(*operations) + self._valid_ops_for_transform_computation: Tuple[Any, ...] = (OperationBase,) + + def _update_transform_matrix_for_valid_op(self, module: nn.Module) -> None: + self._transform_matrices.append(module.transform_matrix) + + def clear_state(self) -> None: + """Reset cached params and transformation-matrix state.""" + self._reset_transform_matrix_state() + return super().clear_state() + + def validate_operations(self, *operations: OperationBase) -> None: + """Ensure all provided modules are :class:`OperationBase` instances. + + Args: + *operations: Operations passed to :class:`PolicySequential`. + + Raises: + ValueError: Any provided operation is not a Kornia auto operation. + """ + invalid_ops: List[OperationBase] = [] + for op in operations: + if not isinstance(op, OperationBase): + invalid_ops.append(op) + if len(invalid_ops) != 0: + raise ValueError(f"All operations must be Kornia Operations. Got {invalid_ops}.") + + def identity_matrix(self, input: torch.Tensor) -> torch.Tensor: + """Return identity matrix.""" + return eye_like(3, input) + + def get_transformation_matrix( + self, + input: torch.Tensor, + params: Optional[List[ParamItem]] = None, + recompute: bool = False, + extra_args: Optional[Dict[str, Any]] = None, + ) -> torch.Tensor: + """Compute the transformation matrix according to the provided parameters. + + Args: + input: the input torch.Tensor. + params: params for the sequence. + recompute: if to recompute the transformation matrix according to the params. + default: False. + extra_args: Optional dictionary of extra arguments with specific options for different input types. + + """ + if params is None: + raise NotImplementedError("requires params to be provided.") + named_modules: Iterator[Tuple[str, nn.Module]] = self.get_forward_sequence(params) + + # Define as 1 for broadcasting + res_mat: torch.Tensor = self.identity_matrix(_transform_input(input)) + for (_, module), param in zip(named_modules, params if params is not None else []): + module = cast(OperationBase, module) + if isinstance(module.op, (K.GeometricAugmentationBase2D,)) and isinstance(param.data, dict): + ori_shape = input.shape + input = module.op.transform_tensor(input) + # Standardize shape + if recompute: + flags = override_parameters(module.op.flags, extra_args, in_place=False) + mat = module.op.generate_transformation_matrix(input, param.data, flags) + elif module.op._transform_matrix is not None: + mat = torch.as_tensor(module.transform_matrix, device=input.device, dtype=input.dtype) + else: + raise RuntimeError(f"{module}.transform_matrix is None while `recompute=False`.") + res_mat = mat @ res_mat + input = module.op.transform_output_tensor(input, ori_shape) + if module.op.keepdim and ori_shape != input.shape: + res_mat = res_mat.squeeze() + return res_mat + + def is_intensity_only(self) -> bool: + """Check whether the policy contains only intensity transforms. + + Returns: + ``True`` when no geometric operation is present. + """ + for module in self.children(): + module = cast(OperationBase, module) + if isinstance(module.op, (K.GeometricAugmentationBase2D,)): + return False + return True + + def get_forward_sequence(self, params: Optional[List[ParamItem]] = None) -> Iterator[Tuple[str, nn.Module]]: + """Return the operation sequence used for execution. + + Args: + params: Optional recorded parameters. When provided, only modules + referenced by those params are returned. + + Returns: + Iterator of ``(name, module)`` pairs. + """ + if params is not None: + return super().get_children_by_params(params) + return self.named_children() + + def forward_parameters(self, batch_shape: Size) -> List[ParamItem]: + """Generate parameter dictionaries for each operation in the policy. + + Args: + batch_shape: Input shape used to sample operation parameters. + + Returns: + List of :class:`ParamItem` objects in execution order. + """ + named_modules: Iterator[Tuple[str, nn.Module]] = self.get_forward_sequence() + + params: List[ParamItem] = [] + mod_param: Union[Dict[str, torch.Tensor], List[ParamItem]] + for name, module in named_modules: + module = cast(OperationBase, module) + mod_param = module.op.forward_parameters(batch_shape) + param = ParamItem(name, mod_param) + params.append(param) + return params + + def transform_inputs( + self, input: torch.Tensor, params: List[ParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Apply all operations using a prepared parameter list. + + Args: + input: Input tensor. + params: Parameters for each operation in this policy. + extra_args: Optional overrides forwarded to child modules. + + Returns: + Transformed tensor. + """ + for param in params: + module = self.get_submodule(param.name) + input = InputSequentialOps.transform(input, module=module, param=param, extra_args=extra_args) + self._update_transform_matrix_by_module(module) + return input diff --git a/kornia/augmentation/auto/rand_augment/__init__.py b/kornia/augmentation/auto/rand_augment/__init__.py new file mode 100644 index 0000000..5781dff --- /dev/null +++ b/kornia/augmentation/auto/rand_augment/__init__.py @@ -0,0 +1,23 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia RandAugment — RandAugment policy implementations for Kornia augmentation. + +This subpackage provides the RandAugment class and related utilities. +""" + +from .rand_augment import RandAugment diff --git a/kornia/augmentation/auto/rand_augment/ops.py b/kornia/augmentation/auto/rand_augment/ops.py new file mode 100644 index 0000000..c8443f3 --- /dev/null +++ b/kornia/augmentation/auto/rand_augment/ops.py @@ -0,0 +1,132 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""RandAugment operation wrapper.""" + +from kornia.augmentation.auto.operations import ( + AutoContrast, + Brightness, + Contrast, + Equalize, + Invert, + OperationBase, + Posterize, + Rotate, + Saturate, + Sharpness, + ShearX, + ShearY, + Solarize, + SolarizeAdd, + TranslateX, + TranslateY, +) + + +def shear_x(min_mag: float, max_mag: float) -> OperationBase: + """Return ShearX op.""" + if min_mag != -max_mag: + raise ValueError( + f"{ShearX.__name__} is a symmetric operation that `- min_mag == max_mag`. Got [{min_mag}, {max_mag}]" + ) + return ShearX(None, 1.0, magnitude_range=(0.0, max_mag), symmetric_megnitude=True) + + +def shear_y(min_mag: float, max_mag: float) -> OperationBase: + """Return ShearY op.""" + if min_mag != -max_mag: + raise ValueError( + f"{ShearY.__name__} is a symmetric operation that `- min_mag == max_mag`. Got [{min_mag}, {max_mag}]" + ) + return ShearY(None, 1.0, magnitude_range=(0.0, max_mag), symmetric_megnitude=True) + + +def translate_x(min_mag: float, max_mag: float) -> OperationBase: + """Return TranslateX op.""" + if min_mag != -max_mag: + raise ValueError( + f"{TranslateX.__name__} is a symmetric operation that `- min_mag == max_mag`. Got [{min_mag}, {max_mag}]" + ) + return TranslateX(None, 1.0, magnitude_range=(0.0, max_mag), symmetric_megnitude=True) + + +def translate_y(min_mag: float, max_mag: float) -> OperationBase: + """Return TranslateY op.""" + if min_mag != -max_mag: + raise ValueError( + f"{TranslateY.__name__} is a symmetric operation that `- min_mag == max_mag`. Got [{min_mag}, {max_mag}]" + ) + return TranslateY(None, 1.0, magnitude_range=(0.0, max_mag), symmetric_megnitude=True) + + +def rotate(min_mag: float, max_mag: float) -> OperationBase: + """Return rotate op.""" + if min_mag != -max_mag: + raise ValueError( + f"{Rotate.__name__} is a symmetric operation that `- min_mag == max_mag`. Got [{min_mag}, {max_mag}]" + ) + return Rotate(None, 1.0, magnitude_range=(0.0, max_mag), symmetric_megnitude=True) + + +def auto_contrast(min_mag: float, max_mag: float) -> OperationBase: + """Return AutoConstrast op.""" + return AutoContrast(1.0) + + +def invert(min_mag: float, max_mag: float) -> OperationBase: + """Return invert op.""" + return Invert(1.0) + + +def equalize(min_mag: float, max_mag: float) -> OperationBase: + """Return equalize op.""" + return Equalize(1.0) + + +def solarize(min_mag: float, max_mag: float) -> OperationBase: + """Return solarize op.""" + return Solarize(None, 1.0, magnitude_range=(min_mag, max_mag)) + + +def solarize_add(min_mag: float, max_mag: float) -> OperationBase: + """Return SolarizeAdd op.""" + return SolarizeAdd(None, 1.0, magnitude_range=(min_mag, max_mag)) + + +def posterize(min_mag: float, max_mag: float) -> OperationBase: + """Return posterize op.""" + return Posterize(None, 1.0, magnitude_range=(min_mag, max_mag)) + + +def contrast(min_mag: float, max_mag: float) -> OperationBase: + """Return contrast op.""" + return Contrast(None, 1.0, magnitude_range=(min_mag, max_mag)) + + +def brightness(min_mag: float, max_mag: float) -> OperationBase: + """Return brightness op.""" + return Brightness(None, 1.0, magnitude_range=(min_mag, max_mag)) + + +def sharpness(min_mag: float, max_mag: float) -> OperationBase: + """Return sharpness op.""" + return Sharpness(None, 1.0, magnitude_range=(min_mag, max_mag)) + + +def color(min_mag: float, max_mag: float) -> OperationBase: + """Return color op.""" + return Saturate(None, 1.0, magnitude_range=(min_mag, max_mag)) diff --git a/kornia/augmentation/auto/rand_augment/rand_augment.py b/kornia/augmentation/auto/rand_augment/rand_augment.py new file mode 100644 index 0000000..20647e4 --- /dev/null +++ b/kornia/augmentation/auto/rand_augment/rand_augment.py @@ -0,0 +1,167 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Iterator, List, Optional, Tuple, Union, cast + +import torch +from torch import nn + +from kornia.augmentation.auto.base import SUBPOLICY_CONFIG, PolicyAugmentBase +from kornia.augmentation.auto.operations import OperationBase +from kornia.augmentation.auto.operations.policy import PolicySequential +from kornia.augmentation.container.params import ParamItem + +from . import ops + +default_policy: List[SUBPOLICY_CONFIG] = [ + [("auto_contrast", 0, 1)], + [("equalize", 0, 1)], + [("invert", 0, 1)], + [("rotate", -30.0, 30.0)], + [("posterize", 0.0, 4)], + [("solarize", 0.0, 1.0)], + [("solarize_add", 0.0, 0.43)], + [("color", 0.1, 1.9)], + [("contrast", 0.1, 1.9)], + [("brightness", 0.1, 1.9)], + [("sharpness", 0.1, 1.9)], + [("shear_x", -0.3, 0.3)], + [("shear_y", -0.3, 0.3)], + # (CutoutAbs, 0, 40), + [("translate_x", -0.1, 0.1)], + [("translate_y", -0.1, 0.1)], +] + + +class RandAugment(PolicyAugmentBase): + """Apply RandAugment :cite:`cubuk2020randaugment` augmentation strategies. + + Args: + n: the number of augmentations to apply sequentially. + m: magnitude for all the augmentations, ranged from [0, 30]. + policy: candidate transformations. If None, a default candidate list will be used. + transformation_matrix_mode: computation mode for the chained transformation matrix, via `.transform_matrix` + attribute. + If `silent`, transformation matrix will be computed silently and the non-rigid + modules will be ignored as identity transformations. + If `rigid`, transformation matrix will be computed silently and the non-rigid + modules will trigger errors. + If `skip`, transformation matrix will be totally ignored. + + Examples: + >>> import kornia.augmentation as K + >>> in_tensor = torch.rand(5, 3, 30, 30) + >>> aug = K.AugmentationSequential(RandAugment(n=2, m=10)) + >>> aug(in_tensor).shape + torch.Size([5, 3, 30, 30]) + + """ + + def __init__( + self, + n: int, + m: int, + policy: Optional[List[SUBPOLICY_CONFIG]] = None, + transformation_matrix_mode: str = "silent", + ) -> None: + if m <= 0 or m >= 30: + raise ValueError(f"Expect `m` in [0, 30]. Got {m}.") + + if policy is None: + _policy = default_policy + else: + _policy = policy + + super().__init__(_policy, transformation_matrix_mode=transformation_matrix_mode) + self.n = n + self.m = m + + def rand_selector(self, n: int) -> torch.Tensor: + """Randomly choose ``n`` policy indices without replacement. + + Args: + n: Number of candidate policies to sample. + + Returns: + Tensor of indices into this module's children. + """ + perm = torch.randperm(len(self._modules)) + idx = perm[:n] + return idx + + def compose_subpolicy_sequential(self, subpolicy: SUBPOLICY_CONFIG) -> PolicySequential: + """Build a :class:`PolicySequential` for one RandAugment candidate op. + + Args: + subpolicy: Single-entry policy as ``[(name, low, high)]``. + + Returns: + Sequential wrapper around that operation. + + Raises: + RuntimeError: ``subpolicy`` contains more than one operation. + """ + if len(subpolicy) != 1: + raise RuntimeError(f"Each policy must have only one operation for RandAugment. Got {len(subpolicy)}.") + name, low, high = subpolicy[0] + return PolicySequential(*[getattr(ops, name)(low, high)]) + + def get_forward_sequence(self, params: Optional[List[ParamItem]] = None) -> Iterator[Tuple[str, nn.Module]]: + """Return the policy modules to execute for this forward call. + + Args: + params: Optional recorded parameters. When provided, the sequence is + reconstructed from them. + + Returns: + Iterator of ``(name, module)`` pairs. + """ + if params is None: + idx = self.rand_selector(self.n) + return self.get_children_by_indices(idx) + + return self.get_children_by_params(params) + + def forward_parameters(self, batch_shape: torch.Size) -> List[ParamItem]: + """Sample parameters for the selected RandAugment operations. + + Args: + batch_shape: Input shape used for parameter sampling. + + Returns: + Parameters for the sampled operations, including magnitude scaled from + the global ``m`` value. + """ + named_modules: Iterator[Tuple[str, nn.Module]] = self.get_forward_sequence() + params: List[ParamItem] = [] + mod_param: Union[Dict[str, torch.Tensor], List[ParamItem]] + m = torch.tensor([self.m / 30] * batch_shape[0]) + + for name, module in named_modules: + # The Input PolicySequential only got one child. + op = cast(PolicySequential, module)[0] + op = cast(OperationBase, op) + mag = None + if op.magnitude_range is not None: + minval, maxval = op.magnitude_range + mag = m * float(maxval - minval) + minval + mod_param = op.forward_parameters(batch_shape, mag=mag) + # Compose it + param = ParamItem(name, [ParamItem(next(iter(module.named_children()))[0], mod_param)]) + params.append(param) + + return params diff --git a/kornia/augmentation/auto/trivial_augment/__init__.py b/kornia/augmentation/auto/trivial_augment/__init__.py new file mode 100644 index 0000000..46caa53 --- /dev/null +++ b/kornia/augmentation/auto/trivial_augment/__init__.py @@ -0,0 +1,23 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia TrivialAugment — TrivialAugment policy implementations for Kornia augmentation. + +This subpackage provides the TrivialAugment class and related utilities. +""" + +from .trivial_augment import TrivialAugment diff --git a/kornia/augmentation/auto/trivial_augment/trivial_augment.py b/kornia/augmentation/auto/trivial_augment/trivial_augment.py new file mode 100644 index 0000000..f280e79 --- /dev/null +++ b/kornia/augmentation/auto/trivial_augment/trivial_augment.py @@ -0,0 +1,112 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Iterator, List, Optional, Tuple + +import torch +from torch import nn +from torch.distributions import Categorical + +from kornia.augmentation.auto.base import SUBPOLICY_CONFIG, PolicyAugmentBase +from kornia.augmentation.auto.operations.policy import PolicySequential +from kornia.augmentation.auto.rand_augment import ops +from kornia.augmentation.container.params import ParamItem + +default_policy: List[SUBPOLICY_CONFIG] = [ + # [("identity", 0, 1)], + [("auto_contrast", 0, 1)], + [("equalize", 0, 1)], + [("rotate", -30.0, 30.0)], + [("posterize", 0.0, 4)], + [("solarize", 0.0, 1.0)], + # (Color, 0.1, 1.9), + [("contrast", 0.1, 1.9)], + [("brightness", 0.1, 1.9)], + [("sharpness", 0.1, 1.9)], + [("shear_x", -0.3, 0.3)], + [("shear_y", -0.3, 0.3)], + [("translate_x", -0.5, 0.5)], + [("translate_y", -0.5, 0.5)], +] + + +class TrivialAugment(PolicyAugmentBase): + """Apply TrivialAugment :cite:`muller2021trivialaugment` augmentation strategies. + + Args: + policy: candidate transformations. If None, a default candidate list will be used. + transformation_matrix_mode: computation mode for the chained transformation matrix, via `.transform_matrix` + attribute. + If `silent`, transformation matrix will be computed silently and the non-rigid + modules will be ignored as identity transformations. + If `rigid`, transformation matrix will be computed silently and the non-rigid + modules will trigger errors. + If `skip`, transformation matrix will be totally ignored. + + Examples: + >>> import kornia.augmentation as K + >>> in_tensor = torch.rand(5, 3, 30, 30) + >>> aug = K.AugmentationSequential(TrivialAugment()) + >>> aug(in_tensor).shape + torch.Size([5, 3, 30, 30]) + + """ + + def __init__( + self, policy: Optional[List[SUBPOLICY_CONFIG]] = None, transformation_matrix_mode: str = "silent" + ) -> None: + if policy is None: + _policy = default_policy + else: + _policy = policy + + super().__init__(_policy, transformation_matrix_mode=transformation_matrix_mode) + selection_weights = torch.tensor([1.0 / len(self)] * len(self)) + self.rand_selector = Categorical(selection_weights) + + def compose_subpolicy_sequential(self, subpolicy: SUBPOLICY_CONFIG) -> PolicySequential: + """Build a :class:`PolicySequential` from one TrivialAugment candidate. + + Args: + subpolicy: Single-entry policy as ``[(name, low, high)]``. + + Returns: + Sequential wrapper around the selected operation. + + Raises: + RuntimeError: ``subpolicy`` contains more than one operation. + """ + if len(subpolicy) != 1: + raise RuntimeError(f"Each policy must have only one operation for TrivialAugment. Got {len(subpolicy)}.") + name, low, high = subpolicy[0] + return PolicySequential(*[getattr(ops, name)(low, high)]) + + def get_forward_sequence(self, params: Optional[List[ParamItem]] = None) -> Iterator[Tuple[str, nn.Module]]: + """Return the operation sequence to execute. + + Args: + params: Optional recorded parameters. When provided, the sequence is + reconstructed from them. + + Returns: + Iterator of ``(name, module)`` pairs. + """ + if params is None: + idx = self.rand_selector.sample((1,)) + return self.get_children_by_indices(idx) + + return self.get_children_by_params(params) diff --git a/kornia/augmentation/base.py b/kornia/augmentation/base.py new file mode 100644 index 0000000..c213baa --- /dev/null +++ b/kornia/augmentation/base.py @@ -0,0 +1,586 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from enum import Enum +from typing import Any, Callable, Dict, Optional, Tuple, Union + +import torch +from torch import nn + +from kornia.augmentation.random_generator import RandomGeneratorBase +from kornia.augmentation.utils import ( + _transform_output_shape, + override_parameters, +) +from kornia.core.utils import is_autocast_enabled +from kornia.geometry.boxes import Boxes +from kornia.geometry.keypoints import Keypoints + +TensorWithTransformMat = Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]] + + +# Trick mypy into not applying contravariance rules to inputs by defining +# forward as a value, rather than a function. See also +# https://github.com/python/mypy/issues/8795 +# Based on the trick that torch.nn.Module does for the forward method +def _apply_transform_unimplemented(self: nn.Module, *input: Any) -> torch.Tensor: + r"""Define the computation performed at every call. + + Should be overridden by all subclasses. + """ + raise NotImplementedError(f'nn.Module [{type(self).__name__}] is missing the required "apply_tranform" function') + + +class _BasicAugmentationBase(nn.Module): + r"""_BasicAugmentationBase base class for customized augmentation implementations. + + Plain augmentation base class without the functionality of transformation matrix calculations. + By default, the random computations will be happened on CPU with ``torch.get_default_dtype()``. + To change this behaviour, please use ``set_rng_device_and_dtype``. + + For automatically generating the corresponding ``__repr__`` with full customized parameters, you may need to + implement ``_param_generator`` by inheriting ``RandomGeneratorBase`` for generating random parameters and + put all static parameters inside ``self.flags``. You may take the advantage of ``PlainUniformGenerator`` to + generate simple uniform parameters with less boilerplate code. + + Args: + p: probability for applying an augmentation. This param controls the augmentation probabilities element-wise. + p_batch: probability for applying an augmentation to a batch. This param controls the augmentation + probabilities batch-wise. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input ``True`` or broadcast it to + the batch form ``False``. + + """ + + # Flag for whether this augmentation supports ONNX export. Override to False in subclasses + # that use ops the legacy tracer can't lower (e.g. ``torch.histc``, + # ``torch.distributions.Beta``). + # Users can introspect via ``aug.exportable``; CI iterates the known-exportable + # subset in ``tests/augmentation/test_onnx_export.py``. + ONNX_EXPORTABLE = True + + @property + def exportable(self) -> bool: + """Whether this augmentation supports ONNX export via the legacy tracer at opset 20. + + Reflects the class-level ``ONNX_EXPORTABLE`` flag. Note that ``True`` here + means *the graph traces and exports* — it does not guarantee the resulting + ONNX runtime output is bit-equivalent to eager. See the categorisation in + ``tests/augmentation/test_onnx_export.py`` for the numerical-correctness + signal per augmentation. + """ + return bool(self.ONNX_EXPORTABLE) + + def __init__( + self, + p: float = 0.5, + p_batch: float = 1.0, + same_on_batch: bool = False, + keepdim: bool = False, + ) -> None: + super().__init__() + self.p = p + self.p_batch = p_batch + self.same_on_batch = same_on_batch + self.keepdim = keepdim + self._params: Dict[str, torch.Tensor] = {} + self._param_generator: Optional[RandomGeneratorBase] = None + self.flags: Dict[str, Any] = {} + self.set_rng_device_and_dtype(torch.device("cpu"), torch.get_default_dtype()) + + apply_transform: Callable[..., torch.Tensor] = _apply_transform_unimplemented + + def to(self, *args: Any, **kwargs: Any) -> "_BasicAugmentationBase": + r"""Set the device and dtype for the random number generator.""" + device, dtype, _, _ = torch._C._nn._parse_to(*args, **kwargs) + self.set_rng_device_and_dtype(device, dtype) + return super().to(*args, **kwargs) + + def __repr__(self) -> str: + txt = f"p={self.p}, p_batch={self.p_batch}, same_on_batch={self.same_on_batch}" + if isinstance(self._param_generator, RandomGeneratorBase): + txt = f"{self._param_generator!s}, {txt}" + for k, v in self.flags.items(): + if isinstance(v, Enum): + txt += f", {k}={v.name.lower()}" + else: + txt += f", {k}={v}" + return f"{self.__class__.__name__}({txt})" + + def __unpack_input__(self, input: torch.Tensor) -> torch.Tensor: + return input + + def transform_tensor( + self, + input: torch.Tensor, + *, + shape: Optional[torch.Tensor] = None, + match_channel: bool = True, + ) -> torch.Tensor: + """Standardize input tensors.""" + raise NotImplementedError + + def validate_tensor(self, input: torch.Tensor) -> None: + """Check if the input torch.Tensor is formatted as expected.""" + raise NotImplementedError + + def transform_output_tensor(self, output: torch.Tensor, output_shape: Tuple[int, ...]) -> torch.Tensor: + """Standardize output tensors.""" + return _transform_output_shape(output, output_shape) if self.keepdim else output + + def generate_parameters(self, batch_shape: Tuple[int, ...]) -> Dict[str, torch.Tensor]: + if self._param_generator is not None: + return self._param_generator(batch_shape, self.same_on_batch) + return {} + + def set_rng_device_and_dtype(self, device: torch.device, dtype: torch.dtype) -> None: + """Change the random generation device and dtype. + + Note: + The generated random numbers are not reproducible across different devices and dtypes. + + """ + self.device = device + self.dtype = dtype + if self._param_generator is not None: + self._param_generator.set_rng_device_and_dtype(device, dtype) + + def __batch_prob_generator__( + self, + batch_shape: Tuple[int, ...], + p: float, + p_batch: float, + same_on_batch: bool, + ) -> torch.Tensor: + batch_prob: torch.Tensor + if p_batch == 1: + batch_prob = torch.ones(1, device=self.device, dtype=self.dtype) + elif p_batch == 0: + batch_prob = torch.zeros(1, device=self.device, dtype=self.dtype) + else: + batch_prob = (torch.rand(1, device=self.device) < p_batch).to(self.dtype) + + if batch_prob.sum() == 1: + elem_prob: torch.Tensor + if p == 1: + elem_prob = torch.ones(batch_shape[0], device=self.device, dtype=self.dtype) + elif p == 0: + elem_prob = torch.zeros(batch_shape[0], device=self.device, dtype=self.dtype) + elif same_on_batch: + elem_prob = (torch.rand(1, device=self.device) < p).to(self.dtype).expand(batch_shape[0]) + else: + elem_prob = (torch.rand(batch_shape[0], device=self.device) < p).to(self.dtype) + batch_prob = batch_prob * elem_prob + else: + batch_prob = batch_prob.repeat(batch_shape[0]) + + if len(batch_prob.shape) == 2: + return batch_prob[..., 0] + return batch_prob + + def _process_kwargs_to_params_and_flags( + self, + params: Optional[Dict[str, torch.Tensor]] = None, + flags: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: + # NOTE: determine how to save self._params + save_kwargs = kwargs["save_kwargs"] if "save_kwargs" in kwargs else False + + params = self._params if params is None else params + flags = self.flags if flags is None else flags + + if save_kwargs: + params = override_parameters(params, kwargs, in_place=True) + self._params = params + else: + self._params = params + params = override_parameters(params, kwargs, in_place=False) + + flags = override_parameters(flags, kwargs, in_place=False) + return params, flags + + def forward_parameters(self, batch_shape: Tuple[int, ...]) -> Dict[str, torch.Tensor]: + batch_prob = self.__batch_prob_generator__(batch_shape, self.p, self.p_batch, self.same_on_batch) + _params = self.generate_parameters(batch_shape) + if _params is None: + _params = {} + _params["batch_prob"] = batch_prob + # Added another input_size parameter for geometric transformations + # This might be needed for correctly inversing. + input_size = torch.tensor(batch_shape, dtype=torch.long) + _params.update({"forward_input_shape": input_size}) + return _params + + def apply_func(self, input: torch.Tensor, params: Dict[str, torch.Tensor], flags: Dict[str, Any]) -> torch.Tensor: + return self.apply_transform(input, params, flags) + + def forward( + self, input: torch.Tensor, params: Optional[Dict[str, torch.Tensor]] = None, **kwargs: Any + ) -> torch.Tensor: + """Perform forward operations. + + Args: + input: the input torch.Tensor. + params: the corresponding parameters for an operation. + If None, a new parameter suite will be generated. + **kwargs: key-value pairs to override the parameters and flags. + + Note: + By default, all the overwriting parameters in kwargs will not be recorded + as in ``self._params``. If you wish it to be recorded, you may pass + ``save_kwargs=True`` additionally. + + """ + in_tensor = self.__unpack_input__(input) + input_shape = in_tensor.shape + in_tensor = self.transform_tensor(in_tensor) + batch_shape = in_tensor.shape + if params is None: + params = self.forward_parameters(batch_shape) + + if "batch_prob" not in params: + params["batch_prob"] = torch.tensor([True] * batch_shape[0]) + + params, flags = self._process_kwargs_to_params_and_flags(params, self.flags, **kwargs) + + output = self.apply_func(in_tensor, params, flags) + return self.transform_output_tensor(output, input_shape) if self.keepdim else output + + +class _AugmentationBase(_BasicAugmentationBase): + r"""_AugmentationBase base class for customized augmentation implementations. + + Advanced augmentation base class with the functionality of transformation matrix calculations. + + Args: + p: probability for applying an augmentation. This param controls the augmentation probabilities + element-wise for a batch. + p_batch: probability for applying an augmentation to a batch. This param controls the augmentation + probabilities batch-wise. + same_on_batch: apply the same transformation across the batch. + keepdim: whether to keep the output shape the same as input ``True`` or broadcast it + to the batch form ``False``. + + """ + + def apply_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + # apply transform for the input image torch.Tensor + raise NotImplementedError + + def apply_non_transform( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + # apply additional transform for the images that are skipped from transformation + # where batch_prob == False. + return input + + def transform_inputs( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + **kwargs: Any, + ) -> torch.Tensor: + params, flags = self._process_kwargs_to_params_and_flags( + self._params if params is None else params, flags, **kwargs + ) + + batch_prob = params["batch_prob"] + to_apply = batch_prob > 0.5 + ori_shape = input.shape + in_tensor = self.transform_tensor(input) + + self.validate_tensor(in_tensor) + + output_transformed = self.apply_transform(in_tensor, params, flags, transform=transform) + output_not_transformed = self.apply_non_transform(in_tensor, params, flags, transform=transform) + + if ( + output_transformed.shape == output_not_transformed.shape + and output_transformed.shape[0] == to_apply.shape[0] + ): + to_apply_expanded = to_apply.view(-1, *([1] * (len(output_transformed.shape) - 1))) + output = torch.where(to_apply_expanded, output_transformed, output_not_transformed) + else: + # Shape-changing augmentations (e.g. RandomCrop, Resize) cannot be where-blended + # because the two outputs differ in spatial size. We fall back to a Python branch + # on to_apply.any(); this is non-onnx-exportable. + output = output_transformed if bool(to_apply.any()) else output_not_transformed + + if is_autocast_enabled(): + output = output.type(input.dtype) + + output = _transform_output_shape(output, ori_shape) if self.keepdim else output + + if is_autocast_enabled(): + output = output.type(input.dtype) + return output + + def transform_masks( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + **kwargs: Any, + ) -> torch.Tensor: + params, flags = self._process_kwargs_to_params_and_flags( + self._params if params is None else params, flags, **kwargs + ) + + batch_prob = params["batch_prob"] + to_apply = batch_prob > 0.5 + ori_shape = input.shape + + shape = params["forward_input_shape"] + in_tensor = self.transform_tensor(input, shape=shape, match_channel=False) + + self.validate_tensor(in_tensor) + + output_transformed = self.apply_transform_mask(in_tensor, params, flags, transform=transform) + output_not_transformed = self.apply_non_transform_mask(in_tensor, params, flags, transform=transform) + + if ( + output_transformed.shape == output_not_transformed.shape + and output_transformed.shape[0] == to_apply.shape[0] + ): + to_apply_expanded = to_apply.view(-1, *([1] * (len(output_transformed.shape) - 1))) + output = torch.where(to_apply_expanded, output_transformed, output_not_transformed) + else: + # Shape-changing augmentations (e.g. RandomCrop, Resize) cannot be where-blended + # because the two outputs differ in spatial size. We fall back to a Python branch + # on to_apply.any(); this is non-onnx-exportable. + output = output_transformed if bool(to_apply.any()) else output_not_transformed + + output = _transform_output_shape(output, ori_shape, reference_shape=shape) if self.keepdim else output + return output + + def transform_boxes( + self, + input: Boxes, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + **kwargs: Any, + ) -> Boxes: + if not isinstance(input, Boxes): + raise RuntimeError(f"Only `Boxes` is supported. Got {type(input)}.") + + params, flags = self._process_kwargs_to_params_and_flags( + self._params if params is None else params, flags, **kwargs + ) + + batch_prob = params["batch_prob"] + to_apply = batch_prob > 0.5 + + output_transformed = self.apply_transform_box(input, params, flags, transform=transform) + output_not_transformed = self.apply_non_transform_box(input, params, flags, transform=transform) + + data_transformed = output_transformed.data + data_not_transformed = output_not_transformed.data + + if is_autocast_enabled(): + data_transformed = data_transformed.type(input.data.dtype) + data_not_transformed = data_not_transformed.type(input.data.dtype) + + if data_transformed.shape == data_not_transformed.shape and data_transformed.shape[0] == to_apply.shape[0]: + to_apply_expanded = to_apply.view(-1, *([1] * (len(data_transformed.shape) - 1))) + blended_data = torch.where(to_apply_expanded, data_transformed, data_not_transformed) + else: + blended_data = data_transformed if bool(to_apply.any()) else data_not_transformed + + # Reuse the not-transformed Boxes container (preserves mode/_N/is_batched/etc.) + # and swap in the blended data, same effect as the index_put on .data. + output = output_not_transformed.clone() + output._data = blended_data + return output + + def transform_keypoints( + self, + input: Keypoints, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + **kwargs: Any, + ) -> Keypoints: + if not isinstance(input, Keypoints): + raise RuntimeError(f"Only `Keypoints` is supported. Got {type(input)}.") + + params, flags = self._process_kwargs_to_params_and_flags( + self._params if params is None else params, flags, **kwargs + ) + + batch_prob = params["batch_prob"] + to_apply = batch_prob > 0.5 + + output_transformed = self.apply_transform_keypoint(input, params, flags, transform=transform) + output_not_transformed = self.apply_non_transform_keypoint(input, params, flags, transform=transform) + + data_transformed = output_transformed.data + data_not_transformed = output_not_transformed.data + + if is_autocast_enabled(): + data_transformed = data_transformed.type(input.data.dtype) + data_not_transformed = data_not_transformed.type(input.data.dtype) + + if data_transformed.shape == data_not_transformed.shape and data_transformed.shape[0] == to_apply.shape[0]: + to_apply_expanded = to_apply.view(-1, *([1] * (len(data_transformed.shape) - 1))) + blended_data = torch.where(to_apply_expanded, data_transformed, data_not_transformed) + else: + blended_data = data_transformed if bool(to_apply.any()) else data_not_transformed + + output = output_not_transformed.clone() + output._data = blended_data + return output + + def transform_classes( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + **kwargs: Any, + ) -> torch.Tensor: + params, flags = self._process_kwargs_to_params_and_flags( + self._params if params is None else params, flags, **kwargs + ) + + batch_prob = params["batch_prob"] + to_apply = batch_prob > 0.5 + + output_transformed = self.apply_transform_class(input, params, flags, transform=transform) + output_not_transformed = self.apply_non_transform_class(input, params, flags, transform=transform) + + if ( + output_transformed.shape == output_not_transformed.shape + and output_transformed.shape[0] == to_apply.shape[0] + ): + to_apply_expanded = to_apply.view(-1, *([1] * (len(output_transformed.shape) - 1))) + output = torch.where(to_apply_expanded, output_transformed, output_not_transformed) + else: + # Shape-changing augmentations (e.g. RandomCrop, Resize) cannot be where-blended + # because the two outputs differ in spatial size. We fall back to a Python branch + # on to_apply.any(); this is non-onnx-exportable. + output = output_transformed if bool(to_apply.any()) else output_not_transformed + + return output + + def apply_non_transform_mask( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Process masks corresponding to the inputs that are no transformation applied.""" + raise NotImplementedError + + def apply_transform_mask( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Process masks corresponding to the inputs that are transformed.""" + raise NotImplementedError + + def apply_non_transform_box( + self, + input: Boxes, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> Boxes: + """Process boxes corresponding to the inputs that are no transformation applied.""" + return input + + def apply_transform_box( + self, + input: Boxes, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> Boxes: + """Process boxes corresponding to the inputs that are transformed.""" + raise NotImplementedError + + def apply_non_transform_keypoint( + self, + input: Keypoints, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> Keypoints: + """Process keypoints corresponding to the inputs that are no transformation applied.""" + return input + + def apply_transform_keypoint( + self, + input: Keypoints, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> Keypoints: + """Process keypoints corresponding to the inputs that are transformed.""" + raise NotImplementedError + + def apply_non_transform_class( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Process class tags corresponding to the inputs that are no transformation applied.""" + return input + + def apply_transform_class( + self, + input: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Dict[str, Any], + transform: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Process class tags corresponding to the inputs that are transformed.""" + raise NotImplementedError + + def apply_func( + self, + in_tensor: torch.Tensor, + params: Dict[str, torch.Tensor], + flags: Optional[Dict[str, Any]] = None, + ) -> torch.Tensor: + if flags is None: + flags = self.flags + + output = self.transform_inputs(in_tensor, params, flags) + + return output diff --git a/kornia/augmentation/container/__init__.py b/kornia/augmentation/container/__init__.py new file mode 100644 index 0000000..8179593 --- /dev/null +++ b/kornia/augmentation/container/__init__.py @@ -0,0 +1,28 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Augmentation Container — Sequential and dispatcher augmentation utilities. + +This subpackage provides augmentation containers and dispatchers for flexible pipelines. +""" + +from kornia.augmentation.container.augment import AugmentationSequential +from kornia.augmentation.container.base import ImageSequentialBase +from kornia.augmentation.container.dispatcher import ManyToManyAugmentationDispather, ManyToOneAugmentationDispather +from kornia.augmentation.container.image import ImageSequential +from kornia.augmentation.container.patch import PatchSequential +from kornia.augmentation.container.video import VideoSequential diff --git a/kornia/augmentation/container/augment.py b/kornia/augmentation/container/augment.py new file mode 100644 index 0000000..55eb182 --- /dev/null +++ b/kornia/augmentation/container/augment.py @@ -0,0 +1,690 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import warnings +from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple, Union, cast + +import torch +from torch import nn + +from kornia.augmentation._2d.base import RigidAffineAugmentationBase2D +from kornia.augmentation._3d.base import AugmentationBase3D, RigidAffineAugmentationBase3D +from kornia.augmentation.base import _AugmentationBase +from kornia.constants import DataKey, Resample +from kornia.core.ops import eye_like +from kornia.core.utils import is_autocast_enabled +from kornia.geometry.boxes import Boxes, VideoBoxes +from kornia.geometry.keypoints import Keypoints, VideoKeypoints + +from .base import TransformMatrixMinIn +from .image import ImageSequential +from .ops import AugmentationSequentialOps, DataType +from .params import ParamItem +from .patch import PatchSequential +from .video import VideoSequential + +__all__ = ["AugmentationSequential"] + +_BOXES_OPTIONS = {DataKey.BBOX, DataKey.BBOX_XYXY, DataKey.BBOX_XYWH} +_KEYPOINTS_OPTIONS = {DataKey.KEYPOINTS} +_IMG_OPTIONS = {DataKey.INPUT, DataKey.IMAGE} +_MSK_OPTIONS = {DataKey.MASK} +_CLS_OPTIONS = {DataKey.CLASS, DataKey.LABEL} + +MaskDataType = Union[torch.Tensor, List[torch.Tensor]] + + +class AugmentationSequential(TransformMatrixMinIn, ImageSequential): + r"""AugmentationSequential for handling multiple input types like inputs, masks, keypoints at once. + + .. image:: _static/img/AugmentationSequential.png + + Args: + *args: a list of kornia augmentation modules. + + data_keys: the input type sequential for applying augmentations. Accepts "input", "image", "mask", + "bbox", "bbox_xyxy", "bbox_xywh", "keypoints", "class", "label". + + same_on_batch: apply the same transformation across the batch. If None, it will not overwrite the function-wise + settings. + + keepdim: whether to keep the output shape the same as input (True) or broadcast it to the batch form (False). + If None, it will not overwrite the function-wise settings. + + random_apply: randomly select a sublist (order agnostic) of args to apply transformation. + If int, a fixed number of transformations will be selected. + If (a,), x number of transformations (a <= x <= len(args)) will be selected. + If (a, b), x number of transformations (a <= x <= b) will be selected. + If True, the whole list of args will be processed as a sequence in a random order. + If False, the whole list of args will be processed as a sequence in original order. + + transformation_matrix_mode: computation mode for the chained transformation matrix, via `.transform_matrix` + attribute. + If `silent`, transformation matrix will be computed silently and the non-rigid + modules will be ignored as identity transformations. + If `rigid`, transformation matrix will be computed silently and the non-rigid + modules will trigger errors. + If `skip`, transformation matrix will be totally ignored. + + extra_args: to control the behaviour for each datakeys. By default, masks are handled by nearest interpolation + strategies. + + .. note:: + Mix augmentations (e.g. RandomMixUp, RandomCutMix) can only be working with "input"/"image" data key. + It is not clear how to deal with the conversions of masks, bounding boxes and keypoints. + + .. note:: + See a working example `here `__. + + Examples: + >>> import kornia + >>> input = torch.randn(2, 3, 5, 6) + >>> mask = torch.ones(2, 3, 5, 6) + >>> bbox = torch.tensor([[ + ... [1., 1.], + ... [2., 1.], + ... [2., 2.], + ... [1., 2.], + ... ]]).expand(2, 1, -1, -1) + >>> points = torch.tensor([[[1., 1.]]]).expand(2, -1, -1) + >>> aug_list = AugmentationSequential( + ... kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + ... kornia.augmentation.RandomAffine(360, p=1.0), + ... data_keys=["input", "mask", "bbox", "keypoints"], + ... same_on_batch=False, + ... random_apply=10, + ... ) + >>> out = aug_list(input, mask, bbox, points) + >>> [o.shape for o in out] + [torch.Size([2, 3, 5, 6]), torch.Size([2, 3, 5, 6]), torch.Size([2, 1, 4, 2]), torch.Size([2, 1, 2])] + >>> # apply the exact augmentation again. + >>> out_rep = aug_list(input, mask, bbox, points, params=aug_list._params) + >>> [(o == o_rep).all() for o, o_rep in zip(out, out_rep)] + [tensor(True), tensor(True), tensor(True), tensor(True)] + >>> # inverse the augmentations + >>> out_inv = aug_list.inverse(*out) + >>> [o.shape for o in out_inv] + [torch.Size([2, 3, 5, 6]), torch.Size([2, 3, 5, 6]), torch.Size([2, 1, 4, 2]), torch.Size([2, 1, 2])] + + This example demonstrates the integration of VideoSequential and AugmentationSequential. + + >>> import kornia + >>> input = torch.randn(2, 3, 5, 6)[None] + >>> mask = torch.ones(2, 3, 5, 6)[None] + >>> bbox = torch.tensor([[ + ... [1., 1.], + ... [2., 1.], + ... [2., 2.], + ... [1., 2.], + ... ]]).expand(2, 1, -1, -1)[None] + >>> points = torch.tensor([[[1., 1.]]]).expand(2, -1, -1)[None] + >>> aug_list = AugmentationSequential( + ... VideoSequential( + ... kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + ... kornia.augmentation.RandomAffine(360, p=1.0), + ... ), + ... data_keys=["input", "mask", "bbox", "keypoints"] + ... ) + >>> out = aug_list(input, mask, bbox, points) + >>> [o.shape for o in out] # doctest: +ELLIPSIS + [torch.Size([1, 2, 3, 5, 6]), torch.Size([1, 2, 3, 5, 6]), ...([1, 2, 1, 4, 2]), torch.Size([1, 2, 1, 2])] + + Perform ``OneOf`` transformation with ``random_apply=1`` and ``random_apply_weights`` + in ``AugmentationSequential``. + + >>> import kornia + >>> input = torch.randn(2, 3, 5, 6)[None] + >>> mask = torch.ones(2, 3, 5, 6)[None] + >>> bbox = torch.tensor([[ + ... [1., 1.], + ... [2., 1.], + ... [2., 2.], + ... [1., 2.], + ... ]]).expand(2, 1, -1, -1)[None] + >>> points = torch.tensor([[[1., 1.]]]).expand(2, -1, -1)[None] + >>> aug_list = AugmentationSequential( + ... VideoSequential( + ... kornia.augmentation.RandomAffine(360, p=1.0), + ... ), + ... VideoSequential( + ... kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + ... ), + ... data_keys=["input", "mask", "bbox", "keypoints"], + ... random_apply=1, + ... random_apply_weights=[0.5, 0.3] + ... ) + >>> out = aug_list(input, mask, bbox, points) + >>> [o.shape for o in out] # doctest: +ELLIPSIS + [torch.Size([1, 2, 3, 5, 6]), torch.Size([1, 2, 3, 5, 6]), ...([1, 2, 1, 4, 2]), torch.Size([1, 2, 1, 2])] + + This example shows how to use a list of masks and boxes within AugmentationSequential + + >>> import kornia.augmentation as K + >>> input = torch.randn(2, 3, 256, 256) + >>> mask = [torch.ones(1, 3, 256, 256), torch.ones(1, 2, 256, 256)] + >>> bbox = [ + ... torch.tensor([[28.0, 53.0, 143.0, 164.0], [254.0, 158.0, 364.0, 290.0], [307.0, 204.0, 413.0, 350.0]]), + ... torch.tensor([[254.0, 158.0, 364.0, 290.0], [307.0, 204.0, 413.0, 350.0]]) + ... ] + >>> bbox = [Boxes.from_tensor(i).data for i in bbox] + + >>> aug_list = K.AugmentationSequential( + ... K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + ... K.RandomHorizontalFlip(p=1.0), + ... K.ImageSequential(K.RandomHorizontalFlip(p=1.0)), + ... K.ImageSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0)), + ... data_keys=["input", "mask", "bbox"], + ... same_on_batch=False, + ... random_apply=10, + ... ) + >>> out = aug_list(input, mask, bbox) + + How to use a dictionary as input with AugmentationSequential? The dictionary keys that start with + one of the available datakeys will be augmented accordingly. Otherwise, the dictionary item is passed + without any augmentation. + + >>> import kornia.augmentation as K + >>> img = torch.randn(1, 3, 256, 256) + >>> mask = [torch.ones(1, 3, 256, 256), torch.ones(1, 2, 256, 256)] + >>> bbox = [ + ... torch.tensor([[28.0, 53.0, 143.0, 164.0], [254.0, 158.0, 364.0, 290.0], [307.0, 204.0, 413.0, 350.0]]), + ... torch.tensor([[254.0, 158.0, 364.0, 290.0], [307.0, 204.0, 413.0, 350.0]]) + ... ] + >>> bbox = [Boxes.from_tensor(i).data for i in bbox] + >>> aug_dict = K.AugmentationSequential( + ... K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + ... K.RandomHorizontalFlip(p=1.0), + ... K.ImageSequential(K.RandomHorizontalFlip(p=1.0)), + ... K.ImageSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0)), + ... data_keys=None, + ... same_on_batch=False, + ... random_apply=10, + ... ) + >>> data = {'image': img, 'mask': mask[0], 'mask-b': mask[1], 'bbox': bbox[0], 'bbox-other':bbox[1]} + >>> out = aug_dict(data) + >>> out.keys() + dict_keys(['image', 'mask', 'mask-b', 'bbox', 'bbox-other']) + + """ + + input_dtype = None + mask_dtype = None + + def __init__( + self, + *args: Union[_AugmentationBase, ImageSequential], + data_keys: Optional[Union[Sequence[str], Sequence[int], Sequence[DataKey]]] = (DataKey.INPUT,), + same_on_batch: Optional[bool] = None, + keepdim: Optional[bool] = None, + random_apply: Union[int, bool, Tuple[int, int]] = False, + random_apply_weights: Optional[List[float]] = None, + transformation_matrix_mode: str = "silent", + extra_args: Optional[Dict[DataKey, Dict[str, Any]]] = None, + ) -> None: + self._transform_matrix: Optional[torch.Tensor] + self._transform_matrices: List[Optional[torch.Tensor]] = [] + + super().__init__( + *args, + same_on_batch=same_on_batch, + keepdim=keepdim, + random_apply=random_apply, + random_apply_weights=random_apply_weights, + ) + + self._parse_transformation_matrix_mode(transformation_matrix_mode) + + self._valid_ops_for_transform_computation: Tuple[Any, ...] = ( + RigidAffineAugmentationBase2D, + RigidAffineAugmentationBase3D, + AugmentationSequential, + ) + + self.data_keys: Optional[List[DataKey]] + if data_keys is not None: + self.data_keys = [DataKey.get(inp) for inp in data_keys] + else: + self.data_keys = data_keys + + if self.data_keys: + if any(in_type not in DataKey for in_type in self.data_keys): + raise AssertionError(f"`data_keys` must be in {DataKey}. Got {self.data_keys}.") + + if self.data_keys[0] != DataKey.INPUT: + raise NotImplementedError(f"The first input must be {DataKey.INPUT}.") + + self.transform_op = AugmentationSequentialOps(self.data_keys) + + self.contains_video_sequential: bool = False + self.contains_3d_augmentation: bool = False + for arg in args: + if isinstance(arg, PatchSequential) and not arg.is_intensity_only(): + warnings.warn( + "Geometric transformation detected in PatchSeqeuntial, which would break bbox, mask.", stacklevel=1 + ) + if isinstance(arg, VideoSequential): + self.contains_video_sequential = True + # NOTE: only for images are supported for 3D. + if isinstance(arg, AugmentationBase3D): + self.contains_3d_augmentation = True + self._transform_matrix = None + self.extra_args = extra_args or {DataKey.MASK: {"resample": Resample.NEAREST, "align_corners": None}} + + def clear_state(self) -> None: + """Reset cached params and transformation-matrix state.""" + self._reset_transform_matrix_state() + return super().clear_state() + + def _update_transform_matrix_for_valid_op(self, module: nn.Module) -> None: + self._transform_matrices.append(module.transform_matrix) + + def identity_matrix(self, input: torch.Tensor) -> torch.Tensor: + """Return identity matrix.""" + if self.contains_3d_augmentation: + return eye_like(4, input) + else: + return eye_like(3, input) + + def inverse( # type: ignore[override] + self, + *args: Union[DataType, Dict[str, DataType]], + params: Optional[List[ParamItem]] = None, + data_keys: Optional[Union[List[str], List[int], List[DataKey]]] = None, + ) -> Union[DataType, List[DataType], Dict[str, DataType]]: + """Reverse the transformation applied. + + Number of input tensors must align with the number of``data_keys``. If ``data_keys`` is not set, use + ``self.data_keys`` by default. + """ + original_keys = None + if len(args) == 1 and isinstance(args[0], dict): + original_keys, data_keys, args, invalid_data = self._preproc_dict_data(cast(Dict[str, DataType], args[0])) + + # args here should already be `DataType` + # NOTE: how to right type to: unpacked args <-> tuple of args to unpack + # issue with `self._preproc_dict_data` return args type + + self.transform_op.data_keys = self.transform_op.preproc_datakeys(data_keys) + + self._validate_args_datakeys(*args, data_keys=self.transform_op.data_keys) # type: ignore + + in_args = self._arguments_preproc(*args, data_keys=self.transform_op.data_keys) # type: ignore + + if params is None: + if self._params is None: + raise ValueError( + "No parameters available for inversing, please run a forward pass first " + "or passing valid params into this function." + ) + params = self._params + + outputs: List[DataType] = in_args + for param in params[::-1]: + module = self.get_submodule(param.name) + outputs = self.transform_op.inverse( # type: ignore + *outputs, module=module, param=param, extra_args=self.extra_args + ) + if not isinstance(outputs, list | tuple): + # Make sure we are unpacking a list whilst post-proc + outputs = [outputs] + + outputs = self._arguments_postproc(args, outputs, data_keys=self.transform_op.data_keys) # type: ignore + + if isinstance(original_keys, tuple): + result = {k: v for v, k in zip(outputs, original_keys)} + if invalid_data: + result.update(invalid_data) + return result + + if len(outputs) == 1 and isinstance(outputs, list): + return outputs[0] + + return outputs + + def _validate_args_datakeys(self, *args: DataType, data_keys: List[DataKey]) -> None: + if len(args) != len(data_keys): + raise AssertionError( + f"The number of inputs must align with the number of data_keys. Got {len(args)} and {len(data_keys)}." + ) + # TODO: validate args batching, and its consistency + + def _arguments_preproc(self, *args: DataType, data_keys: List[DataKey]) -> List[DataType]: + inp: List[DataType] = [] + for arg, dcate in zip(args, data_keys): + if DataKey.get(dcate) in _IMG_OPTIONS: + arg = cast(torch.Tensor, arg) + self.input_dtype = arg.dtype + inp.append(arg) + elif DataKey.get(dcate) in _MSK_OPTIONS: + if isinstance(inp, list): + arg = cast(List[torch.Tensor], arg) + self.mask_dtype = arg[0].dtype + else: + arg = cast(torch.Tensor, arg) + self.mask_dtype = arg.dtype + inp.append(self._preproc_mask(arg)) + elif DataKey.get(dcate) in _KEYPOINTS_OPTIONS: + inp.append(self._preproc_keypoints(arg, dcate)) + elif DataKey.get(dcate) in _BOXES_OPTIONS: + inp.append(self._preproc_boxes(arg, dcate)) + elif DataKey.get(dcate) in _CLS_OPTIONS: + inp.append(arg) + else: + raise NotImplementedError(f"input type of {dcate} is not implemented.") + return inp + + def _arguments_postproc( + self, in_args: List[DataType], out_args: List[DataType], data_keys: List[DataKey] + ) -> List[DataType]: + out: List[DataType] = [] + for in_arg, out_arg, dcate in zip(in_args, out_args, data_keys): + if DataKey.get(dcate) in _IMG_OPTIONS: + # It is torch.Tensor type already. + out.append(out_arg) + # TODO: may add the float to integer (for masks), etc. + elif DataKey.get(dcate) in _MSK_OPTIONS: + _out_m = self._postproc_mask(cast(MaskDataType, out_arg)) + out.append(_out_m) + + elif DataKey.get(dcate) in _KEYPOINTS_OPTIONS: + _out_k = self._postproc_keypoint(in_arg, cast(Keypoints, out_arg), dcate) + if is_autocast_enabled() and isinstance(in_arg, torch.Tensor | Keypoints): + if isinstance(_out_k, list): + _out_k = [i.type(in_arg.dtype) for i in _out_k] + else: + _out_k = _out_k.type(in_arg.dtype) + out.append(_out_k) + + elif DataKey.get(dcate) in _BOXES_OPTIONS: + _out_b = self._postproc_boxes(in_arg, cast(Boxes, out_arg), dcate) + if is_autocast_enabled() and isinstance(in_arg, torch.Tensor | Boxes): + if isinstance(_out_b, list): + _out_b = [i.type(in_arg.dtype) for i in _out_b] + else: + _out_b = _out_b.type(in_arg.dtype) + out.append(_out_b) + + elif DataKey.get(dcate) in _CLS_OPTIONS: + out.append(out_arg) + + else: + raise NotImplementedError(f"input type of {dcate} is not implemented.") + + return out + + def forward( # type: ignore[override] + self, + *args: Union[DataType, Dict[str, DataType]], + params: Optional[List[ParamItem]] = None, + data_keys: Optional[Union[List[str], List[int], List[DataKey]]] = None, + ) -> Union[DataType, List[DataType], Dict[str, DataType]]: + """Compute multiple tensors simultaneously according to ``self.data_keys``.""" + self.clear_state() + + # Strip trailing ``None`` positional args. The legacy torch.onnx.export tracer + # rebinds keyword-only defaults (``params``/``data_keys``) as positional, so + # ``forward(x)`` arrives as ``forward(x, None, None)``. Real data inputs are + # always tensors / Boxes / Keypoints / dicts — never ``None``, so this is safe. + while args and args[-1] is None: + args = args[:-1] + + # Unpack/handle dictionary args + original_keys = None + if len(args) == 1 and isinstance(args[0], dict): + original_keys, data_keys, args, invalid_data = self._preproc_dict_data(cast(Dict[str, DataType], args[0])) + + self.transform_op.data_keys = self.transform_op.preproc_datakeys(data_keys) + + self._validate_args_datakeys(*args, data_keys=self.transform_op.data_keys) # type: ignore + + in_args = self._arguments_preproc(*args, data_keys=self.transform_op.data_keys) # type: ignore + + if params is None: + # image data must exist if params is not provided. + if DataKey.INPUT in self.transform_op.data_keys: + inp = in_args[self.transform_op.data_keys.index(DataKey.INPUT)] + if not isinstance(inp, torch.Tensor): + raise ValueError(f"`INPUT` should be a torch.Tensor but `{type(inp)}` received.") + # A video input shall be BCDHW while an image input shall be BCHW + if self.contains_video_sequential or self.contains_3d_augmentation: + _, out_shape = self.autofill_dim(inp, dim_range=(3, 5)) + else: + _, out_shape = self.autofill_dim(inp, dim_range=(2, 4)) + params = self.forward_parameters(out_shape) + else: + raise ValueError("`params` must be provided whilst INPUT is not in data_keys.") + + outputs: Union[torch.Tensor, List[DataType]] = in_args + for param in params: + module = self.get_submodule(param.name) + outputs = self.transform_op.transform( # type: ignore + *outputs, module=module, param=param, extra_args=self.extra_args + ) + if not isinstance(outputs, list | tuple): + # Make sure we are unpacking a list whilst post-proc + outputs = [outputs] + self._update_transform_matrix_by_module(module) + + outputs = self._arguments_postproc(args, outputs, data_keys=self.transform_op.data_keys) # type: ignore + # Restore it back + self.transform_op.data_keys = self.data_keys + + self._params = params + + if isinstance(original_keys, tuple): + result = {k: v for v, k in zip(outputs, original_keys)} + if invalid_data: + result.update(invalid_data) + return result + + if len(outputs) == 1 and isinstance(outputs, list): + return outputs[0] + + return outputs + + def __call__( + self, + *inputs: Any, + input_names_to_handle: Optional[List[Any]] = None, + output_type: Literal["pt", "numpy", "pil"] = "pt", + **kwargs: Any, + ) -> Any: + """Overwrite the __call__ function to handle various inputs. + + Args: + inputs: Inputs to operate on. + input_names_to_handle: List of input names to convert, if None, handle all inputs. + output_type: Desired output type ('pt', 'numpy', or 'pil'). + kwargs: Additional arguments. + + Returns: + Callable: Decorated function with converted input and output types. + + """ + # Wrap the forward method with the decorator + if not self._disable_features: + # TODO: Some more behaviour for AugmentationSequential needs to be revisited later + # e.g. We convert only images, etc. + decorated_forward = self.convert_input_output( + input_names_to_handle=input_names_to_handle, output_type=output_type + )(super(ImageSequential, self).__call__) + _output_image = decorated_forward(*inputs, **kwargs) + + in_data_keys: Optional[List[DataKey]] + if len(inputs) == 1 and isinstance(inputs[0], dict): + original_keys, in_data_keys, inputs, _invalid_data = self._preproc_dict_data(inputs[0]) + else: + in_data_keys = kwargs.get("data_keys", self.data_keys) + data_keys = self.transform_op.preproc_datakeys(in_data_keys) + + if len(data_keys) > 1 and DataKey.INPUT in data_keys: + # NOTE: we may update it later for more supports of drawing boxes, etc. + idx = data_keys.index(DataKey.INPUT) + if output_type == "pt": + self._output_image = _output_image + if isinstance(_output_image, dict): + self._output_image[original_keys[idx]] = _output_image[original_keys[idx]] + else: + self._output_image[idx] = _output_image[idx] + elif isinstance(_output_image, dict): + self._output_image[original_keys[idx]] = _output_image[original_keys[idx]] + else: + self._output_image[idx] = _output_image[idx] + else: + self._output_image = _output_image + else: + _output_image = super(ImageSequential, self).__call__(*inputs, **kwargs) + return _output_image + + def _preproc_dict_data( + self, data: Dict[str, DataType] + ) -> Tuple[Tuple[str, ...], List[DataKey], Tuple[DataType, ...], Optional[Dict[str, Any]]]: + if self.data_keys is not None: + raise ValueError("If you are using a dictionary as input, the data_keys should be None.") + + keys = tuple(data.keys()) + data_keys, invalid_keys = self._read_datakeys_from_dict(keys) + invalid_data = {i: data.pop(i) for i in invalid_keys} if invalid_keys else None + keys = tuple(k for k in keys if k not in invalid_keys) if invalid_keys else keys + data_unpacked = tuple(data.values()) + + return keys, data_keys, data_unpacked, invalid_data + + def _read_datakeys_from_dict(self, keys: Sequence[str]) -> Tuple[List[DataKey], Optional[List[str]]]: + def retrieve_key(key: str) -> DataKey: + """Try to retrieve the datakey value by matching `*`.""" + # Alias cases, like INPUT, will not be get by the enum iterator. + if key.upper().startswith("INPUT"): + return DataKey.INPUT + + for dk in DataKey: + if key.upper() in {"BBOX_XYXY", "BBOX_XYWH"}: + return DataKey.get(key.upper()) + if key.upper().startswith(dk.name): + return DataKey.get(dk.name) + + allowed_dk = " | ".join(f"`{d.name}`" for d in DataKey) + raise ValueError( + f"Your input data dictionary keys should start with some of datakey values: {allowed_dk}. Got `{key}`" + ) + + valid_data_keys = [] + invalid_keys = [] + for k in keys: + try: + valid_data_keys.append(DataKey.get(retrieve_key(k))) + except ValueError: + invalid_keys.append(k) + + return valid_data_keys, invalid_keys + + def _preproc_mask(self, arg: MaskDataType) -> MaskDataType: + if isinstance(arg, list): + new_arg = [] + for a in arg: + a_new = a.to(self.input_dtype) if self.input_dtype else a.to(torch.float) + new_arg.append(a_new) + return new_arg + + else: + arg = arg.to(self.input_dtype) if self.input_dtype else arg.to(torch.float) + return arg + + def _postproc_mask(self, arg: MaskDataType) -> MaskDataType: + if isinstance(arg, list): + new_arg = [] + for a in arg: + a_new = a.to(self.mask_dtype) if self.mask_dtype else a.to(torch.float) + new_arg.append(a_new) + return new_arg + + else: + arg = arg.to(self.mask_dtype) if self.mask_dtype else arg.to(torch.float) + return arg + + def _preproc_boxes(self, arg: DataType, dcate: DataKey) -> Boxes: + if DataKey.get(dcate) in [DataKey.BBOX]: + mode = "vertices_plus" + elif DataKey.get(dcate) in [DataKey.BBOX_XYXY]: + mode = "xyxy_plus" + elif DataKey.get(dcate) in [DataKey.BBOX_XYWH]: + mode = "xywh" + else: + raise ValueError(f"Unsupported mode `{DataKey.get(dcate).name}`.") + if isinstance(arg, Boxes): + return arg + elif self.contains_video_sequential: + arg = cast(torch.Tensor, arg) + return VideoBoxes.from_tensor(arg) + elif self.contains_3d_augmentation: + raise NotImplementedError("3D box handlers are not yet supported.") + else: + arg = cast(torch.Tensor, arg) + return Boxes.from_tensor(arg, mode=mode) + + def _postproc_boxes( + self, in_arg: DataType, out_arg: Boxes, dcate: DataKey + ) -> Union[torch.Tensor, List[torch.Tensor], Boxes]: + if DataKey.get(dcate) in [DataKey.BBOX]: + mode = "vertices_plus" + elif DataKey.get(dcate) in [DataKey.BBOX_XYXY]: + mode = "xyxy_plus" + elif DataKey.get(dcate) in [DataKey.BBOX_XYWH]: + mode = "xywh" + else: + raise ValueError(f"Unsupported mode `{DataKey.get(dcate).name}`.") + + # TODO: handle 3d scenarios + if isinstance(in_arg, Boxes): + return out_arg + else: + return out_arg.to_tensor(mode=mode) + + def _preproc_keypoints(self, arg: DataType, dcate: DataKey) -> Keypoints: + dtype = None + + if self.contains_video_sequential: + arg = cast(Union[torch.Tensor, List[torch.Tensor]], arg) + if isinstance(arg, list): + if not torch.is_floating_point(arg[0]): + dtype = arg[0].dtype + arg = [a.float() for a in arg] + elif not torch.is_floating_point(arg): + dtype = arg.dtype + arg = arg.float() + video_result = VideoKeypoints.from_tensor(arg) + return video_result.type(dtype) if dtype else video_result + elif self.contains_3d_augmentation: + raise NotImplementedError("3D keypoint handlers are not yet supported.") + elif isinstance(arg, Keypoints): + return arg + else: + arg = cast(torch.Tensor, arg) + if not torch.is_floating_point(arg): + dtype = arg.dtype + arg = arg.float() + # TODO: Add List[torch.Tensor] in the future. + result = Keypoints.from_tensor(arg) + return result.type(dtype) if dtype else result + + def _postproc_keypoint( + self, in_arg: DataType, out_arg: Keypoints, dcate: DataKey + ) -> Union[torch.Tensor, List[torch.Tensor], Keypoints]: + if isinstance(in_arg, Keypoints): + return out_arg + else: + return out_arg.to_tensor() diff --git a/kornia/augmentation/container/base.py b/kornia/augmentation/container/base.py new file mode 100644 index 0000000..91182ec --- /dev/null +++ b/kornia/augmentation/container/base.py @@ -0,0 +1,511 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from collections import OrderedDict +from itertools import zip_longest +from typing import Any, Dict, Iterator, List, Optional, Tuple + +import torch +from torch import nn + +import kornia.augmentation as K +from kornia.augmentation.base import _AugmentationBase +from kornia.geometry.boxes import Boxes +from kornia.geometry.keypoints import Keypoints + +from .ops import BoxSequentialOps, InputSequentialOps, KeypointSequentialOps, MaskSequentialOps +from .params import ParamItem + +__all__ = ["BasicSequentialBase", "ImageSequentialBase", "SequentialBase"] + + +class BasicSequentialBase(nn.Sequential): + r"""BasicSequential for creating kornia modulized processing pipeline. + + Args: + *args : a list of kornia augmentation and image operation modules. + + """ + + def __init__(self, *args: nn.Module) -> None: + # To name the modules properly + _args = OrderedDict() + for idx, mod in enumerate(args): + if not isinstance(mod, nn.Module): + raise NotImplementedError(f"Only nn.Module are supported at this moment. Got {mod}.") + _args.update({f"{mod.__class__.__name__}_{idx}": mod}) + super().__init__(_args) + self._params: Optional[List[ParamItem]] = None + + def get_submodule(self, target: str) -> nn.Module: + """Get submodule. + + This code is taken from torch 1.9.0 since it is not introduced + back to torch 1.7.1. We included this for maintaining more + backward torch versions. + + Args: + target: The fully-qualified string name of the submodule + to look for. (See above example for how to specify a + fully-qualified string.) + + Returns: + nn.Module: The submodule referenced by ``target`` + + Raises: + AttributeError: If the target string references an invalid + path or resolves to something that is not an + ``nn.Module`` + + """ + if len(target) == 0: + return self + + atoms: List[str] = target.split(".") + mod = self + + for item in atoms: + if not hasattr(mod, item): + raise AttributeError(mod._get_name() + " has no attribute `" + item + "`") + + mod = getattr(mod, item) + + if not isinstance(mod, nn.Module): + raise AttributeError("`" + item + "` is not an nn.Module") + + return mod + + def clear_state(self) -> None: + """Reset self._params state to None.""" + self._params = None + + # TODO: Implement this for all submodules. + def forward_parameters(self, batch_shape: torch.Size) -> List[ParamItem]: + """Generate parameters for a forward pass. + + Args: + batch_shape: Input batch shape. + + Returns: + Per-module parameters in execution order. + """ + raise NotImplementedError + + def get_children_by_indices(self, indices: torch.Tensor) -> Iterator[Tuple[str, nn.Module]]: + """Yield child modules selected by index. + + Args: + indices: Indices of children to fetch. + + Yields: + ``(name, module)`` pairs matching ``indices`` order. + """ + modules = list(self.named_children()) + for idx in indices: + yield modules[idx] + + def get_children_by_params(self, params: List[ParamItem]) -> Iterator[Tuple[str, nn.Module]]: + """Yield child modules referenced by a list of params. + + Args: + params: Parameter items whose ``name`` fields identify the children. + + Yields: + ``(name, module)`` pairs matching ``params`` order. + """ + modules = list(self.named_children()) + # TODO: Wrong params passed here when nested ImageSequential + for param in params: + yield modules[list(dict(self.named_children()).keys()).index(param.name)] + + def get_params_by_module(self, named_modules: Iterator[Tuple[str, nn.Module]]) -> Iterator[ParamItem]: + """Create placeholder params for a module iterator. + + Args: + named_modules: Modules that will be executed. + + Yields: + :class:`ParamItem` entries with ``data=None``. + """ + # This will not take module._params + for name, _ in named_modules: + yield ParamItem(name, None) + + +class SequentialBase(BasicSequentialBase): + r"""SequentialBase for creating kornia modulized processing pipeline. + + Args: + *args : a list of kornia augmentation and image operation modules. + same_on_batch: apply the same transformation across the batch. + If None, it will not overwrite the function-wise settings. + return_transform: if ``True`` return the matrix describing the transformation + applied to each. If None, it will not overwrite the function-wise settings. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). If None, it will not overwrite the function-wise settings. + + """ + + def __init__(self, *args: nn.Module, same_on_batch: Optional[bool] = None, keepdim: Optional[bool] = None) -> None: + # To name the modules properly + super().__init__(*args) + self._same_on_batch = same_on_batch + self._keepdim = keepdim + self.update_attribute(same_on_batch, keepdim=keepdim) + + def update_attribute( + self, + same_on_batch: Optional[bool] = None, + return_transform: Optional[bool] = None, + keepdim: Optional[bool] = None, + ) -> None: + """Propagate sequence-level flags to child augmentation modules. + + Args: + same_on_batch: Override for ``same_on_batch``. + return_transform: Reserved for compatibility with older interfaces. + keepdim: Override for ``keepdim``. + """ + for mod in self.children(): + # MixAugmentation does not have return transform + if isinstance(mod, (_AugmentationBase, K.MixAugmentationBaseV2)): + if same_on_batch is not None: + mod.same_on_batch = same_on_batch + if keepdim is not None: + mod.keepdim = keepdim + if isinstance(mod, SequentialBase): + mod.update_attribute(same_on_batch, return_transform, keepdim) + + @property + def same_on_batch(self) -> Optional[bool]: + """Return the sequence-level ``same_on_batch`` setting.""" + return self._same_on_batch + + @same_on_batch.setter + def same_on_batch(self, same_on_batch: Optional[bool]) -> None: + self._same_on_batch = same_on_batch + self.update_attribute(same_on_batch=same_on_batch) + + @property + def keepdim(self) -> Optional[bool]: + """Return the sequence-level ``keepdim`` setting.""" + return self._keepdim + + @keepdim.setter + def keepdim(self, keepdim: Optional[bool]) -> None: + self._keepdim = keepdim + self.update_attribute(keepdim=keepdim) + + def autofill_dim(self, input: torch.Tensor, dim_range: Tuple[int, int] = (2, 4)) -> Tuple[torch.Size, torch.Size]: + """Fill torch.Tensor dim to the upper bound of dim_range. + + If input torch.Tensor dim is smaller than the lower bound of dim_range, an error will be thrown out. + """ + ori_shape = input.shape + if len(ori_shape) < dim_range[0] or len(ori_shape) > dim_range[1]: + raise RuntimeError(f"input shape expected to be in {dim_range} while got {ori_shape}.") + while len(input.shape) < dim_range[1]: + input = input[None] + return ori_shape, input.shape + + +class ImageSequentialBase(SequentialBase): + """Provide a base class for sequential image-only augmentations. + + This class handles the logic for applying a series of transformations + sequentially to input tensors while managing the transformation matrices. + """ + + def identity_matrix(self, input: torch.Tensor) -> torch.Tensor: + """Return identity matrix.""" + raise NotImplementedError + + def get_transformation_matrix( + self, + input: torch.Tensor, + params: Optional[List[ParamItem]] = None, + recompute: bool = False, + extra_args: Optional[Dict[str, Any]] = None, + ) -> Optional[torch.Tensor]: + """Compute the transformation matrix according to the provided parameters. + + Args: + input: the input torch.Tensor. + params: params for the sequence. + recompute: if to recompute the transformation matrix according to the params. + default: False. + extra_args: Optional dictionary of extra arguments with specific options for different input types. + """ + raise NotImplementedError + + def forward_parameters(self, batch_shape: torch.Size) -> List[ParamItem]: + """Generate parameters for child modules. + + Args: + batch_shape: Input batch shape. + + Returns: + Per-module parameter list. + """ + raise NotImplementedError + + def get_forward_sequence(self, params: Optional[List[ParamItem]] = None) -> Iterator[Tuple[str, nn.Module]]: + """Get module sequence by input params.""" + raise NotImplementedError + + def transform_inputs( + self, input: torch.Tensor, params: List[ParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Apply the sequence to an input tensor. + + Args: + input: Input tensor. + params: Parameters aligned with the execution sequence. + extra_args: Optional per-input-type overrides. + + Returns: + Transformed tensor. + """ + for param in params: + module = self.get_submodule(param.name) + input = InputSequentialOps.transform(input, module=module, param=param, extra_args=extra_args) + return input + + def inverse_inputs( + self, input: torch.Tensor, params: List[ParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Apply inverse transforms for an input tensor. + + Args: + input: Tensor produced by :meth:`transform_inputs`. + params: Parameters used during forward execution. + extra_args: Optional per-input-type overrides. + + Returns: + Tensor mapped back through inverse operations. + """ + for (_, module), param in zip_longest(list(self.get_forward_sequence(params))[::-1], params[::-1]): + input = InputSequentialOps.inverse(input, module=module, param=param, extra_args=extra_args) + return input + + def transform_masks( + self, input: torch.Tensor, params: List[ParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Apply the sequence to mask tensors. + + Args: + input: Mask tensor. + params: Parameters aligned with the execution sequence. + extra_args: Optional per-input-type overrides. + + Returns: + Transformed mask tensor. + """ + for param in params: + module = self.get_submodule(param.name) + input = MaskSequentialOps.transform(input, module=module, param=param, extra_args=extra_args) + return input + + def inverse_masks( + self, input: torch.Tensor, params: List[ParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Apply inverse transforms for mask tensors. + + Args: + input: Mask tensor produced by :meth:`transform_masks`. + params: Parameters used during forward execution. + extra_args: Optional per-input-type overrides. + + Returns: + Inverse-transformed mask tensor. + """ + for (_, module), param in zip_longest(list(self.get_forward_sequence(params))[::-1], params[::-1]): + input = MaskSequentialOps.inverse(input, module=module, param=param, extra_args=extra_args) + return input + + def transform_boxes( + self, input: Boxes, params: List[ParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> Boxes: + """Apply the sequence to bounding boxes. + + Args: + input: Bounding boxes. + params: Parameters aligned with the execution sequence. + extra_args: Optional per-input-type overrides. + + Returns: + Transformed boxes. + """ + for param in params: + module = self.get_submodule(param.name) + input = BoxSequentialOps.transform(input, module=module, param=param, extra_args=extra_args) + return input + + def inverse_boxes( + self, input: Boxes, params: List[ParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> Boxes: + """Apply inverse transforms for bounding boxes. + + Args: + input: Boxes produced by :meth:`transform_boxes`. + params: Parameters used during forward execution. + extra_args: Optional per-input-type overrides. + + Returns: + Inverse-transformed boxes. + """ + for (_, module), param in zip_longest(list(self.get_forward_sequence(params))[::-1], params[::-1]): + input = BoxSequentialOps.inverse(input, module=module, param=param, extra_args=extra_args) + return input + + def transform_keypoints( + self, input: Keypoints, params: List[ParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> Keypoints: + """Apply the sequence to keypoints. + + Args: + input: Keypoints. + params: Parameters aligned with the execution sequence. + extra_args: Optional per-input-type overrides. + + Returns: + Transformed keypoints. + """ + for param in params: + module = self.get_submodule(param.name) + input = KeypointSequentialOps.transform(input, module=module, param=param, extra_args=extra_args) + return input + + def inverse_keypoints( + self, input: Keypoints, params: List[ParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> Keypoints: + """Apply inverse transforms for keypoints. + + Args: + input: Keypoints produced by :meth:`transform_keypoints` (conceptually + ``(B, N, 2)`` coordinates, where the last dimension stores + ``(x, y)``. + params: Parameters used during forward execution. + extra_args: Optional per-input-type overrides. + + Returns: + Inverse-transformed keypoints. + """ + for (_, module), param in zip_longest(list(self.get_forward_sequence(params))[::-1], params[::-1]): + input = KeypointSequentialOps.inverse(input, module=module, param=param, extra_args=extra_args) + return input + + def inverse( + self, input: torch.Tensor, params: Optional[List[ParamItem]] = None, extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Inverse transformation. + + Used to inverse a torch.Tensor according to the performed transformation by a forward pass, or with respect to + provided parameters. + """ + if params is None: + if self._params is None: + raise ValueError( + "No parameters available for inversing, please run a forward pass first " + "or passing valid params into this function." + ) + params = self._params + + input = self.inverse_inputs(input, params, extra_args=extra_args) + + return input + + def forward( + self, input: torch.Tensor, params: Optional[List[ParamItem]] = None, extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Run the sequential augmentation pipeline. + + Args: + input: Input tensor. + params: Optional precomputed parameters. If omitted, parameters are + sampled from ``input``. + extra_args: Optional per-input-type overrides. + + Returns: + Augmented tensor. + """ + self.clear_state() + + if params is None: + inp = input + _, out_shape = self.autofill_dim(inp, dim_range=(2, 4)) + params = self.forward_parameters(out_shape) + + input = self.transform_inputs(input, params=params, extra_args=extra_args) + + self._params = params + return input + + +class TransformMatrixMinIn: + """Enables computation matrix computation.""" + + _valid_ops_for_transform_computation: Tuple[Any, ...] = () + _transformation_matrix_arg: str = "silent" + + def __init__(self, *args, **kwargs) -> None: # type:ignore + super().__init__(*args, **kwargs) + self._transform_matrix: Optional[torch.Tensor] = None + self._transform_matrices: List[Optional[torch.Tensor]] = [] + + def _parse_transformation_matrix_mode(self, transformation_matrix_mode: str) -> None: + _valid_transformation_matrix_args = {"silence", "silent", "rigid", "skip"} + if transformation_matrix_mode not in _valid_transformation_matrix_args: + raise ValueError( + f"`transformation_matrix` has to be one of {_valid_transformation_matrix_args}. " + f"Got {transformation_matrix_mode}." + ) + self._transformation_matrix_arg = transformation_matrix_mode + + @property + def transform_matrix(self) -> Optional[torch.Tensor]: + # In AugmentationSequential, the parent class is accessed first. + # So that it was None in the beginning. We hereby use lazy computation here. + if self._transform_matrix is None and len(self._transform_matrices) != 0: + self._transform_matrix = self._transform_matrices[0] + for mat in self._transform_matrices[1:]: + self._update_transform_matrix(mat) + return self._transform_matrix + + def _update_transform_matrix_for_valid_op(self, module: nn.Module) -> None: + raise NotImplementedError(module) + + def _update_transform_matrix_by_module(self, module: nn.Module) -> None: + if self._transformation_matrix_arg == "skip": + return + if isinstance(module, self._valid_ops_for_transform_computation): + self._update_transform_matrix_for_valid_op(module) + elif self._transformation_matrix_arg == "rigid": + raise RuntimeError( + f"Non-rigid module `{module}` is not supported under `rigid` computation mode. " + "Please either update the module or change the `transformation_matrix` argument." + ) + + def _update_transform_matrix(self, transform_matrix: Optional[torch.Tensor]) -> None: + if self._transform_matrix is None: + self._transform_matrix = transform_matrix + else: + self._transform_matrix = transform_matrix @ self._transform_matrix + + def _reset_transform_matrix_state(self) -> None: + self._transform_matrix = None + self._transform_matrices = [] diff --git a/kornia/augmentation/container/dispatcher.py b/kornia/augmentation/container/dispatcher.py new file mode 100644 index 0000000..72787a6 --- /dev/null +++ b/kornia/augmentation/container/dispatcher.py @@ -0,0 +1,130 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import List, Tuple, Union + +from torch import Tensor, nn + +from .augment import AugmentationSequential + + +class ManyToManyAugmentationDispather(nn.Module): + r"""Dispatches different augmentations to different inputs element-wisely. + + Args: + augmentations: a list or a sequence of kornia AugmentationSequential modules. + + Examples: + >>> import torch + >>> input_1, input_2 = torch.randn(2, 3, 5, 6), torch.randn(2, 3, 5, 6) + >>> mask_1, mask_2 = torch.ones(2, 3, 5, 6), torch.ones(2, 3, 5, 6) + >>> aug_list = ManyToManyAugmentationDispather( + ... AugmentationSequential( + ... kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + ... kornia.augmentation.RandomAffine(360, p=1.0), + ... data_keys=["input", "mask",], + ... ), + ... AugmentationSequential( + ... kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + ... kornia.augmentation.RandomAffine(360, p=1.0), + ... data_keys=["input", "mask",], + ... ) + ... ) + >>> output = aug_list((input_1, mask_1), (input_2, mask_2)) + + """ + + def __init__(self, *augmentations: AugmentationSequential) -> None: + super().__init__() + self._check_consistency(*augmentations) + self.augmentations = augmentations + + def _check_consistency(self, *augmentations: AugmentationSequential) -> bool: + for i, aug in enumerate(augmentations): + if not isinstance(aug, AugmentationSequential): + raise ValueError(f"Please wrap your augmentations[`{i}`] with `AugmentationSequentials`.") + return True + + def forward(self, *input: Union[List[Tensor], List[Tuple[Tensor]]]) -> Union[List[Tensor], List[Tuple[Tensor]]]: + """Apply each augmentation to the matching input tuple. + + Args: + *input: One input bundle per configured augmentation. + + Returns: + Outputs from each augmentation, preserving input order. + """ + return [aug(*inp) for inp, aug in zip(input, self.augmentations)] + + +class ManyToOneAugmentationDispather(nn.Module): + r"""Dispatches different augmentations to a single input and returns a list. + + Same `datakeys` must be applied across different augmentations. By default, with input + (image, mask), the augmentations must not mess it as (mask, image) to avoid unexpected + errors. This check can be cancelled with `strict=False` if needed. + + Args: + augmentations: a list or a sequence of kornia AugmentationSequential modules. + + Examples: + >>> import torch + >>> input = torch.randn(2, 3, 5, 6) + >>> mask = torch.ones(2, 3, 5, 6) + >>> aug_list = ManyToOneAugmentationDispather( + ... AugmentationSequential( + ... kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + ... kornia.augmentation.RandomAffine(360, p=1.0), + ... data_keys=["input", "mask",], + ... ), + ... AugmentationSequential( + ... kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + ... kornia.augmentation.RandomAffine(360, p=1.0), + ... data_keys=["input", "mask",], + ... ) + ... ) + >>> output = aug_list(input, mask) + + """ + + def __init__(self, *augmentations: AugmentationSequential, strict: bool = True) -> None: + super().__init__() + self.strict = strict + self._check_consistency(*augmentations) + self.augmentations = augmentations + + def _check_consistency(self, *augmentations: AugmentationSequential) -> bool: + for i, aug in enumerate(augmentations): + if not isinstance(aug, AugmentationSequential): + raise ValueError(f"Please wrap your augmentations[`{i}`] with `AugmentationSequentials`.") + if self.strict and i != 0 and aug.data_keys != augmentations[i - 1].data_keys: + raise RuntimeError( + f"Different `data_keys` between {i - 1} and {i} elements, " + f"got {aug.data_keys} and {augmentations[i - 1].data_keys}." + ) + return True + + def forward(self, *input: Union[Tensor, Tuple[Tensor]]) -> Union[List[Tensor], List[Tuple[Tensor]]]: + """Apply all augmentations to the same input payload. + + Args: + *input: Shared input payload passed to every augmentation. + + Returns: + One output per augmentation sequence. + """ + return [aug(*input) for aug in self.augmentations] diff --git a/kornia/augmentation/container/image.py b/kornia/augmentation/container/image.py new file mode 100644 index 0000000..0d61436 --- /dev/null +++ b/kornia/augmentation/container/image.py @@ -0,0 +1,434 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Iterator, List, Literal, Optional, Tuple, Union, cast + +import torch +from torch import nn + +import kornia.augmentation as K +from kornia.augmentation.base import _AugmentationBase +from kornia.augmentation.utils import override_parameters +from kornia.core import ImageModule +from kornia.core.mixin.image_module import ImageModuleMixIn +from kornia.core.ops import eye_like + +from .base import ImageSequentialBase +from .params import ParamItem + +__all__ = ["ImageSequential"] + + +class ImageModuleForSequentialMixIn(ImageModuleMixIn): + _disable_features: bool = False + + @property + def disable_features(self) -> bool: + return self._disable_features + + @disable_features.setter + def disable_features(self, value: bool = True) -> None: + self._disable_features = value + + def disable_item_features(self, *args: nn.Module) -> None: + for arg in args: + if isinstance(arg, ImageModule): + arg.disable_features = True + + +class ImageSequential(ImageSequentialBase, ImageModuleForSequentialMixIn): + r"""nn.Sequential for creating kornia image processing pipeline. + + Args: + *args : a list of kornia augmentation and image operation modules. + same_on_batch: apply the same transformation across the batch. + If None, it will not overwrite the function-wise settings. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). If None, it will not overwrite the function-wise settings. + random_apply: randomly select a sublist (order agnostic) of args to + apply transformation. The selection probability aligns to the ``random_apply_weights``. + If int, a fixed number of transformations will be selected. + If (a,), x number of transformations (a <= x <= len(args)) will be selected. + If (a, b), x number of transformations (a <= x <= b) will be selected. + If True, the whole list of args will be processed as a sequence in a random order. + If False, the whole list of args will be processed as a sequence in original order. + random_apply_weights: a list of selection weights for each operation. The length shall be as + same as the number of operations. By default, operations are sampled uniformly. + + .. note:: + Transformation matrix returned only considers the transformation applied in ``kornia.augmentation`` module. + Those transformations in ``kornia.geometry`` will not be taken into account. + + Examples: + >>> _ = torch.manual_seed(77) + >>> import kornia + >>> input = torch.randn(2, 3, 5, 6) + >>> aug_list = ImageSequential( + ... kornia.color.BgrToRgb(), + ... kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + ... kornia.filters.MedianBlur((3, 3)), + ... kornia.augmentation.RandomAffine(360, p=1.0), + ... kornia.enhance.Invert(), + ... kornia.augmentation.RandomMixUpV2(p=1.0), + ... same_on_batch=True, + ... random_apply=10, + ... ) + >>> out = aug_list(input) + >>> out.shape + torch.Size([2, 3, 5, 6]) + + Reproduce with provided params. + >>> out2 = aug_list(input, params=aug_list._params) + >>> torch.equal(out, out2) + True + + Perform ``OneOf`` transformation with ``random_apply=1`` and ``random_apply_weights`` in ``ImageSequential``. + + >>> import kornia + >>> input = torch.randn(2, 3, 5, 6) + >>> aug_list = ImageSequential( + ... kornia.color.BgrToRgb(), + ... kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + ... kornia.filters.MedianBlur((3, 3)), + ... kornia.augmentation.RandomAffine(360, p=1.0), + ... random_apply=1, + ... random_apply_weights=[0.5, 0.3, 0.2, 0.5] + ... ) + >>> out= aug_list(input) + >>> out.shape + torch.Size([2, 3, 5, 6]) + + """ + + def __init__( + self, + *args: nn.Module, + same_on_batch: Optional[bool] = None, + keepdim: Optional[bool] = None, + random_apply: Union[int, bool, Tuple[int, int]] = False, + random_apply_weights: Optional[List[float]] = None, + if_unsupported_ops: str = "raise", + disable_item_features: bool = True, + disable_sequential_features: bool = False, + ) -> None: + if disable_item_features: + self.disable_item_features(*args) + if disable_sequential_features: + self.disable_features = True + super().__init__(*args, same_on_batch=same_on_batch, keepdim=keepdim) + + self.random_apply = self._read_random_apply(random_apply, len(args)) + if random_apply_weights is not None and len(random_apply_weights) != len(self): + raise ValueError( + "The length of `random_apply_weights` must be as same as the number of operations." + f"Got {len(random_apply_weights)} and {len(self)}." + ) + self.random_apply_weights = torch.as_tensor(random_apply_weights or torch.ones((len(self),))) + self.if_unsupported_ops = if_unsupported_ops + + def _read_random_apply( + self, random_apply: Union[int, bool, Tuple[int, int]], max_length: int + ) -> Union[Tuple[int, int], bool]: + """Process the scenarios for random apply.""" + if isinstance(random_apply, bool) and random_apply is False: + random_apply = False + elif isinstance(random_apply, bool) and random_apply is True: + random_apply = (max_length, max_length + 1) + elif isinstance(random_apply, int): + random_apply = (random_apply, random_apply + 1) + elif ( + isinstance(random_apply, tuple) + and len(random_apply) == 2 + and isinstance(random_apply[0], int) + and isinstance(random_apply[1], int) + ): + random_apply = (random_apply[0], random_apply[1] + 1) + elif isinstance(random_apply, tuple) and len(random_apply) == 1 and isinstance(random_apply[0], int): + random_apply = (random_apply[0], max_length + 1) + else: + raise ValueError(f"Non-readable random_apply. Got {random_apply}.") + if random_apply is not False and not ( + isinstance(random_apply, tuple) + and len(random_apply) == 2 + and isinstance(random_apply[0], int) + and isinstance(random_apply[1], int) + ): + raise AssertionError(f"Expect a tuple of (int, int). Got {random_apply}.") + return random_apply + + def get_random_forward_sequence(self, with_mix: bool = True) -> Tuple[Iterator[Tuple[str, nn.Module]], bool]: + """Get a forward sequence when random apply is in need. + + Args: + with_mix: if to require a mix augmentation for the sequence. + + Note: + Mix augmentations (e.g. RandomMixUp) will be only applied once even in a random forward. + + """ + if isinstance(self.random_apply, tuple): + num_samples = int(torch.randint(*self.random_apply, (1,)).item()) + else: + raise TypeError(f"random apply should be a tuple. Gotcha {type(self.random_apply)}") + + multinomial_weights = self.random_apply_weights.clone() + # Mix augmentation can only be applied once per forward + mix_indices = self.get_mix_augmentation_indices(self.named_children()) + + multinomial_weights[mix_indices] = 0 + + total_weight = multinomial_weights.sum() + if total_weight == 0: + indices = torch.tensor([], device=multinomial_weights.device, dtype=torch.long) + else: + indices = torch.multinomial( + multinomial_weights, + num_samples, + replacement=num_samples > total_weight.item(), + ) + + mix_added = False + if with_mix and len(mix_indices) != 0: + # Make the selection fair. + if (torch.rand(1) < ((len(mix_indices) + len(indices)) / len(self))).item(): + mix_idx = torch.multinomial((~multinomial_weights.bool()).float(), 1) + if len(indices) == 0: + indices = mix_idx + else: + indices[-1] = mix_idx + indices = indices[torch.randperm(len(indices))] + mix_added = True + + return self.get_children_by_indices(indices), mix_added + + def get_mix_augmentation_indices(self, named_modules: Iterator[Tuple[str, nn.Module]]) -> List[int]: + """Get all the mix augmentations since they are label-involved. + + Special operations needed for label-involved augmentations. + """ + # NOTE: MixV2 will not be a special op in the future. + return [idx for idx, (_, child) in enumerate(named_modules) if isinstance(child, K.MixAugmentationBaseV2)] + + def get_forward_sequence(self, params: Optional[List[ParamItem]] = None) -> Iterator[Tuple[str, nn.Module]]: + """Return the module sequence used for the current forward call. + + Args: + params: Optional recorded parameters. When provided, the sequence is + reconstructed from them. + + Returns: + Iterator of ``(name, module)`` pairs. + + Raises: + ValueError: More than one mix augmentation is present while + ``random_apply`` is disabled. + """ + if params is None: + # Mix augmentation can only be applied once per forward + mix_indices = self.get_mix_augmentation_indices(self.named_children()) + + if self.random_apply: + return self.get_random_forward_sequence()[0] + + if len(mix_indices) > 1: + raise ValueError( + "Multiple mix augmentation is prohibited without enabling random_apply." + f"Detected {len(mix_indices)} mix augmentations." + ) + + return self.named_children() + + return self.get_children_by_params(params) + + def forward_parameters(self, batch_shape: torch.Size) -> List[ParamItem]: + """Generate parameters for all modules in the chosen sequence. + + Args: + batch_shape: Input batch shape. + + Returns: + Parameter list aligned with the execution order. + """ + named_modules: Iterator[Tuple[str, nn.Module]] = self.get_forward_sequence() + + params: List[ParamItem] = [] + mod_param: Union[Dict[str, torch.Tensor], List[ParamItem]] + for name, module in named_modules: + if isinstance(module, (_AugmentationBase | K.MixAugmentationBaseV2 | ImageSequentialBase)): + mod_param = module.forward_parameters(batch_shape) + param = ParamItem(name, mod_param) + else: + param = ParamItem(name, None) + batch_shape = _get_new_batch_shape(param, batch_shape) + params.append(param) + return params + + def identity_matrix(self, input: torch.Tensor) -> torch.Tensor: + """Return identity matrix.""" + return eye_like(3, input) + + def get_transformation_matrix( + self, + input: torch.Tensor, + params: Optional[List[ParamItem]] = None, + recompute: bool = False, + extra_args: Optional[Dict[str, Any]] = None, + ) -> Optional[torch.Tensor]: + """Compute the transformation matrix according to the provided parameters. + + Args: + input: the input torch.Tensor. + params: params for the sequence. + recompute: if to recompute the transformation matrix according to the params. + default: False. + extra_args: Optional dictionary of extra arguments with specific options for different input types. + """ + if params is None: + raise NotImplementedError("requires params to be provided.") + if extra_args is None: + extra_args = {} + named_modules: Iterator[Tuple[str, nn.Module]] = self.get_forward_sequence(params) + + # Define as 1 for broadcasting + res_mat: Optional[torch.Tensor] = None + for (_, module), param in zip(named_modules, params if params is not None else []): + if isinstance(module, K.GeometricAugmentationBase2D) and isinstance(param.data, dict): + ori_shape = input.shape + try: + input = module.transform_tensor(input) + except ValueError: + # Ignore error for 5-dim video + pass + # Standardize shape + if recompute: + flags = override_parameters(module.flags, extra_args, in_place=False) + mat = module.generate_transformation_matrix(input, param.data, flags) + elif module._transform_matrix is not None: + mat = torch.as_tensor(module._transform_matrix, device=input.device, dtype=input.dtype) + else: + raise RuntimeError(f"{module}._transform_matrix is None while `recompute=False`.") + res_mat = mat if res_mat is None else mat @ res_mat + input = module.transform_output_tensor(input, ori_shape) + if module.keepdim and ori_shape != input.shape: + res_mat = res_mat.squeeze() + elif isinstance(module, ImageSequentialBase): + # If not augmentationSequential + if isinstance(module, K.AugmentationSequential) and not recompute: + mat = torch.as_tensor(module._transform_matrix, device=input.device, dtype=input.dtype) + else: + maybe_param_data = cast(Optional[List[ParamItem]], param.data) + _mat = module.get_transformation_matrix( + input, maybe_param_data, recompute=recompute, extra_args=extra_args + ) + mat = module.identity_matrix(input) if _mat is None else _mat + res_mat = mat if res_mat is None else mat @ res_mat + return res_mat + + # TODO: Make this as a class property to avoid running every time. + def is_intensity_only(self, strict: bool = True) -> bool: + """Check if all transformations are intensity-based. + + Args: + strict: if strict is False, it will allow non-augmentation Modules to be passed. + e.g. `kornia.enhance.AdjustBrightness` will be recognized as non-intensity module + if strict is set to True. + + Note: patch processing would break the continuity of labels (e.g. bbounding boxes, masks). + + """ + for arg in self.children(): + if isinstance(arg, ImageSequential) and not arg.is_intensity_only(strict): + return False + elif isinstance(arg, ImageSequential): + pass + elif isinstance(arg, K.IntensityAugmentationBase2D): + pass + elif strict: + # disallow non-registered ops if in strict mode + # TODO: add an ops register module + return False + return True + + def __call__( + self, + *inputs: Any, + input_names_to_handle: Optional[List[Any]] = None, + output_type: Literal["pt", "numpy", "pil"] = "pt", + **kwargs: Any, + ) -> Any: + """Overwrite the __call__ function to handle various inputs. + + Args: + inputs: Inputs to operate on. + input_names_to_handle: List of input names to convert, if None, handle all inputs. + output_type: Desired output type ('pt', 'numpy', or 'pil'). + kwargs: Additional arguments. + + Returns: + Callable: Decorated function with converted input and output types. + + """ + # Wrap the forward method with the decorator + if not self._disable_features: + decorated_forward = self.convert_input_output( + input_names_to_handle=input_names_to_handle, output_type=output_type + )(super().__call__) + _output_image = decorated_forward(*inputs, **kwargs) + if output_type == "pt": + self._output_image = self._detach_tensor_to_cpu(_output_image) + else: + self._output_image = _output_image + else: + _output_image = super().__call__(*inputs, **kwargs) + return _output_image + + +def _get_new_batch_shape(param: ParamItem, batch_shape: torch.Size) -> torch.Size: + """Get the new batch shape if the augmentation changes the image size. + + Note: + Augmentations that change the image size must provide the parameter `output_size`. + + """ + data = param.data + if data is None: + return batch_shape + + # If data is a list, process all subitems (exit early if all subitems are None) + if isinstance(data, list): + for p in data: + batch_shape = _get_new_batch_shape(p, batch_shape) + return batch_shape + + # Carefully avoid evaluating expression multiple times; batch_prob is often a 1-element torch.Tensor + if "output_size" in data: + # Inline check for common PyTorch float torch.Tensor case + batch_prob = data.get("batch_prob", None) + if batch_prob is not None: + # Avoid repeated indexing, always fetch scalar efficiently + prob = batch_prob.item() if batch_prob.numel() == 1 else batch_prob[0].item() + if prob <= 0.5: + return batch_shape + else: + # batch_prob missing, fallback do not update shape + return batch_shape + # Mutate only last two dims + new_batch_shape = list(batch_shape) + new_batch_shape[-2:] = data["output_size"][0] + return torch.Size(new_batch_shape) + + return batch_shape diff --git a/kornia/augmentation/container/ops.py b/kornia/augmentation/container/ops.py new file mode 100644 index 0000000..31d3eab --- /dev/null +++ b/kornia/augmentation/container/ops.py @@ -0,0 +1,752 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import copy +from abc import ABCMeta, abstractmethod +from typing import Any, Callable, Dict, Generic, List, Optional, Type, TypeVar, Union + +import torch +from torch import nn +from typing_extensions import ParamSpec + +import kornia.augmentation as K +from kornia.augmentation.base import _AugmentationBase +from kornia.constants import DataKey +from kornia.geometry.boxes import Boxes +from kornia.geometry.keypoints import Keypoints + +from .params import ParamItem + +DataType = Union[torch.Tensor, List[torch.Tensor], Boxes, Keypoints] + +# NOTE: shouldn't this SequenceDataType alias be equals to List[DataType]? +SequenceDataType = Union[List[torch.Tensor], List[List[torch.Tensor]], List[Boxes], List[Keypoints]] + +T = TypeVar("T") + + +class SequentialOpsInterface(Generic[T], metaclass=ABCMeta): + """Abstract interface for applying and inversing transformations.""" + + @classmethod + def get_instance_module_param(cls, param: ParamItem) -> Dict[str, torch.Tensor]: + """Extract per-module parameter dict from a :class:`ParamItem`. + + Args: + param: Parameter wrapper produced by sequential containers. + + Returns: + Dictionary of tensor parameters for one module call. + + Raises: + TypeError: ``param.data`` is not a dictionary. + """ + if isinstance(param, ParamItem) and isinstance(param.data, dict): + _params = param.data + else: + raise TypeError(f"Expected param (ParamItem.data) be a dictionary. Gotcha {param}.") + return _params + + @classmethod + def get_sequential_module_param(cls, param: ParamItem) -> List[ParamItem]: + """Extract nested sequential parameters from a :class:`ParamItem`. + + Args: + param: Parameter wrapper produced by sequential containers. + + Returns: + List of :class:`ParamItem` values for nested modules. + + Raises: + TypeError: ``param.data`` is not a list. + """ + if isinstance(param, ParamItem) and isinstance(param.data, list): + _params = param.data + else: + raise TypeError(f"Expected param (ParamItem.data) be a list. Gotcha {param}.") + return _params + + @classmethod + @abstractmethod + def transform(cls, input: T, module: nn.Module, param: ParamItem, extra_args: Optional[Dict[str, Any]] = None) -> T: + """Apply a transformation with respect to the parameters. + + Args: + input: the input torch.Tensor. + module: any torch nn.Module but only kornia augmentation modules will count + to apply transformations. + param: the corresponding parameters to the module. + extra_args: Optional dictionary of extra arguments with specific options for different input types. + """ + raise NotImplementedError + + @classmethod + @abstractmethod + def inverse(cls, input: T, module: nn.Module, param: ParamItem, extra_args: Optional[Dict[str, Any]] = None) -> T: + """Inverse a transformation with respect to the parameters. + + Args: + input: the input torch.Tensor. + module: any torch nn.Module but only kornia augmentation modules will count + to apply transformations. + param: the corresponding parameters to the module. + extra_args: Optional dictionary of extra arguments with specific options for different input types. + """ + raise NotImplementedError + + +class AugmentationSequentialOps: + """Implement the operational logic for the Augmentation Sequential container. + + This class manages how data keys (e.g., IMAGE, MASK, BBOX) are handled + during the execution of an augmentation pipeline. + + Args: + data_keys: A list of :class:`DataKey` defining the types of data to process. + """ + + def __init__(self, data_keys: Optional[List[DataKey]]) -> None: + self._data_keys = data_keys + + @property + def data_keys(self) -> Optional[List[DataKey]]: + """Return currently configured data keys.""" + return self._data_keys + + @data_keys.setter + def data_keys(self, data_keys: Optional[Union[List[DataKey], List[str], List[int]]]) -> None: + if data_keys: + self._data_keys = [DataKey.get(inp) for inp in data_keys] + else: + self._data_keys = None + + def preproc_datakeys(self, data_keys: Optional[Union[List[str], List[int], List[DataKey]]] = None) -> List[DataKey]: + """Normalize user-provided data keys into :class:`DataKey` values. + + Args: + data_keys: Optional keys passed by the caller. If omitted, this method + uses ``self.data_keys``. + + Returns: + Normalized list of data keys. + + Raises: + ValueError: Neither argument nor instance-level keys are available. + """ + if data_keys is None: + if isinstance(self.data_keys, list): + return self.data_keys + raise ValueError("nn.Sequential ops needs data keys to be able to process.") + else: + return [DataKey.get(inp) for inp in data_keys] + + def _get_op(self, data_key: DataKey) -> Type[SequentialOpsInterface[Any]]: + """Return the corresponding operation given a data key.""" + if data_key == DataKey.INPUT: + return InputSequentialOps + if data_key == DataKey.MASK: + return MaskSequentialOps + if data_key in {DataKey.BBOX, DataKey.BBOX_XYWH, DataKey.BBOX_XYXY}: + return BoxSequentialOps + if data_key == DataKey.KEYPOINTS: + return KeypointSequentialOps + if data_key == DataKey.CLASS: + return ClassSequentialOps + raise RuntimeError(f"Operation for `{data_key.name}` is not found.") + + def transform( + self, + *arg: DataType, + module: nn.Module, + param: ParamItem, + extra_args: Dict[DataKey, Dict[str, Any]], + data_keys: Optional[Union[List[str], List[int], List[DataKey]]] = None, + ) -> Union[DataType, SequenceDataType]: + """Apply one module to all inputs according to their data keys. + + Args: + *arg: Inputs to transform (image, mask, boxes, keypoints, ...). + module: Module to execute. + param: Parameters associated with ``module``. + extra_args: Optional runtime overrides keyed by :class:`DataKey`. + data_keys: Optional key override for this call. + + Returns: + Transformed output(s). A single value is returned for one input, + otherwise a list is returned. + """ + _data_keys = self.preproc_datakeys(data_keys) + + if isinstance(module, K.RandomTransplantation): + # For transforms which require the full input to calculate the parameters (e.g. RandomTransplantation) + param = ParamItem( + name=param.name, + data=module.params_from_input( + *arg, # type: ignore[arg-type] + data_keys=_data_keys, + params=param.data, # type: ignore[arg-type] + extra_args=extra_args, + ), + ) + + outputs = [] + for inp, dcate in zip(arg, _data_keys): + op = self._get_op(dcate) + extra_arg = extra_args.get(dcate, {}) + if dcate.name == "MASK" and isinstance(inp, list): + outputs.append(MaskSequentialOps.transform_list(inp, module, param=param, extra_args=extra_arg)) + else: + outputs.append(op.transform(inp, module, param=param, extra_args=extra_arg)) + if len(outputs) == 1 and isinstance(outputs, (list, tuple)): + return outputs[0] + return outputs + + def inverse( + self, + *arg: DataType, + module: nn.Module, + param: ParamItem, + extra_args: Dict[DataKey, Dict[str, Any]], + data_keys: Optional[Union[List[str], List[int], List[DataKey]]] = None, + ) -> Union[DataType, SequenceDataType]: + """Apply inverse transformation dispatch for one module step. + + Args: + *arg: Inputs to invert. + module: Module used in the forward pass. + param: Parameters captured for ``module`` during forward. + extra_args: Optional runtime overrides keyed by :class:`DataKey`. + data_keys: Optional key override for this call. + + Returns: + Inverse-transformed output(s). A single value is returned for one + input, otherwise a list is returned. + """ + _data_keys = self.preproc_datakeys(data_keys) + outputs = [] + for inp, dcate in zip(arg, _data_keys): + op = self._get_op(dcate) + extra_arg = extra_args[dcate] if dcate in extra_args else {} + outputs.append(op.inverse(inp, module, param=param, extra_args=extra_arg)) + if len(outputs) == 1 and isinstance(outputs, (list, tuple)): + return outputs[0] + return outputs + + +P = ParamSpec("P") + + +def make_input_only_sequential(module: "K.container.ImageSequentialBase") -> Callable[P, torch.Tensor]: + """Disable all other additional inputs (e.g. ) for ImageSequential.""" + + def f(*args: P.args, **kwargs: P.kwargs) -> torch.Tensor: + return module(*args, **kwargs) + + return f + + +def get_geometric_only_param(module: "K.container.ImageSequentialBase", param: List[ParamItem]) -> List[ParamItem]: + """Return geometry param.""" + named_modules = module.get_forward_sequence(param) + + res: List[ParamItem] = [] + for (_, mod), p in zip(named_modules, param): + if isinstance(mod, (K.GeometricAugmentationBase2D, K.GeometricAugmentationBase3D)): + res.append(p) + return res + + +class InputSequentialOps(SequentialOpsInterface[torch.Tensor]): + """Implement the operations for processing input tensors within a sequential container. + + This class provides class methods to apply transformations and manage the + flow of data through the augmentation pipeline. + """ + + @classmethod + def transform( + cls, input: torch.Tensor, module: nn.Module, param: ParamItem, extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Apply one module step to an image tensor. + + Args: + input: Input image tensor. + module: Module to execute. + param: Parameters for ``module``. + extra_args: Optional runtime overrides. + + Returns: + Transformed tensor. + + Raises: + AssertionError: A non-augmentation module receives non-empty params. + """ + if extra_args is None: + extra_args = {} + if isinstance(module, (_AugmentationBase, K.MixAugmentationBaseV2)): + input = module(input, params=cls.get_instance_module_param(param), data_keys=[DataKey.INPUT], **extra_args) + elif isinstance(module, (K.container.ImageSequentialBase,)): + input = module.transform_inputs(input, params=cls.get_sequential_module_param(param), extra_args=extra_args) + elif isinstance(module, (K.auto.operations.OperationBase,)): + input = module(input, params=cls.get_instance_module_param(param)) + else: + if param.data is not None: + raise AssertionError(f"Non-augmentaion operation {param.name} require empty parameters. Got {param}.") + input = module(input) + return input + + @classmethod + def inverse( + cls, input: torch.Tensor, module: nn.Module, param: ParamItem, extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Apply one inverse module step to an image tensor. + + Args: + input: Tensor to invert. + module: Module used in the forward path. + param: Forward parameters for ``module``. + extra_args: Optional runtime overrides. + + Returns: + Inverse-transformed tensor. + + Raises: + NotImplementedError: Inverse for 3D geometric ops is not supported. + """ + if extra_args is None: + extra_args = {} + if isinstance(module, K.GeometricAugmentationBase2D): + input = module.inverse(input, params=cls.get_instance_module_param(param), **extra_args) + elif isinstance(module, (K.GeometricAugmentationBase3D,)): + raise NotImplementedError( + "The support for 3d inverse operations are not yet supported. You are welcome to file a PR in our repo." + ) + elif isinstance(module, (K.auto.operations.OperationBase,)): + return InputSequentialOps.inverse(input, module=module.op, param=param, extra_args=extra_args) + elif isinstance(module, K.ImageSequential) and not module.is_intensity_only(): + input = module.inverse_inputs(input, params=cls.get_sequential_module_param(param), extra_args=extra_args) + elif isinstance(module, K.container.ImageSequentialBase): + input = module.inverse_inputs(input, params=cls.get_sequential_module_param(param), extra_args=extra_args) + return input + + +class ClassSequentialOps(SequentialOpsInterface[torch.Tensor]): + """Apply and inverse transformations for class labels if needed.""" + + @classmethod + def transform( + cls, input: torch.Tensor, module: nn.Module, param: ParamItem, extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Apply class-label handling for one module step. + + Args: + input: Class-label tensor. + module: Module to execute. + param: Parameters for ``module``. + extra_args: Optional runtime overrides. + + Returns: + Class labels after transformation handling. + + Raises: + NotImplementedError: Label-changing mix ops are not supported yet. + """ + if isinstance(module, K.MixAugmentationBaseV2): + raise NotImplementedError( + "The support for class labels for mix augmentations that change the class label is not yet supported." + ) + return input + + @classmethod + def inverse( + cls, input: torch.Tensor, module: nn.Module, param: ParamItem, extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Return class labels unchanged during inverse dispatch. + + Note: + Class labels do not have a geometric inverse in this pipeline. + + Args: + input: Class-label tensor. + module: Module used in forward (unused). + param: Forward parameters (unused). + extra_args: Optional runtime overrides (unused). + + Returns: + Unmodified class labels. + """ + return input + + +class MaskSequentialOps(SequentialOpsInterface[torch.Tensor]): + """Apply and inverse transformations for mask tensors.""" + + @classmethod + def transform( + cls, input: torch.Tensor, module: nn.Module, param: ParamItem, extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Apply a transformation with respect to the parameters. + + Args: + input: the input torch.Tensor. + module: any torch nn.Module but only kornia augmentation modules will count + to apply transformations. + param: the corresponding parameters to the module. + extra_args: Optional dictionary of extra arguments with specific options for different input types. + """ + if extra_args is None: + extra_args = {} + + if isinstance(module, (K.GeometricAugmentationBase2D,)): + input = module.transform_masks( + input, + params=cls.get_instance_module_param(param), + flags=module.flags, + transform=module.transform_matrix, + **extra_args, + ) + + elif isinstance(module, (K.GeometricAugmentationBase3D,)): + raise NotImplementedError( + "The support for 3d mask operations are not yet supported. You are welcome to file a PR in our repo." + ) + + elif isinstance(module, K.RandomTransplantation): + input = module(input, params=cls.get_instance_module_param(param), data_keys=[DataKey.MASK], **extra_args) + + elif isinstance(module, (_AugmentationBase)): + input = module.transform_masks( + input, params=cls.get_instance_module_param(param), flags=module.flags, **extra_args + ) + + elif isinstance(module, K.ImageSequential) and not module.is_intensity_only(): + input = module.transform_masks(input, params=cls.get_sequential_module_param(param), extra_args=extra_args) + + elif isinstance(module, K.container.ImageSequentialBase): + input = module.transform_masks(input, params=cls.get_sequential_module_param(param), extra_args=extra_args) + + elif isinstance(module, (K.auto.operations.OperationBase,)): + input = MaskSequentialOps.transform(input, module=module.op, param=param, extra_args=extra_args) + + return input + + @classmethod + def transform_list( + cls, input: List[torch.Tensor], module: nn.Module, param: ParamItem, extra_args: Optional[Dict[str, Any]] = None + ) -> List[torch.Tensor]: + """Apply a transformation with respect to the parameters. + + Args: + input: list of input tensors. + module: any torch nn.Module but only kornia augmentation modules will count + to apply transformations. + param: the corresponding parameters to the module. + extra_args: Optional dictionary of extra arguments with specific options for different input types. + """ + if extra_args is None: + extra_args = {} + if isinstance(module, (K.GeometricAugmentationBase2D,)): + tfm_input = [] + params = cls.get_instance_module_param(param) + params_i = copy.deepcopy(params) + for i, inp in enumerate(input): + params_i["batch_prob"] = params["batch_prob"][i] + tfm_inp = module.transform_masks( + inp, params=params_i, flags=module.flags, transform=module.transform_matrix, **extra_args + ) + tfm_input.append(tfm_inp) + input = tfm_input + + elif isinstance(module, (K.GeometricAugmentationBase3D,)): + raise NotImplementedError( + "The support for 3d mask operations are not yet supported. You are welcome to file a PR in our repo." + ) + + elif isinstance(module, (_AugmentationBase)): + tfm_input = [] + params = cls.get_instance_module_param(param) + params_i = copy.deepcopy(params) + for i, inp in enumerate(input): + params_i["batch_prob"] = params["batch_prob"][i] + tfm_inp = module.transform_masks(inp, params=params_i, flags=module.flags, **extra_args) + tfm_input.append(tfm_inp) + input = tfm_input + + elif isinstance(module, K.ImageSequential) and not module.is_intensity_only(): + tfm_input = [] + seq_params = cls.get_sequential_module_param(param) + for inp in input: + tfm_inp = module.transform_masks(inp, params=seq_params, extra_args=extra_args) + tfm_input.append(tfm_inp) + input = tfm_input + + elif isinstance(module, K.container.ImageSequentialBase): + tfm_input = [] + seq_params = cls.get_sequential_module_param(param) + for inp in input: + tfm_inp = module.transform_masks(inp, params=seq_params, extra_args=extra_args) + tfm_input.append(tfm_inp) + input = tfm_input + + elif isinstance(module, (K.auto.operations.OperationBase,)): + raise NotImplementedError( + "The support for list of masks under auto operations are not yet supported. You are welcome to file a" + " PR in our repo." + ) + return input + + @classmethod + def inverse( + cls, input: torch.Tensor, module: nn.Module, param: ParamItem, extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Inverse a transformation with respect to the parameters. + + Args: + input: the input torch.Tensor. + module: any torch nn.Module but only kornia augmentation modules will count + to apply transformations. + param: the corresponding parameters to the module. + extra_args: Optional dictionary of extra arguments with specific options for different input types. + """ + if extra_args is None: + extra_args = {} + + if isinstance(module, (K.GeometricAugmentationBase2D,)): + if module.transform_matrix is None: + raise ValueError(f"No valid transformation matrix found in {module.__class__}.") + transform = module.compute_inverse_transformation(module.transform_matrix) + input = module.inverse_masks( + input, + params=cls.get_instance_module_param(param), + flags=module.flags, + transform=transform, + **extra_args, + ) + + elif isinstance(module, (K.GeometricAugmentationBase3D,)): + raise NotImplementedError( + "The support for 3d mask operations are not yet supported. You are welcome to file a PR in our repo." + ) + + elif isinstance(module, K.container.ImageSequentialBase): + input = module.inverse_masks(input, params=cls.get_sequential_module_param(param), extra_args=extra_args) + + elif isinstance(module, (K.auto.operations.OperationBase,)): + input = MaskSequentialOps.inverse(input, module=module.op, param=param, extra_args=extra_args) + + return input + + +class BoxSequentialOps(SequentialOpsInterface[Boxes]): + """Apply and inverse transformations for bounding box tensors. + + This is for transform boxes in the format (B, N, 4, 2). + """ + + @classmethod + def transform( + cls, input: Boxes, module: nn.Module, param: ParamItem, extra_args: Optional[Dict[str, Any]] = None + ) -> Boxes: + """Apply a transformation with respect to the parameters. + + Args: + input: the input torch.Tensor, (B, N, 4, 2) or (B, 4, 2). + module: any torch nn.Module but only kornia augmentation modules will count + to apply transformations. + param: the corresponding parameters to the module. + extra_args: Optional dictionary of extra arguments with specific options for different input types. + """ + if extra_args is None: + extra_args = {} + _input = input.clone() + + if isinstance(module, (K.GeometricAugmentationBase2D,)): + _input = module.transform_boxes( + _input, + cls.get_instance_module_param(param), + module.flags, + transform=module.transform_matrix, + **extra_args, + ) + + elif isinstance(module, (K.GeometricAugmentationBase3D,)): + raise NotImplementedError( + "The support for 3d box operations are not yet supported. You are welcome to file a PR in our repo." + ) + + elif isinstance(module, K.ImageSequential) and not module.is_intensity_only(): + _input = module.transform_boxes( + _input, params=cls.get_sequential_module_param(param), extra_args=extra_args + ) + + elif isinstance(module, K.container.ImageSequentialBase): + _input = module.transform_boxes( + _input, params=cls.get_sequential_module_param(param), extra_args=extra_args + ) + + elif isinstance(module, (K.auto.operations.OperationBase,)): + return BoxSequentialOps.transform(input, module=module.op, param=param, extra_args=extra_args) + + return _input + + @classmethod + def inverse( + cls, input: Boxes, module: nn.Module, param: ParamItem, extra_args: Optional[Dict[str, Any]] = None + ) -> Boxes: + """Inverse a transformation with respect to the parameters. + + Args: + input: the input torch.Tensor. + module: any torch nn.Module but only kornia augmentation modules will count + to apply transformations. + param: the corresponding parameters to the module. + extra_args: Optional dictionary of extra arguments with specific options for different input types. + """ + if extra_args is None: + extra_args = {} + _input = input.clone() + + if isinstance(module, (K.GeometricAugmentationBase2D,)): + if module.transform_matrix is None: + raise ValueError(f"No valid transformation matrix found in {module.__class__}.") + transform = module.compute_inverse_transformation(module.transform_matrix) + _input = module.inverse_boxes( + _input, + param.data, # type: ignore[arg-type] + module.flags, + transform=transform, + **extra_args, + ) + + elif isinstance(module, (K.GeometricAugmentationBase3D,)): + raise NotImplementedError( + "The support for 3d box operations are not yet supported. You are welcome to file a PR in our repo." + ) + + elif isinstance(module, K.ImageSequential) and not module.is_intensity_only(): + _input = module.inverse_boxes(_input, params=cls.get_sequential_module_param(param), extra_args=extra_args) + + elif isinstance(module, K.container.ImageSequentialBase): + _input = module.inverse_boxes(_input, params=cls.get_sequential_module_param(param), extra_args=extra_args) + + elif isinstance(module, (K.auto.operations.OperationBase,)): + return BoxSequentialOps.inverse(input, module=module.op, param=param, extra_args=extra_args) + return _input + + +class KeypointSequentialOps(SequentialOpsInterface[Keypoints]): + """Apply and inverse transformations for keypoints tensors. + + This is for transform keypoints in the format (B, N, 2). + """ + + @classmethod + def transform( + cls, input: Keypoints, module: nn.Module, param: ParamItem, extra_args: Optional[Dict[str, Any]] = None + ) -> Keypoints: + """Apply a transformation with respect to the parameters. + + Args: + input: the input torch.Tensor, (B, N, 4, 2) or (B, 4, 2). + module: any torch nn.Module but only kornia augmentation modules will count + to apply transformations. + param: the corresponding parameters to the module. + extra_args: Optional dictionary of extra arguments with specific options for different input types. + """ + if extra_args is None: + extra_args = {} + _input = input.clone() + + if isinstance(module, (K.GeometricAugmentationBase2D,)): + _input = module.transform_keypoints( + _input, + cls.get_instance_module_param(param), + module.flags, + transform=module.transform_matrix, + **extra_args, + ) + + elif isinstance(module, (K.GeometricAugmentationBase3D,)): + raise NotImplementedError( + "The support for 3d keypoint operations are not yet supported. " + "You are welcome to file a PR in our repo." + ) + + elif isinstance(module, K.ImageSequential) and not module.is_intensity_only(): + _input = module.transform_keypoints( + _input, params=cls.get_sequential_module_param(param), extra_args=extra_args + ) + + elif isinstance(module, K.container.ImageSequentialBase): + _input = module.transform_keypoints( + _input, params=cls.get_sequential_module_param(param), extra_args=extra_args + ) + + elif isinstance(module, (K.auto.operations.OperationBase,)): + return KeypointSequentialOps.transform(input, module=module.op, param=param, extra_args=extra_args) + + return _input + + @classmethod + def inverse( + cls, input: Keypoints, module: nn.Module, param: ParamItem, extra_args: Optional[Dict[str, Any]] = None + ) -> Keypoints: + """Inverse a transformation with respect to the parameters. + + Args: + input: Input keypoints. Coordinates are conceptually stored as + ``(B, N, 2)``, where the last dimension stores ``(x, y)``. + module: any torch nn.Module but only kornia augmentation modules will count + to apply transformations. + param: the corresponding parameters to the module. + extra_args: Optional dictionary of extra arguments with specific options for different input types. + + Returns: + Keypoints after inverse transformation. + """ + if extra_args is None: + extra_args = {} + _input = input.clone() + + if isinstance(module, (K.GeometricAugmentationBase2D,)): + if module.transform_matrix is None: + raise ValueError(f"No valid transformation matrix found in {module.__class__}.") + transform = module.compute_inverse_transformation(module.transform_matrix) + _input = module.inverse_keypoints( + _input, cls.get_instance_module_param(param), module.flags, transform=transform, **extra_args + ) + + elif isinstance(module, (K.GeometricAugmentationBase3D,)): + raise NotImplementedError( + "The support for 3d keypoint operations are not yet supported. " + "You are welcome to file a PR in our repo." + ) + + elif isinstance(module, K.ImageSequential) and not module.is_intensity_only(): + _input = module.inverse_keypoints( + _input, params=cls.get_sequential_module_param(param), extra_args=extra_args + ) + + elif isinstance(module, K.container.ImageSequentialBase): + _input = module.inverse_keypoints( + _input, params=cls.get_sequential_module_param(param), extra_args=extra_args + ) + + elif isinstance(module, (K.auto.operations.OperationBase,)): + return KeypointSequentialOps.inverse(input, module=module.op, param=param, extra_args=extra_args) + + return _input diff --git a/kornia/augmentation/container/params.py b/kornia/augmentation/container/params.py new file mode 100644 index 0000000..b138609 --- /dev/null +++ b/kornia/augmentation/container/params.py @@ -0,0 +1,46 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, List, NamedTuple, Optional, Union + +import torch + +__all__ = ["ParamItem", "PatchParamItem"] + + +class ParamItem(NamedTuple): + """Store the parameters for a single augmentation operation. + + Attributes: + name: The name of the augmentation operation. + data: The dictionary of parameters or a list of nested parameters. + """ + + name: str + data: Optional[Union[Dict[str, torch.Tensor], List["ParamItem"]]] + + +class PatchParamItem(NamedTuple): + """Store parameters for patch-based augmentation operations. + + Attributes: + indices: The list of patch indices where the augmentation is applied. + param: The specific :class:`ParamItem` containing the operation parameters. + """ + + indices: List[int] + param: ParamItem diff --git a/kornia/augmentation/container/patch.py b/kornia/augmentation/container/patch.py new file mode 100644 index 0000000..f8be768 --- /dev/null +++ b/kornia/augmentation/container/patch.py @@ -0,0 +1,565 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from itertools import cycle, islice +from typing import Any, Dict, Iterator, List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +from torch import nn + +import kornia.augmentation as K +from kornia.augmentation.base import _AugmentationBase +from kornia.contrib.extract_patches import extract_tensor_patches +from kornia.geometry.boxes import Boxes +from kornia.geometry.keypoints import Keypoints + +from .base import SequentialBase +from .image import ImageSequential +from .ops import InputSequentialOps +from .params import ParamItem, PatchParamItem + +__all__ = ["PatchSequential"] + + +class PatchSequential(ImageSequential): + r"""Container for performing patch-level image data augmentation. + + .. image:: _static/img/PatchSequential.png + + PatchSequential breaks input images into patches by a given grid size, which will be resembled back + afterwards. + + Different image processing and augmentation methods will be performed on each patch region as + in :cite:`lin2021patch`. + + Args: + *args: a list of processing modules. + grid_size: controls the grid board separation. + padding: same or valid padding. If same padding, it will F.pad to include all pixels if the input + torch.Tensor cannot be divisible by grid_size. If valid padding, the redundant border will be removed. + same_on_batch: apply the same transformation across the batch. + If None, it will not overwrite the function-wise settings. + keepdim: whether to keep the output shape the same as input (True) or broadcast it + to the batch form (False). If None, it will not overwrite the function-wise settings. + patchwise_apply: apply image processing args will be applied patch-wisely. + if ``True``, the number of args must be equal to grid number. + if ``False``, the image processing args will be applied as a sequence to all patches. + random_apply: randomly select a sublist (order agnostic) of args to + apply transformation. + If ``int`` (batchwise mode only), a fixed number of transformations will be selected. + If ``(a,)`` (batchwise mode only), x number of transformations (a <= x <= len(args)) will be selected. + If ``(a, b)`` (batchwise mode only), x number of transformations (a <= x <= b) will be selected. + If ``True``, the whole list of args will be processed in a random order. + If ``False`` and not ``patchwise_apply``, the whole list of args will be processed in original order. + If ``False`` and ``patchwise_apply``, the whole list of args will be processed in original order + location-wisely. + + .. note:: + Transformation matrix returned only considers the transformation applied in ``kornia.augmentation`` module. + Those transformations in ``kornia.geometry`` will not be taken into account. + + .. note:: + See a working example `here `__. + + Examples: + >>> import kornia.augmentation as K + >>> input = torch.randn(2, 3, 224, 224) + >>> seq = PatchSequential( + ... ImageSequential( + ... K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=0.5), + ... K.RandomPerspective(0.2, p=0.5), + ... K.RandomSolarize(0.1, 0.1, p=0.5), + ... ), + ... K.RandomAffine(360, p=1.0), + ... ImageSequential( + ... K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=0.5), + ... K.RandomPerspective(0.2, p=0.5), + ... K.RandomSolarize(0.1, 0.1, p=0.5), + ... ), + ... K.RandomSolarize(0.1, 0.1, p=0.1), + ... grid_size=(2,2), + ... patchwise_apply=True, + ... same_on_batch=True, + ... random_apply=False, + ... ) + >>> out = seq(input) + >>> out.shape + torch.Size([2, 3, 224, 224]) + >>> out1 = seq(input, params=seq._params) + >>> torch.equal(out, out1) + True + + Perform ``OneOf`` transformation with ``random_apply=1`` and ``random_apply_weights`` in ``PatchSequential``. + + >>> import kornia + >>> input = torch.randn(2, 3, 224, 224) + >>> seq = PatchSequential( + ... ImageSequential( + ... K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=0.5), + ... K.RandomPerspective(0.2, p=0.5), + ... K.RandomSolarize(0.1, 0.1, p=0.5), + ... ), + ... K.RandomAffine(360, p=1.0), + ... K.RandomSolarize(0.1, 0.1, p=0.1), + ... grid_size=(2,2), + ... patchwise_apply=False, + ... random_apply=1, + ... random_apply_weights=[0.5, 0.3, 0.8] + ... ) + >>> out = seq(input) + >>> out.shape + torch.Size([2, 3, 224, 224]) + + """ + + def __init__( + self, + *args: nn.Module, + grid_size: Tuple[int, int] = (4, 4), + padding: str = "same", + same_on_batch: Optional[bool] = None, + keepdim: Optional[bool] = None, + patchwise_apply: bool = True, + random_apply: Union[int, bool, Tuple[int, int]] = False, + random_apply_weights: Optional[List[float]] = None, + ) -> None: + _random_apply: Optional[Union[int, Tuple[int, int]]] + + if patchwise_apply and random_apply is True: + # will only apply [1, 4] augmentations per patch + _random_apply = (1, 4) + elif patchwise_apply and random_apply is False: + if len(args) != grid_size[0] * grid_size[1]: + raise ValueError( + "The number of processing modules must be equal with grid size." + f"Got {len(args)} and {grid_size[0] * grid_size[1]}. " + "Please set random_apply = True or patchwise_apply = False." + ) + _random_apply = random_apply + elif patchwise_apply and isinstance(random_apply, (int, tuple)): + raise ValueError(f"Only boolean value allowed when `patchwise_apply` is set to True. Got {random_apply}.") + else: + _random_apply = random_apply + super().__init__( + *args, + same_on_batch=same_on_batch, + keepdim=keepdim, + random_apply=_random_apply, + random_apply_weights=random_apply_weights, + ) + if padding not in ("same", "valid"): + raise ValueError(f"`padding` must be either `same` or `valid`. Got {padding}.") + self.grid_size = grid_size + self.padding = padding + self.patchwise_apply = patchwise_apply + self._params: Optional[List[PatchParamItem]] # type: ignore[assignment] + + def compute_padding( + self, input: torch.Tensor, padding: str, grid_size: Optional[Tuple[int, int]] = None + ) -> Tuple[int, int, int, int]: + """Compute spatial padding needed before patch extraction. + + Args: + input: Input image tensor. + padding: Padding mode, either ``"same"`` or ``"valid"``. + grid_size: Optional grid override. Uses ``self.grid_size`` when omitted. + + Returns: + Padding tuple ``(left, right, top, bottom)``. + + Raises: + NotImplementedError: ``padding`` is neither ``"same"`` nor ``"valid"``. + """ + if grid_size is None: + grid_size = self.grid_size + if padding == "valid": + ph, pw = input.size(-2) // grid_size[0], input.size(-1) // grid_size[1] + return (-pw // 2, pw // 2 - pw, -ph // 2, ph // 2 - ph) + if padding == "same": + ph = input.size(-2) - input.size(-2) // grid_size[0] * grid_size[0] + pw = input.size(-1) - input.size(-1) // grid_size[1] * grid_size[1] + return (pw // 2, pw - pw // 2, ph // 2, ph - ph // 2) + raise NotImplementedError(f"Expect `padding` as either 'valid' or 'same'. Got {padding}.") + + def extract_patches( + self, + input: torch.Tensor, + grid_size: Optional[Tuple[int, int]] = None, + pad: Optional[Tuple[int, int, int, int]] = None, + ) -> torch.Tensor: + """Extract patches from torch.Tensor. + + Example: + >>> import kornia.augmentation as K + >>> pas = PatchSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), patchwise_apply=False) + >>> pas.extract_patches(torch.arange(16).view(1, 1, 4, 4), grid_size=(2, 2)) + tensor([[[[[ 0, 1], + [ 4, 5]]], + + + [[[ 2, 3], + [ 6, 7]]], + + + [[[ 8, 9], + [12, 13]]], + + + [[[10, 11], + [14, 15]]]]]) + >>> pas.extract_patches(torch.arange(54).view(1, 1, 6, 9), grid_size=(2, 2), pad=(-1, -1, -2, -2)) + tensor([[[[[19, 20, 21]]], + + + [[[22, 23, 24]]], + + + [[[28, 29, 30]]], + + + [[[31, 32, 33]]]]]) + + """ + if pad is not None: + input = F.pad(input, list(pad)) + if grid_size is None: + grid_size = self.grid_size + window_size = (input.size(-2) // grid_size[-2], input.size(-1) // grid_size[-1]) + stride = window_size + return extract_tensor_patches(input, window_size, stride) + + def restore_from_patches( + self, + patches: torch.Tensor, + grid_size: Tuple[int, int] = (4, 4), + pad: Optional[Tuple[int, int, int, int]] = None, + ) -> torch.Tensor: + """Restore input from patches. + + Example: + >>> import kornia.augmentation as K + >>> pas = PatchSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), patchwise_apply=False) + >>> out = pas.extract_patches(torch.arange(16).view(1, 1, 4, 4), grid_size=(2, 2)) + >>> pas.restore_from_patches(out, grid_size=(2, 2)) + tensor([[[[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11], + [12, 13, 14, 15]]]]) + + """ + if grid_size is None: + grid_size = self.grid_size + patches_tensor = patches.view(-1, grid_size[0], grid_size[1], *patches.shape[-3:]) + restored_tensor = torch.cat(torch.chunk(patches_tensor, grid_size[0], 1), -2).squeeze(1) + restored_tensor = torch.cat(torch.chunk(restored_tensor, grid_size[1], 1), -1).squeeze(1) + + if pad is not None: + restored_tensor = F.pad(restored_tensor, [-i for i in pad]) + return restored_tensor + + def forward_parameters(self, batch_shape: torch.Size) -> List[PatchParamItem]: # type: ignore[override] + """Generate patch-level parameters for a forward pass. + + Args: + batch_shape: Shape of the extracted patch batch. + + Returns: + List of :class:`PatchParamItem` entries, including patch indices and + operation params. + """ + out_param: List[PatchParamItem] = [] + if not self.patchwise_apply: + params = self.generate_parameters(torch.Size([1, batch_shape[0] * batch_shape[1], *batch_shape[2:]])) + indices = torch.arange(0, batch_shape[0] * batch_shape[1]) + out_param = [PatchParamItem(indices.tolist(), p) for p, _ in params] + # "append" of "list" does not return a value + elif not self.same_on_batch: + params = self.generate_parameters(torch.Size([batch_shape[0] * batch_shape[1], 1, *batch_shape[2:]])) + out_param = [PatchParamItem([i], p) for p, i in params] + # "append" of "list" does not return a value + else: + params = self.generate_parameters(torch.Size([batch_shape[1], batch_shape[0], *batch_shape[2:]])) + indices = torch.arange(0, batch_shape[0] * batch_shape[1], step=batch_shape[1]) + out_param = [PatchParamItem((indices + i).tolist(), p) for p, i in params] + # "append" of "list" does not return a value + return out_param + + def generate_parameters(self, batch_shape: torch.Size) -> Iterator[Tuple[ParamItem, int]]: + """Get multiple forward sequence but maximumly one mix augmentation in between. + + Args: + batch_shape: 5-dim shape arranged as :math:``(N, B, C, H, W)``, in which N represents + the number of sequence. + + """ + if not self.same_on_batch and self.random_apply: + # diff_on_batch and random_apply => patch-wise augmentation + with_mix = False + for i in range(batch_shape[0]): + seq, mix_added = self.get_random_forward_sequence(with_mix=with_mix) + with_mix = mix_added + for s in seq: + if isinstance(s[1], (_AugmentationBase, SequentialBase, K.MixAugmentationBaseV2)): + yield ParamItem(s[0], s[1].forward_parameters(torch.Size(batch_shape[1:]))), i + else: + yield ParamItem(s[0], None), i + elif not self.same_on_batch and not self.random_apply: + for i, nchild in enumerate(self.named_children()): + if isinstance(nchild[1], (_AugmentationBase, SequentialBase, K.MixAugmentationBaseV2)): + yield ParamItem(nchild[0], nchild[1].forward_parameters(torch.Size(batch_shape[1:]))), i + else: + yield ParamItem(nchild[0], None), i + elif not self.random_apply: + # same_on_batch + not random_apply => location-wise augmentation + for i, nchild in enumerate(islice(cycle(self.named_children()), batch_shape[0])): + if isinstance(nchild[1], (_AugmentationBase, SequentialBase, K.MixAugmentationBaseV2)): + yield ParamItem(nchild[0], nchild[1].forward_parameters(torch.Size(batch_shape[1:]))), i + else: + yield ParamItem(nchild[0], None), i + else: + # same_on_batch + random_apply => location-wise augmentation + with_mix = False + for i in range(batch_shape[0]): + seq, mix_added = self.get_random_forward_sequence(with_mix=with_mix) + with_mix = mix_added + for s in seq: + if isinstance(s[1], (_AugmentationBase, SequentialBase, K.MixAugmentationBaseV2)): + yield ParamItem(s[0], s[1].forward_parameters(torch.Size(batch_shape[1:]))), i + else: + yield ParamItem(s[0], None), i + + def forward_by_params(self, input: torch.Tensor, params: List[PatchParamItem]) -> torch.Tensor: + """Apply module parameters to selected patch indices. + + Args: + input: Tensor of extracted patches. + params: Patch operation payloads with ``indices`` and per-module params. + + Returns: + Augmented patch tensor. + """ + in_shape = input.shape + input = input.reshape(-1, *in_shape[-3:]) + + for patch_param in params: + # input, out_param = self.apply_by_param(input, params=patch_param) + module = self.get_submodule(patch_param.param.name) + _input = input[patch_param.indices] + output = InputSequentialOps.transform(_input, module, patch_param.param, extra_args={}) + input[patch_param.indices] = output + + return input.reshape(in_shape) + + def transform_inputs( # type: ignore[override] + self, input: torch.Tensor, params: List[PatchParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Apply patch-wise augmentation to an input tensor. + + Args: + input: Input image tensor. + params: Patch-level parameters from :meth:`forward_parameters`. + extra_args: Optional runtime overrides (unused in this implementation). + + Returns: + Augmented image tensor after extract-transform-restore. + """ + pad = self.compute_padding(input, self.padding) + input = self.extract_patches(input, self.grid_size, pad) + input = self.forward_by_params(input, params) + input = self.restore_from_patches(input, self.grid_size, pad=pad) + + return input + + def inverse_inputs( # type: ignore[override] + self, input: torch.Tensor, params: List[PatchParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Inverse the patch pipeline when supported. + + Args: + input: Augmented image tensor. + params: Parameters used during forward. + extra_args: Optional runtime overrides. + + Returns: + Original tensor for intensity-only pipelines. + + Raises: + NotImplementedError: Geometric patch pipelines cannot be inverted. + """ + if self.is_intensity_only(): + return input + + raise NotImplementedError("PatchSequential inverse cannot be used with geometric transformations.") + + def transform_masks( # type: ignore[override] + self, input: torch.Tensor, params: List[PatchParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Transform masks for intensity-only patch pipelines. + + Args: + input: Input mask tensor. + params: Parameters used during forward. + extra_args: Optional runtime overrides. + + Returns: + Unchanged mask tensor when the pipeline is intensity-only. + + Raises: + NotImplementedError: Geometric patch pipelines are unsupported for masks. + """ + if self.is_intensity_only(): + return input + + raise NotImplementedError("PatchSequential for boxes cannot be used with geometric transformations.") + + def inverse_masks( # type: ignore[override] + self, input: torch.Tensor, params: List[PatchParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Inverse mask transforms for supported patch pipelines. + + Args: + input: Augmented mask tensor. + params: Parameters used during forward. + extra_args: Optional runtime overrides. + + Returns: + Original mask tensor for intensity-only pipelines. + + Raises: + NotImplementedError: Geometric patch pipelines cannot be inverted. + """ + if self.is_intensity_only(): + return input + + raise NotImplementedError("PatchSequential inverse cannot be used with geometric transformations.") + + def transform_boxes( # type: ignore[override] + self, input: Boxes, params: List[PatchParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> Boxes: + """Transform boxes for intensity-only patch pipelines. + + Args: + input: Input boxes. + params: Parameters used during forward. + extra_args: Optional runtime overrides. + + Returns: + Unchanged boxes when the pipeline is intensity-only. + + Raises: + NotImplementedError: Geometric patch pipelines are unsupported for boxes. + """ + if self.is_intensity_only(): + return input + + raise NotImplementedError("PatchSequential for boxes cannot be used with geometric transformations.") + + def inverse_boxes( # type: ignore[override] + self, input: Boxes, params: List[PatchParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> Boxes: + """Inverse box transforms for supported patch pipelines. + + Args: + input: Augmented boxes. + params: Parameters used during forward. + extra_args: Optional runtime overrides. + + Returns: + Original boxes for intensity-only pipelines. + + Raises: + NotImplementedError: Geometric patch pipelines cannot be inverted. + """ + if self.is_intensity_only(): + return input + + raise NotImplementedError("PatchSequential inverse cannot be used with geometric transformations.") + + def transform_keypoints( # type: ignore[override] + self, input: Keypoints, params: List[PatchParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> Keypoints: + """Transform keypoints for intensity-only patch pipelines. + + Args: + input: Input keypoints. + params: Parameters used during forward. + extra_args: Optional runtime overrides. + + Returns: + Unchanged keypoints when the pipeline is intensity-only. + + Raises: + NotImplementedError: Geometric patch pipelines are unsupported for keypoints. + """ + if self.is_intensity_only(): + return input + + raise NotImplementedError("PatchSequential for keypoints cannot be used with geometric transformations.") + + def inverse_keypoints( # type: ignore[override] + self, input: Keypoints, params: List[PatchParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> Keypoints: + """Inverse keypoint transforms for supported patch pipelines. + + Args: + input: Augmented keypoints (conceptually ``(B, N, 2)`` coordinates, + where the last dimension stores ``(x, y)``). + params: Parameters used during forward. + extra_args: Optional runtime overrides. + + Returns: + Original keypoints for intensity-only pipelines. + + Raises: + NotImplementedError: Geometric patch pipelines cannot be inverted. + """ + if self.is_intensity_only(): + return input + + raise NotImplementedError("PatchSequential inverse cannot be used with geometric transformations.") + + def inverse( # type: ignore[override] + self, + input: torch.Tensor, + params: Optional[List[PatchParamItem]] = None, + extra_args: Optional[Dict[str, Any]] = None, + ) -> torch.Tensor: + """Inverse transformation. + + Used to inverse a torch.Tensor according to the performed transformation by a forward pass, or with respect to + provided parameters. + """ + if self.is_intensity_only(): + return input + + raise NotImplementedError("PatchSequential inverse cannot be used with geometric transformations.") + + def forward(self, input: torch.Tensor, params: Optional[List[PatchParamItem]] = None) -> torch.Tensor: # type: ignore[override] + """Input transformation will be returned if input is a tuple.""" + # BCHW -> B(patch)CHW + if isinstance(input, (tuple,)): + raise ValueError("tuple input is not currently supported.") + + if params is None: + params = self.forward_parameters(input.shape) + + output = self.transform_inputs(input, params=params) + + self._params = params + + return output diff --git a/kornia/augmentation/container/video.py b/kornia/augmentation/container/video.py new file mode 100644 index 0000000..8baadc4 --- /dev/null +++ b/kornia/augmentation/container/video.py @@ -0,0 +1,452 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, List, Optional, Tuple, Union, cast + +import torch +from torch import nn + +import kornia.augmentation as K +from kornia.augmentation.base import _AugmentationBase +from kornia.augmentation.container.base import SequentialBase +from kornia.augmentation.container.image import ImageSequential, _get_new_batch_shape +from kornia.geometry.boxes import Boxes +from kornia.geometry.keypoints import Keypoints + +from .params import ParamItem + +__all__ = ["VideoSequential"] + + +class VideoSequential(ImageSequential): + r"""VideoSequential for processing 5-dim video data like (B, T, C, H, W) and (B, C, T, H, W). + + `VideoSequential` is used to replace `nn.Sequential` for processing video data augmentations. + By default, `VideoSequential` enabled `same_on_frame` to make sure the same augmentations happen + across temporal dimension. Meanwhile, it will not affect other augmentation behaviours like the + settings on `same_on_batch`, etc. + + Args: + *args: a list of augmentation module. + data_format: only BCTHW and BTCHW are supported. + same_on_frame: apply the same transformation across the channel per frame. + random_apply: randomly select a sublist (order agnostic) of args to + apply transformation. + If int, a fixed number of transformations will be selected. + If (a,), x number of transformations (a <= x <= len(args)) will be selected. + If (a, b), x number of transformations (a <= x <= b) will be selected. + If None, the whole list of args will be processed as a sequence. + + Note: + Transformation matrix returned only considers the transformation applied in ``kornia.augmentation`` module. + Those transformations in ``kornia.geometry`` will not be taken into account. + + Example: + If set `same_on_frame` to True, we would expect the same augmentation has been applied to each + timeframe. + + >>> import kornia + >>> input = torch.randn(2, 3, 1, 5, 6).repeat(1, 1, 4, 1, 1) + >>> aug_list = VideoSequential( + ... kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + ... kornia.color.BgrToRgb(), + ... kornia.augmentation.RandomAffine(360, p=1.0), + ... random_apply=10, + ... data_format="BCTHW", + ... same_on_frame=True) + >>> output = aug_list(input) + >>> (output[0, :, 0] == output[0, :, 1]).all() + tensor(True) + >>> (output[0, :, 1] == output[0, :, 2]).all() + tensor(True) + >>> (output[0, :, 2] == output[0, :, 3]).all() + tensor(True) + + If set `same_on_frame` to False: + + >>> aug_list = VideoSequential( + ... kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + ... kornia.augmentation.RandomAffine(360, p=1.0), + ... kornia.augmentation.RandomMixUpV2(p=1.0), + ... data_format="BCTHW", + ... same_on_frame=False) + >>> output = aug_list(input) + >>> output.shape + torch.Size([2, 3, 4, 5, 6]) + >>> (output[0, :, 0] == output[0, :, 1]).all() + tensor(False) + + Reproduce with provided params. + >>> out2 = aug_list(input, params=aug_list._params) + >>> torch.equal(output, out2) + True + + Perform ``OneOf`` transformation with ``random_apply=1`` and ``random_apply_weights`` in ``VideoSequential``. + + >>> import kornia + >>> input, label = torch.randn(2, 3, 1, 5, 6).repeat(1, 1, 4, 1, 1), torch.tensor([0, 1]) + >>> aug_list = VideoSequential( + ... kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + ... kornia.augmentation.RandomAffine(360, p=1.0), + ... kornia.augmentation.RandomMixUpV2(p=1.0), + ... data_format="BCTHW", + ... same_on_frame=False, + ... random_apply=1, + ... random_apply_weights=[0.5, 0.3, 0.8] + ... ) + >>> out = aug_list(input) + >>> out.shape + torch.Size([2, 3, 4, 5, 6]) + + """ + + # TODO: implement transform_matrix + + def __init__( + self, + *args: nn.Module, + data_format: str = "BTCHW", + same_on_frame: bool = True, + random_apply: Union[int, bool, Tuple[int, int]] = False, + random_apply_weights: Optional[List[float]] = None, + ) -> None: + super().__init__( + *args, + same_on_batch=None, + keepdim=None, + random_apply=random_apply, + random_apply_weights=random_apply_weights, + ) + self.same_on_frame = same_on_frame + self.data_format = data_format.upper() + if self.data_format not in ["BCTHW", "BTCHW"]: + raise AssertionError(f"Only `BCTHW` and `BTCHW` are supported. Got `{data_format}`.") + self._temporal_channel: int + if self.data_format == "BCTHW": + self._temporal_channel = 2 + elif self.data_format == "BTCHW": + self._temporal_channel = 1 + + def __infer_channel_exclusive_batch_shape__(self, batch_shape: torch.Size, chennel_index: int) -> torch.Size: + # Fix mypy complains: error: Incompatible return value type (got "Tuple[int, ...]", expected "Size") + return cast(torch.Size, batch_shape[:chennel_index] + batch_shape[chennel_index + 1 :]) + + def __repeat_param_across_channels__(self, param: torch.Tensor, frame_num: int) -> torch.Tensor: + """Repeat parameters across channels. + + The input is shaped as (B, ...), while to output (B * same_on_frame, ...), which + to guarantee that the same transformation would happen for each frame. + + (B1, B2, ..., Bn) => (B1, ... B1, B2, ..., B2, ..., Bn, ..., Bn) + | ch_size | | ch_size | ..., | ch_size | + """ + repeated = param[:, None, ...].repeat(1, frame_num, *([1] * len(param.shape[1:]))) + return repeated.reshape(-1, *list(param.shape[1:])) + + def __broadcast_param__( + self, v: torch.Tensor, batch_shape: torch.Size, frame_num: int, same_on_frame: bool, same_on_batch: bool + ) -> torch.Tensor: + if not v.numel(): + return v + + if same_on_frame and same_on_batch: + return v.repeat(batch_shape[0] * frame_num, *([1] * (v.ndim - 1))) + elif same_on_frame: + return self.__repeat_param_across_channels__(v, frame_num) + elif same_on_batch: + return v.unsqueeze(1).repeat(1, batch_shape[0], *([1] * (v.ndim - 1))).reshape(-1, *v.shape[1:]) + return v + + def _input_shape_convert_in(self, input: torch.Tensor, frame_num: int) -> torch.Tensor: + # Convert any shape to (B, T, C, H, W) + if self.data_format == "BCTHW": + # Convert (B, C, T, H, W) to (B, T, C, H, W) + input = input.transpose(1, 2) + if self.data_format == "BTCHW": + pass + + input = input.reshape(-1, *input.shape[2:]) + return input + + def _input_shape_convert_back(self, input: torch.Tensor, frame_num: int) -> torch.Tensor: + input = input.view(-1, frame_num, *input.shape[1:]) + if self.data_format == "BCTHW": + input = input.transpose(1, 2) + if self.data_format == "BTCHW": + pass + + return input + + def forward_parameters(self, batch_shape: torch.Size) -> List[ParamItem]: + """Generate per-module parameters for video inputs. + + Args: + batch_shape: Video batch shape, typically 5D. + + Returns: + Parameter list aligned with execution order, with values broadcast + across frames depending on ``same_on_frame`` and module settings. + """ + frame_num = batch_shape[self._temporal_channel] + named_modules = self.get_forward_sequence() + # Got param generation shape to (B, C, H, W). Ignoring T. + batch_shape = self.__infer_channel_exclusive_batch_shape__(batch_shape, self._temporal_channel) + + params = [] + for name, module in named_modules: + if isinstance(module, (K.RandomCrop, _AugmentationBase, K.MixAugmentationBaseV2)): + is_same_on_batch = getattr(module, "same_on_batch", False) + + if self.same_on_frame and is_same_on_batch: + mod_shape = torch.Size([1, *batch_shape[1:]]) + elif self.same_on_frame: + mod_shape = batch_shape + elif is_same_on_batch: + mod_shape = torch.Size([frame_num, *batch_shape[1:]]) + else: + mod_shape = torch.Size([batch_shape[0] * frame_num, *batch_shape[1:]]) + + mod_param = module.forward_parameters(mod_shape) + + if isinstance(mod_param, dict): + for k, v in mod_param.items(): + # TODO: revise ColorJiggle and ColorJitter order param in the future to align the standard. + if k == "order" and isinstance(module, (K.ColorJiggle, K.ColorJitter)): + continue + if k == "forward_input_shape": + mod_param.update({k: v}) + continue + mod_param[k] = self.__broadcast_param__( + v, batch_shape, frame_num, self.same_on_frame, is_same_on_batch + ) + + param = ParamItem(name, mod_param) + + elif isinstance(module, (SequentialBase,)): + seq_param = module.forward_parameters(batch_shape) + if self.same_on_frame: + raise ValueError("nn.Sequential is currently unsupported for ``same_on_frame``.") + param = ParamItem(name, seq_param) + + else: + param = ParamItem(name, None) + + batch_shape = _get_new_batch_shape(param, batch_shape) + params.append(param) + + return params + + def transform_inputs( + self, input: torch.Tensor, params: List[ParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Apply sequence transforms to video tensors. + + Args: + input: Input video tensor. + params: Parameters aligned with the execution sequence. + extra_args: Optional per-input-type overrides. + + Returns: + Transformed video tensor in the original video layout. + """ + frame_num: int = input.size(self._temporal_channel) + input = self._input_shape_convert_in(input, frame_num) + + input = super().transform_inputs(input, params, extra_args=extra_args) + + input = self._input_shape_convert_back(input, frame_num) + return input + + def inverse_inputs( + self, input: torch.Tensor, params: List[ParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Apply inverse sequence transforms to video tensors. + + Args: + input: Augmented video tensor. + params: Parameters used during forward execution. + extra_args: Optional per-input-type overrides. + + Returns: + Inverse-transformed video tensor in the original layout. + """ + frame_num: int = input.size(self._temporal_channel) + input = self._input_shape_convert_in(input, frame_num) + + input = super().inverse_inputs(input, params, extra_args=extra_args) + + input = self._input_shape_convert_back(input, frame_num) + return input + + def transform_masks( + self, input: torch.Tensor, params: List[ParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Apply sequence transforms to video mask tensors. + + Args: + input: Input video mask tensor. + params: Parameters aligned with the execution sequence. + extra_args: Optional per-input-type overrides. + + Returns: + Transformed mask tensor in the original video layout. + """ + frame_num: int = input.size(self._temporal_channel) + input = self._input_shape_convert_in(input, frame_num) + + input = super().transform_masks(input, params, extra_args=extra_args) + + input = self._input_shape_convert_back(input, frame_num) + return input + + def inverse_masks( + self, input: torch.Tensor, params: List[ParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Apply inverse sequence transforms to video masks. + + Args: + input: Augmented video mask tensor. + params: Parameters used during forward execution. + extra_args: Optional per-input-type overrides. + + Returns: + Inverse-transformed mask tensor in the original layout. + """ + frame_num: int = input.size(self._temporal_channel) + input = self._input_shape_convert_in(input, frame_num) + + input = super().inverse_masks(input, params, extra_args=extra_args) + + input = self._input_shape_convert_back(input, frame_num) + return input + + def transform_boxes( # type: ignore[override] + self, input: Union[torch.Tensor, Boxes], params: List[ParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> Union[torch.Tensor, Boxes]: + """Transform bounding boxes. + + Args: + input: torch.Tensor with shape :math:`(B, T, N, 4, 2)`. + If input is a `Keypoints` type, the internal shape is :math:`(B * T, N, 4, 2)`. + params: params for the sequence. + extra_args: Optional dictionary of extra arguments with specific options for different input types. + """ + if isinstance(input, torch.Tensor): + batchsize, frame_num = input.size(0), input.size(1) + input = Boxes.from_tensor(input.view(-1, input.size(2), input.size(3), input.size(4)), mode="vertices_plus") + input = super().transform_boxes(input, params, extra_args=extra_args) + input = input.data.view(batchsize, frame_num, -1, 4, 2) + else: + input = super().transform_boxes(input, params, extra_args=extra_args) + return input + + def inverse_boxes( # type: ignore[override] + self, input: Union[torch.Tensor, Boxes], params: List[ParamItem], extra_args: Optional[Dict[str, Any]] = None + ) -> Union[torch.Tensor, Boxes]: + """Transform bounding boxes. + + Args: + input: torch.Tensor with shape :math:`(B, T, N, 4, 2)`. + If input is a `Keypoints` type, the internal shape is :math:`(B * T, N, 4, 2)`. + params: params for the sequence. + extra_args: Optional dictionary of extra arguments with specific options for different input types. + """ + if isinstance(input, torch.Tensor): + batchsize, frame_num = input.size(0), input.size(1) + input = Boxes.from_tensor(input.view(-1, input.size(2), input.size(3), input.size(4)), mode="vertices_plus") + input = super().inverse_boxes(input, params, extra_args=extra_args) + input = input.data.view(batchsize, frame_num, -1, 4, 2) + else: + input = super().inverse_boxes(input, params, extra_args=extra_args) + return input + + def transform_keypoints( # type: ignore[override] + self, + input: Union[torch.Tensor, Keypoints], + params: List[ParamItem], + extra_args: Optional[Dict[str, Any]] = None, + ) -> Union[torch.Tensor, Keypoints]: + """Transform bounding boxes. + + Args: + input: torch.Tensor with shape :math:`(B, T, N, 2)`. + If input is a `Keypoints` type, the internal shape is :math:`(B * T, N, 2)`. + params: params for the sequence. + extra_args: Optional dictionary of extra arguments with specific options for different input types. + """ + if isinstance(input, torch.Tensor): + batchsize, frame_num = input.size(0), input.size(1) + input = Keypoints(input.view(-1, input.size(2), input.size(3))) + input = super().transform_keypoints(input, params, extra_args=extra_args) + input = input.data.view(batchsize, frame_num, -1, 2) + else: + input = super().transform_keypoints(input, params, extra_args=extra_args) + return input + + def inverse_keypoints( # type: ignore[override] + self, + input: Union[torch.Tensor, Keypoints], + params: List[ParamItem], + extra_args: Optional[Dict[str, Any]] = None, + ) -> Union[torch.Tensor, Keypoints]: + """Transform bounding boxes. + + Args: + input: torch.Tensor with shape :math:`(B, T, N, 2)`. + If input is a `Keypoints` type, the internal shape is :math:`(B * T, N, 2)`. + params: params for the sequence. + extra_args: Optional dictionary of extra arguments with specific options for different input types. + """ + if isinstance(input, torch.Tensor): + frame_num, batchsize = input.size(0), input.size(1) + input = Keypoints(input.view(-1, input.size(2), input.size(3))) + input = super().inverse_keypoints(input, params, extra_args=extra_args) + input = input.data.view(batchsize, frame_num, -1, 2) + else: + input = super().inverse_keypoints(input, params, extra_args=extra_args) + return input + + def inverse( + self, input: torch.Tensor, params: Optional[List[ParamItem]] = None, extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Inverse transformation. + + Used to inverse a torch.Tensor according to the performed transformation by a forward pass, or with respect to + provided parameters. + """ + if params is None: + if self._params is not None: + params = self._params + else: + raise RuntimeError("No valid params to inverse the transformation.") + + return self.inverse_inputs(input, params, extra_args=extra_args) + + def forward( + self, input: torch.Tensor, params: Optional[List[ParamItem]] = None, extra_args: Optional[Dict[str, Any]] = None + ) -> torch.Tensor: + """Define the video computation performed.""" + if len(input.shape) != 5: + raise AssertionError(f"Input must be a 5-dim torch.Tensor. Got {input.shape}.") + + if params is None: + self._params = self.forward_parameters(input.shape) + params = self._params + + output = self.transform_inputs(input, params, extra_args=extra_args) + + return output diff --git a/kornia/augmentation/presets/__init__.py b/kornia/augmentation/presets/__init__.py new file mode 100644 index 0000000..58ddd5d --- /dev/null +++ b/kornia/augmentation/presets/__init__.py @@ -0,0 +1,21 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Augmentation Presets — Predefined augmentation pipelines and configurations. + +This subpackage provides ready-to-use augmentation presets for common tasks. +""" diff --git a/kornia/augmentation/presets/ada.py b/kornia/augmentation/presets/ada.py new file mode 100644 index 0000000..8aa73d0 --- /dev/null +++ b/kornia/augmentation/presets/ada.py @@ -0,0 +1,306 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import warnings +from typing import Any, Dict, List, Optional, Tuple, Union, cast + +import torch + +from .. import ( # noqa: TID252 + AugmentationSequential, + ColorJitter, + ImageSequential, + RandomAffine, + RandomErasing, + RandomGaussianNoise, + RandomHorizontalFlip, + RandomRotation90, +) +from ..base import _AugmentationBase # noqa: TID252 +from ..container.params import ParamItem # noqa: TID252 + +_data_keys_type = List[str] +_inputs_type = Union[torch.Tensor, Dict[str, torch.Tensor]] + + +class AdaptiveDiscriminatorAugmentation(AugmentationSequential): + r"""Implementation of Adaptive Discriminator Augmentation for GANs training as introduced in :cite:`Karras2020ada`. + + adjust a global probability p over all augmentations list to select a subset of images to augment + based on an exponential moving average of the Discriminator's accuracy labeling real samples. + + + Args: + *args: a list of kornia augmentation modules, set to a default list if not specified. + + initial_p: initial global probability `p` for applying the augmentations + + adjustment_speed: float + step size for updating the global probability `p` + + max_p: maximum allowed value for `p` + + target_real_acc: target Discriminator accuracy to guide `p` adjustments + + + ema_lambda: EMA smoothing factor. The real accuracy EMA is what's used to determine the `p` update + + update_every: `p` update frequency (in steps) + + erasing_scale: scale range used for `RandomErasing` if default augmentations are used + + erasing_ratio: aspect ratio range used for `RandomErasing` if default augmentations are used + + erasing_fill_value: fill value used in `RandomErasing` + + same_on_batch: apply the same transformation across the batch + + data_keys: input types to apply augmentations on + + + **kwargs: Additional keyword arguments passed to `AugmentationSequential` + + + Examples: + >>> from kornia.augmentation.presets.ada import AdaptiveDiscriminatorAugmentation + >>> original = torch.randn(2, 3, 16, 16) + >>> ada = AdaptiveDiscriminatorAugmentation() + >>> augmented = ada(original) + + This example demonstrates using default augmentations with AdaptiveDiscriminatorAugmentation in a GAN training loop. + + + >>> import kornia.augmentation as K + >>> from kornia.augmentation.presets.ada import AdaptiveDiscriminatorAugmentation + >>> originals = torch.randn(2, 3, 5, 6) + >>> aug_list = [ + ... K.RandomRotation90(times=(0, 3), p=1), + ... K.RandomAffine(degrees=10, translate=(.1, .1), scale=(.9, 1.1), p=1), + ... K.ColorJitter(brightness=.2, contrast=.2, saturation=.2, hue=.1, p=1), + ... ] + + >>> ada = AdaptiveDiscriminatorAugmentation(*aug_list) + >>> augmented = ada(original) + + This example demonstrates using custom augmentations with AdaptiveDiscriminatorAugmentation. + """ + + def __init__( + self, + *args: Union[_AugmentationBase, ImageSequential], + initial_p: float = 1e-5, + adjustment_speed: float = 1e-2, + max_p: float = 0.8, + target_real_acc: float = 0.85, + ema_lambda: float = 0.99, + update_every: int = 5, + erasing_scale: Union[torch.Tensor, Tuple[float, float]] = (0.02, 0.33), + erasing_ratio: Union[torch.Tensor, Tuple[float, float]] = (0.3, 3.3), + erasing_fill_value: float = 0.0, + data_keys: Optional[_data_keys_type] = None, + same_on_batch: Optional[bool] = False, + **kwargs: Any, + ) -> None: + if not args: + args = self.default_ada_transfroms(erasing_scale, erasing_ratio, erasing_fill_value) + + super().__init__( + *args, + data_keys=data_keys + if data_keys is not None + else [ + "input", + ], + same_on_batch=same_on_batch, + **kwargs, + ) + + if adjustment_speed <= 0: + raise ValueError(f"Invalid `adjustment_speed` ({adjustment_speed}) — must be greater than 0") + + if not 0 <= target_real_acc <= 1: + raise ValueError(f"Invalid `target_real_acc` ({target_real_acc}) — must be in [0, 1]") + + if not 0 <= ema_lambda <= 1: + raise ValueError(f"Invalid `ema_lambda` ({ema_lambda}) — must be in [0, 1]") + + if update_every < 1: + raise ValueError(f"Invalid `update_every` ({update_every}) — must be at least 1") + + if not 0 <= max_p <= 1: + raise ValueError(f"Invalid `max_p` ({max_p}) — must be in [0, 1]") + + if not 0 <= initial_p <= 1: + raise ValueError(f"Invalid `initial_p` ({initial_p}) — must be in [0, 1]") + + if initial_p > max_p: + warnings.warn( + f"`initial_p` ({initial_p}) is greater than `max_p` ({max_p}), resetting `initial_p` to `max_p`", + stacklevel=2, + ) + initial_p = max_p + + self.p = initial_p + self.adjustment_speed = adjustment_speed + self.max_p = max_p + self.target_real_acc = target_real_acc + self.ema_lambda = ema_lambda + self.update_every = update_every + self.real_acc_ema: float = 0.5 + self._num_calls = 0 # -update_every # to avoid updating in the first `update_every` steps + + def default_ada_transfroms( + self, + scale: Union[torch.Tensor, Tuple[float, float]], + ratio: Union[torch.Tensor, Tuple[float, float]], + value: float, + ) -> Tuple[Union[_AugmentationBase, ImageSequential], ...]: + """Return the default augmentation stack used by ADA. + + Args: + scale: Erase-area range passed to :class:`RandomErasing`. + ratio: Erase-aspect-ratio range passed to :class:`RandomErasing`. + value: Fill value used by :class:`RandomErasing`. + + Returns: + Tuple of augmentation modules applied inside ADA. + """ + # if changed in the future, please change the expected transforms list in test_presets.py + return ( + RandomHorizontalFlip(p=1), + RandomRotation90(times=(0, 3), p=1.0), + RandomErasing(scale=scale, ratio=ratio, value=value, p=0.9), + RandomAffine(degrees=10, translate=(0.1, 0.1), scale=(0.9, 1.1), p=1.0), + ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1, p=1.0), + RandomGaussianNoise(std=0.1, p=1.0), + ) + + def update(self, real_acc: float) -> None: + r"""Update internal params `p` once every `update_every` calls based on discriminator accuracy. + + the update is based on an exponential moving average of `real_acc` + `p` is updated by adding or subtracting `adjustment_speed` from it and clamp it at [0, `max_p`] + + Args: + real_acc: the Discriminator's accuracy labeling real samples. + """ + self._num_calls += 1 + + if self._num_calls < self.update_every: + return + self._num_calls = 0 + + self.real_acc_ema = self.ema_lambda * self.real_acc_ema + (1 - self.ema_lambda) * real_acc + + if self.real_acc_ema < self.target_real_acc: + self.p = max(0, self.p - self.adjustment_speed) + else: + self.p = min(self.p + self.adjustment_speed, self.max_p) + + def _get_inputs_metadata( + self, inputs: _inputs_type, data_keys: _data_keys_type + ) -> Tuple[int, Optional[Union[str, torch.device]]]: + if isinstance(inputs, dict): + key = data_keys[0] + batch_size = inputs[key].size(0) + device = inputs[key].device + else: + batch_size = inputs.size(0) + device = inputs.device + + return batch_size, device + + def _sample_inputs(self, inputs: _inputs_type, data_keys: _data_keys_type, p_tensor: torch.Tensor) -> _inputs_type: + if isinstance(inputs, dict): + return {key: inputs[key][p_tensor] for key in data_keys} + else: + return inputs[p_tensor] + + def _merge_inputs( + self, + original: _inputs_type, + augmented: _inputs_type, + p_tensor: torch.Tensor, + ) -> _inputs_type: + merged: _inputs_type + if isinstance(original, dict) and isinstance(augmented, dict): + merged = {} + for key in original.keys(): + merged_tensor = original[key].clone() + merged_tensor[p_tensor] = augmented[key] + merged[key] = merged_tensor + elif isinstance(original, torch.Tensor) and isinstance(augmented, torch.Tensor): + merged = original.clone() + merged[p_tensor] = augmented + else: + raise TypeError( + f"original inputs and augmented inputs aren't of the same type " + f"(type({type(original)}), type({type(augmented)}))" + ) + return merged + + def forward( # type: ignore[override] + self, + inputs: _inputs_type, + params: Optional[List[ParamItem]] = None, + data_keys: Optional[_data_keys_type] = None, + real_acc: Optional[float] = None, + ) -> _inputs_type: + r"""Apply augmentations to a subset of input tensors with global probability `p`. + + This method applies the augmentation pipeline to a subset of input samples, randomly selected + via a Bernoulli distribution with probability `p` + + if `real_acc` is provided, the internal probability `p` is updated via the `update` method. + Non-augmented samples retain their original values, and the output matches the input structure. + + `real_acc` is the Discriminator's accuracy on real images; for example, + `(real_logits > 0).float().mean().item()` if using logits andn assuming real labels are positive. + """ + if real_acc is not None: + self.update(real_acc) + + if self.p == 0: + return inputs + + if data_keys is None: + data_keys = ( + [k.name for k in self.data_keys] + if self.data_keys is not None + else [ + "input", + ] + ) + + batch_size, device = self._get_inputs_metadata(inputs, data_keys=data_keys) + + p_tensor = torch.bernoulli(torch.full((batch_size,), self.p, dtype=torch.float32, device=device)).bool() + + if not p_tensor.any(): + return inputs + + selected_inputs: _inputs_type = self._sample_inputs(inputs, data_keys=data_keys, p_tensor=p_tensor) + augmented_inputs = cast( + _inputs_type, + super().forward( + selected_inputs, # type: ignore[arg-type] + params=params, + data_keys=data_keys, + ), + ) + + return self._merge_inputs(inputs, augmented_inputs, p_tensor) diff --git a/kornia/augmentation/random_generator/_2d/__init__.py b/kornia/augmentation/random_generator/_2d/__init__.py new file mode 100644 index 0000000..b43eda0 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/__init__.py @@ -0,0 +1,45 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from kornia.augmentation.random_generator._2d.affine import * +from kornia.augmentation.random_generator._2d.channel_dropout import * +from kornia.augmentation.random_generator._2d.channel_shuffle import * +from kornia.augmentation.random_generator._2d.color_jiggle import * +from kornia.augmentation.random_generator._2d.color_jitter import * +from kornia.augmentation.random_generator._2d.crop import * +from kornia.augmentation.random_generator._2d.cutmix import * +from kornia.augmentation.random_generator._2d.gaussian_blur import * +from kornia.augmentation.random_generator._2d.gaussian_illumination import * +from kornia.augmentation.random_generator._2d.gaussian_noise import * +from kornia.augmentation.random_generator._2d.jigsaw import * +from kornia.augmentation.random_generator._2d.jpeg import * +from kornia.augmentation.random_generator._2d.linear_illumination import * +from kornia.augmentation.random_generator._2d.mixup import * +from kornia.augmentation.random_generator._2d.mosaic import * +from kornia.augmentation.random_generator._2d.motion_blur import * +from kornia.augmentation.random_generator._2d.patchmix import * +from kornia.augmentation.random_generator._2d.perspective import * +from kornia.augmentation.random_generator._2d.plain_uniform import * +from kornia.augmentation.random_generator._2d.planckian_jitter import * +from kornia.augmentation.random_generator._2d.posterize import * +from kornia.augmentation.random_generator._2d.probability import * +from kornia.augmentation.random_generator._2d.random_rain import * +from kornia.augmentation.random_generator._2d.rectangle_earase import * +from kornia.augmentation.random_generator._2d.resize import * +from kornia.augmentation.random_generator._2d.salt_pepper_noise import * +from kornia.augmentation.random_generator._2d.shear import * +from kornia.augmentation.random_generator._2d.translate import * diff --git a/kornia/augmentation/random_generator/_2d/affine.py b/kornia/augmentation/random_generator/_2d/affine.py new file mode 100644 index 0000000..c22e88a --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/affine.py @@ -0,0 +1,213 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import ( + _adapted_rsampling, + _check_positive_int_or_traced, + _common_param_check, + _joint_range_check, + _range_bound, +) +from kornia.core.utils import _extract_device_dtype + +__all__ = ["AffineGenerator"] + + +class AffineGenerator(RandomGeneratorBase): + r"""Get parameters for ``affine`` for a random affine transform. + + Args: + degrees: Range of degrees to select from like (min, max). + translate: tuple of maximum absolute fraction for horizontal + and vertical translations. For example translate=(a, b), then horizontal shift + is randomly sampled in the range -img_width * a < dx < img_width * a and vertical shift is + randomly sampled in the range -img_height * b < dy < img_height * b. Will not translate by default. + scale: scaling factor interval, e.g (a, b), then scale is + randomly sampled from the range a <= scale <= b. Will keep original scale by default. + shear: Range of degrees to select from. + If float, a shear parallel to the x axis in the range (-shear, +shear) will be applied. + If (a, b), a shear parallel to the x axis in the range (-shear, +shear) will be applied. + If (a, b, c, d), then x-axis shear in (shear[0], shear[1]) and y-axis shear in (shear[2], shear[3]) + will be applied. Will not apply shear by default. + If torch.Tensor, shear is a 2x2 torch.Tensor, a x-axis shear in (shear[0][0], shear[0][1]) and + y-axis shear in + (shear[1][0], shear[1][1]) will be applied. Will not apply shear by default. + + Returns: + A dict of parameters to be passed for transformation. + - translations (torch.Tensor): element-wise translations with a shape of (B, 2). + - center (torch.Tensor): element-wise center with a shape of (B, 2). + - scale (torch.Tensor): element-wise scales with a shape of (B, 2). + - angle (torch.Tensor): element-wise rotation angles with a shape of (B,). + - shear_x (torch.Tensor): element-wise x-axis shears with a shape of (B,). + - shear_y (torch.Tensor): element-wise y-axis shears with a shape of (B,). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, + degrees: Union[torch.Tensor, float, Tuple[float, float]], + translate: Optional[Union[torch.Tensor, Tuple[float, float]]] = None, + scale: Optional[Union[torch.Tensor, Tuple[float, float], Tuple[float, float, float, float]]] = None, + shear: Optional[Union[torch.Tensor, float, Tuple[float, float]]] = None, + ) -> None: + super().__init__() + self.degrees = degrees + self.translate = translate + self.scale = scale + self.shear = shear + + def __repr__(self) -> str: + repr = f"degrees={self.degrees}, translate={self.translate}, scale={self.scale}, shear={self.shear}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + _degrees = _range_bound(self.degrees, "degrees", 0, (-360, 360)).to(device=device, dtype=dtype) + _translate = ( + self.translate + if self.translate is None + else _range_bound(self.translate, "translate", bounds=(0, 1), check="singular").to( + device=device, dtype=dtype + ) + ) + _scale: Optional[torch.Tensor] = None + if self.scale is not None: + if len(self.scale) == 2: + _scale = _range_bound(self.scale[:2], "scale", bounds=(0, float("inf")), check="singular").to( + device=device, dtype=dtype + ) + elif len(self.scale) == 4: + _scale = torch.cat( + [ + _range_bound(self.scale[:2], "scale_x", bounds=(0, float("inf")), check="singular"), + _range_bound(self.scale[-2:], "scale_y", bounds=(0, float("inf")), check="singular"), + ] + ).to(device=device, dtype=dtype) + else: + raise ValueError(f"'scale' expected to be either 2 or 4 elements. Got {self.scale}") + _shear: Optional[torch.Tensor] = None + if self.shear is not None: + shear = torch.as_tensor(self.shear, device=device, dtype=dtype) + if shear.shape == torch.Size([2, 2]): + _shear = shear + else: + _shear = torch.stack( + [ + _range_bound(shear if shear.dim() == 0 else shear[:2], "shear-x", 0, (-360, 360)), + ( + torch.tensor([0, 0], device=device, dtype=dtype) + if shear.dim() == 0 or len(shear) == 2 + else _range_bound(shear[2:], "shear-y", 0, (-360, 360)) + ), + ] + ) + + translate_x_sampler: Optional[UniformDistribution] = None + translate_y_sampler: Optional[UniformDistribution] = None + scale_2_sampler: Optional[UniformDistribution] = None + scale_4_sampler: Optional[UniformDistribution] = None + shear_x_sampler: Optional[UniformDistribution] = None + shear_y_sampler: Optional[UniformDistribution] = None + + if _translate is not None: + translate_x_sampler = UniformDistribution(-_translate[0], _translate[0], validate_args=False) + translate_y_sampler = UniformDistribution(-_translate[1], _translate[1], validate_args=False) + if _scale is not None: + if len(_scale) == 2: + scale_2_sampler = UniformDistribution(_scale[0], _scale[1], validate_args=False) + elif len(_scale) == 4: + scale_2_sampler = UniformDistribution(_scale[0], _scale[1], validate_args=False) + scale_4_sampler = UniformDistribution(_scale[2], _scale[3], validate_args=False) + else: + raise ValueError(f"'scale' expected to be either 2 or 4 elements. Got {self.scale}") + if _shear is not None: + _joint_range_check(_shear[0], "shear") + _joint_range_check(_shear[1], "shear") + shear_x_sampler = UniformDistribution(_shear[0][0], _shear[0][1], validate_args=False) + shear_y_sampler = UniformDistribution(_shear[1][0], _shear[1][1], validate_args=False) + + self.degree_sampler = UniformDistribution(_degrees[0], _degrees[1], validate_args=False) + self.translate_x_sampler = translate_x_sampler + self.translate_y_sampler = translate_y_sampler + self.scale_2_sampler = scale_2_sampler + self.scale_4_sampler = scale_4_sampler + self.shear_x_sampler = shear_x_sampler + self.shear_y_sampler = shear_y_sampler + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + height = batch_shape[-2] + width = batch_shape[-1] + + _device, _dtype = _extract_device_dtype([self.degrees, self.translate, self.scale, self.shear]) + _common_param_check(batch_size, same_on_batch) + _check_positive_int_or_traced(width, "width") + _check_positive_int_or_traced(height, "height") + + angle = _adapted_rsampling((batch_size,), self.degree_sampler, same_on_batch).to(device=_device, dtype=_dtype) + + # compute torch.Tensor ranges + if self.scale_2_sampler is not None: + _scale = _adapted_rsampling((batch_size,), self.scale_2_sampler, same_on_batch).unsqueeze(1).repeat(1, 2) + if self.scale_4_sampler is not None: + _scale[:, 1] = _adapted_rsampling((batch_size,), self.scale_4_sampler, same_on_batch) + _scale = _scale.to(device=_device, dtype=_dtype) + else: + _scale = torch.ones((batch_size, 2), device=_device, dtype=_dtype) + + if self.translate_x_sampler is not None and self.translate_y_sampler is not None: + translations = torch.stack( + [ + _adapted_rsampling((batch_size,), self.translate_x_sampler, same_on_batch) * width, + _adapted_rsampling((batch_size,), self.translate_y_sampler, same_on_batch) * height, + ], + dim=-1, + ) + translations = translations.to(device=_device, dtype=_dtype) + else: + translations = torch.zeros((batch_size, 2), device=_device, dtype=_dtype) + + center: torch.Tensor = torch.tensor([width, height], device=_device, dtype=_dtype).view(1, 2) / 2.0 - 0.5 + center = center.expand(batch_size, -1) + + if self.shear_x_sampler is not None and self.shear_y_sampler is not None: + sx = _adapted_rsampling((batch_size,), self.shear_x_sampler, same_on_batch) + sy = _adapted_rsampling((batch_size,), self.shear_y_sampler, same_on_batch) + sx = sx.to(device=_device, dtype=_dtype) + sy = sy.to(device=_device, dtype=_dtype) + else: + sx = torch.tensor([0] * batch_size, device=_device, dtype=_dtype) + sy = torch.tensor([0] * batch_size, device=_device, dtype=_dtype) + + return { + "translations": translations, + "center": center, + "scale": _scale, + "angle": angle, + "shear_x": sx, + "shear_y": sy, + } diff --git a/kornia/augmentation/random_generator/_2d/channel_dropout.py b/kornia/augmentation/random_generator/_2d/channel_dropout.py new file mode 100644 index 0000000..27cf406 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/channel_dropout.py @@ -0,0 +1,75 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import _adapted_rsampling, _common_param_check, _range_bound + + +class ChannelDropoutGenerator(RandomGeneratorBase): + r"""Generate random dropout masks for channels in a batch of images. + + Args: + num_drop_channels: The number of channels to drop randomly. + + Returns: + A dictionary containing the dropout mask. + - dropout_mask: Binary masks (bool) indicating the dropped channels with a shape of (B, C, H, W). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, + num_drop_channels: int, + ) -> None: + super().__init__() + self.num_drop_channels = num_drop_channels + self.drop_sampler: UniformDistribution + + def __repr__(self) -> str: + r"""Return a string representation of the object.""" + repr_buf = f"num_drop_channels={self.num_drop_channels}" + return repr_buf + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + r"""Create samplers for generating random dropout parameters.""" + drop = _range_bound((0.0, 1.0), "drop", device=device, dtype=dtype) + self.drop_sampler = UniformDistribution(drop[0], drop[1], validate_args=False) + + def forward(self, batch_shape: tuple[int, ...], same_on_batch: bool = False) -> dict[str, torch.Tensor]: + r"""Generate a mask for dropout channels.""" + batch_size, channels, _, _ = batch_shape + _common_param_check(batch_size, same_on_batch) + _device, _dtype = self.device, self.dtype + + batch_idx = torch.arange(batch_size, device=_device, dtype=torch.long).reshape(batch_size, 1) + channel_idx = torch.argsort( + _adapted_rsampling((batch_size, channels), self.drop_sampler, same_on_batch), dim=1 + )[:, : self.num_drop_channels].to(torch.long) + + return { + "batch_idx": batch_idx, + "channel_idx": channel_idx, + } diff --git a/kornia/augmentation/random_generator/_2d/channel_shuffle.py b/kornia/augmentation/random_generator/_2d/channel_shuffle.py new file mode 100644 index 0000000..50f3cc9 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/channel_shuffle.py @@ -0,0 +1,70 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import _adapted_rsampling, _common_param_check + +__all__ = ["ChannelShuffleGenerator"] + + +class ChannelShuffleGenerator(RandomGeneratorBase): + r"""Generate random channel permutation indices for a batch of images. + + Returns: + A dictionary containing the channel permutation indices. + - ``channels``: Long tensor of permuted channel indices with a shape of ``(B, C)``. + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __repr__(self) -> str: + return "" + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + """Create a Uniform(0, 1) sampler used to generate random channel scores for argsort shuffling.""" + self.shuffle_sampler = UniformDistribution( + torch.tensor(0.0, device=device, dtype=dtype), + torch.tensor(1.0, device=device, dtype=dtype), + validate_args=False, + ) + + def forward(self, batch_shape: tuple[int, ...], same_on_batch: bool = False) -> dict[str, torch.Tensor]: + """Generate channel permutation indices. + + Args: + batch_shape: Shape of the input batch ``(B, C, H, W)``. + same_on_batch: If ``True``, the same permutation is applied to all items in the batch. + + Returns: + Dictionary with key ``"channels"`` mapping to a ``(B, C)`` long tensor of permuted indices. + """ + batch_size, C = batch_shape[0], batch_shape[1] + _common_param_check(batch_size, same_on_batch) + + # Sample uniform scores of shape (B, C); _adapted_rsampling handles same_on_batch by + # generating (1, C) and repeating across the batch dimension when same_on_batch=True. + scores = _adapted_rsampling((batch_size, C), self.shuffle_sampler, same_on_batch) + channels = scores.argsort(dim=1).to(dtype=torch.long) + return {"channels": channels} diff --git a/kornia/augmentation/random_generator/_2d/color_jiggle.py b/kornia/augmentation/random_generator/_2d/color_jiggle.py new file mode 100644 index 0000000..7733cbc --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/color_jiggle.py @@ -0,0 +1,103 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from functools import partial +from typing import Dict, List, Tuple, Union + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import _adapted_rsampling, _common_param_check, _joint_range_check, _range_bound +from kornia.core.utils import _extract_device_dtype + +__all__ = ["ColorJiggleGenerator"] + + +class ColorJiggleGenerator(RandomGeneratorBase): + r"""Generate random color jiter parameters for a batch of images following OpenCV. + + Args: + brightness: The brightness factor to apply. + contrast: The contrast factor to apply. + saturation: The saturation factor to apply. + hue: The hue factor to apply. + + Returns: + A dict of parameters to be passed for transformation. + - brightness_factor: element-wise brightness factors with a shape of (B,). + - contrast_factor: element-wise contrast factors with a shape of (B,). + - hue_factor: element-wise hue factors with a shape of (B,). + - saturation_factor: element-wise saturation factors with a shape of (B,). + - order: applying orders of the color adjustments with a shape of (4). In which, + 0 is brightness adjustment; 1 is contrast adjustment; + 2 is saturation adjustment; 3 is hue adjustment. + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, + brightness: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 0.0, + contrast: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 0.0, + saturation: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 0.0, + hue: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 0.0, + ) -> None: + super().__init__() + self.brightness = brightness + self.contrast = contrast + self.saturation = saturation + self.hue = hue + + def __repr__(self) -> str: + return f"brightness={self.brightness}, contrast={self.contrast}, saturation={self.saturation}, hue={self.hue}" + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + brightness = _range_bound(self.brightness, "brightness", center=1.0, bounds=(0, 2), device=device, dtype=dtype) + contrast: torch.Tensor = _range_bound(self.contrast, "contrast", center=1.0, device=device, dtype=dtype) + saturation: torch.Tensor = _range_bound(self.saturation, "saturation", center=1.0, device=device, dtype=dtype) + hue: torch.Tensor = _range_bound(self.hue, "hue", bounds=(-0.5, 0.5), device=device, dtype=dtype) + + _joint_range_check(brightness, "brightness", (0, 2)) + _joint_range_check(contrast, "contrast", (0, float("inf"))) + _joint_range_check(hue, "hue", (-0.5, 0.5)) + _joint_range_check(saturation, "saturation", (0, float("inf"))) + + self.brightness_sampler = UniformDistribution(brightness[0], brightness[1], validate_args=False) + self.contrast_sampler = UniformDistribution(contrast[0], contrast[1], validate_args=False) + self.hue_sampler = UniformDistribution(hue[0], hue[1], validate_args=False) + self.saturation_sampler = UniformDistribution(saturation[0], saturation[1], validate_args=False) + self.randperm = partial(torch.randperm, device=device, dtype=dtype) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + _common_param_check(batch_size, same_on_batch) + _device, _dtype = _extract_device_dtype([self.brightness, self.contrast, self.hue, self.saturation]) + brightness_factor = _adapted_rsampling((batch_size,), self.brightness_sampler, same_on_batch) + contrast_factor = _adapted_rsampling((batch_size,), self.contrast_sampler, same_on_batch) + hue_factor = _adapted_rsampling((batch_size,), self.hue_sampler, same_on_batch) + saturation_factor = _adapted_rsampling((batch_size,), self.saturation_sampler, same_on_batch) + return { + "brightness_factor": brightness_factor.to(device=_device, dtype=_dtype), + "contrast_factor": contrast_factor.to(device=_device, dtype=_dtype), + "hue_factor": hue_factor.to(device=_device, dtype=_dtype), + "saturation_factor": saturation_factor.to(device=_device, dtype=_dtype), + "order": self.randperm(4).to(device=_device, dtype=_dtype).long(), + } diff --git a/kornia/augmentation/random_generator/_2d/color_jitter.py b/kornia/augmentation/random_generator/_2d/color_jitter.py new file mode 100644 index 0000000..48c046e --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/color_jitter.py @@ -0,0 +1,110 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, List, Tuple, Union + +import torch + +from kornia.augmentation.random_generator.base import ( + RandomGeneratorBase, + UniformDistribution, +) +from kornia.augmentation.utils import ( + _adapted_rsampling, + _joint_range_check, + _range_bound, +) + +__all__ = ["ColorJitterGenerator"] + + +class ColorJitterGenerator(RandomGeneratorBase): + r"""Generate random color jiter parameters for a batch of images following Pil. + + This implementation is for maintaining compatibility with torchvision. It does not + follow the color theory and is not be actively maintained. Prefer using + :func:`kornia.augmentation.ColorJiggleGenerator` + + Args: + brightness: The brightness factor to apply. + contrast: The contrast factor to apply. + saturation: The saturation factor to apply. + hue: The hue factor to apply. + + Returns: + A dict of parameters to be passed for transformation. + - brightness_factor: element-wise brightness factors with a shape of (B,). + - contrast_factor: element-wise contrast factors with a shape of (B,). + - hue_factor: element-wise hue factors with a shape of (B,). + - saturation_factor: element-wise saturation factors with a shape of (B,). + - order: applying orders of the color adjustments with a shape of (4). In which, + 0 is brightness adjustment; 1 is contrast adjustment; + 2 is saturation adjustment; 3 is hue adjustment. + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, + brightness: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 0.0, + contrast: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 0.0, + saturation: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 0.0, + hue: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 0.0, + ) -> None: + super().__init__() + self.brightness = brightness + self.contrast = contrast + self.saturation = saturation + self.hue = hue + + def __repr__(self) -> str: + return f"brightness={self.brightness}, contrast={self.contrast}, saturation={self.saturation}, hue={self.hue}" + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + brightness: torch.Tensor = _range_bound(self.brightness, "brightness", center=1.0, device=device, dtype=dtype) + contrast: torch.Tensor = _range_bound(self.contrast, "contrast", center=1.0, device=device, dtype=dtype) + saturation: torch.Tensor = _range_bound(self.saturation, "saturation", center=1.0, device=device, dtype=dtype) + hue: torch.Tensor = _range_bound(self.hue, "hue", bounds=(-0.5, 0.5), device=device, dtype=dtype) + + _joint_range_check(brightness, "brightness", (0, float("inf"))) + _joint_range_check(contrast, "contrast", (0, float("inf"))) + _joint_range_check(hue, "hue", (-0.5, 0.5)) + _joint_range_check(saturation, "saturation", (0, float("inf"))) + + self.brightness_sampler = UniformDistribution(brightness[0], brightness[1], validate_args=False) + self.contrast_sampler = UniformDistribution(contrast[0], contrast[1], validate_args=False) + self.hue_sampler = UniformDistribution(hue[0], hue[1], validate_args=False) + self.saturation_sampler = UniformDistribution(saturation[0], saturation[1], validate_args=False) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + brightness_factor = _adapted_rsampling((batch_size,), self.brightness_sampler, same_on_batch) + contrast_factor = _adapted_rsampling((batch_size,), self.contrast_sampler, same_on_batch) + hue_factor = _adapted_rsampling((batch_size,), self.hue_sampler, same_on_batch) + saturation_factor = _adapted_rsampling((batch_size,), self.saturation_sampler, same_on_batch) + + return { + "brightness_factor": brightness_factor, + "contrast_factor": contrast_factor, + "hue_factor": hue_factor, + "saturation_factor": saturation_factor, + "order": torch.randperm(4, dtype=torch.long), + } diff --git a/kornia/augmentation/random_generator/_2d/crop.py b/kornia/augmentation/random_generator/_2d/crop.py new file mode 100644 index 0000000..5c100e9 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/crop.py @@ -0,0 +1,354 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Optional, Tuple, Union + +import torch +from torch.distributions import Uniform + +from kornia.augmentation.random_generator.base import RandomGeneratorBase +from kornia.augmentation.utils import ( + _adapted_rsampling, + _check_positive_int_or_traced, + _common_param_check, + _joint_range_check, +) +from kornia.core.utils import _extract_device_dtype +from kornia.geometry.bbox import bbox_generator + +__all__ = ["CropGenerator", "ResizedCropGenerator", "center_crop_generator"] + + +class CropGenerator(RandomGeneratorBase): + r"""Get parameters for ```crop``` transformation for crop transform. + + Args: + size (tuple): Desired size of the crop operation, like (h, w). + If torch.Tensor, it must be (B, 2). + resize_to (tuple): Desired output size of the crop, like (h, w). If None, no resize will be performed. + + Returns: + params Dict[str, torch.Tensor]: parameters to be passed for transformation. + - src (torch.Tensor): cropping bounding boxes with a shape of (B, 4, 2). + - dst (torch.Tensor): output bounding boxes with a shape (B, 4, 2). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__(self, size: Union[Tuple[int, int], torch.Tensor], resize_to: Optional[Tuple[int, int]] = None) -> None: + super().__init__() + self.size = size + self.resize_to = resize_to + + def __repr__(self) -> str: + repr = f"crop_size={self.size}" + if self.resize_to is not None: + repr += f", resize_to={self.resize_to}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + self.rand_sampler = Uniform( + torch.tensor(0.0, device=device, dtype=dtype), torch.tensor(1.0, device=device, dtype=dtype) + ) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + _common_param_check(batch_size, same_on_batch) + _device, _dtype = _extract_device_dtype([self.size if isinstance(self.size, torch.Tensor) else None]) + + if batch_size == 0: + return { + "src": torch.zeros([0, 4, 2], device=_device, dtype=_dtype), + "dst": torch.zeros([0, 4, 2], device=_device, dtype=_dtype), + } + + input_size = (batch_shape[-2], batch_shape[-1]) + if not isinstance(self.size, torch.Tensor): + size = torch.tensor(self.size, device=_device, dtype=_dtype).repeat(batch_size, 1) + else: + size = self.size.to(device=_device, dtype=_dtype) + if size.shape != torch.Size([batch_size, 2]): + raise AssertionError( + "If `size` is a torch.Tensor, it must be shaped as (B, 2). " + f"Got {size.shape} while expecting {torch.Size([batch_size, 2])}." + ) + if not (input_size[0] > 0 and input_size[1] > 0 and (size > 0).all()): + raise AssertionError(f"Got non-positive input size or size. {input_size}, {size}.") + size = size.floor() + + x_diff = input_size[1] - size[:, 1] + 1 + y_diff = input_size[0] - size[:, 0] + 1 + + # Start point will be 0 if diff < 0 + x_diff = x_diff.clamp(0) + y_diff = y_diff.clamp(0) + + if same_on_batch: + # If same_on_batch, select the first then repeat. + x_start = ( + _adapted_rsampling((batch_size,), self.rand_sampler, same_on_batch).to(x_diff) * x_diff[0] + ).floor() + y_start = ( + _adapted_rsampling((batch_size,), self.rand_sampler, same_on_batch).to(y_diff) * y_diff[0] + ).floor() + else: + x_start = (_adapted_rsampling((batch_size,), self.rand_sampler, same_on_batch).to(x_diff) * x_diff).floor() + y_start = (_adapted_rsampling((batch_size,), self.rand_sampler, same_on_batch).to(y_diff) * y_diff).floor() + crop_src = bbox_generator( + x_start.view(-1).to(device=_device, dtype=_dtype), + y_start.view(-1).to(device=_device, dtype=_dtype), + torch.where(size[:, 1] == 0, torch.tensor(input_size[1], device=_device, dtype=_dtype), size[:, 1]), + torch.where(size[:, 0] == 0, torch.tensor(input_size[0], device=_device, dtype=_dtype), size[:, 0]), + ) + + if self.resize_to is None: + crop_dst = bbox_generator( + torch.tensor([0] * batch_size, device=_device, dtype=_dtype), + torch.tensor([0] * batch_size, device=_device, dtype=_dtype), + size[:, 1], + size[:, 0], + ) + _output_size = size.to(dtype=torch.long) + else: + if not ( + len(self.resize_to) == 2 + and isinstance(self.resize_to[0], (int,)) + and isinstance(self.resize_to[1], (int,)) + and self.resize_to[0] > 0 + and self.resize_to[1] > 0 + ): + raise AssertionError(f"`resize_to` must be a tuple of 2 positive integers. Got {self.resize_to}.") + crop_dst = torch.tensor( + [ + [ + [0, 0], + [self.resize_to[1] - 1, 0], + [self.resize_to[1] - 1, self.resize_to[0] - 1], + [0, self.resize_to[0] - 1], + ] + ], + device=_device, + dtype=_dtype, + ).repeat(batch_size, 1, 1) + _output_size = torch.tensor(self.resize_to, device=_device, dtype=torch.long).expand(batch_size, -1) + + _input_size = torch.tensor(input_size, device=_device, dtype=torch.long).expand(batch_size, -1) + + return {"src": crop_src, "dst": crop_dst, "input_size": _input_size, "output_size": _output_size} + + +class ResizedCropGenerator(CropGenerator): + r"""Get cropping heights and widths for ```crop``` transformation for resized crop transform. + + Args: + output_size (Tuple[int, int]): expected output size of each edge. + scale (torch.Tensor): range of size of the origin size cropped with (2,) shape. + ratio (torch.Tensor): range of aspect ratio of the origin aspect ratio cropped with (2,) shape. + + Returns: + params Dict[str, torch.Tensor]: parameters to be passed for transformation. + - size (torch.Tensor): element-wise cropping sizes with a shape of (B, 2). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. + + Examples: + >>> _ = torch.manual_seed(42) + >>> rcg = ResizedCropGenerator((30, 30), scale=torch.tensor([.7, 1.3]), ratio=torch.tensor([.9, 1.])) + >>> out = rcg(torch.Size([1, 3, 3])) + >>> out["src"] + tensor([[[0., 0.], + [2., 0.], + [2., 2.], + [0., 2.]]]) + >>> out["dst"] + tensor([[[ 0., 0.], + [29., 0.], + [29., 29.], + [ 0., 29.]]]) + >>> out["input_size"] + tensor([[3, 3]]) + >>> out["output_size"] + tensor([[30, 30]]) + + """ + + def __init__( + self, + output_size: Tuple[int, int], + scale: Union[torch.Tensor, Tuple[float, float]], + ratio: Union[torch.Tensor, Tuple[float, float]], + ) -> None: + if not ( + len(output_size) == 2 + and isinstance(output_size[0], (int,)) + and isinstance(output_size[1], (int,)) + and output_size[0] > 0 + and output_size[1] > 0 + ): + raise AssertionError(f"`output_size` must be a tuple of 2 positive integers. Got {output_size}.") + super().__init__(size=output_size, resize_to=output_size) # fake an intermedia crop size + self.scale = scale + self.ratio = ratio + self.output_size = output_size + + def __repr__(self) -> str: + repr = f"scale={self.scale}, resize_to={self.ratio}, output_size={self.output_size}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + scale = torch.as_tensor(self.scale, device=device, dtype=dtype) + ratio = torch.as_tensor(self.ratio, device=device, dtype=dtype) + _joint_range_check(scale, "scale") + _joint_range_check(ratio, "ratio") + self.rand_sampler = Uniform( + torch.tensor(0.0, device=device, dtype=dtype), torch.tensor(1.0, device=device, dtype=dtype) + ) + self.log_ratio_sampler = Uniform(torch.log(ratio[0]), torch.log(ratio[1]), validate_args=False) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + size = (batch_shape[-2], batch_shape[-1]) + _device, _dtype = _extract_device_dtype([self.scale, self.ratio]) + + if batch_size == 0: + return { + "src": torch.zeros([0, 4, 2], device=_device, dtype=_dtype), + "dst": torch.zeros([0, 4, 2], device=_device, dtype=_dtype), + "size": torch.zeros([0, 2], device=_device, dtype=_dtype), + } + + rand_tensor = _adapted_rsampling((batch_size, 10), self.rand_sampler, same_on_batch).to( + device=_device, dtype=_dtype + ) + scale_tensor = torch.as_tensor(self.scale, device=_device, dtype=_dtype) + area = (rand_tensor * (scale_tensor[1] - scale_tensor[0]) + scale_tensor[0]) * size[0] * size[1] + log_ratio = _adapted_rsampling((batch_size, 10), self.log_ratio_sampler, same_on_batch).to( + device=_device, dtype=_dtype + ) + aspect_ratio = torch.exp(log_ratio) + + w = torch.sqrt(area * aspect_ratio).round().floor() + h = torch.sqrt(area / aspect_ratio).round().floor() + # Element-wise w, h condition + cond = ((0 < w) * (w < size[1]) * (0 < h) * (h < size[0])).int() + + # torch.argmax is not reproducible across devices: https://github.com/pytorch/pytorch/issues/17738 + # Here, we will select the first occurrence of the duplicated elements. + cond_bool, argmax_dim1 = ((cond.cumsum(1) == 1) & cond.bool()).max(1) + h_out = h[torch.arange(0, batch_size, device=_device, dtype=torch.long), argmax_dim1] + w_out = w[torch.arange(0, batch_size, device=_device, dtype=torch.long), argmax_dim1] + + if not cond_bool.all(): + # Fallback to center crop + in_ratio = float(size[0]) / float(size[1]) + _min = float(self.ratio.min()) if isinstance(self.ratio, torch.Tensor) else min(self.ratio) + if in_ratio < _min: + h_ct = torch.tensor(size[0], device=_device, dtype=_dtype) + w_ct = torch.round(h_ct / _min) + elif in_ratio > _min: + w_ct = torch.tensor(size[1], device=_device, dtype=_dtype) + h_ct = torch.round(w_ct * _min) + else: # whole image + h_ct = torch.tensor(size[0], device=_device, dtype=_dtype) + w_ct = torch.tensor(size[1], device=_device, dtype=_dtype) + h_ct = h_ct.floor() + w_ct = w_ct.floor() + + h_out = torch.where(cond_bool, h_out, h_ct) + w_out = torch.where(cond_bool, w_out, w_ct) + + # Clamp crop size to input size to prevent out-of-bounds crops + h_out = torch.clamp(h_out, min=1, max=size[0]) + w_out = torch.clamp(w_out, min=1, max=size[1]) + + # Update the crop size. + self.size = torch.stack([h_out, w_out], dim=1) + return super().forward(batch_shape, same_on_batch) + + +def center_crop_generator( + batch_size: int, + height: int, + width: int, + size: Tuple[int, int], + device: Union[None, str, torch.device] = None, +) -> Dict[str, torch.Tensor]: + r"""Get parameters for ```center_crop``` transformation for center crop transform. + + Args: + batch_size (int): the torch.Tensor batch size. + height (int) : height of the image. + width (int): width of the image. + size (tuple): Desired output size of the crop, like (h, w). + device (Union[str, torch.device, None]): the device on which the random numbers will be generated. Default: cpu. + + Returns: + params Dict[str, torch.Tensor]: parameters to be passed for transformation. + - src (torch.Tensor): cropping bounding boxes with a shape of (B, 4, 2). + - dst (torch.Tensor): output bounding boxes with a shape (B, 4, 2). + + Note: + No random number will be generated. + + """ + if device is None: + device = torch.device("cpu") + _common_param_check(batch_size) + if not isinstance(size, (tuple, list)) or len(size) != 2: + raise ValueError(f"Input size must be a tuple/list of length 2. Got {size}") + _check_positive_int_or_traced(height, "height") + _check_positive_int_or_traced(width, "width") + if isinstance(height, int) and isinstance(width, int) and not (height >= size[0] and width >= size[1]): + raise AssertionError(f"Crop size must be smaller than input size. Got ({height}, {width}) and {size}.") + + # unpack input sizes + dst_h, dst_w = size + src_h, src_w = height, width + + # compute start/end offsets + dst_h_half = dst_h / 2 + dst_w_half = dst_w / 2 + src_h_half = src_h / 2 + src_w_half = src_w / 2 + + start_x = int(src_w_half - dst_w_half) + start_y = int(src_h_half - dst_h_half) + + end_x = start_x + dst_w - 1 + end_y = start_y + dst_h - 1 + + # [y, x] origin + # top-left, top-right, bottom-right, bottom-left + points_src: torch.Tensor = torch.tensor( + [[[start_x, start_y], [end_x, start_y], [end_x, end_y], [start_x, end_y]]], device=device, dtype=torch.long + ).expand(batch_size, -1, -1) + + # [y, x] destination + # top-left, top-right, bottom-right, bottom-left + points_dst: torch.Tensor = torch.tensor( + [[[0, 0], [dst_w - 1, 0], [dst_w - 1, dst_h - 1], [0, dst_h - 1]]], device=device, dtype=torch.long + ).expand(batch_size, -1, -1) + + _input_size = torch.tensor((height, width), device=device, dtype=torch.long).expand(batch_size, -1) + _output_size = torch.tensor(size, device=device, dtype=torch.long).expand(batch_size, -1) + + return {"src": points_src, "dst": points_dst, "input_size": _input_size, "output_size": _output_size} diff --git a/kornia/augmentation/random_generator/_2d/cutmix.py b/kornia/augmentation/random_generator/_2d/cutmix.py new file mode 100644 index 0000000..fbcb968 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/cutmix.py @@ -0,0 +1,166 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Optional, Tuple, Union + +import torch +from torch.distributions import Bernoulli, Beta, Uniform + +from kornia.augmentation.random_generator.base import RandomGeneratorBase +from kornia.augmentation.utils import ( + _adapted_rsampling, + _adapted_sampling, + _check_positive_int_or_traced, + _common_param_check, + _joint_range_check, +) +from kornia.core.utils import _extract_device_dtype +from kornia.geometry.bbox import bbox_generator + +__all__ = ["CutmixGenerator"] + + +class CutmixGenerator(RandomGeneratorBase): + r"""Generate cutmix indexes and lambdas for a batch of inputs. + + Args: + p (float): probability of applying cutmix. + num_mix (int): number of images to mix with. Default is 1. + beta (torch.Tensor, optional): hyperparameter for generating cut size from beta distribution. + If None, it will be set to 1. + cut_size (torch.Tensor, optional): controlling the minimum and maximum cut ratio from [0, 1]. + If None, it will be set to [0, 1], which means no restriction. + + Returns: + params Dict[str, torch.Tensor]: parameters to be passed for transformation. + - mix_pairs (torch.Tensor): element-wise probabilities with a shape of (num_mix, B). + - crop_src (torch.Tensor): element-wise probabilities with a shape of (num_mix, B, 4, 2). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, + cut_size: Optional[Union[torch.Tensor, Tuple[float, float]]] = None, + beta: Optional[Union[torch.Tensor, float]] = None, + num_mix: int = 1, + p: float = 1.0, + ) -> None: + super().__init__() + self.cut_size = cut_size + self.beta = beta + self.num_mix = num_mix + self.p = p + + if not (num_mix >= 1 and isinstance(num_mix, (int,))): + raise AssertionError(f"`num_mix` must be an integer greater than 1. Got {num_mix}.") + + def __repr__(self) -> str: + repr = f"cut_size={self.cut_size}, beta={self.beta}, num_mix={self.num_mix}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + if self.beta is None: + self._beta = torch.tensor(1.0, device=device, dtype=dtype) + else: + self._beta = torch.as_tensor(self.beta, device=device, dtype=dtype) + if self.cut_size is None: + self._cut_size = torch.tensor([0.0, 1.0], device=device, dtype=dtype) + else: + self._cut_size = torch.as_tensor(self.cut_size, device=device, dtype=dtype) + + _joint_range_check(self._cut_size, "cut_size", bounds=(0, 1)) + + self.beta_sampler = Beta(self._beta, self._beta) + self.prob_sampler = Bernoulli(torch.tensor(float(self.p), device=device, dtype=dtype)) + self.rand_sampler = Uniform( + torch.tensor(0.0, device=device, dtype=dtype), + torch.tensor(1.0, device=device, dtype=dtype), + validate_args=False, + ) + self.pair_sampler = Uniform( + torch.tensor(0.0, device=device, dtype=dtype), + torch.tensor(1.0, device=device, dtype=dtype), + validate_args=False, + ) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + height = batch_shape[-2] + width = batch_shape[-1] + + _check_positive_int_or_traced(height, "height") + _check_positive_int_or_traced(width, "width") + _device, _dtype = _extract_device_dtype([self.beta, self.cut_size]) + _common_param_check(batch_size, same_on_batch) + + if batch_size == 0: + return { + "mix_pairs": torch.zeros([0, 3], device=_device, dtype=torch.long), + "crop_src": torch.zeros([0, 4, 2], device=_device, dtype=_dtype), + } + + with torch.no_grad(): + batch_probs: torch.Tensor = _adapted_sampling( + (batch_size * self.num_mix,), self.prob_sampler, same_on_batch + ) + mix_pairs: torch.Tensor = ( + _adapted_sampling((self.num_mix, batch_size), self.pair_sampler, same_on_batch) + .to(device=_device, dtype=_dtype) + .argsort(dim=1) + ) + + cutmix_betas: torch.Tensor = _adapted_rsampling((batch_size * self.num_mix,), self.beta_sampler, same_on_batch) + + # Note: torch.clamp does not accept torch.Tensor, cutmix_betas.clamp(cut_size[0], cut_size[1]) throws: + # Argument 1 to "clamp" of "_TensorBase" has incompatible type "torch.Tensor"; expected "float" + cutmix_betas = torch.min(torch.max(cutmix_betas, self._cut_size[0]), self._cut_size[1]) + cutmix_rate = torch.sqrt(1.0 - cutmix_betas) * batch_probs + + cut_height = (cutmix_rate * height).floor().to(device=_device, dtype=_dtype) + cut_width = (cutmix_rate * width).floor().to(device=_device, dtype=_dtype) + _gen_shape = (1,) + + if same_on_batch: + _gen_shape = (cut_height.size(0),) + cut_height = cut_height[0] + cut_width = cut_width[0] + + # Reserve at least 1 pixel for cropping. + x_start = _adapted_rsampling(_gen_shape, self.rand_sampler, same_on_batch).to(device=_device, dtype=_dtype) * ( + width - cut_width - 1 + ) + y_start = _adapted_rsampling(_gen_shape, self.rand_sampler, same_on_batch).to(device=_device, dtype=_dtype) * ( + height - cut_height - 1 + ) + x_start = x_start.floor() + y_start = y_start.floor() + + crop_src = bbox_generator(x_start.squeeze(), y_start.squeeze(), cut_width, cut_height) + + # (B * num_mix, 4, 2) => (num_mix, batch_size, 4, 2) + crop_src = crop_src.view(self.num_mix, batch_size, 4, 2) + + return { + "mix_pairs": mix_pairs.to(device=_device, dtype=torch.long), + "crop_src": crop_src.floor().to(device=_device, dtype=_dtype), + "image_shape": torch.as_tensor(batch_shape[-2:], device=_device, dtype=_dtype), + } diff --git a/kornia/augmentation/random_generator/_2d/gaussian_blur.py b/kornia/augmentation/random_generator/_2d/gaussian_blur.py new file mode 100644 index 0000000..e355ee8 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/gaussian_blur.py @@ -0,0 +1,79 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Tuple, Union + +import torch +from torch import Tensor + +from kornia.augmentation.random_generator.base import ( + RandomGeneratorBase, + UniformDistribution, +) +from kornia.augmentation.utils import ( + _adapted_rsampling, + _common_param_check, + _joint_range_check, +) + +__all__ = ["RandomGaussianBlurGenerator"] + + +class RandomGaussianBlurGenerator(RandomGeneratorBase): + r"""Generate random gaussian blur parameters for a batch of images. + + Args: + sigma: The range to uniformly sample the standard deviation for the Gaussian kernel. + + Returns: + A dict of parameters to be passed for transformation. + - sigma: element-wise standard deviation with a shape of (B,). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__(self, sigma: Union[Tuple[float, float], Tensor] = (0.1, 2.0)) -> None: + super().__init__() + if sigma[1] < sigma[0]: + raise TypeError(f"sigma_max should be higher than sigma_min: {sigma} passed.") + + self.sigma = sigma + self.sigma_sampler: UniformDistribution + + def __repr__(self) -> str: + repr_buf = f"sigma={self.sigma}" + return repr_buf + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + if not isinstance(self.sigma, (torch.Tensor)): + sigma = torch.tensor(self.sigma, device=device, dtype=dtype) + else: + sigma = self.sigma.to(device=device, dtype=dtype) + + _joint_range_check(sigma, "sigma", (0, float("inf"))) + + self.sigma_sampler = UniformDistribution(sigma[0], sigma[1], validate_args=False) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, Tensor]: + batch_size = batch_shape[0] + _common_param_check(batch_size, same_on_batch) + sigma = _adapted_rsampling((batch_size,), self.sigma_sampler, same_on_batch) + return {"sigma": sigma} diff --git a/kornia/augmentation/random_generator/_2d/gaussian_illumination.py b/kornia/augmentation/random_generator/_2d/gaussian_illumination.py new file mode 100644 index 0000000..610382f --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/gaussian_illumination.py @@ -0,0 +1,137 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch + +from kornia.augmentation.random_generator.base import ( + RandomGeneratorBase, + UniformDistribution, +) +from kornia.augmentation.utils import ( + _adapted_rsampling, + _common_param_check, + _range_bound, +) +from kornia.enhance import normalize_min_max +from kornia.filters.kernels import gaussian + + +class GaussianIlluminationGenerator(RandomGeneratorBase): + r"""Generates random 2D Gaussian illumination patterns for image augmentation. + + Args: + gain: Range for the gain factor applied to the generated illumination. + center: Range for the center coordinates of the Gaussian distribution. + sigma: Range for the standard deviation of the Gaussian distribution. + sign: Range for the sign of the Gaussian distribution. + + Returns: + A dictionary of parameters to be passed for transformation. + - gradient: : Generated 2D Gaussian illumination pattern with shape (B, C, H, W). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, + gain: tuple[float, float], + center: tuple[float, float], + sigma: tuple[float, float], + sign: tuple[float, float], + ) -> None: + super().__init__() + self.gain = gain + self.center = center + self.sigma = sigma + self.sign = sign + + self.gain_sampler: UniformDistribution + self.center_sampler: UniformDistribution + self.sigma_sampler: UniformDistribution + self.sign_sampler: UniformDistribution + + def __repr__(self) -> str: + r"""Return a string representation of the object.""" + repr_buf = f"gain={self.gain}, center={self.center}, sigma={self.sigma}, sign={self.sign}" + return repr_buf + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + r"""Create samplers for generating random gaussian illumination parameters.""" + gain = _range_bound(self.gain, "gain", device=device, dtype=dtype) + self.gain_sampler = UniformDistribution(gain[0], gain[1], validate_args=False) + + center = _range_bound(self.center, "center", device=device, dtype=dtype) + self.center_sampler = UniformDistribution(center[0], center[1], validate_args=False) + + sigma = _range_bound(self.sigma, "sigma", device=device, dtype=dtype) + self.sigma_sampler = UniformDistribution(sigma[0], sigma[1], validate_args=False) + + sign = _range_bound( + self.sign, + "sign", + bounds=(-1.0, 1.0), + center=0.0, + device=device, + dtype=dtype, + ) + self.sign_sampler = UniformDistribution(sign[0], sign[1], validate_args=False) + + def forward(self, batch_shape: tuple[int, ...], same_on_batch: bool = False) -> dict[str, torch.Tensor]: + r"""Generate random 2D Gaussian illumination patterns.""" + batch_size, channels, height, width = batch_shape + + _common_param_check(batch_size, same_on_batch) + _device, _dtype = self.device, self.dtype + + # TODO: check whether we need generate all the parameters at once + + gain_factor = _adapted_rsampling((batch_size, 1, 1, 1), self.gain_sampler, same_on_batch) + + sigma_x = width * _adapted_rsampling((batch_size, 1), self.sigma_sampler, same_on_batch) + + center_x = torch.round(width * _adapted_rsampling((batch_size, 1), self.center_sampler, same_on_batch)) + + sigma_y = height * _adapted_rsampling((batch_size, 1), self.sigma_sampler, same_on_batch) + + center_y = torch.round(height * _adapted_rsampling((batch_size, 1), self.center_sampler, same_on_batch)) + + sign = torch.where( + _adapted_rsampling((batch_size, 1, 1, 1), self.sign_sampler, same_on_batch) >= 0.0, + torch.tensor(1.0, device=_device, dtype=_dtype), + torch.tensor(-1.0, device=_device, dtype=_dtype), + ) + + # Generate random gaussian for create a 2D gaussian image. + gauss_x = gaussian(width, sigma_x, mean=center_x, device=_device, dtype=_dtype).unsqueeze(1) + + gauss_y = gaussian(height, sigma_y, mean=center_y, device=_device, dtype=_dtype).unsqueeze(2) + + # gradient = (batch_size, channels, height, width) + gradient = (gauss_y @ gauss_x).unsqueeze_(1).repeat(1, channels, 1, 1) + + # TODO: this will crash if the shape is not batched + # Normalize between 0-1 to apply the gain factor. + gradient = normalize_min_max(gradient, min_val=0.0, max_val=1.0) + gradient = sign.mul_(gain_factor).mul(gradient) + + return {"gradient": gradient} diff --git a/kornia/augmentation/random_generator/_2d/gaussian_noise.py b/kornia/augmentation/random_generator/_2d/gaussian_noise.py new file mode 100644 index 0000000..0014ce4 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/gaussian_noise.py @@ -0,0 +1,88 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +from torch.distributions import Normal + +from kornia.augmentation.random_generator.base import RandomGeneratorBase +from kornia.augmentation.utils import _common_param_check + +__all__ = ["GaussianNoiseGenerator"] + + +class GaussianNoiseGenerator(RandomGeneratorBase): + r"""Generate random Gaussian noise tensors for a batch of images. + + Args: + mean: Mean of the Gaussian distribution. + std: Standard deviation of the Gaussian distribution. + + Returns: + A dictionary containing the pre-generated noise tensor. + - ``gaussian_noise``: Float tensor of noise values with a shape of ``(B, C, H, W)`` + (or ``(1, C, H, W)`` when ``same_on_batch=True``, to be expanded in ``apply_transform``). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__(self, mean: float, std: float) -> None: + super().__init__() + self.mean = mean + self.std = std + + def __repr__(self) -> str: + return f"mean={self.mean}, std={self.std}" + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + """Store device and dtype for the Normal distribution (created on demand in forward).""" + # Normal is created fresh in forward() so device/dtype are always consistent. + # No persistent sampler is needed beyond what the base class tracks. + + def forward(self, batch_shape: tuple[int, ...], same_on_batch: bool = False) -> dict[str, torch.Tensor]: + """Generate a Gaussian noise tensor matching the input batch shape. + + Args: + batch_shape: Shape of the input batch ``(B, C, H, W)``. + same_on_batch: If ``True``, generate a single noise map ``(1, C, H, W)`` shared + across all batch items. The augmentation's ``apply_transform`` is responsible + for expanding it to the full batch size. + + Returns: + Dictionary with key ``"gaussian_noise"`` containing the noise tensor. + """ + batch_size, C, H, W = batch_shape + _common_param_check(batch_size, same_on_batch) + + device = self.device if self.device is not None else torch.device("cpu") + dtype = self.dtype if self.dtype is not None else torch.get_default_dtype() + + dist = Normal( + torch.tensor(self.mean, device=device, dtype=dtype), + torch.tensor(self.std, device=device, dtype=dtype), + ) + + # When same_on_batch, generate a single (1, C, H, W) map — apply_transform will expand it. + sample_B = 1 if same_on_batch else batch_size + gaussian_noise = dist.rsample((sample_B, C, H, W)) + + return {"gaussian_noise": gaussian_noise} diff --git a/kornia/augmentation/random_generator/_2d/jigsaw.py b/kornia/augmentation/random_generator/_2d/jigsaw.py new file mode 100644 index 0000000..96a4ec1 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/jigsaw.py @@ -0,0 +1,75 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Tuple + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase +from kornia.augmentation.random_generator.utils import randperm +from kornia.augmentation.utils import _common_param_check + +__all__ = ["JigsawGenerator"] + + +class JigsawGenerator(RandomGeneratorBase): + r"""Generate Jigsaw permutation indices for a batch of inputs. + + Args: + grid: the Jigsaw puzzle grid. e.g. (2, 2) means + each output will mix image patches in a 2x2 grid. + + Returns: + A dict of parameters to be passed for transformation. + - permutation (Tensor): Jigsaw permutation arrangement. + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__(self, grid: Tuple[int, int] = (4, 4), ensure_perm: bool = True) -> None: + super().__init__() + self.grid = grid + self.ensure_perm = ensure_perm + + def __repr__(self) -> str: + repr = f"grid={self.grid}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + self._device = device + self._dtype = dtype + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + _common_param_check(batch_size, same_on_batch) + + perm_times = self.grid[0] * self.grid[1] + # Generate mosiac order in one shot + if batch_size == 0: + rand_ids = torch.zeros([0, perm_times], device=self._device) + elif same_on_batch: + rand_ids = randperm(perm_times, ensure_perm=self.ensure_perm, device=self._device) + rand_ids = torch.stack([rand_ids] * batch_size) + else: + rand_ids = torch.stack( + [randperm(perm_times, ensure_perm=self.ensure_perm, device=self._device) for _ in range(batch_size)] + ) + return {"permutation": rand_ids} diff --git a/kornia/augmentation/random_generator/_2d/jpeg.py b/kornia/augmentation/random_generator/_2d/jpeg.py new file mode 100644 index 0000000..99f7d96 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/jpeg.py @@ -0,0 +1,70 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, List, Tuple, Union + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import _adapted_rsampling, _common_param_check, _joint_range_check, _range_bound +from kornia.core.utils import _extract_device_dtype + +__all__ = ["JPEGGenerator"] + + +class JPEGGenerator(RandomGeneratorBase): + r"""Generate random JPEG augmentation parameters for a batch. + + Args: + jpeg_quality: The RandomJPEG quality to apply + + Returns: + A dict of parameters to be passed for transformation. + - jpeg_quality: element-wise contrast factors with a shape of (B,). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, + jpeg_quality: Union[torch.Tensor, float, Tuple[float, float], List[float]] = 50.0, + ) -> None: + super().__init__() + self.jpeg_quality: Union[torch.Tensor, float, Tuple[float, float], List[float]] = jpeg_quality + + def __repr__(self) -> str: + return f"RandomJPEG quality={self.jpeg_quality}" + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + jpeg_quality = _range_bound( + self.jpeg_quality, "jpeg_quality", center=50.0, bounds=(1, 100), device=device, dtype=dtype + ) + + _joint_range_check(jpeg_quality, "jpeg_quality", (1, 100)) + + self.jpeg_quality_sampler = UniformDistribution(jpeg_quality[0], jpeg_quality[1], validate_args=False) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + _common_param_check(batch_size, same_on_batch) + _device, _dtype = _extract_device_dtype([self.jpeg_quality]) + jpeg_quality_value = _adapted_rsampling((batch_size,), self.jpeg_quality_sampler, same_on_batch) + return {"jpeg_quality": jpeg_quality_value.to(device=_device, dtype=_dtype)} diff --git a/kornia/augmentation/random_generator/_2d/linear_illumination.py b/kornia/augmentation/random_generator/_2d/linear_illumination.py new file mode 100644 index 0000000..7d6b024 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/linear_illumination.py @@ -0,0 +1,208 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import _adapted_rsampling, _common_param_check, _range_bound +from kornia.core.utils import _extract_device_dtype +from kornia.enhance.normalize import normalize_min_max + + +class LinearIlluminationGenerator(RandomGeneratorBase): + r"""Generates random 2D Linear illumination patterns for image augmentation. + + Args: + gain: Range for the gain factor applied to the generated illumination. + sign: Range for the sign of the Linear distribution. + + Returns: + A dictionary of parameters to be passed for transformation. + - gradient: : Generated 2D Linear illumination pattern with shape (B, C, H, W). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, + gain: tuple[float, float], + sign: tuple[float, float], + ) -> None: + super().__init__() + self.gain = gain + self.sign = sign + + def __repr__(self) -> str: + r"""Return a string representation of the object.""" + repr = f"gain={self.gain}, sign={self.sign}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + r"""Create samplers for generating random gaussian illumination parameters.""" + gain = _range_bound( + self.gain, + "gain", + ).to(device, dtype) + self.gain_sampler = UniformDistribution(gain[0], gain[1], validate_args=False) + + sign = _range_bound(self.sign, "sign", bounds=(-1.0, 1.0), center=0.0).to(device, dtype) + self.sign_sampler = UniformDistribution(sign[0], sign[1], validate_args=False) + + self.directions_sampler = UniformDistribution(0, 4, validate_args=False) + + def forward(self, batch_shape: tuple[int, ...], same_on_batch: bool = False) -> dict[str, torch.Tensor]: + r"""Generate random 2D Gaussian illumination patterns.""" + batch_size, channels, height, width = batch_shape + _common_param_check(batch_size, same_on_batch) + _device, _dtype = _extract_device_dtype([self.gain, self.sign]) + + # Random gain and sign + gain_factor = _adapted_rsampling((batch_size, 1, 1, 1), self.gain_sampler, same_on_batch).to( + device=_device, dtype=_dtype + ) + sign = torch.where( + _adapted_rsampling((batch_size, 1, 1, 1), self.sign_sampler, same_on_batch) >= 0.0, + torch.tensor(1, device=_device, dtype=_dtype), + torch.tensor(-1, device=_device, dtype=_dtype), + ) + + # Directions (0=lower,1=upper,2=left,3=right), shape [B,1,1,1] + directions = _adapted_rsampling((batch_size, 1, 1, 1), self.directions_sampler, same_on_batch).to( + device=_device, dtype=torch.int8 + ) + + # Precompute 1D ramps + ramp_h = torch.linspace(0, 1, height, device=_device, dtype=_dtype).view(1, 1, height, 1) + ramp_h_rev = torch.linspace(1, 0, height, device=_device, dtype=_dtype).view(1, 1, height, 1) + ramp_w = torch.linspace(0, 1, width, device=_device, dtype=_dtype).view(1, 1, 1, width) + ramp_w_rev = torch.linspace(1, 0, width, device=_device, dtype=_dtype).view(1, 1, 1, width) + + # Broadcast masks for each direction + d = directions.to(torch.int64) # [B,1,1,1] + m0 = (d == 0).to(_dtype) # lower + m1 = (d == 1).to(_dtype) # upper + m2 = (d == 2).to(_dtype) # left + m3 = (d == 3).to(_dtype) # right + + # Build [B,1,H,W] gradient in one shot + grad_b1 = m0 * ramp_h + m1 * ramp_h_rev + m2 * ramp_w + m3 * ramp_w_rev + + # Expand to [B,C,H,W] + gradient = grad_b1.expand(batch_size, channels, height, width) + + # Apply sign and gain + gradient = sign * gain_factor * gradient + + return {"gradient": gradient.to(device=_device, dtype=_dtype)} + + +class LinearCornerIlluminationGenerator(RandomGeneratorBase): + r"""Generates random 2D Linear (from corner) illumination patterns for image augmentation. + + Args: + gain: Range for the gain factor applied to the generated illumination. + sign: Range for the sign of the linear distribution. + + Returns: + A dictionary of parameters to be passed for transformation. + - gradient: : Generated 2D Linear illumination pattern with shape (B, C, H, W). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, + gain: tuple[float, float], + sign: tuple[float, float], + ) -> None: + super().__init__() + self.gain = gain + self.sign = sign + + def __repr__(self) -> str: + r"""Return a string representation of the object.""" + repr = f"gain={self.gain}, sign={self.sign}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + r"""Create samplers for generating random gaussian illumination parameters.""" + gain = _range_bound( + self.gain, + "gain", + ).to(device, dtype) + self.gain_sampler = UniformDistribution(gain[0], gain[1], validate_args=False) + + sign = _range_bound(self.sign, "sign", bounds=(-1.0, 1.0), center=0.0).to(device, dtype) + self.sign_sampler = UniformDistribution(sign[0], sign[1], validate_args=False) + + self.directions_sampler = UniformDistribution(0, 4, validate_args=False) + + def forward(self, batch_shape: tuple[int, ...], same_on_batch: bool = False) -> dict[str, torch.Tensor]: + r"""Generate random 2D Gaussian illumination patterns.""" + batch_size, channels, height, width = batch_shape + _common_param_check(batch_size, same_on_batch) + _device, _dtype = _extract_device_dtype([self.gain, self.sign]) + + gain_factor = _adapted_rsampling((batch_size, 1, 1, 1), self.gain_sampler, same_on_batch).to( + device=_device, dtype=_dtype + ) + sign = torch.where( + _adapted_rsampling((batch_size, 1, 1, 1), self.sign_sampler, same_on_batch) >= 0.0, + torch.tensor(1, device=_device, dtype=_dtype), + torch.tensor(-1, device=_device, dtype=_dtype), + ) + + directions = _adapted_rsampling((batch_size, 1, 1, 1), self.directions_sampler, same_on_batch).to( + device=_device, + dtype=torch.long, # int8 → long for gather + ) + + # Compute base gradients + y_grad = torch.linspace(0, 1, height, device=_device, dtype=_dtype).unsqueeze(1).expand(height, width) + x_grad = torch.linspace(0, 1, width, device=_device, dtype=_dtype).unsqueeze(0).expand(height, width) + + base = torch.stack( + [ + x_grad + y_grad, # 0: Bottom right + -x_grad + y_grad, # 1: Bottom left + x_grad - y_grad, # 2: Upper right + 1 - (x_grad + y_grad), # 3: Upper left + ], + dim=0, + ) # (4, H, W) + + # Expand to (4, C, H, W) + base = base.unsqueeze(1).expand(-1, channels, -1, -1) + + # Index according to directions + gradient = base[directions.view(-1), :, :, :] # (B, C, H, W) + + gradient = sign * gain_factor * normalize_min_max(gradient) + + return { + "gradient": gradient.to(device=_device, dtype=_dtype), + } diff --git a/kornia/augmentation/random_generator/_2d/mixup.py b/kornia/augmentation/random_generator/_2d/mixup.py new file mode 100644 index 0000000..949c814 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/mixup.py @@ -0,0 +1,83 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Optional, Tuple, Union + +import torch +from torch.distributions import Bernoulli + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import _adapted_rsampling, _adapted_sampling, _common_param_check, _joint_range_check +from kornia.core.utils import _extract_device_dtype + +__all__ = ["MixupGenerator"] + + +class MixupGenerator(RandomGeneratorBase): + r"""Generate mixup indexes and lambdas for a batch of inputs. + + Args: + lambda_val (torch.Tensor, optional): min-max strength for mixup images, ranged from [0., 1.]. + If None, it will be set to tensor([0., 1.]), which means no restrictions. + + Returns: + A dict of parameters to be passed for transformation. + - mix_pairs (torch.Tensor): element-wise probabilities with a shape of (B,). + - mixup_lambdas (torch.Tensor): element-wise probabilities with a shape of (B,). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__(self, lambda_val: Optional[Union[torch.Tensor, Tuple[float, float]]] = None, p: float = 1.0) -> None: + super().__init__() + self.lambda_val = lambda_val + self.p = p + + def __repr__(self) -> str: + repr = f"lambda_val={self.lambda_val}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + if self.lambda_val is None: + lambda_val = torch.tensor([0.0, 1.0], device=device, dtype=dtype) + else: + lambda_val = torch.as_tensor(self.lambda_val, device=device, dtype=dtype) + + _joint_range_check(lambda_val, "lambda_val", bounds=(0, 1)) + self.lambda_sampler = UniformDistribution(lambda_val[0], lambda_val[1], validate_args=False) + self.prob_sampler = Bernoulli(torch.tensor(float(self.p), device=device, dtype=dtype)) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + + _common_param_check(batch_size, same_on_batch) + _device, _dtype = _extract_device_dtype([self.lambda_val]) + + with torch.no_grad(): + batch_probs: torch.Tensor = _adapted_sampling((batch_size,), self.prob_sampler, same_on_batch) + mixup_pairs: torch.Tensor = torch.randperm(batch_size, device=_device, dtype=_dtype).long() + mixup_lambdas: torch.Tensor = _adapted_rsampling((batch_size,), self.lambda_sampler, same_on_batch) + mixup_lambdas = mixup_lambdas * batch_probs + + return { + "mixup_pairs": mixup_pairs.to(device=_device, dtype=torch.long), + "mixup_lambdas": mixup_lambdas.to(device=_device, dtype=_dtype), + } diff --git a/kornia/augmentation/random_generator/_2d/mosaic.py b/kornia/augmentation/random_generator/_2d/mosaic.py new file mode 100644 index 0000000..26e7176 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/mosaic.py @@ -0,0 +1,125 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Optional, Tuple + +import torch +from torch.distributions import Uniform + +from kornia.augmentation.random_generator.base import RandomGeneratorBase +from kornia.augmentation.utils import _adapted_rsampling, _common_param_check +from kornia.core.utils import _extract_device_dtype +from kornia.geometry.bbox import bbox_generator + +__all__ = ["MosaicGenerator"] + + +class MosaicGenerator(RandomGeneratorBase): + r"""Generate mixup indexes and lambdas for a batch of inputs. + + Args: + output_size: the output torch.Tensor width and height after mosaicing. + mosaic_grid: the number of images and image arrangement. e.g. (2, 2) means + each output will mix 4 images in a 2x2 grid. + start_ratio_range: top-left (x, y) position for cropping the mosaic images. + + Returns: + A dict of parameters to be passed for transformation. + - mosaic_ids (torch.Tensor): a shape of (B, N) torch.tensor, where n is the number of mosaic images. + - src (torch.Tensor): cropping bounding boxes with a shape of (B, 4, 2). + - dst (torch.Tensor): output bounding boxes with a shape (B, 4, 2). + - batch_shapes (torch.Tensor): image shapes in the batch with a shape of (B, 3). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, + output_size: Optional[Tuple[int, int]] = None, + mosaic_grid: Tuple[int, int] = (2, 2), + start_ratio_range: Tuple[float, float] = (0.3, 0.7), + ) -> None: + super().__init__() + self.output_size = output_size + self.mosaic_grid = mosaic_grid + self.start_ratio_range = start_ratio_range + + def __repr__(self) -> str: + repr = ( + f"output_size={self.output_size}, mosaic_grid={self.mosaic_grid}, " + f"start_ratio_range={self.start_ratio_range}" + ) + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + self.start_ratio_range_sampler = Uniform( + torch.tensor(self.start_ratio_range[0], device=device, dtype=dtype), + torch.tensor(self.start_ratio_range[1], device=device, dtype=dtype), + validate_args=False, + ) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + input_sizes = (batch_shape[-2], batch_shape[-1]) + # output_size = input_sizes if self.output_size is None else self.output_size + + _common_param_check(batch_size, same_on_batch) + _device, _dtype = _extract_device_dtype([self.mosaic_grid]) + + perm_times = self.mosaic_grid[0] * self.mosaic_grid[1] + # Generate mosiac order in one shot + rand_ids = torch.randperm(batch_size * (perm_times - 1), device=_device) % batch_size + mosiac_ids = ( + torch.cat([torch.arange(0, batch_size, device=_device), rand_ids]) + .reshape(perm_times, batch_size) + .permute(1, 0) + ) + + start_corner_factor = _adapted_rsampling( + (batch_size, 2), self.start_ratio_range_sampler, same_on_batch=False + ).to(device=_device, dtype=_dtype) + start_corner_x = start_corner_factor[:, 0] * batch_shape[-2] + start_corner_y = start_corner_factor[:, 1] * batch_shape[-1] + crop_src = bbox_generator( + start_corner_x, + start_corner_y, + start_corner_x.clone().fill_(input_sizes[0]), + start_corner_y.clone().fill_(input_sizes[1]), + ) + crop_dst = torch.tensor( + [[[0, 0], [input_sizes[1] - 1, 0], [input_sizes[1] - 1, input_sizes[0] - 1], [0, input_sizes[0] - 1]]], + device=_device, + dtype=_dtype, + ).repeat(batch_size, 1, 1) + # NOTE: In case we support a list of torch.Tensor images later. For a better consistency. + # B x 3 + + batch_shapes: torch.Tensor + if batch_size == 0: + batch_shapes = torch.zeros([0, 3], device=_device, dtype=torch.long) + else: + batch_shapes = torch.stack([torch.as_tensor(batch_shape[1:], device=_device) for _ in range(batch_size)]) + return { + "permutation": mosiac_ids.to(device=_device, dtype=torch.long), + "src": crop_src, + "dst": crop_dst, + "batch_shapes": batch_shapes, + } diff --git a/kornia/augmentation/random_generator/_2d/motion_blur.py b/kornia/augmentation/random_generator/_2d/motion_blur.py new file mode 100644 index 0000000..5752f5d --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/motion_blur.py @@ -0,0 +1,106 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Tuple, Union + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import _adapted_rsampling, _common_param_check, _range_bound +from kornia.core.utils import _extract_device_dtype + +__all__ = ["MotionBlurGenerator"] + + +class MotionBlurGenerator(RandomGeneratorBase): + r"""Get parameters for motion blur. + + Args: + kernel_size: motion kernel size (odd and positive). + If int, the kernel will have a fixed size. + If Tuple[int, int], it will randomly generate the value from the range batch-wisely. + angle: angle of the motion blur in degrees (anti-clockwise rotation). + If float, it will generate the value from (-angle, angle). + direction: forward/backward direction of the motion blur. + Lower values towards -1.0 will point the motion blur towards the back (with angle provided via angle), + while higher values towards 1.0 will point the motion blur forward. A value of 0.0 leads to a + uniformly (but still angled) motion blur. + If float, it will generate the value from (-direction, direction). + If Tuple[int, int], it will randomly generate the value from the range. + + Returns: + A dict of parameters to be passed for transformation. + - ksize_factor (torch.Tensor): element-wise kernel size factors with a shape of (B,). + - angle_factor (torch.Tensor): element-wise angle factors with a shape of (B,). + - direction_factor (torch.Tensor): element-wise direction factors with a shape of (B,). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, + kernel_size: Union[int, Tuple[int, int]], + angle: Union[torch.Tensor, float, Tuple[float, float]], + direction: Union[torch.Tensor, float, Tuple[float, float]], + ) -> None: + super().__init__() + self.kernel_size = kernel_size + self.angle = angle + self.direction = direction + + def __repr__(self) -> str: + repr = f"kernel_size={self.kernel_size}, angle={self.angle}, direction={self.direction}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + angle = _range_bound(self.angle, "angle", center=0.0, bounds=(-360, 360)).to(device=device, dtype=dtype) + direction = _range_bound(self.direction, "direction", center=0.0, bounds=(-1, 1)).to(device=device, dtype=dtype) + if isinstance(self.kernel_size, int): + if not (self.kernel_size >= 3 and self.kernel_size % 2 == 1): + raise AssertionError(f"`kernel_size` must be odd and greater than 3. Got {self.kernel_size}.") + self.ksize_sampler = UniformDistribution(self.kernel_size // 2, self.kernel_size // 2, validate_args=False) + elif isinstance(self.kernel_size, tuple): + # kernel_size is fixed across the batch + if len(self.kernel_size) != 2: + raise AssertionError(f"`kernel_size` must be (2,) if it is a tuple. Got {self.kernel_size}.") + self.ksize_sampler = UniformDistribution( + self.kernel_size[0] // 2, self.kernel_size[1] // 2, validate_args=False + ) + else: + raise TypeError(f"Unsupported type: {type(self.kernel_size)}") + + self.angle_sampler = UniformDistribution(angle[0], angle[1], validate_args=False) + self.direction_sampler = UniformDistribution(direction[0], direction[1], validate_args=False) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + _common_param_check(batch_size, same_on_batch) + # self.ksize_factor.expand((batch_size, -1)) + _device, _dtype = _extract_device_dtype([self.angle, self.direction]) + angle_factor = _adapted_rsampling((batch_size,), self.angle_sampler, same_on_batch) + direction_factor = _adapted_rsampling((batch_size,), self.direction_sampler, same_on_batch) + ksize_factor = _adapted_rsampling((batch_size,), self.ksize_sampler, same_on_batch).int() * 2 + 1 + + return { + "ksize_factor": ksize_factor.to(device=_device, dtype=torch.int32), + "angle_factor": angle_factor.to(device=_device, dtype=_dtype), + "direction_factor": direction_factor.to(device=_device, dtype=_dtype), + } diff --git a/kornia/augmentation/random_generator/_2d/patchmix.py b/kornia/augmentation/random_generator/_2d/patchmix.py new file mode 100644 index 0000000..6b37911 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/patchmix.py @@ -0,0 +1,123 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Tuple + +import torch +from torch.distributions import Beta, Uniform + +from kornia.augmentation.random_generator.base import RandomGeneratorBase +from kornia.augmentation.utils import _adapted_sampling, _common_param_check +from kornia.core.utils import _extract_device_dtype + +__all__ = ["PatchMixGenerator"] + + +class PatchMixGenerator(RandomGeneratorBase): + r"""Generate patchmix indexes and lambdas for a batch of inputs. + + Args: + alpha (float): hyperparameter for generating cut size from beta distribution. + patch_size (int): size of the patch to be swapped. + p (float): probability of applying patchmix. + + Returns: + params Dict[str, torch.Tensor]: parameters to be passed for transformation. + - mix_pairs (torch.Tensor): element-wise probabilities with a shape of (B). + - patch_coords (torch.Tensor): top-left coordinates of the patch (B, 2). + - lam (torch.Tensor): mixing parameter (B). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + """ + + def __init__( + self, + alpha: float = 1.0, + patch_size: int = 16, + p: float = 1.0, + ) -> None: + super().__init__() + self.alpha = alpha + self.patch_size = patch_size + self.p = p + + def __repr__(self) -> str: + repr = f"alpha={self.alpha}, patch_size={self.patch_size}, p={self.p}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + self.beta_sampler = Beta( + torch.tensor(self.alpha, device=device, dtype=dtype), + torch.tensor(self.alpha, device=device, dtype=dtype), + ) + self.rand_sampler = Uniform( + torch.tensor(0.0, device=device, dtype=dtype), + torch.tensor(1.0, device=device, dtype=dtype), + ) + self.pair_sampler = Uniform( + torch.tensor(0.0, device=device, dtype=dtype), + torch.tensor(1.0, device=device, dtype=dtype), + ) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + height = batch_shape[-2] + width = batch_shape[-1] + + _device, _dtype = _extract_device_dtype([self.alpha]) + _common_param_check(batch_size, same_on_batch) + + if batch_size == 0: + return { + "mix_pairs": torch.zeros([0], device=_device, dtype=torch.long), + "patch_coords": torch.zeros([0, 2], device=_device, dtype=torch.long), + "lam": torch.zeros([0], device=_device, dtype=_dtype), + } + + with torch.no_grad(): + mix_pairs: torch.Tensor = ( + _adapted_sampling((batch_size,), self.pair_sampler, same_on_batch) + .to(device=_device, dtype=_dtype) + .argsort(dim=0) + ) + # Sample lam from beta distribution + lam = _adapted_sampling((batch_size,), self.beta_sampler, same_on_batch).to(device=_device, dtype=_dtype) + + # Sample patch coordinates + # height - patch_size + 1 + max_y = height - self.patch_size + max_x = width - self.patch_size + + y = ( + (_adapted_sampling((batch_size,), self.rand_sampler, same_on_batch) * (max_y + 1)) + .floor() + .to(torch.long) + ) + x = ( + (_adapted_sampling((batch_size,), self.rand_sampler, same_on_batch) * (max_x + 1)) + .floor() + .to(torch.long) + ) + + return { + "mix_pairs": mix_pairs.to(device=_device, dtype=torch.long), + "patch_coords": torch.stack([x, y], dim=1).to(device=_device, dtype=torch.long), + "lam": lam, + } diff --git a/kornia/augmentation/random_generator/_2d/perspective.py b/kornia/augmentation/random_generator/_2d/perspective.py new file mode 100644 index 0000000..a6cabf4 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/perspective.py @@ -0,0 +1,105 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Tuple, Union + +import torch +from torch.distributions import Uniform + +from kornia.augmentation.random_generator.base import RandomGeneratorBase +from kornia.augmentation.utils import _adapted_rsampling, _check_positive_int_or_traced, _common_param_check +from kornia.core.utils import _extract_device_dtype + +__all__ = ["PerspectiveGenerator"] + + +class PerspectiveGenerator(RandomGeneratorBase): + r"""Get parameters for ``perspective`` for a random perspective transform. + + Args: + distortion_scale: the degree of distortion, ranged from 0 to 1. + sampling_method: ``'basic'`` | ``'area_preserving'``. Default: ``'basic'`` + If ``'basic'``, samples by translating the image corners randomly inwards. + If ``'area_preserving'``, samples by randomly translating the image corners in any direction. + Preserves area on average. See https://arxiv.org/abs/2104.03308 for further details. + + Returns: + A dict of parameters to be passed for transformation. + - start_points (torch.Tensor): element-wise perspective source areas with a shape of (B, 4, 2). + - end_points (torch.Tensor): element-wise perspective target areas with a shape of (B, 4, 2). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__(self, distortion_scale: Union[torch.Tensor, float] = 0.5, sampling_method: str = "basic") -> None: + super().__init__() + if sampling_method not in ("basic", "area_preserving"): + raise NotImplementedError(f"Sampling method {sampling_method} not yet implemented.") + self.distortion_scale = distortion_scale + self.sampling_method = sampling_method + + def __repr__(self) -> str: + repr = f"distortion_scale={self.distortion_scale}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + self._distortion_scale = torch.as_tensor(self.distortion_scale, device=device, dtype=dtype) + if not (self._distortion_scale.dim() == 0 and 0 <= self._distortion_scale <= 1): + raise AssertionError(f"'distortion_scale' must be a scalar within [0, 1]. Got {self._distortion_scale}.") + self.rand_val_sampler = Uniform( + torch.tensor(0, device=device, dtype=dtype), + torch.tensor(1, device=device, dtype=dtype), + validate_args=False, + ) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + height = batch_shape[-2] + width = batch_shape[-1] + + _device, _dtype = _extract_device_dtype([self.distortion_scale]) + _common_param_check(batch_size, same_on_batch) + _check_positive_int_or_traced(height, "height") + _check_positive_int_or_traced(width, "width") + + start_points: torch.Tensor = torch.tensor( + [[[0.0, 0], [width - 1, 0], [width - 1, height - 1], [0, height - 1]]], device=_device, dtype=_dtype + ).expand(batch_size, -1, -1) + + # generate random offset not larger than half of the image + fx = self._distortion_scale * width / 2 + fy = self._distortion_scale * height / 2 + + factor = torch.stack([fx, fy], dim=0).view(-1, 1, 2).to(device=_device, dtype=_dtype) + + # TODO: This line somehow breaks the gradcheck + rand_val: torch.Tensor = _adapted_rsampling(start_points.shape, self.rand_val_sampler, same_on_batch).to( + device=_device, dtype=_dtype + ) + if self.sampling_method == "basic": + pts_norm = torch.tensor([[[1, 1], [-1, 1], [-1, -1], [1, -1]]], device=_device, dtype=_dtype) + offset = factor * rand_val * pts_norm + elif self.sampling_method == "area_preserving": + offset = 2 * factor * (rand_val - 0.5) + + end_points = start_points + offset + + return {"start_points": start_points, "end_points": end_points} diff --git a/kornia/augmentation/random_generator/_2d/plain_uniform.py b/kornia/augmentation/random_generator/_2d/plain_uniform.py new file mode 100644 index 0000000..5981f8e --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/plain_uniform.py @@ -0,0 +1,101 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple + +import torch +from torch.distributions import Distribution + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import _adapted_rsampling, _common_param_check, _range_bound +from kornia.core.utils import _extract_device_dtype + +__all__ = ["ParameterBound", "PlainUniformGenerator"] + +# factor, name, center, range +ParameterBound = Tuple[Any, str, Optional[float], Optional[Tuple[float, float]]] + + +class PlainUniformGenerator(RandomGeneratorBase): + r"""Generate random parameters that distributed uniformly. + + Args: + *samplers: a list of tuple in a pattern of ``(factor, name, center, range)``, in which + the factor can be a two-numbered tuple, or a ``(2,)`` shaped torch torch.Tensor. The name + will be the corresponding key of the returning dict. The center and range must be + both provided worked as a validator to the given factor. + + Returns: + A dict of parameters to be passed for transformation according the number of samplers + and the pointed returning name of each tuple. + - ``name``: element-wise probabilities with a shape of (B,). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + Example: + >>> _ = torch.manual_seed(44) + >>> PlainUniformGenerator( + ... ((0., 1.), "factor_1", None, None), + ... (torch.tensor([-0.5, 0.5]), "factor_2", 0.1, (-1., 1.)), + ... )(torch.Size([2])) + {'factor_1': tensor([0.7196, 0.7307]), 'factor_2': tensor([ 0.3278, -0.3657])} + + """ + + def __init__(self, *samplers: ParameterBound) -> None: + super().__init__() + self.samplers = samplers + names = [] + for factor, name, center, bound in samplers: + if name in names: + raise RuntimeError(f"factor name `{name}` has already been registered. Please check the duplication.") + names.append(name) + if isinstance(factor, torch.nn.Parameter): + self.register_parameter(name, factor) + elif isinstance(factor, torch.Tensor): + self.register_buffer(name, factor) + else: + factor = _range_bound(factor, name, center=center, bounds=bound) + self.register_buffer(name, factor) + + def __repr__(self) -> str: + repr = ", ".join([f"{name}={factor}" for factor, name, _, _ in self.samplers]) + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + self.sampler_dict: Dict[str, Distribution] = {} + for factor, name, center, bound in self.samplers: + if center is None and bound is None: + factor = torch.as_tensor(factor, device=device, dtype=dtype) + elif center is None or bound is None: + raise ValueError(f"`center` and `bound` should be both None or provided. Got {center} and {bound}.") + else: + factor = _range_bound(factor, name, center=center, bounds=bound, device=device, dtype=dtype) + self.sampler_dict.update({name: UniformDistribution(factor[0], factor[1], validate_args=False)}) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + _common_param_check(batch_size, same_on_batch) + _device, _dtype = _extract_device_dtype([t for t, _, _, _ in self.samplers]) + + return { + name: _adapted_rsampling((batch_size,), dist, same_on_batch).to(device=_device, dtype=_dtype) + for name, dist in self.sampler_dict.items() + } diff --git a/kornia/augmentation/random_generator/_2d/planckian_jitter.py b/kornia/augmentation/random_generator/_2d/planckian_jitter.py new file mode 100644 index 0000000..b3b41be --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/planckian_jitter.py @@ -0,0 +1,46 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, List, Tuple + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import _adapted_rsampling, _common_param_check, _joint_range_check, _range_bound + +__all__ = ["PlanckianJitterGenerator"] + + +class PlanckianJitterGenerator(RandomGeneratorBase): + r"""Generate random planckian jitter parameters for a batch of images.""" + + def __init__(self, domain: List[float]) -> None: + super().__init__() + self.domain = domain + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + idx_range = _range_bound(self.domain, "idx_range", device=device, dtype=dtype) + + _joint_range_check(idx_range, "idx_range", (0, self.domain[1])) + self.pl_idx_dist = UniformDistribution(idx_range[0], idx_range[1], validate_args=False) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + _common_param_check(batch_size, same_on_batch) + pl_idx = _adapted_rsampling((batch_size,), self.pl_idx_dist, same_on_batch) + + return {"idx": pl_idx.long()} diff --git a/kornia/augmentation/random_generator/_2d/posterize.py b/kornia/augmentation/random_generator/_2d/posterize.py new file mode 100644 index 0000000..11b1be7 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/posterize.py @@ -0,0 +1,71 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Tuple, Union + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import _adapted_rsampling, _common_param_check, _joint_range_check +from kornia.core.utils import _extract_device_dtype + +__all__ = ["PosterizeGenerator"] + + +class PosterizeGenerator(RandomGeneratorBase): + r"""Generate random posterize parameters for a batch of images. + + Args: + bits: floats that ranged from (0, 8], in which 0 gives black image and 8 gives the original. + If float x, bits will be generated from (x, 8). + If tuple (x, y), bits will be generated from (x, y). + + Returns: + A dict of parameters to be passed for transformation. + - bits_factor (torch.Tensor): element-wise bit factors with a shape of (B,). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__(self, bits: Union[float, Tuple[float, float], torch.Tensor]) -> None: + super().__init__() + self.bits_factor = bits + + def __repr__(self) -> str: + repr = f"bits={self.bits_factor}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + bits = torch.as_tensor(self.bits_factor, device=device, dtype=dtype) + if len(bits.size()) == 0: + bits = bits.repeat(2) + bits[1] = 8 + elif not (len(bits.size()) == 1 and bits.size(0) == 2): + raise ValueError(f"'bits' shall be either a scalar or a length 2 torch.Tensor. Got {bits}.") + _joint_range_check(bits, "bits", (0, 8)) + self.bit_sampler = UniformDistribution(bits[0], bits[1], validate_args=False) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + _common_param_check(batch_size, same_on_batch) + _device, _ = _extract_device_dtype([self.bits_factor if isinstance(self.bits_factor, torch.Tensor) else None]) + bits_factor = _adapted_rsampling((batch_size,), self.bit_sampler, same_on_batch) + return {"bits_factor": bits_factor.round().to(device=_device, dtype=torch.int32)} diff --git a/kornia/augmentation/random_generator/_2d/probability.py b/kornia/augmentation/random_generator/_2d/probability.py new file mode 100644 index 0000000..079fe64 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/probability.py @@ -0,0 +1,97 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Optional, Tuple + +import torch +from torch.distributions import Bernoulli + +from kornia.augmentation.random_generator.base import RandomGeneratorBase +from kornia.augmentation.utils import _adapted_sampling, _common_param_check + +__all__ = ["ProbabilityGenerator"] + + +class ProbabilityGenerator(RandomGeneratorBase): + r"""Generate random probabilities for a batch of inputs. + + Args: + p: probability to generate an 1-d binary mask. Default value is 0.5. + + Returns: + A dict of parameters to be passed for transformation. + - probs (torch.Tensor): element-wise probabilities with a shape of (B,). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__(self, p: float = 0.5) -> None: + super().__init__() + self.p = p + + def __repr__(self) -> str: + repr = f"p={self.p}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + p = torch.tensor(float(self.p), device=device, dtype=dtype) + self.sampler = Bernoulli(p) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + probs_mask: torch.Tensor = _adapted_sampling((batch_size,), self.sampler, same_on_batch).bool() + return {"probs": probs_mask} + + +def random_prob_generator( + batch_size: int, + p: float = 0.5, + same_on_batch: bool = False, + device: Optional[torch.device] = None, + dtype: torch.dtype = torch.float32, +) -> torch.Tensor: + r"""Generate random probabilities for a batch of inputs. + + Args: + batch_size (int): the number of images. + p (float): probability to generate an 1-d binary mask. Default value is 0.5. + same_on_batch (bool): apply the same transformation across the batch. Default: False. + device (torch.device): the device on which the random numbers will be generated. Default: cpu. + dtype (torch.dtype): the data type of the generated random numbers. Default: float32. + + Returns: + torch.Tensor: parameters to be passed for transformation. + - probs (torch.Tensor): element-wise probabilities with a shape of (B,). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. + + """ + if device is None: + device = torch.device("cpu") + _common_param_check(batch_size, same_on_batch) + if not isinstance(p, (int, float)) or p > 1 or p < 0: + raise TypeError(f"The probability should be a float number within [0, 1]. Got {type(p)}.") + + _bernoulli = Bernoulli(torch.tensor(float(p), device=device, dtype=dtype)) + probs_mask: torch.Tensor = _adapted_sampling((batch_size,), _bernoulli, same_on_batch).bool() + + return probs_mask diff --git a/kornia/augmentation/random_generator/_2d/random_rain.py b/kornia/augmentation/random_generator/_2d/random_rain.py new file mode 100644 index 0000000..9d60776 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/random_rain.py @@ -0,0 +1,92 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import _adapted_rsampling, _common_param_check, _range_bound +from kornia.core.utils import _extract_device_dtype + + +class RainGenerator(RandomGeneratorBase): + def __init__( + self, number_of_drops: tuple[int, int], drop_height: tuple[int, int], drop_width: tuple[int, int] + ) -> None: + super().__init__() + self.number_of_drops = number_of_drops + self.drop_height = drop_height + self.drop_width = drop_width + + def __repr__(self) -> str: + repr = f"number_of_drops={self.number_of_drops}, drop_height={self.drop_height}, drop_width={self.drop_width}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + number_of_drops = _range_bound( + self.number_of_drops, + "number_of_drops", + center=self.number_of_drops[0] / 2 + self.number_of_drops[1] / 2, + bounds=(self.number_of_drops[0], self.number_of_drops[1] + 1), + ).to(device) + drop_height = _range_bound( + self.drop_height, + "drop_height", + center=self.drop_height[0] / 2 + self.drop_height[1] / 2, + bounds=(self.drop_height[0], self.drop_height[1] + 1), + ).to(device) + drop_width = _range_bound( + self.drop_width, + "drop_width", + center=self.drop_width[0] / 2 + self.drop_width[1] / 2, + bounds=(self.drop_width[0], self.drop_width[1] + 1), + ).to(device) + + drop_coordinates = _range_bound((0, 1), "drops_coordinate", center=0.5, bounds=(0, 1)).to( + device=device, dtype=dtype + ) + self.number_of_drops_sampler = UniformDistribution(number_of_drops[0], number_of_drops[1], validate_args=False) + self.drop_height_sampler = UniformDistribution(drop_height[0], drop_height[1], validate_args=False) + self.drop_width_sampler = UniformDistribution(drop_width[0], drop_width[1], validate_args=False) + self.coordinates_sampler = UniformDistribution(drop_coordinates[0], drop_coordinates[1], validate_args=False) + + def forward(self, batch_shape: tuple[int, ...], same_on_batch: bool = False) -> dict[str, torch.Tensor]: + batch_size = batch_shape[0] + _common_param_check(batch_size, same_on_batch) + _device, _dtype = _extract_device_dtype([self.drop_width, self.drop_height, self.number_of_drops]) + # self.ksize_factor.expand((batch_size, -1)) + number_of_drops_factor = _adapted_rsampling((batch_size,), self.number_of_drops_sampler).to( + device=_device, dtype=torch.long + ) + drop_height_factor = _adapted_rsampling((batch_size,), self.drop_height_sampler, same_on_batch).to( + device=_device, dtype=torch.long + ) + drop_width_factor = _adapted_rsampling((batch_size,), self.drop_width_sampler, same_on_batch).to( + device=_device, dtype=torch.long + ) + coordinates_factor = _adapted_rsampling( + (batch_size, int(number_of_drops_factor.max().item()) if number_of_drops_factor.numel() > 0 else 0, 2), + self.coordinates_sampler, + same_on_batch=same_on_batch, + ).to(device=_device) + return { + "number_of_drops_factor": number_of_drops_factor, + "coordinates_factor": coordinates_factor, + "drop_height_factor": drop_height_factor, + "drop_width_factor": drop_width_factor, + } diff --git a/kornia/augmentation/random_generator/_2d/rectangle_earase.py b/kornia/augmentation/random_generator/_2d/rectangle_earase.py new file mode 100644 index 0000000..c089e5e --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/rectangle_earase.py @@ -0,0 +1,160 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Tuple, Union + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import ( + _adapted_rsampling, + _check_positive_int_or_traced, + _common_param_check, + _joint_range_check, +) +from kornia.core.utils import _extract_device_dtype + +__all__ = ["RectangleEraseGenerator"] + + +class RectangleEraseGenerator(RandomGeneratorBase): + r"""Get parameters for ```erasing``` transformation for erasing transform. + + Args: + scale (torch.Tensor): range of size of the origin size cropped. Shape (2). + ratio (torch.Tensor): range of aspect ratio of the origin aspect ratio cropped. Shape (2). + value (float): value to be filled in the erased area. + + Returns: + A dict of parameters to be passed for transformation. + - widths (torch.Tensor): element-wise erasing widths with a shape of (B,). + - heights (torch.Tensor): element-wise erasing heights with a shape of (B,). + - xs (torch.Tensor): element-wise erasing x coordinates with a shape of (B,). + - ys (torch.Tensor): element-wise erasing y coordinates with a shape of (B,). + - values (torch.Tensor): element-wise filling values with a shape of (B,). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, + scale: Union[torch.Tensor, Tuple[float, float]] = (0.02, 0.33), + ratio: Union[torch.Tensor, Tuple[float, float]] = (0.3, 3.3), + value: float = 0.0, + ) -> None: + super().__init__() + self.scale = scale + self.ratio = ratio + self.value = value + + def __repr__(self) -> str: + repr = f"scale={self.scale}, resize_to={self.ratio}, value={self.value}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + scale = torch.as_tensor(self.scale, device=device, dtype=dtype) + ratio = torch.as_tensor(self.ratio, device=device, dtype=dtype) + + if not (isinstance(self.value, (int, float)) and self.value >= 0 and self.value <= 1): + raise AssertionError(f"'value' must be a number between 0 - 1. Got {self.value}.") + _joint_range_check(scale, "scale", bounds=(0, float("inf"))) + _joint_range_check(ratio, "ratio", bounds=(0, float("inf"))) + + self.scale_sampler = UniformDistribution(scale[0], scale[1], validate_args=False) + + if ratio[0] < 1.0 and ratio[1] > 1.0: + self.ratio_sampler1 = UniformDistribution(ratio[0], 1, validate_args=False) + self.ratio_sampler2 = UniformDistribution(1, ratio[1], validate_args=False) + self.index_sampler = UniformDistribution( + torch.tensor(0, device=device, dtype=dtype), + torch.tensor(1, device=device, dtype=dtype), + validate_args=False, + ) + else: + self.ratio_sampler = UniformDistribution(ratio[0], ratio[1], validate_args=False) + self.uniform_sampler = UniformDistribution( + torch.tensor(0, device=device, dtype=dtype), + torch.tensor(1, device=device, dtype=dtype), + validate_args=False, + ) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + height = batch_shape[-2] + width = batch_shape[-1] + _check_positive_int_or_traced(height, "height") + _check_positive_int_or_traced(width, "width") + + _common_param_check(batch_size, same_on_batch) + _device, _dtype = _extract_device_dtype([self.ratio, self.scale]) + images_area = height * width + target_areas = ( + _adapted_rsampling((batch_size,), self.scale_sampler, same_on_batch).to(device=_device, dtype=_dtype) + * images_area + ) + + if self.ratio[0] < 1.0 and self.ratio[1] > 1.0: + aspect_ratios1 = _adapted_rsampling((batch_size,), self.ratio_sampler1, same_on_batch) + aspect_ratios2 = _adapted_rsampling((batch_size,), self.ratio_sampler2, same_on_batch) + if same_on_batch: + rand_idxs = ( + torch.round(_adapted_rsampling((1,), self.index_sampler, same_on_batch)).repeat(batch_size).bool() + ) + else: + rand_idxs = torch.round(_adapted_rsampling((batch_size,), self.index_sampler, same_on_batch)).bool() + aspect_ratios = torch.where(rand_idxs, aspect_ratios1, aspect_ratios2) + else: + aspect_ratios = _adapted_rsampling((batch_size,), self.ratio_sampler, same_on_batch) + + aspect_ratios = aspect_ratios.to(device=_device, dtype=_dtype) + + # based on target areas and aspect ratios, rectangle params are computed + heights = torch.min( + torch.max( + torch.round((target_areas * aspect_ratios) ** (1 / 2)), torch.tensor(1.0, device=_device, dtype=_dtype) + ), + torch.tensor(height, device=_device, dtype=_dtype), + ) + + widths = torch.min( + torch.max( + torch.round((target_areas / aspect_ratios) ** (1 / 2)), torch.tensor(1.0, device=_device, dtype=_dtype) + ), + torch.tensor(width, device=_device, dtype=_dtype), + ) + + xs_ratio = _adapted_rsampling((batch_size,), self.uniform_sampler, same_on_batch).to( + device=_device, dtype=_dtype + ) + ys_ratio = _adapted_rsampling((batch_size,), self.uniform_sampler, same_on_batch).to( + device=_device, dtype=_dtype + ) + + xs = xs_ratio * (width - widths + 1) + ys = ys_ratio * (height - heights + 1) + + return { + "widths": widths.floor(), + "heights": heights.floor(), + "xs": xs.floor(), + "ys": ys.floor(), + "values": torch.tensor([self.value] * batch_size, device=_device, dtype=_dtype), + } diff --git a/kornia/augmentation/random_generator/_2d/resize.py b/kornia/augmentation/random_generator/_2d/resize.py new file mode 100644 index 0000000..ef8dad7 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/resize.py @@ -0,0 +1,111 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Tuple, Union + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase +from kornia.augmentation.utils import _common_param_check +from kornia.geometry.bbox import bbox_generator +from kornia.geometry.transform.affwarp import _side_to_image_size + +__all__ = ["ResizeGenerator"] + + +class ResizeGenerator(RandomGeneratorBase): + r"""Get parameters for ```resize``` transformation for resize transform. + + Args: + resize_to: Desired output size of the crop, like (h, w). + side: Which side to resize if `resize_to` is only of type int. + + Returns: + parameters to be passed for transformation. + - src (torch.Tensor): cropping bounding boxes with a shape of (B, 4, 2). + - dst (torch.Tensor): output bounding boxes with a shape (B, 4, 2). + - input_size (torch.Tensor): (h, w) from batch input. + - resize_to (tuple): new (h, w) for batch input. + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__(self, resize_to: Union[int, Tuple[int, int]], side: str = "short") -> None: + super().__init__() + self.output_size = resize_to + self.side = side + + def __repr__(self) -> str: + repr = f"output_size={self.output_size}" + return repr + + def make_samplers(self, device: Union[str, torch.device, None], dtype: torch.dtype) -> None: + self.device = device + self.dtype = dtype + pass + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + _common_param_check(batch_size, same_on_batch) + _device = self.device + _dtype = self.dtype + + if batch_size == 0: + return { + "src": torch.zeros([0, 4, 2], device=_device, dtype=_dtype), + "dst": torch.zeros([0, 4, 2], device=_device, dtype=_dtype), + } + + input_size = h, w = (batch_shape[-2], batch_shape[-1]) + + src = bbox_generator( + torch.tensor(0, device=_device, dtype=_dtype), + torch.tensor(0, device=_device, dtype=_dtype), + torch.tensor(input_size[1], device=_device, dtype=_dtype), + torch.tensor(input_size[0], device=_device, dtype=_dtype), + ).repeat(batch_size, 1, 1) + + if isinstance(self.output_size, int): + aspect_ratio = w / h + output_size = _side_to_image_size(self.output_size, aspect_ratio, self.side) + else: + output_size = self.output_size + + if not ( + len(output_size) == 2 + and isinstance(output_size[0], (int,)) + and isinstance(output_size[1], (int,)) + and output_size[0] > 0 + and output_size[1] > 0 + ): + raise AssertionError(f"`resize_to` must be a tuple of 2 positive integers. Got {output_size}.") + + dst = bbox_generator( + torch.tensor(0, device=_device, dtype=_dtype), + torch.tensor(0, device=_device, dtype=_dtype), + torch.tensor(output_size[1], device=_device, dtype=_dtype), + torch.tensor(output_size[0], device=_device, dtype=_dtype), + ).repeat(batch_size, 1, 1) + + _input_size = torch.tensor(input_size, device=_device, dtype=torch.long).expand(batch_size, -1) + _output_size = torch.tensor(output_size, device=_device, dtype=torch.long).expand(batch_size, -1) + + return {"src": src, "dst": dst, "input_size": _input_size, "output_size": _output_size} diff --git a/kornia/augmentation/random_generator/_2d/salt_pepper_noise.py b/kornia/augmentation/random_generator/_2d/salt_pepper_noise.py new file mode 100644 index 0000000..f803209 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/salt_pepper_noise.py @@ -0,0 +1,110 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import _adapted_rsampling, _adapted_uniform, _common_param_check, _range_bound +from kornia.core.utils import _extract_device_dtype + + +class SaltAndPepperGenerator(RandomGeneratorBase): + r"""Generate random Salt and Pepper noise parameters for a batch of images. + + Args: + amount: A tuple representing the range for the amount of noise to apply. + salt_and_pepper: A tuple representing the range for the ratio of Salt and Pepper noise. + + Returns: + A dictionary of parameters to be passed for transformation. + - amount_factor: Element-wise factors determining the amount of noise with a shape of (B,). + - salt_and_pepper_factor: Element-wise factors determining the ratio of Salt and Pepper noise + with a shape of (B,). + - mask_salt: Binary masks (bool) indicating the presence of Salt noise with a shape of (B, C, H, W). + - mask_pepper: Binary masks (bool) indicating the presence of Pepper noise with a shape of (B, C, H, W). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, + amount: tuple[float, float], + salt_and_pepper: tuple[float, float], + ) -> None: + super().__init__() + self.amount = amount + self.salt_and_pepper = salt_and_pepper + + def __repr__(self) -> str: + r"""Return a string representation of the object.""" + repr = f"amount={self.amount}, salt_and_pepper={self.salt_and_pepper}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + r"""Create samplers for generating random noise parameters.""" + amount = _range_bound( + self.amount, + "amount", + ).to(device, dtype) + salt_and_pepper = _range_bound( + self.salt_and_pepper, + "salt_and_pepper", + ).to(device, dtype) + + self.amount_sampler = UniformDistribution(amount[0], amount[1], validate_args=False) + self.salt_and_pepper_sampler = UniformDistribution(salt_and_pepper[0], salt_and_pepper[1], validate_args=False) + + def forward(self, batch_shape: tuple[int, ...], same_on_batch: bool = False) -> dict[str, torch.Tensor]: + r"""Generate Salt and Pepper noise masks for a batch of images.""" + batch_size, C, H, W = batch_shape + _common_param_check(batch_size, same_on_batch) + _device, _dtype = _extract_device_dtype([self.amount, self.salt_and_pepper]) + + amount_factor = _adapted_rsampling((batch_size,), self.amount_sampler, same_on_batch).to( + device=_device, dtype=_dtype + ) + salt_and_pepper_factor = _adapted_rsampling((batch_size,), self.salt_and_pepper_sampler, same_on_batch).to( + device=_device, dtype=_dtype + ) + + ## Generate noise masks. + mask_noise = _adapted_uniform( + (batch_size, 1, H, W), low=0.0, high=1.0, same_on_batch=same_on_batch + ) < amount_factor.view(batch_size, 1, 1, 1) + mask_salt = _adapted_uniform( + (batch_size, 1, H, W), low=0.0, high=1.0, same_on_batch=same_on_batch + ) < salt_and_pepper_factor.view(batch_size, 1, 1, 1) + + # If the number of channels is greater than one (3), replicate the generated mask for each channel. + if C > 1: + mask_noise = mask_noise.repeat(1, C, 1, 1) + mask_salt = mask_salt.repeat(1, C, 1, 1) + mask_pepper = (~mask_salt & mask_noise).to(device=_device) + mask_salt = (mask_salt & mask_noise).to(device=_device) + + return { + "amount_factor": amount_factor, + "salt_and_pepper_factor": salt_and_pepper_factor, + "mask_salt": mask_salt, + "mask_pepper": mask_pepper, + } diff --git a/kornia/augmentation/random_generator/_2d/shear.py b/kornia/augmentation/random_generator/_2d/shear.py new file mode 100644 index 0000000..8a1fe5f --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/shear.py @@ -0,0 +1,115 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Tuple, Union + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import ( + _adapted_rsampling, + _check_positive_int_or_traced, + _common_param_check, + _joint_range_check, + _range_bound, +) +from kornia.core.utils import _extract_device_dtype + +__all__ = ["ShearGenerator"] + + +class ShearGenerator(RandomGeneratorBase): + r"""Get parameters for ``shear`` for a random shear transform. + + Args: + shear: Range of degrees to select from. + If float, a shear parallel to the x axis in the range (-shear, +shear) will be applied. + If (a, b), a shear parallel to the x axis in the range (-shear, +shear) will be applied. + If (a, b, c, d), then x-axis shear in (shear[0], shear[1]) and y-axis shear in (shear[2], shear[3]) + will be applied. Will not apply shear by default. + If torch.Tensor, shear is a 2x2 torch.Tensor, a x-axis shear in (shear[0][0], shear[0][1]) and + y-axis shear in + (shear[1][0], shear[1][1]) will be applied. Will not apply shear by default. + + Returns: + A dict of parameters to be passed for transformation. + - shear_x (torch.Tensor): element-wise x-axis shears with a shape of (B,). + - shear_y (torch.Tensor): element-wise y-axis shears with a shape of (B,). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, shear: Union[torch.Tensor, float, Tuple[float, float], Tuple[float, float, float, float]] + ) -> None: + super().__init__() + self.shear = shear + + def __repr__(self) -> str: + repr = f"shear={self.shear}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + shear = torch.as_tensor(self.shear, device=device, dtype=dtype) + if shear.shape == torch.Size([2, 2]): + _shear = shear + else: + _shear = torch.stack( + [ + _range_bound(shear if shear.dim() == 0 else shear[:2], "shear-x", 0, (-360, 360)), + ( + torch.tensor([0, 0], device=device, dtype=dtype) + if shear.dim() == 0 or len(shear) == 2 + else _range_bound(shear[2:], "shear-y", 0, (-360, 360)) + ), + ] + ) + + _joint_range_check(_shear[0], "shear") + _joint_range_check(_shear[1], "shear") + self.shear_x = _shear[0].clone() + self.shear_y = _shear[1].clone() + + shear_x_sampler = UniformDistribution(_shear[0][0], _shear[0][1], validate_args=False) + shear_y_sampler = UniformDistribution(_shear[1][0], _shear[1][1], validate_args=False) + + self.shear_x_sampler = shear_x_sampler + self.shear_y_sampler = shear_y_sampler + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + height = batch_shape[-2] + width = batch_shape[-1] + + _device, _dtype = _extract_device_dtype([self.shear]) + _common_param_check(batch_size, same_on_batch) + _check_positive_int_or_traced(width, "width") + _check_positive_int_or_traced(height, "height") + + center: torch.Tensor = torch.tensor([width, height], device=_device, dtype=_dtype).view(1, 2) / 2.0 - 0.5 + center = center.expand(batch_size, -1) + + sx = _adapted_rsampling((batch_size,), self.shear_x_sampler, same_on_batch) + sy = _adapted_rsampling((batch_size,), self.shear_y_sampler, same_on_batch) + sx = sx.to(device=_device, dtype=_dtype) + sy = sy.to(device=_device, dtype=_dtype) + + return {"center": center, "shear_x": sx, "shear_y": sy} diff --git a/kornia/augmentation/random_generator/_2d/translate.py b/kornia/augmentation/random_generator/_2d/translate.py new file mode 100644 index 0000000..1d808b9 --- /dev/null +++ b/kornia/augmentation/random_generator/_2d/translate.py @@ -0,0 +1,119 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import ( + _adapted_rsampling, + _check_positive_int_or_traced, + _common_param_check, + _range_bound, +) +from kornia.core.utils import _extract_device_dtype + +__all__ = ["TranslateGenerator"] + + +class TranslateGenerator(RandomGeneratorBase): + r"""Get parameters for ``translate`` for a random translate transform. + + Args: + translate: tuple of maximum absolute fraction for horizontal + and vertical translations. For example translate=(a, b), then horizontal shift + is randomly sampled in the range -img_width * a < dx < img_width * a and vertical shift is + randomly sampled in the range -img_height * b < dy < img_height * b. Will not translate by default. + + Returns: + A dict of parameters to be passed for transformation. + - translations (torch.Tensor): element-wise translations with a shape of (B, 2). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, + translate_x: Optional[Union[torch.Tensor, Tuple[float, float]]] = None, + translate_y: Optional[Union[torch.Tensor, Tuple[float, float]]] = None, + ) -> None: + super().__init__() + self.translate_x = translate_x + self.translate_y = translate_y + + def __repr__(self) -> str: + repr = f"translate_x={self.translate_x}, translate_y={self.translate_y}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + self.translate_x_sampler = None + self.translate_y_sampler = None + + if self.translate_x is not None: + _translate_x = _range_bound(self.translate_x, "translate_x", bounds=(-1, 1), check="joint").to( + device=device, dtype=dtype + ) + + self.translate_x_sampler = UniformDistribution( + _translate_x[..., 0], _translate_x[..., 1], validate_args=False + ) + + if self.translate_y is not None: + _translate_y = _range_bound(self.translate_y, "translate_y", bounds=(-1, 1), check="joint").to( + device=device, dtype=dtype + ) + + self.translate_y_sampler = UniformDistribution( + _translate_y[..., 0], _translate_y[..., 1], validate_args=False + ) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + height = batch_shape[-2] + width = batch_shape[-1] + + _device, _dtype = _extract_device_dtype([self.translate_x, self.translate_y]) + _common_param_check(batch_size, same_on_batch) + _check_positive_int_or_traced(width, "width") + _check_positive_int_or_traced(height, "height") + + if self.translate_x_sampler is not None: + translate_x = ( + _adapted_rsampling((batch_size,), self.translate_x_sampler, same_on_batch).to( + device=_device, dtype=_dtype + ) + * width + ) + else: + translate_x = torch.zeros((batch_size,), device=_device, dtype=_dtype) + + if self.translate_y_sampler is not None: + translate_y = ( + _adapted_rsampling((batch_size,), self.translate_y_sampler, same_on_batch).to( + device=_device, dtype=_dtype + ) + * height + ) + else: + translate_y = torch.zeros((batch_size,), device=_device, dtype=_dtype) + + return {"translate_x": translate_x, "translate_y": translate_y} diff --git a/kornia/augmentation/random_generator/_3d/__init__.py b/kornia/augmentation/random_generator/_3d/__init__.py new file mode 100644 index 0000000..4987f56 --- /dev/null +++ b/kornia/augmentation/random_generator/_3d/__init__.py @@ -0,0 +1,22 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from kornia.augmentation.random_generator._3d.affine import * +from kornia.augmentation.random_generator._3d.crop import * +from kornia.augmentation.random_generator._3d.motion_blur import * +from kornia.augmentation.random_generator._3d.perspective import * +from kornia.augmentation.random_generator._3d.rotation import * diff --git a/kornia/augmentation/random_generator/_3d/affine.py b/kornia/augmentation/random_generator/_3d/affine.py new file mode 100644 index 0000000..b90c041 --- /dev/null +++ b/kornia/augmentation/random_generator/_3d/affine.py @@ -0,0 +1,240 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Optional, Tuple, Union + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import ( + _adapted_rsampling, + _check_positive_int_or_traced, + _singular_range_check, + _tuple_range_reader, +) +from kornia.core.utils import _extract_device_dtype + + +class AffineGenerator3D(RandomGeneratorBase): + r"""Get parameters for ```3d affine``` transformation random affine transform. + + Args: + degrees: Range of yaw (x-axis), pitch (y-axis), roll (z-axis) to select from. + If degrees is a number, then yaw, pitch, roll will be generated from the range of (-degrees, +degrees). + If degrees is a tuple of (min, max), then yaw, pitch, roll will be generated from the range of (min, max). + If degrees is a list of floats [a, b, c], then yaw, pitch, roll will be generated from (-a, a), (-b, b) + and (-c, c). + If degrees is a list of tuple ((a, b), (m, n), (x, y)), then yaw, pitch, roll will be generated from + (a, b), (m, n) and (x, y). + Set to 0 to deactivate rotations. + translate: tuple of maximum absolute fraction for horizontal, vertical and + depthical translations (dx,dy,dz). For example translate=(a, b, c), then + horizontal shift will be randomly sampled in the range -img_width * a < dx < img_width * a + vertical shift will be randomly sampled in the range -img_height * b < dy < img_height * b. + depthical shift will be randomly sampled in the range -img_depth * c < dz < img_depth * c. + Will not translate by default. + scale: scaling factor interval. + If (a, b) represents isotropic scaling, the scale is randomly sampled from the range a <= scale <= b. + If ((a, b), (c, d), (e, f)), the scale is randomly sampled from the range a <= scale_x <= b, + c <= scale_y <= d, e <= scale_z <= f. Will keep original scale by default. + shears: Range of degrees to select from. + If shear is a number, a shear to the 6 facets in the range (-shear, +shear) will be applied. + If shear is a tuple of 2 values, a shear to the 6 facets in the range (shear[0], shear[1]) will be applied. + If shear is a tuple of 6 values, a shear to the i-th facet in the range (-shear[i], shear[i]) + will be applied. + If shear is a tuple of 6 tuples, a shear to the i-th facet in the range (-shear[i, 0], shear[i, 1]) + will be applied. + + Returns: + A dict of parameters to be passed for transformation. + - translations (torch.Tensor): element-wise translations with a shape of (B, 3). + - center (torch.Tensor): element-wise center with a shape of (B, 3). + - scale (torch.Tensor): element-wise scales with a shape of (B, 3). + - angle (torch.Tensor): element-wise rotation angles with a shape of (B, 3). + - sxy (torch.Tensor): element-wise x-y-facet shears with a shape of (B,). + - sxz (torch.Tensor): element-wise x-z-facet shears with a shape of (B,). + - syx (torch.Tensor): element-wise y-x-facet shears with a shape of (B,). + - syz (torch.Tensor): element-wise y-z-facet shears with a shape of (B,). + - szx (torch.Tensor): element-wise z-x-facet shears with a shape of (B,). + - szy (torch.Tensor): element-wise z-y-facet shears with a shape of (B,). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, + degrees: Union[ + torch.Tensor, + float, + Tuple[float, float], + Tuple[float, float, float], + Tuple[Tuple[float, float], Tuple[float, float], Tuple[float, float]], + ], + translate: Optional[Union[torch.Tensor, Tuple[float, float, float]]] = None, + scale: Optional[ + Union[ + torch.Tensor, Tuple[float, float], Tuple[Tuple[float, float], Tuple[float, float], Tuple[float, float]] + ] + ] = None, + shears: Union[ + None, + torch.Tensor, + float, + Tuple[float, float], + Tuple[float, float, float, float, float, float], + Tuple[ + Tuple[float, float], + Tuple[float, float], + Tuple[float, float], + Tuple[float, float], + Tuple[float, float], + Tuple[float, float], + ], + ] = None, + ) -> None: + super().__init__() + self.degrees = degrees + self.shears = shears + self.translate = translate + self.scale = scale + + def __repr__(self) -> str: + repr = f"degrees={self.degrees}, shears={self.shears}, translate={self.translate}, scale={self.scale}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + degrees = _tuple_range_reader(self.degrees, 3, device, dtype) + shear: Optional[torch.Tensor] = None + if self.shears is not None: + shear = _tuple_range_reader(self.shears, 6, device, dtype) + self.sxy_sampler = UniformDistribution(shear[0, 0], shear[0, 1], validate_args=False) + self.sxz_sampler = UniformDistribution(shear[1, 0], shear[1, 1], validate_args=False) + self.syx_sampler = UniformDistribution(shear[2, 0], shear[2, 1], validate_args=False) + self.syz_sampler = UniformDistribution(shear[3, 0], shear[3, 1], validate_args=False) + self.szx_sampler = UniformDistribution(shear[4, 0], shear[4, 1], validate_args=False) + self.szy_sampler = UniformDistribution(shear[5, 0], shear[5, 1], validate_args=False) + + # check translation range + self._translate: Optional[torch.Tensor] = None + if self.translate is not None: + self._translate = torch.as_tensor(self.translate, device=device, dtype=dtype) + _singular_range_check(self._translate, "translate", bounds=(0, 1), mode="3d") + + # check scale range + self._scale: Optional[torch.Tensor] = None + if self.scale is not None: + _scale = torch.as_tensor(self.scale, device=device, dtype=dtype) + if _scale.shape == torch.Size([2]): + self._scale = _scale.unsqueeze(0).repeat(3, 1) + elif _scale.shape != torch.Size([3, 2]): + raise ValueError(f"'scale' shall be either shape (2) or (3, 2). Got {self.scale}.") + else: + self._scale = _scale + _singular_range_check(self._scale[0], "scale-x", bounds=(0, float("inf")), mode="2d") + _singular_range_check(self._scale[1], "scale-y", bounds=(0, float("inf")), mode="2d") + _singular_range_check(self._scale[2], "scale-z", bounds=(0, float("inf")), mode="2d") + self.scale_1_sampler = UniformDistribution(self._scale[0, 0], self._scale[0, 1], validate_args=False) + self.scale_2_sampler = UniformDistribution(self._scale[1, 0], self._scale[1, 1], validate_args=False) + self.scale_3_sampler = UniformDistribution(self._scale[2, 0], self._scale[2, 1], validate_args=False) + + self.yaw_sampler = UniformDistribution(degrees[0][0], degrees[0][1], validate_args=False) + self.pitch_sampler = UniformDistribution(degrees[1][0], degrees[1][1], validate_args=False) + self.roll_sampler = UniformDistribution(degrees[2][0], degrees[2][1], validate_args=False) + + self.uniform_sampler = UniformDistribution( + torch.tensor(0, device=device, dtype=dtype), + torch.tensor(1, device=device, dtype=dtype), + validate_args=False, + ) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + depth = batch_shape[-3] + height = batch_shape[-2] + width = batch_shape[-1] + + _check_positive_int_or_traced(depth, "depth") + _check_positive_int_or_traced(height, "height") + _check_positive_int_or_traced(width, "width") + + _device, _dtype = _extract_device_dtype([self.degrees, self.translate, self.scale, self.shears]) + + # degrees = degrees.to(device=device, dtype=dtype) + yaw = _adapted_rsampling((batch_size,), self.yaw_sampler, same_on_batch) + pitch = _adapted_rsampling((batch_size,), self.pitch_sampler, same_on_batch) + roll = _adapted_rsampling((batch_size,), self.roll_sampler, same_on_batch) + angles = torch.stack([yaw, pitch, roll], dim=1) + + # compute tensor ranges + if self._scale is not None: + scale = torch.stack( + [ + _adapted_rsampling((batch_size,), self.scale_1_sampler, same_on_batch), + _adapted_rsampling((batch_size,), self.scale_2_sampler, same_on_batch), + _adapted_rsampling((batch_size,), self.scale_3_sampler, same_on_batch), + ], + dim=1, + ) + else: + scale = torch.ones(batch_size, device=_device, dtype=_dtype).reshape(batch_size, 1).repeat(1, 3) + + if self._translate is not None: + max_dx: torch.Tensor = self._translate[0] * width + max_dy: torch.Tensor = self._translate[1] * height + max_dz: torch.Tensor = self._translate[2] * depth + # translations should be in x,y,z + translations = torch.stack( + [ + (_adapted_rsampling((batch_size,), self.uniform_sampler, same_on_batch) - 0.5) * max_dx * 2, + (_adapted_rsampling((batch_size,), self.uniform_sampler, same_on_batch) - 0.5) * max_dy * 2, + (_adapted_rsampling((batch_size,), self.uniform_sampler, same_on_batch) - 0.5) * max_dz * 2, + ], + dim=1, + ) + else: + translations = torch.zeros((batch_size, 3), device=_device, dtype=_dtype) + + # center should be in x,y,z + center: torch.Tensor = torch.tensor([width, height, depth], device=_device, dtype=_dtype).view(1, 3) / 2.0 - 0.5 + center = center.expand(batch_size, -1) + + if self.shears is not None: + sxy = _adapted_rsampling((batch_size,), self.sxy_sampler, same_on_batch) + sxz = _adapted_rsampling((batch_size,), self.sxz_sampler, same_on_batch) + syx = _adapted_rsampling((batch_size,), self.syx_sampler, same_on_batch) + syz = _adapted_rsampling((batch_size,), self.syz_sampler, same_on_batch) + szx = _adapted_rsampling((batch_size,), self.szx_sampler, same_on_batch) + szy = _adapted_rsampling((batch_size,), self.szy_sampler, same_on_batch) + else: + sxy = sxz = syx = syz = szx = szy = torch.tensor([0] * batch_size, device=_device, dtype=_dtype) + + return { + "translations": torch.as_tensor(translations, device=_device, dtype=_dtype), + "center": torch.as_tensor(center, device=_device, dtype=_dtype), + "scale": torch.as_tensor(scale, device=_device, dtype=_dtype), + "angles": torch.as_tensor(angles, device=_device, dtype=_dtype), + "sxy": torch.as_tensor(sxy, device=_device, dtype=_dtype), + "sxz": torch.as_tensor(sxz, device=_device, dtype=_dtype), + "syx": torch.as_tensor(syx, device=_device, dtype=_dtype), + "syz": torch.as_tensor(syz, device=_device, dtype=_dtype), + "szx": torch.as_tensor(szx, device=_device, dtype=_dtype), + "szy": torch.as_tensor(szy, device=_device, dtype=_dtype), + } diff --git a/kornia/augmentation/random_generator/_3d/crop.py b/kornia/augmentation/random_generator/_3d/crop.py new file mode 100644 index 0000000..0434982 --- /dev/null +++ b/kornia/augmentation/random_generator/_3d/crop.py @@ -0,0 +1,262 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Optional, Tuple, Union + +import torch +from torch.distributions import Uniform + +from kornia.augmentation.random_generator.base import RandomGeneratorBase +from kornia.augmentation.utils import _adapted_rsampling, _check_positive_int_or_traced, _common_param_check +from kornia.core.utils import _extract_device_dtype +from kornia.geometry.bbox import bbox_generator3d + + +class CropGenerator3D(RandomGeneratorBase): + r"""Get parameters for ```crop``` transformation for crop transform. + + Args: + size (tuple): Desired size of the crop operation, like (d, h, w). + If torch.Tensor, it must be (B, 3). + resize_to (tuple): Desired output size of the crop, like (d, h, w). If None, no resize will be performed. + + Returns: + A dict of parameters to be passed for transformation. + - src (torch.Tensor): cropping bounding boxes with a shape of (B, 8, 3). + - dst (torch.Tensor): output bounding boxes with a shape (B, 8, 3). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, size: Union[Tuple[int, int, int], torch.Tensor], resize_to: Optional[Tuple[int, int, int]] = None + ) -> None: + super().__init__() + self.size = size + self.resize_to = resize_to + + def __repr__(self) -> str: + repr = f"crop_size={self.size}" + if self.resize_to is not None: + repr += f", resize_to={self.resize_to}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + self.rand_sampler = Uniform( + torch.tensor(0.0, device=device, dtype=dtype), torch.tensor(1.0, device=device, dtype=dtype) + ) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size, _, depth, height, width = batch_shape + _common_param_check(batch_size, same_on_batch) + _device, _dtype = _extract_device_dtype([self.size if isinstance(self.size, torch.Tensor) else None]) + + if not isinstance(self.size, torch.Tensor): + size = torch.tensor(self.size, device=_device, dtype=_dtype).repeat(batch_size, 1) + else: + size = self.size.to(device=_device, dtype=_dtype) + if size.shape != torch.Size([batch_size, 3]): + raise AssertionError( + "If `size` is a torch.Tensor, it must be shaped as (B, 3). " + f"Got {size.shape} while expecting {torch.Size([batch_size, 3])}." + ) + if not ( + isinstance(depth, (int,)) + and isinstance(height, (int,)) + and isinstance(width, (int,)) + and depth > 0 + and height > 0 + and width > 0 + ): + raise AssertionError(f"`batch_shape` should not contain negative values. Got {(batch_shape)}.") + + x_diff = width - size[:, 2] + 1 + y_diff = height - size[:, 1] + 1 + z_diff = depth - size[:, 0] + 1 + + if (x_diff < 0).any() or (y_diff < 0).any() or (z_diff < 0).any(): + raise ValueError( + f"input_size {(depth, height, width)} cannot be smaller than crop size {size!s} in any dimension." + ) + + if batch_size == 0: + return { + "src": torch.zeros([0, 8, 3], device=_device, dtype=_dtype), + "dst": torch.zeros([0, 8, 3], device=_device, dtype=_dtype), + } + + x_start = _adapted_rsampling((batch_size,), self.rand_sampler, same_on_batch).to(device=_device, dtype=_dtype) + y_start = _adapted_rsampling((batch_size,), self.rand_sampler, same_on_batch).to(device=_device, dtype=_dtype) + z_start = _adapted_rsampling((batch_size,), self.rand_sampler, same_on_batch).to(device=_device, dtype=_dtype) + + x_start = (x_start * x_diff).floor() + y_start = (y_start * y_diff).floor() + z_start = (z_start * z_diff).floor() + + crop_src = bbox_generator3d( + x_start.view(-1), y_start.view(-1), z_start.view(-1), size[:, 2] - 1, size[:, 1] - 1, size[:, 0] - 1 + ) + + if self.resize_to is None: + crop_dst = bbox_generator3d( + torch.tensor([0] * batch_size, device=_device, dtype=_dtype), + torch.tensor([0] * batch_size, device=_device, dtype=_dtype), + torch.tensor([0] * batch_size, device=_device, dtype=_dtype), + size[:, 2] - 1, + size[:, 1] - 1, + size[:, 0] - 1, + ) + else: + if not ( + len(self.resize_to) == 3 + and isinstance(self.resize_to[0], (int,)) + and isinstance(self.resize_to[1], (int,)) + and isinstance(self.resize_to[2], (int,)) + and self.resize_to[0] > 0 + and self.resize_to[1] > 0 + and self.resize_to[2] > 0 + ): + raise AssertionError(f"`resize_to` must be a tuple of 3 positive integers. Got {self.resize_to}.") + crop_dst = torch.tensor( + [ + [ + [0, 0, 0], + [self.resize_to[-1] - 1, 0, 0], + [self.resize_to[-1] - 1, self.resize_to[-2] - 1, 0], + [0, self.resize_to[-2] - 1, 0], + [0, 0, self.resize_to[-3] - 1], + [self.resize_to[-1] - 1, 0, self.resize_to[-3] - 1], + [self.resize_to[-1] - 1, self.resize_to[-2] - 1, self.resize_to[-3] - 1], + [0, self.resize_to[-2] - 1, self.resize_to[-3] - 1], + ] + ], + device=_device, + dtype=_dtype, + ).repeat(batch_size, 1, 1) + + return {"src": crop_src.to(device=_device), "dst": crop_dst.to(device=_device)} + + +def center_crop_generator3d( + batch_size: int, + depth: int, + height: int, + width: int, + size: Tuple[int, int, int], + device: Union[None, str, torch.device] = None, +) -> Dict[str, torch.Tensor]: + r"""Get parameters for ```center_crop3d``` transformation for center crop transform. + + Args: + batch_size (int): the torch.Tensor batch size. + depth (int) : depth of the image. + height (int) : height of the image. + width (int): width of the image. + size (tuple): Desired output size of the crop, like (d, h, w). + device (Union[str, torch.device, None]): the device on which the random numbers will be generated. Default: cpu. + + Returns: + params Dict[str, torch.Tensor]: parameters to be passed for transformation. + - src (torch.Tensor): cropping bounding boxes with a shape of (B, 8, 3). + - dst (torch.Tensor): output bounding boxes with a shape (B, 8, 3). + + Note: + No random number will be generated. + + """ + if device is None: + device = torch.device("cpu") + if not isinstance(size, (tuple, list)) or len(size) != 3: + raise ValueError(f"Input size must be a tuple/list of length 3. Got {size}") + _check_positive_int_or_traced(depth, "depth") + _check_positive_int_or_traced(height, "height") + _check_positive_int_or_traced(width, "width") + if ( + isinstance(depth, int) + and isinstance(height, int) + and isinstance(width, int) + and not (depth >= size[0] and height >= size[1] and width >= size[2]) + ): + raise AssertionError(f"Crop size must be smaller than input size. Got ({depth}, {height}, {width}) and {size}.") + + if batch_size == 0: + return {"src": torch.zeros([0, 8, 3]), "dst": torch.zeros([0, 8, 3])} + # unpack input sizes + dst_d, dst_h, dst_w = size + src_d, src_h, src_w = (depth, height, width) + + # compute start/end offsets + dst_d_half = dst_d / 2 + dst_h_half = dst_h / 2 + dst_w_half = dst_w / 2 + src_d_half = src_d / 2 + src_h_half = src_h / 2 + src_w_half = src_w / 2 + + start_x = src_w_half - dst_w_half + start_y = src_h_half - dst_h_half + start_z = src_d_half - dst_d_half + + end_x = start_x + dst_w - 1 + end_y = start_y + dst_h - 1 + end_z = start_z + dst_d - 1 + # [x, y, z] origin + # top-left-front, top-right-front, bottom-right-front, bottom-left-front + # top-left-back, top-right-back, bottom-right-back, bottom-left-back + # Note: DeprecationWarning: an integer is required (got type float). + # Implicit conversion to integers using __int__ is deprecated, and may be removed in a future version of Python. + points_src: torch.Tensor = torch.tensor( + [ + [ + [int(start_x), int(start_y), int(start_z)], + [int(end_x), int(start_y), int(start_z)], + [int(end_x), int(end_y), int(start_z)], + [int(start_x), int(end_y), int(start_z)], + [int(start_x), int(start_y), int(end_z)], + [int(end_x), int(start_y), int(end_z)], + [int(end_x), int(end_y), int(end_z)], + [int(start_x), int(end_y), int(end_z)], + ] + ], + device=device, + dtype=torch.long, + ).expand(batch_size, -1, -1) + + # [x, y, z] destination + # top-left-front, top-right-front, bottom-right-front, bottom-left-front + # top-left-back, top-right-back, bottom-right-back, bottom-left-back + points_dst: torch.Tensor = torch.tensor( + [ + [ + [0, 0, 0], + [dst_w - 1, 0, 0], + [dst_w - 1, dst_h - 1, 0], + [0, dst_h - 1, 0], + [0, 0, dst_d - 1], + [dst_w - 1, 0, dst_d - 1], + [dst_w - 1, dst_h - 1, dst_d - 1], + [0, dst_h - 1, dst_d - 1], + ] + ], + device=device, + dtype=torch.long, + ).expand(batch_size, -1, -1) + return {"src": points_src, "dst": points_dst} diff --git a/kornia/augmentation/random_generator/_3d/motion_blur.py b/kornia/augmentation/random_generator/_3d/motion_blur.py new file mode 100644 index 0000000..50e8d3b --- /dev/null +++ b/kornia/augmentation/random_generator/_3d/motion_blur.py @@ -0,0 +1,115 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Tuple, Union + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import _adapted_rsampling, _common_param_check, _range_bound, _tuple_range_reader +from kornia.core.utils import _extract_device_dtype + + +class MotionBlurGenerator3D(RandomGeneratorBase): + r"""Get parameters for motion blur. + + Args: + kernel_size: motion kernel size (odd and positive). + If int, the kernel will have a fixed size. + If Tuple[int, int], it will randomly generate the value from the range batch-wisely. + angle: angle of the motion blur in degrees (anti-clockwise rotation). + If float, it will generate the value from (-angle, angle). + direction: forward/backward direction of the motion blur. + Lower values towards -1.0 will point the motion blur towards the back (with angle provided via angle), + while higher values towards 1.0 will point the motion blur forward. A value of 0.0 leads to a + uniformly (but still angled) motion blur. + If float, it will generate the value from (-direction, direction). + If Tuple[int, int], it will randomly generate the value from the range. + + Returns: + A dict of parameters to be passed for transformation. + - ksize_factor (torch.Tensor): element-wise kernel size factors with a shape of (B,). + - angle_factor (torch.Tensor): element-wise angle factors with a shape of (B,). + - direction_factor (torch.Tensor): element-wise direction factors with a shape of (B,). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, + kernel_size: Union[int, Tuple[int, int]], + angle: Union[ + torch.Tensor, + float, + Tuple[float, float, float], + Tuple[Tuple[float, float], Tuple[float, float], Tuple[float, float]], + ], + direction: Union[torch.Tensor, float, Tuple[float, float]], + ) -> None: + super().__init__() + self.kernel_size = kernel_size + self.angle = angle + self.direction = direction + + def __repr__(self) -> str: + repr = f"kernel_size={self.kernel_size}, angle={self.angle}, direction={self.direction}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + angle: torch.Tensor = _tuple_range_reader(self.angle, 3, device=device, dtype=dtype) + direction = _range_bound(self.direction, "direction", center=0.0, bounds=(-1, 1)).to(device=device, dtype=dtype) + if isinstance(self.kernel_size, int): + if not (self.kernel_size >= 3 and self.kernel_size % 2 == 1): + raise AssertionError(f"`kernel_size` must be odd and greater than 3. Got {self.kernel_size}.") + self.ksize_sampler = UniformDistribution(self.kernel_size // 2, self.kernel_size // 2, validate_args=False) + elif isinstance(self.kernel_size, tuple): + # kernel_size is fixed across the batch + if len(self.kernel_size) != 2: + raise AssertionError(f"`kernel_size` must be (2,) if it is a tuple. Got {self.kernel_size}.") + self.ksize_sampler = UniformDistribution( + self.kernel_size[0] // 2, self.kernel_size[1] // 2, validate_args=False + ) + else: + raise TypeError(f"Unsupported type: {type(self.kernel_size)}") + + self.yaw_sampler = UniformDistribution(angle[0][0], angle[0][1], validate_args=False) + self.pitch_sampler = UniformDistribution(angle[1][0], angle[1][1], validate_args=False) + self.roll_sampler = UniformDistribution(angle[2][0], angle[2][1], validate_args=False) + self.direction_sampler = UniformDistribution(direction[0], direction[1], validate_args=False) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + _common_param_check(batch_size, same_on_batch) + # self.ksize_factor.expand((batch_size, -1)) + _device, _dtype = _extract_device_dtype([self.angle, self.direction]) + yaw_factor = _adapted_rsampling((batch_size,), self.yaw_sampler, same_on_batch) + pitch_factor = _adapted_rsampling((batch_size,), self.pitch_sampler, same_on_batch) + roll_factor = _adapted_rsampling((batch_size,), self.roll_sampler, same_on_batch) + angle_factor = torch.stack([yaw_factor, pitch_factor, roll_factor], 1) + + direction_factor = _adapted_rsampling((batch_size,), self.direction_sampler, same_on_batch) + ksize_factor = _adapted_rsampling((batch_size,), self.ksize_sampler, same_on_batch).int() * 2 + 1 + + return { + "ksize_factor": ksize_factor.to(device=_device, dtype=torch.int32), + "angle_factor": angle_factor.to(device=_device, dtype=_dtype), + "direction_factor": direction_factor.to(device=_device, dtype=_dtype), + } diff --git a/kornia/augmentation/random_generator/_3d/perspective.py b/kornia/augmentation/random_generator/_3d/perspective.py new file mode 100644 index 0000000..b5e2a75 --- /dev/null +++ b/kornia/augmentation/random_generator/_3d/perspective.py @@ -0,0 +1,108 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Tuple, Union + +import torch +from torch.distributions import Uniform + +from kornia.augmentation.random_generator.base import RandomGeneratorBase +from kornia.augmentation.utils import _adapted_rsampling, _common_param_check +from kornia.core.utils import _extract_device_dtype + + +class PerspectiveGenerator3D(RandomGeneratorBase): + r"""Get parameters for ``perspective`` for a random perspective transform. + + Args: + distortion_scale: controls the degree of distortion and ranges from 0 to 1. + + Returns: + A dict of parameters to be passed for transformation. + - src (torch.Tensor): perspective source bounding boxes with a shape of (B, 8, 3). + - dst (torch.Tensor): perspective target bounding boxes with a shape (B, 8, 3). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__(self, distortion_scale: Union[torch.Tensor, float] = 0.5) -> None: + super().__init__() + self.distortion_scale = distortion_scale + + def __repr__(self) -> str: + repr = f"distortion_scale={self.distortion_scale}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + self._distortion_scale = torch.as_tensor(self.distortion_scale, device=device, dtype=dtype) + if not (self._distortion_scale.dim() == 0 and 0 <= self._distortion_scale <= 1): + raise AssertionError(f"'distortion_scale' must be a scalar within [0, 1]. Got {self._distortion_scale}") + self.rand_sampler = Uniform( + torch.tensor(0, device=device, dtype=dtype), + torch.tensor(1, device=device, dtype=dtype), + validate_args=False, + ) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + depth = batch_shape[-3] + height = batch_shape[-2] + width = batch_shape[-1] + + _common_param_check(batch_size, same_on_batch) + _device, _dtype = _extract_device_dtype([self.distortion_scale]) + + start_points: torch.Tensor = torch.tensor( + [ + [ + [0.0, 0, 0], + [width - 1, 0, 0], + [width - 1, height - 1, 0], + [0, height - 1, 0], + [0.0, 0, depth - 1], + [width - 1, 0, depth - 1], + [width - 1, height - 1, depth - 1], + [0, height - 1, depth - 1], + ] + ], + device=_device, + dtype=_dtype, + ).expand(batch_size, -1, -1) + + # generate random offset not larger than half of the image + fx = self._distortion_scale * width / 2 + fy = self._distortion_scale * height / 2 + fz = self._distortion_scale * depth / 2 + + factor = torch.stack([fx, fy, fz], 0).view(-1, 1, 3).to(device=_device, dtype=_dtype) + + rand_val: torch.Tensor = _adapted_rsampling(start_points.shape, self.rand_sampler, same_on_batch).to( + device=_device, dtype=_dtype + ) + + pts_norm = torch.tensor( + [[[1, 1, 1], [-1, 1, 1], [-1, -1, 1], [1, -1, 1], [1, 1, -1], [-1, 1, -1], [-1, -1, -1], [1, -1, -1]]], + device=_device, + dtype=_dtype, + ) + end_points = start_points + factor * rand_val * pts_norm + + return {"start_points": start_points, "end_points": end_points} diff --git a/kornia/augmentation/random_generator/_3d/rotation.py b/kornia/augmentation/random_generator/_3d/rotation.py new file mode 100644 index 0000000..fba2d00 --- /dev/null +++ b/kornia/augmentation/random_generator/_3d/rotation.py @@ -0,0 +1,88 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Tuple, Union + +import torch + +from kornia.augmentation.random_generator.base import RandomGeneratorBase, UniformDistribution +from kornia.augmentation.utils import _adapted_rsampling, _common_param_check, _tuple_range_reader +from kornia.core.utils import _extract_device_dtype + + +class RotationGenerator3D(RandomGeneratorBase): + r"""Get parameters for ``rotate`` for a random 3D rotate transform. + + Args: + degrees: Range of yaw (x-axis), pitch (y-axis), roll (z-axis) to select from. + If degrees is a number, then yaw, pitch, roll will be generated from the range of (-degrees, +degrees). + If degrees is a tuple of (min, max), then yaw, pitch, roll will be generated from the range of (min, max). + If degrees is a list of floats [a, b, c], then yaw, pitch, roll will be generated from (-a, a), (-b, b) + and (-c, c). + If degrees is a list of tuple ((a, b), (m, n), (x, y)), then yaw, pitch, roll will be generated from + (a, b), (m, n) and (x, y). + Set to 0 to deactivate rotations. + + Returns: + A dict of parameters to be passed for transformation. + - yaw (torch.Tensor): element-wise rotation yaws with a shape of (B,). + - pitch (torch.Tensor): element-wise rotation pitches with a shape of (B,). + - roll (torch.Tensor): element-wise rotation rolls with a shape of (B,). + + Note: + The generated random numbers are not reproducible across different devices and dtypes. By default, + the parameters will be generated on CPU in float32. This can be changed by calling + ``self.set_rng_device_and_dtype(device="cuda", dtype=torch.float64)``. + + """ + + def __init__( + self, + degrees: Union[ + torch.Tensor, + float, + Tuple[float, float, float], + Tuple[Tuple[float, float], Tuple[float, float], Tuple[float, float]], + ], + ) -> None: + super().__init__() + self.degrees = degrees + + def __repr__(self) -> str: + repr = f"degrees={self.degrees}" + return repr + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + degrees = _tuple_range_reader(self.degrees, 3, device, dtype) + self.yaw_sampler = UniformDistribution(degrees[0][0], degrees[0][1], validate_args=False) + self.pitch_sampler = UniformDistribution(degrees[1][0], degrees[1][1], validate_args=False) + self.roll_sampler = UniformDistribution(degrees[2][0], degrees[2][1], validate_args=False) + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + batch_size = batch_shape[0] + _common_param_check(batch_size, same_on_batch) + _device, _dtype = _extract_device_dtype([self.degrees]) + + return { + "yaw": _adapted_rsampling((batch_size,), self.yaw_sampler, same_on_batch).to(device=_device, dtype=_dtype), + "pitch": _adapted_rsampling((batch_size,), self.pitch_sampler, same_on_batch).to( + device=_device, dtype=_dtype + ), + "roll": _adapted_rsampling((batch_size,), self.roll_sampler, same_on_batch).to( + device=_device, dtype=_dtype + ), + } diff --git a/kornia/augmentation/random_generator/__init__.py b/kornia/augmentation/random_generator/__init__.py new file mode 100644 index 0000000..bea7a1d --- /dev/null +++ b/kornia/augmentation/random_generator/__init__.py @@ -0,0 +1,26 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Augmentation Random Generator — Random parameter generators for augmentations. + +This subpackage provides random generators for 2D and 3D augmentation modules. +""" + +from kornia.augmentation.random_generator._2d import * +from kornia.augmentation.random_generator._3d import * + +from .base import DistributionWithMapper, UniformDistribution diff --git a/kornia/augmentation/random_generator/base.py b/kornia/augmentation/random_generator/base.py new file mode 100644 index 0000000..ad230c8 --- /dev/null +++ b/kornia/augmentation/random_generator/base.py @@ -0,0 +1,192 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Callable, Dict, Optional, Tuple, Type, TypeVar, Union + +import torch +from torch import nn +from torch.distributions import Distribution, Uniform + +from kornia.augmentation.utils.helpers import MultiprocessWrapper + +T = TypeVar("T") + + +class _PostInitInjectionMetaClass(type): + """To inject the ``__post_init__`` function after the creation of each instance.""" + + def __call__(cls: Type[T], *args: Any, **kwargs: Any) -> T: + obj = type.__call__(cls, *args, **kwargs) + obj.__post_init__() + return obj + + +class RandomGeneratorBase(nn.Module, metaclass=_PostInitInjectionMetaClass): + """Base class for generating random augmentation parameters.""" + + device: Union[None, str, torch.device] = None + dtype: torch.dtype + + def __init__(self) -> None: + super().__init__() + + def __post_init__(self) -> None: + self.set_rng_device_and_dtype() + + def set_rng_device_and_dtype( + self, + device: Optional[torch.device] = None, + dtype: torch.dtype = torch.float32, + ) -> None: + """Change the random generation device and dtype. + + Note: + The generated random numbers are not reproducible across different devices and dtypes. + + """ + if device is None: + device = torch.device("cpu") + if self.device != device or self.dtype != dtype: + self.make_samplers(device, dtype) + self.device = device + self.dtype = dtype + + # TODO: refine the logic with module.to() + def to(self, *args: Any, **kwargs: Any) -> "RandomGeneratorBase": + """Update sampler device and dtype using ``torch.nn.Module.to`` semantics. + + Args: + *args: Positional arguments accepted by ``Module.to``. + **kwargs: Keyword arguments accepted by ``Module.to``. + + Returns: + This generator instance. + """ + device, dtype, _, _ = torch._C._nn._parse_to(*args, **kwargs) + self.set_rng_device_and_dtype(device=device, dtype=dtype) + return self + + def make_samplers(self, device: torch.device, dtype: torch.dtype) -> None: + """Create distribution samplers for the given device and dtype. + + Args: + device: Target device. + dtype: Target floating-point dtype. + + Raises: + NotImplementedError: Subclass did not implement sampler creation. + """ + raise NotImplementedError + + def forward(self, batch_shape: Tuple[int, ...], same_on_batch: bool = False) -> Dict[str, torch.Tensor]: + """Sample random augmentation parameters. + + Args: + batch_shape: Target batch shape. + same_on_batch: If ``True``, use one sample and broadcast it across the + batch dimension. + + Returns: + Dictionary of tensors consumed by augmentation modules. + + Raises: + NotImplementedError: Subclass did not implement parameter sampling. + """ + raise NotImplementedError + + +class DistributionWithMapper(Distribution): + """Wraps a distribution with a value mapper function. + + This is used to restrict the output values of a given distribution by a value mapper function. + The value mapper function can be functions like sigmoid, tanh, etc. + + Args: + dist: the target distribution. + map_fn: the callable function to adjust the output from distributions. + + Example: + >>> from torch.distributions import Normal + >>> import torch.nn as nn + >>> # without mapper + >>> dist = DistributionWithMapper(Normal(0., 1.,), map_fn=None) + >>> _ = torch.manual_seed(0) + >>> dist.rsample((8,)) + tensor([ 1.5410, -0.2934, -2.1788, 0.5684, -1.0845, -1.3986, 0.4033, 0.8380]) + >>> # with sigmoid mapper + >>> dist = DistributionWithMapper(Normal(0., 1.,), map_fn=nn.Sigmoid()) + >>> _ = torch.manual_seed(0) + >>> dist.rsample((8,)) + tensor([0.8236, 0.4272, 0.1017, 0.6384, 0.2527, 0.1980, 0.5995, 0.6980]) + + """ + + def __init__(self, dist: Distribution, map_fn: Optional[Callable[[torch.Tensor], torch.Tensor]] = None) -> None: + self.dist = dist + self.map_fn = map_fn + + def rsample(self, sample_shape: Tuple[int, ...]) -> torch.Tensor: # type: ignore[override] + """Draw a reparameterized sample and apply ``map_fn`` when provided. + + Args: + sample_shape: Desired sample shape. + + Returns: + Sample tensor after optional mapping. + """ + out = self.dist.rsample(torch.Size(sample_shape)) + if self.map_fn is not None: + out = self.map_fn(out) + return out + + def sample(self, sample_shape: Tuple[int, ...]) -> torch.Tensor: # type: ignore[override] + """Draw a sample and apply ``map_fn`` when provided. + + Args: + sample_shape: Desired sample shape. + + Returns: + Sample tensor after optional mapping. + """ + out = self.dist.sample(torch.Size(sample_shape)) + if self.map_fn is not None: + out = self.map_fn(out) + return out + + def sample_n(self, n: int) -> torch.Tensor: + """Draw ``n`` samples and apply ``map_fn`` when provided. + + Args: + n: Number of samples. + + Returns: + Sample tensor after optional mapping. + """ + out = self.dist.sample_n(n) + if self.map_fn is not None: + out = self.map_fn(out) + return out + + def __getattr__(self, attr: str) -> Any: + try: + return getattr(self, attr) + except AttributeError: + return getattr(self.dist, attr) + + +class UniformDistribution(MultiprocessWrapper, Uniform): + """Wrapper around torch Uniform distribution which makes it work with the 'spawn' multiprocessing context.""" diff --git a/kornia/augmentation/random_generator/utils.py b/kornia/augmentation/random_generator/utils.py new file mode 100644 index 0000000..eb67e97 --- /dev/null +++ b/kornia/augmentation/random_generator/utils.py @@ -0,0 +1,29 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any + +import torch + + +def randperm(n: int, ensure_perm: bool = True, **kwargs: Any) -> torch.Tensor: + """`randomperm` with the ability to ensure the different arrangement generated.""" + perm = torch.randperm(n, **kwargs) + if ensure_perm: + while torch.all(torch.eq(perm, torch.arange(n, device=perm.device))): + perm = torch.randperm(n, **kwargs) + return perm diff --git a/kornia/augmentation/utils/__init__.py b/kornia/augmentation/utils/__init__.py new file mode 100644 index 0000000..23bda9f --- /dev/null +++ b/kornia/augmentation/utils/__init__.py @@ -0,0 +1,79 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Augmentation Utils — Helper functions and utilities for augmentations. + +This subpackage provides internal helpers and utility functions for augmentation modules. +""" + +from kornia.augmentation.utils.helpers import ( + _adapted_beta, + _adapted_rsampling, + _adapted_sampling, + _adapted_uniform, + _infer_batch_shape, + _infer_batch_shape3d, + _shape_validation, + _transform_input, + _transform_input3d, + _transform_input3d_by_shape, + _transform_input_by_shape, + _transform_output_shape, + _validate_input, + _validate_input3d, + _validate_input_dtype, + _validate_input_shape, + _validate_shape, + deepcopy_dict, + override_parameters, +) +from kornia.augmentation.utils.param_validation import ( + _check_positive_int_or_traced, + _common_param_check, + _joint_range_check, + _range_bound, + _singular_range_check, + _tuple_range_reader, +) + +__all__ = [ + "_adapted_beta", + "_adapted_rsampling", + "_adapted_sampling", + "_adapted_uniform", + "_check_positive_int_or_traced", + "_common_param_check", + "_infer_batch_shape", + "_infer_batch_shape3d", + "_joint_range_check", + "_range_bound", + "_shape_validation", + "_singular_range_check", + "_transform_input", + "_transform_input3d", + "_transform_input3d_by_shape", + "_transform_input_by_shape", + "_transform_output_shape", + "_tuple_range_reader", + "_validate_input", + "_validate_input3d", + "_validate_input_dtype", + "_validate_input_shape", + "_validate_shape", + "deepcopy_dict", + "override_parameters", +] diff --git a/kornia/augmentation/utils/helpers.py b/kornia/augmentation/utils/helpers.py new file mode 100644 index 0000000..0173265 --- /dev/null +++ b/kornia/augmentation/utils/helpers.py @@ -0,0 +1,495 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from functools import wraps +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import torch +from torch.distributions import Beta, Uniform + +from kornia.core.utils import _extract_device_dtype +from kornia.geometry.boxes import Boxes +from kornia.geometry.keypoints import Keypoints + + +def _validate_input(f: Callable[..., Any]) -> Callable[..., Any]: + r"""Validate the 2D input of the wrapped function. + + Args: + f: a function that takes the first argument as torch.Tensor. + + Returns: + the wrapped function after input is validated. + + """ + + @wraps(f) + def wrapper(input: torch.Tensor, *args: Any, **kwargs: Any) -> Any: + if not torch.is_tensor(input): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(input)}") + + _validate_shape(input.shape, required_shapes=("BCHW",)) + _validate_input_dtype(input, accepted_dtypes=[torch.bfloat16, torch.float16, torch.float32, torch.float64]) + + return f(input, *args, **kwargs) + + return wrapper + + +def _validate_input3d(f: Callable[..., Any]) -> Callable[..., Any]: + r"""Validate the 3D input of the wrapped function. + + Args: + f: a function that takes the first argument as torch.Tensor. + + Returns: + the wrapped function after input is validated. + + """ + + @wraps(f) + def wrapper(input: torch.Tensor, *args: Any, **kwargs: Any) -> Any: + if not torch.is_tensor(input): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(input)}") + + input_shape = len(input.shape) + if input_shape != 5: + raise AssertionError(f"Expect input of 5 dimensions, got {input_shape} instead") + _validate_input_dtype(input, accepted_dtypes=[torch.bfloat16, torch.float16, torch.float32, torch.float64]) + + return f(input, *args, **kwargs) + + return wrapper + + +def _infer_batch_shape(input: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]) -> torch.Size: + r"""Infer input shape. + + Input may be either (torch.Tensor,) or (torch.Tensor, transform_matrix) + """ + if isinstance(input, tuple): + tensor_var = _transform_input(input[0]) + else: + tensor_var = _transform_input(input) + return tensor_var.shape + + +def _infer_batch_shape3d(input: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]) -> torch.Size: + r"""Infer input shape. + + Input may be either (torch.Tensor,) or (torch.Tensor, transform_matrix) + """ + if isinstance(input, tuple): + tensor_var = _transform_input3d(input[0]) + else: + tensor_var = _transform_input3d(input) + return tensor_var.shape + + +def _transform_input_by_shape( + input: torch.Tensor, reference_shape: torch.Tensor, match_channel: bool = True +) -> torch.Tensor: + """Reshape an input torch.Tensor to have the same dimensions as the reference_shape. + + Arguments: + input: torch.Tensor to be transformed + reference_shape: shape used as reference + match_channel: if True, C_{src} == C_{ref}. otherwise, no constrain. C =1 by default + + """ + B = reference_shape[-4] if len(reference_shape) >= 4 else None + C = reference_shape[-3] if len(reference_shape) >= 3 else None + + if len(input.shape) == 2: + input = input.unsqueeze(0) + + if len(input.shape) == 3: + # If the first dim matches within the batch_size, add a `C` dim + # Useful to handler Masks without `C` dimensions + input = input.unsqueeze(1) if B == input.shape[-3] else input.unsqueeze(0) + + if match_channel and C: + if not input.shape[-3] == C: + raise ValueError("The C dimension of torch.Tensor did not match with the reference torch.Tensor.") + elif match_channel and C is None: + raise ValueError("The reference torch.Tensor do not have a C dimension!") + + return input + + +def _transform_input3d_by_shape( + input: torch.Tensor, reference_shape: torch.Tensor, match_channel: bool = True +) -> torch.Tensor: + """Reshape an input torch.Tensor to have the same dimensions as the reference_shape. + + Arguments: + input: torch.Tensor to be transformed + reference_shape: shape used as reference + match_channel: if True, C_{src} == C_{ref}. otherwise, no constrain. C =1 by default + + """ + B = reference_shape[-5] if len(reference_shape) >= 5 else None + C = reference_shape[-4] if len(reference_shape) >= 4 else None + + if len(input.shape) == 3: + input = input.unsqueeze(0) + + if len(input.shape) == 4 and B == input.shape[-4]: + # If the first dim matches within the batch_size, add a `C` dim + # Useful to handler Masks without `C` dimensions + input = input.unsqueeze(2) + + if match_channel and C: + if not input.shape[-4] == C: + raise ValueError("The C dimension of torch.Tensor did not match with the reference torch.Tensor.") + elif match_channel and C is None: + raise ValueError("The reference torch.Tensor do not have a C dimension!") + + return input + + +def _transform_input(input: torch.Tensor) -> torch.Tensor: + r"""Reshape an input torch.Tensor to be (*, C, H, W). Accept either (H, W), (C, H, W) or (*, C, H, W). + + Args: + input: torch.Tensor + + Returns: + torch.Tensor + + """ + if not torch.is_tensor(input): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(input)}") + + if len(input.shape) not in [2, 3, 4]: + raise ValueError(f"Input size must have a shape of either (H, W), (C, H, W) or (*, C, H, W). Got {input.shape}") + + if len(input.shape) == 2: + input = input.unsqueeze(0) + + if len(input.shape) == 3: + input = input.unsqueeze(0) + + return input + + +def _transform_input3d(input: torch.Tensor) -> torch.Tensor: + r"""Reshape an input torch.Tensor to be (*, C, D, H, W). Accept either (D, H, W), (C, D, H, W) or (*, C, D, H, W). + + Args: + input: torch.Tensor + + Returns: + torch.Tensor + + """ + if not torch.is_tensor(input): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(input)}") + + if len(input.shape) not in [3, 4, 5]: + raise ValueError( + f"Input size must have a shape of either (D, H, W), (C, D, H, W) or (*, C, D, H, W). Got {input.shape}" + ) + + if len(input.shape) == 3: + input = input.unsqueeze(0) + + if len(input.shape) == 4: + input = input.unsqueeze(0) + + return input + + +def _validate_input_dtype(input: torch.Tensor, accepted_dtypes: List[torch.dtype]) -> None: + r"""Check if the dtype of the input torch.Tensor is in the range of accepted_dtypes. + + Args: + input: torch.Tensor + accepted_dtypes: List. e.g. [torch.float32, torch.float64] + """ + if input.dtype not in accepted_dtypes: + raise TypeError(f"Expected input of {accepted_dtypes}. Got {input.dtype}") + + +def _transform_output_shape( + output: torch.Tensor, shape: Tuple[int, ...], *, reference_shape: Optional[torch.Tensor] = None +) -> torch.Tensor: + r"""Collapse the broadcasted batch dimensions an input torch.Tensor to be the specified shape. + + Args: + output: torch.Tensor + shape: List/tuple of int + reference_shape: torch.Tensor representation of shape to control which dimensions are collapsed. + + Returns: + torch.Tensor + + """ + out_tensor = output + + for dim in range(len(out_tensor.shape) - len(shape)): + idx = 0 + if reference_shape is not None and out_tensor.shape[0] == reference_shape[0] != 1 and len(shape) > 2: + idx = 1 + if out_tensor.shape[idx] != 1: + raise AssertionError(f"Dimension {dim} of input is expected to be 1, got {out_tensor.shape[idx]}") + out_tensor = out_tensor.squeeze(idx) + + return out_tensor + + +def _validate_shape(shape: Union[Tuple[int, ...], torch.Size], required_shapes: Tuple[str, ...] = ("BCHW",)) -> None: + r"""Check if the dtype of the input torch.Tensor is in the range of accepted_dtypes. + + Args: + shape: torch.Tensor shape + required_shapes: List. e.g. ["BCHW", "BCDHW"] + """ + passed = False + for required_shape in required_shapes: + if len(shape) == len(required_shape): + passed = True + break + if not passed: + raise TypeError(f"Expected input shape in {required_shape}. Got {shape}.") + + +def _validate_input_shape(input: torch.Tensor, channel_index: int, number: int) -> bool: + r"""Validate if an input has the right shape. + + e.g. to check if an input is channel first. + If channel first, the second channel of an RGB input shall be fixed to 3. To verify using: + _validate_input_shape(input, 1, 3) + + Args: + input: torch.Tensor + channel_index: int + number: int + Returns: + bool + + """ + return input.shape[channel_index] == number + + +def _adapted_rsampling( + shape: Union[Tuple[int, ...], torch.Size], + dist: torch.distributions.Distribution, + same_on_batch: Optional[bool] = False, +) -> torch.Tensor: + r"""Sample from a uniform reparameterized sampling function that accepts 'same_on_batch'. + + If same_on_batch is True, all values generated will be exactly same given a batch_size (shape[0]). By default, + same_on_batch is set to False. + """ + if isinstance(shape, tuple): + shape = torch.Size(shape) + + if same_on_batch: + rsample_size = torch.Size((1, *shape[1:])) + rsample = dist.rsample(rsample_size) + return rsample.repeat(shape[0], *[1] * (len(rsample.shape) - 1)) + return dist.rsample(shape) + + +def _adapted_sampling( + shape: Union[Tuple[int, ...], torch.Size], + dist: torch.distributions.Distribution, + same_on_batch: Optional[bool] = False, +) -> torch.Tensor: + r"""Sample from a uniform sampling function that accepts 'same_on_batch'. + + If same_on_batch is True, all values generated will be exactly same given a batch_size (shape[0]). By default, + same_on_batch is set to False. + """ + if isinstance(shape, tuple): + shape = torch.Size(shape) + + if same_on_batch: + return dist.sample(torch.Size((1, *shape[1:]))).repeat(shape[0], *[1] * (len(shape) - 1)) + return dist.sample(shape) + + +def _adapted_uniform( + shape: Union[Tuple[int, ...], torch.Size], + low: Union[float, torch.Tensor], + high: Union[float, torch.Tensor], + same_on_batch: bool = False, +) -> torch.Tensor: + r"""Sample from a uniform sampling function that accepts 'same_on_batch'. + + If same_on_batch is True, all values generated will be exactly same given a batch_size (shape[0]). By default, + same_on_batch is set to False. + + By default, sampling happens on the default device and dtype. If low/high is a torch.Tensor, + sampling will happen in the same device/dtype as low/high torch.Tensor. + """ + device, dtype = _extract_device_dtype( + [low if isinstance(low, torch.Tensor) else None, high if isinstance(high, torch.Tensor) else None] + ) + low = torch.as_tensor(low, device=device, dtype=dtype) + high = torch.as_tensor(high, device=device, dtype=dtype) + # validate_args=False to fix pytorch 1.7.1 error: + # ValueError: Uniform is not defined when low>= high. + dist = Uniform(low, high, validate_args=False) + return _adapted_rsampling(shape, dist, same_on_batch) + + +def _adapted_beta( + shape: Union[Tuple[int, ...], torch.Size], + a: Union[float, torch.Tensor], + b: Union[float, torch.Tensor], + same_on_batch: bool = False, +) -> torch.Tensor: + r"""Sample from a beta sampling function that accepts 'same_on_batch'. + + If same_on_batch is True, all values generated will be exactly same given a batch_size (shape[0]). By default, + same_on_batch is set to False. + + By default, sampling happens on the default device and dtype. If a/b is a torch.Tensor, + sampling will happen in the same device/dtype as a/b torch.Tensor. + """ + device, dtype = _extract_device_dtype( + [a if isinstance(a, torch.Tensor) else None, b if isinstance(b, torch.Tensor) else None] + ) + a = torch.as_tensor(a, device=device, dtype=dtype) + b = torch.as_tensor(b, device=device, dtype=dtype) + dist = Beta(a, b, validate_args=False) + return _adapted_rsampling(shape, dist, same_on_batch) + + +def _shape_validation(param: torch.Tensor, shape: Union[Tuple[int, ...], List[int]], name: str) -> None: + if param.shape != torch.Size(shape): + raise AssertionError(f"Invalid shape for {name}. Expected {shape}. Got {param.shape}") + + +def deepcopy_dict(params: Dict[str, Any]) -> Dict[str, Any]: + """Perform deep copy on any dict. + + Support torch.Tensor copying here. + """ + out = {} + for k, v in params.items(): + # NOTE: Only Tensors created explicitly by the user (graph leaves) support the deepcopy protocol + if isinstance(v, torch.Tensor): + out.update({k: v.clone()}) + else: + out.update({k: v}) + return out + + +def override_parameters( + params: Dict[str, Any], + params_override: Optional[Dict[str, Any]] = None, + if_none_exist: str = "ignore", + in_place: bool = False, +) -> Dict[str, Any]: + """Override params dict w.r.t params_override. + + Args: + params: source parameters. + params_override: key-values to override the source parameters. + if_none_exist: behaviour if the key in `params_override` does not exist in `params`. + 'raise' | 'ignore'. + in_place: if to override in-place or not. + + """ + if params_override is None: + return params + out = params if in_place else deepcopy_dict(params) + for k, v in params_override.items(): + if k in params_override: + out[k] = v + elif if_none_exist == "ignore": + pass + elif if_none_exist == "raise": + raise RuntimeError(f"Param `{k}` not existed in `{params_override}`.") + else: + raise ValueError(f"`{if_none_exist}` is not a valid option.") + return out + + +def preprocess_boxes(input: Union[torch.Tensor, Boxes], mode: str = "vertices_plus") -> Boxes: + r"""Preprocess input boxes. + + Args: + input: 2D boxes, shape of :math:`(N, 4, 2)`, :math:`(B, N, 4, 2)` or a list of :math:`(N, 4, 2)`. + See below for more details. + mode: The format in which the boxes are provided. + + * 'xyxy': boxes are assumed to be in the format ``xmin, ymin, xmax, ymax`` where + ``width = xmax - xmin`` + and ``height = ymax - ymin``. With shape :math:`(N, 4)`, :math:`(B, N, 4)`. + * 'xyxy_plus': similar to 'xyxy' mode but where box width and length are defined as + ``width = xmax - xmin + 1`` and ``height = ymax - ymin + 1``. + With shape :math:`(N, 4)`, :math:`(B, N, 4)`. + * 'xywh': boxes are assumed to be in the format ``xmin, ymin, width, height`` where + ``width = xmax - xmin`` and ``height = ymax - ymin``. With shape :math:`(N, 4)`, :math:`(B, N, 4)`. + * 'vertices': boxes are defined by their vertices points in the following ``clockwise`` order: + *top-left, top-right, bottom-right, bottom-left*. Vertices coordinates are in (x,y) order. Finally, + box width and height are defined as ``width = xmax - xmin`` and ``height = ymax - ymin``. + With shape :math:`(N, 4, 2)` or :math:`(B, N, 4, 2)`. + * 'vertices_plus': similar to 'vertices' mode but where box width and length are defined as + ``width = xmax - xmin + 1`` and ``height = ymax - ymin + 1``. ymin + 1``. + With shape :math:`(N, 4, 2)` or :math:`(B, N, 4, 2)`. + + Note: + **2D boxes format** is defined as a floating data type torch.Tensor of shape ``Nx4x2`` or ``BxNx4x2`` + where each box is a `quadrilateral `_ defined by + it's 4 vertices + coordinates (A, B, C, D). Coordinates must be in ``x, y`` order. The height and width of a box is defined as + ``width = xmax - xmin + 1`` and ``height = ymax - ymin + 1``. Examples of + `quadrilaterals `_ are rectangles, rhombus and trapezoids. + + """ + # TODO: We may allow list here. + # input is BxNx4x2 or Boxes. + if isinstance(input, torch.Tensor): + if not (len(input.shape) == 4 and input.shape[2:] == torch.Size([4, 2])): + raise RuntimeError(f"Only BxNx4x2 torch.Tensor is supported. Got {input.shape}.") + input = Boxes.from_tensor(input, mode=mode) + if not isinstance(input, Boxes): + raise RuntimeError(f"Expect `Boxes` type. Got {type(input)}.") + return input + + +def preprocess_keypoints(input: Union[torch.Tensor, Keypoints]) -> Keypoints: + """Preprocess input keypoints.""" + # TODO: We may allow list here. + if isinstance(input, torch.Tensor): + if not (len(input.shape) == 3 and input.shape[1:] == torch.Size([2])): + raise RuntimeError(f"Only BxNx2 torch.Tensor is supported. Got {input.shape}.") + input = Keypoints(input, False) + if isinstance(input, Keypoints): + raise RuntimeError(f"Expect `Keypoints` type. Got {type(input)}.") + return input + + +def preprocess_classes(input: torch.Tensor) -> torch.Tensor: + """Preprocess input class tags.""" + # TODO: We may allow list here. + return input + + +class MultiprocessWrapper: + """When used as a base class, makes the class work with the 'spawn' multiprocessing context.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + args = tuple(arg.clone() if isinstance(arg, torch.Tensor) else arg for arg in args) + kwargs = {key: val.clone() if isinstance(val, torch.Tensor) else val for key, val in kwargs.items()} + + super().__init__(*args, **kwargs) diff --git a/kornia/augmentation/utils/param_validation.py b/kornia/augmentation/utils/param_validation.py new file mode 100644 index 0000000..1c09a3c --- /dev/null +++ b/kornia/augmentation/utils/param_validation.py @@ -0,0 +1,232 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, List, Optional, Tuple, Union + +import torch + + +def _check_positive_int_or_traced(value: Any, name: str) -> None: + """Assert ``value`` is a non-negative Python int, or accept it as a traced dim. + + Used by random generators to validate ``height``/``width``/``depth`` derived from + ``tensor.shape``. Under tracing those are 0-d tensors; we accept them rather than + aborting export, since the surrounding ops will fail loudly if the value is + actually invalid. + """ + if _is_traced_dim(value): + return + if not (isinstance(value, int) and value > 0): + raise AssertionError(f"`{name}` must be a positive integer. Got {value}.") + + +def _is_traced_dim(value: Any) -> bool: + """Return True if ``value`` looks like a shape dim that came from tensor tracing. + + Under torch.onnx.export's TorchScript tracer (and torch.fx), accessing + ``tensor.shape[i]`` can yield a 0-d tensor instead of a Python ``int``. The + runtime validators in this module are redundant during tracing, the tensor + constructors that follow will catch any real shape problem. Treat such + "traced ints" as opaque-but-valid so the trace can continue. + """ + return isinstance(value, torch.Tensor) and value.numel() == 1 and value.dim() <= 1 + + +def _common_param_check(batch_size: int, same_on_batch: Optional[bool] = None) -> None: + """Check valid batch_size and same_on_batch params. + + Skips the integer-type check when ``batch_size`` is a 0-d / 1-element tensor + (the shape of a traced tensor), since static configuration is already + validated at module construction time. + """ + if _is_traced_dim(batch_size): + return + if not (isinstance(batch_size, int) and batch_size >= 0): + raise AssertionError(f"`batch_size` shall be a positive integer. Got {batch_size}.") + if same_on_batch is not None and not isinstance(same_on_batch, bool): + raise AssertionError(f"`same_on_batch` shall be boolean. Got {same_on_batch}.") + + +def _range_bound( + factor: Union[torch.Tensor, float, Tuple[float, float], List[float]], + name: str, + center: Optional[float] = 0.0, + bounds: Optional[Tuple[float, float]] = (0, float("inf")), + check: Optional[str] = "joint", + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + r"""Check inputs and compute the corresponding factor bounds.""" + if device is None: + device = torch.device("cpu") + if dtype is None: + dtype = torch.get_default_dtype() + if not isinstance(factor, (torch.Tensor)): + factor = torch.tensor(factor, device=device, dtype=dtype) + factor_bound: torch.Tensor + + if factor.dim() == 0: + if factor < 0: + raise ValueError(f"If {name} is a single number, it must be non negative. Got {factor}.") + if center is None or bounds is None: + raise ValueError(f"`center` and `bounds` cannot be None for single number. Got {center}, {bounds}.") + # Should be something other than clamp + # Currently, single value factor will not out of scope as long as the user provided it. + # Note: I personally think throw an error will be better than a coarse clamp. + factor_bound = factor.repeat(2) * torch.tensor([-1.0, 1.0], device=factor.device, dtype=factor.dtype) + center + factor_bound = factor_bound.clamp(bounds[0], bounds[1]).to(device=device, dtype=dtype) + else: + factor_bound = torch.as_tensor(factor, device=device, dtype=dtype) + + if check is not None: + if check == "joint": + _joint_range_check(factor_bound, name, bounds) + elif check == "singular": + _singular_range_check(factor_bound, name, bounds) + else: + raise NotImplementedError(f"methods '{check}' not implemented.") + + return factor_bound + + +def _joint_range_check(ranged_factor: torch.Tensor, name: str, bounds: Optional[Tuple[float, float]] = None) -> None: + """Check if bounds[0] <= ranged_factor[0] <= ranged_factor[1] <= bounds[1].""" + if bounds is None: + bounds = (float("-inf"), float("inf")) + if ranged_factor.dim() == 1 and len(ranged_factor) == 2: + if not bounds[0] <= ranged_factor[0] or not bounds[1] >= ranged_factor[1]: + raise ValueError(f"{name} out of bounds. Expected inside {bounds}, got {ranged_factor}.") + + if not bounds[0] <= ranged_factor[0] <= ranged_factor[1] <= bounds[1]: + raise ValueError(f"{name}[0] should be smaller than {name}[1] got {ranged_factor}") + else: + raise TypeError( + f"{name} should be a torch.Tensor with length 2 whose values between {bounds}. Got {ranged_factor}." + ) + + +def _singular_range_check( + ranged_factor: torch.Tensor, + name: str, + bounds: Optional[Tuple[float, float]] = None, + skip_none: bool = False, + mode: str = "2d", +) -> None: + """Check if bounds[0] <= ranged_factor[0] <= bounds[1] and bounds[0] <= ranged_factor[1] <= bounds[1].""" + if mode == "2d": + dim_size = 2 + elif mode == "3d": + dim_size = 3 + else: + raise ValueError(f"'mode' shall be either 2d or 3d. Got {mode}") + + if skip_none and ranged_factor is None: + return + if bounds is None: + bounds = (float("-inf"), float("inf")) + if ranged_factor.dim() == 1 and len(ranged_factor) == dim_size: + for f in ranged_factor: + if not bounds[0] <= f <= bounds[1]: + raise ValueError(f"{name} out of bounds. Expected inside {bounds}, got {ranged_factor}.") + else: + raise TypeError( + f"{name} should be a float number or a tuple with length {dim_size} whose values between {bounds}." + f"Got {ranged_factor}" + ) + + +def _tuple_range_reader( + input_range: Union[torch.Tensor, float, Tuple[Any, ...]], + target_size: int, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + """Given target_size, it will generate the corresponding (target_size, 2) range torch.Tensor. + + This is for element-wise params. + + Example: + >>> degree = torch.tensor([0.2, 0.3]) + >>> _tuple_range_reader(degree, 3) # read degree for yaw, pitch and roll. + tensor([[0.2000, 0.3000], + [0.2000, 0.3000], + [0.2000, 0.3000]]) + + """ + target_shape = torch.Size([target_size, 2]) + + if isinstance(input_range, torch.Tensor): + if (len(input_range.shape) == 0) or (len(input_range.shape) == 1 and len(input_range) == 1): + if input_range < 0: + raise ValueError(f"If input_range is only one number it must be a positive number. Got{input_range}") + input_range_tmp = input_range.repeat(2).to(device=device, dtype=dtype) * torch.tensor( + [-1, 1], device=device, dtype=dtype + ) + input_range_tmp = input_range_tmp.repeat(target_shape[0], 1) + + elif len(input_range.shape) == 1 and len(input_range) == 2: + input_range_tmp = input_range.repeat(target_shape[0], 1).to(device=device, dtype=dtype) + + elif len(input_range.shape) == 1 and len(input_range) == target_shape[0]: + input_range_tmp = input_range.unsqueeze(1).repeat(1, 2).to(device=device, dtype=dtype) * torch.tensor( + [-1, 1], device=device, dtype=dtype + ) + + elif input_range.shape == target_shape: + input_range_tmp = input_range.to(device=device, dtype=dtype) + + else: + raise ValueError( + f"Degrees must be a {list(target_shape)} torch.tensor for the degree range for independent operation." + f"Got {input_range}" + ) + elif isinstance(input_range, (float, int)): + if input_range < 0: + raise ValueError(f"If input_range is only one number it must be a positive number. Got{input_range}") + input_range_tmp = torch.tensor([-input_range, input_range], device=device, dtype=dtype).repeat( + target_shape[0], 1 + ) + + elif ( + isinstance(input_range, (tuple, list)) + and len(input_range) == 2 + and isinstance(input_range[0], (float, int)) + and isinstance(input_range[1], (float, int)) + ): + input_range_tmp = torch.tensor(input_range, device=device, dtype=dtype).repeat(target_shape[0], 1) + + elif ( + isinstance(input_range, (tuple, list)) + and len(input_range) == target_shape[0] + and all(isinstance(x, (float, int)) for x in input_range) + ): + input_range_tmp = torch.tensor([(-s, s) for s in input_range], device=device, dtype=dtype) + + elif ( + isinstance(input_range, (tuple, list)) + and len(input_range) == target_shape[0] + and all(isinstance(x, (tuple, list)) for x in input_range) + ): + input_range_tmp = torch.tensor(input_range, device=device, dtype=dtype) + + else: + raise TypeError( + "If not pass a torch.tensor, it must be float, (float, float) for isotropic operation or a tuple of " + f"{target_size} floats or {target_size} (float, float) for independent operation. Got {input_range}." + ) + + return input_range_tmp diff --git a/kornia/color/__init__.py b/kornia/color/__init__.py new file mode 100644 index 0000000..eca2845 --- /dev/null +++ b/kornia/color/__init__.py @@ -0,0 +1,159 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Color — Color space conversions and color-related utilities for PyTorch. + +This subpackage provides modules for color transformations, colormaps, and conversions. +""" + +from .colormap import ApplyColorMap, ColorMap, ColorMapType, RGBColor, apply_colormap +from .gray import BgrToGrayscale, GrayscaleToRgb, RgbToGrayscale, bgr_to_grayscale, grayscale_to_rgb, rgb_to_grayscale +from .hls import HlsToRgb, RgbToHls, hls_to_rgb, rgb_to_hls +from .hsv import HsvToRgb, RgbToHsv, hsv_to_rgb, rgb_to_hsv +from .lab import LabToRgb, RgbToLab, lab_to_rgb, rgb_to_lab +from .luv import LuvToRgb, RgbToLuv, luv_to_rgb, rgb_to_luv +from .raw import CFA, RawToRgb, RawToRgb2x2Downscaled, RgbToRaw, raw_to_rgb, raw_to_rgb_2x2_downscaled, rgb_to_raw +from .rgb import ( + BgrToRgb, + BgrToRgba, + LinearRgbToRgb, + NormalsToRgb255, + Rgb255ToNormals, + Rgb255ToRgb, + RgbaToBgr, + RgbaToRgb, + RgbToBgr, + RgbToLinearRgb, + RgbToRgb255, + RgbToRgba, + bgr_to_rgb, + bgr_to_rgba, + linear_rgb_to_rgb, + normals_to_rgb255, + rgb255_to_normals, + rgb255_to_rgb, + rgb_to_bgr, + rgb_to_linear_rgb, + rgb_to_rgb255, + rgb_to_rgba, + rgba_to_bgr, + rgba_to_rgb, +) +from .sepia import Sepia, sepia_from_rgb +from .xyz import RgbToXyz, XyzToRgb, rgb_to_xyz, xyz_to_rgb +from .ycbcr import RgbToYcbcr, YcbcrToRgb, rgb_to_y, rgb_to_ycbcr, ycbcr_to_rgb +from .yuv import ( + RgbToYuv, + RgbToYuv420, + RgbToYuv422, + Yuv420ToRgb, + Yuv422ToRgb, + YuvToRgb, + rgb_to_yuv, + rgb_to_yuv420, + rgb_to_yuv422, + yuv420_to_rgb, + yuv422_to_rgb, + yuv_to_rgb, +) + +__all__ = [ + "AUTUMN", + "CFA", + "ApplyColorMap", + "BgrToGrayscale", + "BgrToRgb", + "BgrToRgba", + "ColorMap", + "ColorMapType", + "GrayscaleToRgb", + "HlsToRgb", + "HsvToRgb", + "LabToRgb", + "LinearRgbToRgb", + "LuvToRgb", + "NormalsToRgb255", + "RGBColor", + "RawToRgb", + "RawToRgb2x2Downscaled", + "Rgb255ToNormals", + "Rgb255ToRgb", + "RgbToBgr", + "RgbToGrayscale", + "RgbToHls", + "RgbToHsv", + "RgbToLab", + "RgbToLinearRgb", + "RgbToLuv", + "RgbToRaw", + "RgbToRgb255", + "RgbToRgba", + "RgbToXyz", + "RgbToYcbcr", + "RgbToYuv", + "RgbToYuv420", + "RgbToYuv422", + "RgbaToBgr", + "RgbaToRgb", + "Sepia", + "XyzToRgb", + "YcbcrToRgb", + "Yuv420ToRgb", + "Yuv422ToRgb", + "YuvToRgb", + "apply_colormap", + "bgr_to_grayscale", + "bgr_to_rgb", + "bgr_to_rgba", + "grayscale_to_rgb", + "hls_to_rgb", + "hsv_to_rgb", + "lab_to_rgb", + "linear_rgb_to_rgb", + "luv_to_rgb", + "normals_to_rgb255", + "raw_to_rgb", + "raw_to_rgb_2x2_downscaled", + "rgb255_to_normals", + "rgb255_to_rgb", + "rgb_to_bgr", + "rgb_to_grayscale", + "rgb_to_hls", + "rgb_to_hsv", + "rgb_to_lab", + "rgb_to_linear_rgb", + "rgb_to_luv", + "rgb_to_raw", + "rgb_to_rgb255", + "rgb_to_rgba", + "rgb_to_xyz", + "rgb_to_y", + "rgb_to_ycbcr", + "rgb_to_yuv", + "rgb_to_yuv420", + "rgb_to_yuv422", + "rgba_to_bgr", + "rgba_to_rgb", + "sepia", + "xyz_to_rgb", + "ycbcr_to_rgb", + "yuv420_to_rgb", + "yuv422_to_rgb", + "yuv_to_rgb", +] + +sepia = sepia_from_rgb diff --git a/kornia/color/_colormap_data.py b/kornia/color/_colormap_data.py new file mode 100644 index 0000000..a9c487f --- /dev/null +++ b/kornia/color/_colormap_data.py @@ -0,0 +1,1337 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import List + +# All colormaps has 64 base colors. Extracted from matplotlib. +# https://matplotlib.org/stable/users/explain/colors/colormaps.html + + +RGBColor = List[float] + + +def get_autumn_base() -> list[RGBColor]: + return [ + [1.0, 0.0, 0.0], + [1.0, 0.015873015873015872, 0.0], + [1.0, 0.031746031746031744, 0.0], + [1.0, 0.047619047619047616, 0.0], + [1.0, 0.06349206349206349, 0.0], + [1.0, 0.07936507936507936, 0.0], + [1.0, 0.09523809523809523, 0.0], + [1.0, 0.1111111111111111, 0.0], + [1.0, 0.12698412698412698, 0.0], + [1.0, 0.14285714285714285, 0.0], + [1.0, 0.15873015873015872, 0.0], + [1.0, 0.1746031746031746, 0.0], + [1.0, 0.19047619047619047, 0.0], + [1.0, 0.20634920634920634, 0.0], + [1.0, 0.2222222222222222, 0.0], + [1.0, 0.23809523809523808, 0.0], + [1.0, 0.25396825396825395, 0.0], + [1.0, 0.2698412698412698, 0.0], + [1.0, 0.2857142857142857, 0.0], + [1.0, 0.30158730158730157, 0.0], + [1.0, 0.31746031746031744, 0.0], + [1.0, 0.3333333333333333, 0.0], + [1.0, 0.3492063492063492, 0.0], + [1.0, 0.36507936507936506, 0.0], + [1.0, 0.38095238095238093, 0.0], + [1.0, 0.3968253968253968, 0.0], + [1.0, 0.4126984126984127, 0.0], + [1.0, 0.42857142857142855, 0.0], + [1.0, 0.4444444444444444, 0.0], + [1.0, 0.4603174603174603, 0.0], + [1.0, 0.47619047619047616, 0.0], + [1.0, 0.49206349206349204, 0.0], + [1.0, 0.5079365079365079, 0.0], + [1.0, 0.5238095238095237, 0.0], + [1.0, 0.5396825396825397, 0.0], + [1.0, 0.5555555555555556, 0.0], + [1.0, 0.5714285714285714, 0.0], + [1.0, 0.5873015873015872, 0.0], + [1.0, 0.6031746031746031, 0.0], + [1.0, 0.6190476190476191, 0.0], + [1.0, 0.6349206349206349, 0.0], + [1.0, 0.6507936507936507, 0.0], + [1.0, 0.6666666666666666, 0.0], + [1.0, 0.6825396825396826, 0.0], + [1.0, 0.6984126984126984, 0.0], + [1.0, 0.7142857142857142, 0.0], + [1.0, 0.7301587301587301, 0.0], + [1.0, 0.746031746031746, 0.0], + [1.0, 0.7619047619047619, 0.0], + [1.0, 0.7777777777777777, 0.0], + [1.0, 0.7936507936507936, 0.0], + [1.0, 0.8095238095238095, 0.0], + [1.0, 0.8253968253968254, 0.0], + [1.0, 0.8412698412698412, 0.0], + [1.0, 0.8571428571428571, 0.0], + [1.0, 0.873015873015873, 0.0], + [1.0, 0.8888888888888888, 0.0], + [1.0, 0.9047619047619047, 0.0], + [1.0, 0.9206349206349206, 0.0], + [1.0, 0.9365079365079365, 0.0], + [1.0, 0.9523809523809523, 0.0], + [1.0, 0.9682539682539681, 0.0], + [1.0, 0.9841269841269841, 0.0], + [1.0, 1.0, 0.0], + ] + + +def get_bone_base() -> list[RGBColor]: + return [ + [0.0, 0.0, 0.0], + [0.013888888888888888, 0.013888883454100847, 0.019323671497584544], + [0.027777777777777776, 0.027777766908201693, 0.03864734299516909], + [0.041666666666666664, 0.04166665036230254, 0.05797101449275363], + [0.05555555555555555, 0.05555553381640339, 0.07729468599033817], + [0.06944444444444445, 0.06944441727050424, 0.09661835748792272], + [0.08333333333333333, 0.08333330072460508, 0.11594202898550726], + [0.09722222222222221, 0.09722218417870591, 0.13526570048309178], + [0.1111111111111111, 0.11111106763280677, 0.15458937198067635], + [0.125, 0.12499995108690763, 0.1739130434782609], + [0.1388888888888889, 0.13888883454100848, 0.19323671497584544], + [0.15277777777777776, 0.1527777179951093, 0.21256038647342998], + [0.16666666666666666, 0.16666660144921017, 0.23188405797101452], + [0.18055555555555555, 0.18055548490331103, 0.2512077294685991], + [0.19444444444444442, 0.19444436835741183, 0.27053140096618356], + [0.2083333333333333, 0.2083332518115127, 0.2898550724637681], + [0.2222222222222222, 0.22222213526561355, 0.3091787439613527], + [0.23611111111111108, 0.2361110187197144, 0.3285024154589372], + [0.25, 0.24999990217381526, 0.3478260869565218], + [0.26388888888888884, 0.2638887856279161, 0.36714975845410636], + [0.2777777777777778, 0.27777766908201695, 0.38647342995169087], + [0.29166666666666663, 0.2916665525361178, 0.4057971014492754], + [0.3055555555555555, 0.3055554359902186, 0.42512077294685996], + [0.3194444444444444, 0.31944443923603627, 0.44444431944451634], + [0.3333333333333333, 0.3385416582030555, 0.45833321145840344], + [0.34722222222222215, 0.35763887717007464, 0.47222210347229054], + [0.3611111111111111, 0.3767360961370938, 0.48611099548617764], + [0.375, 0.395833315104113, 0.49999988750006474], + [0.38888888888888884, 0.4149305340711322, 0.5138887795139518], + [0.40277777777777773, 0.43402775303815133, 0.5277776715278389], + [0.4166666666666666, 0.4531249720051705, 0.5416665635417259], + [0.4305555555555555, 0.47222219097218965, 0.555555455555613], + [0.4444444444444444, 0.49131940993920886, 0.5694443475695001], + [0.4583333333333332, 0.5104166289062279, 0.5833332395833872], + [0.47222222222222215, 0.5295138478732472, 0.5972221315972743], + [0.48611111111111105, 0.5486110668402664, 0.6111110236111614], + [0.5, 0.5677082858072856, 0.6249999156250485], + [0.5138888888888887, 0.5868055047743046, 0.6388888076389355], + [0.5277777777777777, 0.6059027237413239, 0.6527776996528227], + [0.5416666666666666, 0.624999942708343, 0.6666665916667098], + [0.5555555555555556, 0.6440971616753622, 0.6805554836805969], + [0.5694444444444443, 0.6631943806423812, 0.6944443756944839], + [0.5833333333333333, 0.6822915996094006, 0.7083332677083711], + [0.5972222222222221, 0.7013888185764198, 0.7222221597222582], + [0.611111111111111, 0.720486037543439, 0.7361110517361452], + [0.6249999999999998, 0.739583256510458, 0.7499999437500322], + [0.6388888888888888, 0.7586804754774773, 0.7638888357639194], + [0.6527777777777777, 0.7777776944444965, 0.7777777277778065], + [0.6744790494790494, 0.7916666666666666, 0.7916666197916936], + [0.6961804461804459, 0.8055555555555554, 0.8055555118055806], + [0.7178818428818429, 0.8194444444444444, 0.8194444038194678], + [0.7395832395832396, 0.8333333333333333, 0.8333332958333549], + [0.7612846362846363, 0.8472222222222222, 0.8472221878472419], + [0.7829860329860328, 0.8611111111111109, 0.861111079861129], + [0.8046874296874297, 0.875, 0.8749999718750161], + [0.8263888263888264, 0.8888888888888888, 0.8888888638889032], + [0.8480902230902231, 0.9027777777777778, 0.9027777559027903], + [0.8697916197916196, 0.9166666666666665, 0.9166666479166774], + [0.8914930164930165, 0.9305555555555556, 0.9305555399305645], + [0.9131944131944132, 0.9444444444444444, 0.9444444319444516], + [0.9348958098958099, 0.9583333333333334, 0.9583333239583387], + [0.9565972065972064, 0.9722222222222221, 0.9722222159722258], + [0.9782986032986033, 0.9861111111111112, 0.9861111079861129], + [1.0, 1.0, 1.0], + ] + + +def get_jet_base() -> list[RGBColor]: + return [ + [0.0, 0.0, 0.5], + [0.0, 0.0, 0.5721500721500722], + [0.0, 0.0, 0.6443001443001444], + [0.0, 0.0, 0.7164502164502164], + [0.0, 0.0, 0.7886002886002886], + [0.0, 0.0, 0.8607503607503608], + [0.0, 0.0, 0.9329004329004329], + [0.0, 0.0, 1.0], + [0.0, 0.007936507936507936, 1.0], + [0.0, 0.07142857142857142, 1.0], + [0.0, 0.1349206349206349, 1.0], + [0.0, 0.1984126984126984, 1.0], + [0.0, 0.2619047619047619, 1.0], + [0.0, 0.3253968253968254, 1.0], + [0.0, 0.3888888888888889, 1.0], + [0.0, 0.4523809523809524, 1.0], + [0.0, 0.5158730158730159, 1.0], + [0.0, 0.5793650793650794, 1.0], + [0.0, 0.6428571428571429, 1.0], + [0.0, 0.7063492063492064, 1.0], + [0.0, 0.7698412698412699, 1.0], + [0.0, 0.8333333333333334, 1.0], + [0.0, 0.8968253968253969, 0.9703020993343575], + [0.048643113159242315, 0.9603174603174603, 0.9190988223246289], + [0.09984639016897091, 1.0, 0.8678955453149002], + [0.15104966717869953, 1.0, 0.8166922683051716], + [0.20225294418842812, 1.0, 0.765488991295443], + [0.2534562211981567, 1.0, 0.7142857142857144], + [0.30465949820788535, 1.0, 0.6630824372759858], + [0.35586277521761395, 1.0, 0.6118791602662572], + [0.40706605222734255, 1.0, 0.5606758832565285], + [0.45826932923707114, 1.0, 0.5094726062467999], + [0.5094726062467997, 1.0, 0.4582693292370713], + [0.560675883256528, 1.0, 0.40706605222734304], + [0.6118791602662569, 1.0, 0.355862775217614], + [0.6630824372759855, 1.0, 0.3046594982078854], + [0.7142857142857141, 1.0, 0.2534562211981568], + [0.7654889912954423, 1.0, 0.20225294418842854], + [0.8166922683051714, 1.0, 0.1510496671786996], + [0.8678955453149, 1.0, 0.0998463901689709], + [0.9190988223246286, 1.0, 0.0486431131592423], + [0.9703020993343567, 0.9600235155790716, 0.0], + [1.0, 0.9012345679012346, 0.0], + [1.0, 0.8424456202233981, 0.0], + [1.0, 0.7836566725455615, 0.0], + [1.0, 0.7248677248677253, 0.0], + [1.0, 0.6660787771898884, 0.0], + [1.0, 0.6072898295120519, 0.0], + [1.0, 0.5485008818342153, 0.0], + [1.0, 0.48971193415637915, 0.0], + [1.0, 0.4309229864785422, 0.0], + [1.0, 0.3721340388007057, 0.0], + [1.0, 0.3133450911228691, 0.0], + [1.0, 0.25455614344503297, 0.0], + [1.0, 0.19576719576719603, 0.0], + [1.0, 0.13697824808935943, 0.0], + [1.0, 0.07818930041152294, 0.0], + [0.9329004329004335, 0.019400352733686788, 0.0], + [0.8607503607503608, 0.0, 0.0], + [0.7886002886002886, 0.0, 0.0], + [0.7164502164502164, 0.0, 0.0], + [0.6443001443001448, 0.0, 0.0], + [0.5721500721500721, 0.0, 0.0], + [0.5, 0.0, 0.0], + ] + + +def get_winter_base() -> list[RGBColor]: + return [ + [0.0, 0.0, 1.0], + [0.0, 0.015873015873015872, 0.9920634920634921], + [0.0, 0.031746031746031744, 0.9841269841269842], + [0.0, 0.047619047619047616, 0.9761904761904762], + [0.0, 0.06349206349206349, 0.9682539682539683], + [0.0, 0.07936507936507936, 0.9603174603174603], + [0.0, 0.09523809523809523, 0.9523809523809523], + [0.0, 0.1111111111111111, 0.9444444444444444], + [0.0, 0.12698412698412698, 0.9365079365079365], + [0.0, 0.14285714285714285, 0.9285714285714286], + [0.0, 0.15873015873015872, 0.9206349206349207], + [0.0, 0.1746031746031746, 0.9126984126984127], + [0.0, 0.19047619047619047, 0.9047619047619048], + [0.0, 0.20634920634920634, 0.8968253968253969], + [0.0, 0.2222222222222222, 0.8888888888888888], + [0.0, 0.23809523809523808, 0.8809523809523809], + [0.0, 0.25396825396825395, 0.873015873015873], + [0.0, 0.2698412698412698, 0.8650793650793651], + [0.0, 0.2857142857142857, 0.8571428571428572], + [0.0, 0.30158730158730157, 0.8492063492063492], + [0.0, 0.31746031746031744, 0.8412698412698413], + [0.0, 0.3333333333333333, 0.8333333333333334], + [0.0, 0.3492063492063492, 0.8253968253968254], + [0.0, 0.36507936507936506, 0.8174603174603174], + [0.0, 0.38095238095238093, 0.8095238095238095], + [0.0, 0.3968253968253968, 0.8015873015873016], + [0.0, 0.4126984126984127, 0.7936507936507937], + [0.0, 0.42857142857142855, 0.7857142857142857], + [0.0, 0.4444444444444444, 0.7777777777777778], + [0.0, 0.4603174603174603, 0.7698412698412699], + [0.0, 0.47619047619047616, 0.7619047619047619], + [0.0, 0.49206349206349204, 0.753968253968254], + [0.0, 0.5079365079365079, 0.746031746031746], + [0.0, 0.5238095238095237, 0.7380952380952381], + [0.0, 0.5396825396825397, 0.7301587301587302], + [0.0, 0.5555555555555556, 0.7222222222222222], + [0.0, 0.5714285714285714, 0.7142857142857143], + [0.0, 0.5873015873015872, 0.7063492063492064], + [0.0, 0.6031746031746031, 0.6984126984126984], + [0.0, 0.6190476190476191, 0.6904761904761905], + [0.0, 0.6349206349206349, 0.6825396825396826], + [0.0, 0.6507936507936507, 0.6746031746031746], + [0.0, 0.6666666666666666, 0.6666666666666667], + [0.0, 0.6825396825396826, 0.6587301587301587], + [0.0, 0.6984126984126984, 0.6507936507936508], + [0.0, 0.7142857142857142, 0.6428571428571429], + [0.0, 0.7301587301587301, 0.6349206349206349], + [0.0, 0.746031746031746, 0.626984126984127], + [0.0, 0.7619047619047619, 0.6190476190476191], + [0.0, 0.7777777777777777, 0.6111111111111112], + [0.0, 0.7936507936507936, 0.6031746031746033], + [0.0, 0.8095238095238095, 0.5952380952380952], + [0.0, 0.8253968253968254, 0.5873015873015873], + [0.0, 0.8412698412698412, 0.5793650793650794], + [0.0, 0.8571428571428571, 0.5714285714285714], + [0.0, 0.873015873015873, 0.5634920634920635], + [0.0, 0.8888888888888888, 0.5555555555555556], + [0.0, 0.9047619047619047, 0.5476190476190477], + [0.0, 0.9206349206349206, 0.5396825396825398], + [0.0, 0.9365079365079365, 0.5317460317460317], + [0.0, 0.9523809523809523, 0.5238095238095238], + [0.0, 0.9682539682539681, 0.5158730158730159], + [0.0, 0.9841269841269841, 0.5079365079365079], + [0.0, 1.0, 0.5], + ] + + +def get_rainbow_base() -> list[RGBColor]: + return [ + [0.5, 0.0, 1.0], + [0.46825396825396826, 0.049845885660697156, 0.9996891820008162], + [0.4365079365079365, 0.09956784659581665, 0.9987569212189223], + [0.40476190476190477, 0.14904226617617441, 0.9972037971811801], + [0.373015873015873, 0.19814614319939755, 0.9950307753654014], + [0.3412698412698413, 0.24675739769029362, 0.9922392066001721], + [0.30952380952380953, 0.29475517441090415, 0.9888308262251285], + [0.2777777777777778, 0.3420201433256687, 0.984807753012208], + [0.24603174603174605, 0.3884347962746947, 0.9801724878485438], + [0.2142857142857143, 0.4338837391175581, 0.9749279121818236], + [0.18253968253968256, 0.4782539786213182, 0.969077286229078], + [0.1507936507936508, 0.521435203379498, 0.962624246950012], + [0.11904761904761907, 0.563320058063622, 0.9555728057861408], + [0.08730158730158732, 0.6038044103254774, 0.9479273461671318], + [0.05555555555555558, 0.6427876096865393, 0.9396926207859084], + [0.023809523809523836, 0.6801727377709194, 0.9308737486442042], + [0.007936507936507908, 0.7158668492597183, 0.9214762118704076], + [0.03968253968253965, 0.7497812029677341, 0.9115058523116731], + [0.0714285714285714, 0.7818314824680298, 0.9009688679024191], + [0.10317460317460314, 0.8119380057158565, 0.8898718088114687], + [0.13492063492063489, 0.8400259231507714, 0.8782215733702285], + [0.16666666666666663, 0.8660254037844386, 0.8660254037844387], + [0.19841269841269837, 0.8898718088114685, 0.8532908816321557], + [0.23015873015873012, 0.9115058523116731, 0.8400259231507715], + [0.26190476190476186, 0.9308737486442041, 0.8262387743159949], + [0.2936507936507936, 0.9479273461671317, 0.8119380057158566], + [0.32539682539682535, 0.962624246950012, 0.7971325072229225], + [0.3571428571428571, 0.9749279121818236, 0.7818314824680298], + [0.38888888888888884, 0.984807753012208, 0.766044443118978], + [0.4206349206349206, 0.9922392066001721, 0.7497812029677342], + [0.45238095238095233, 0.9972037971811801, 0.7330518718298263], + [0.4841269841269841, 0.9996891820008162, 0.7158668492597184], + [0.5158730158730158, 0.9996891820008162, 0.6982368180860729], + [0.5476190476190474, 0.9972037971811801, 0.6801727377709196], + [0.5793650793650793, 0.9922392066001721, 0.6616858375968595], + [0.6111111111111112, 0.984807753012208, 0.6427876096865394], + [0.6428571428571428, 0.9749279121818236, 0.6234898018587336], + [0.6746031746031744, 0.9626242469500121, 0.6038044103254775], + [0.7063492063492063, 0.9479273461671318, 0.58374367223479], + [0.7380952380952381, 0.9308737486442042, 0.563320058063622], + [0.7698412698412698, 0.9115058523116732, 0.5425462638657594], + [0.8015873015873014, 0.8898718088114688, 0.5214352033794982], + [0.8333333333333333, 0.8660254037844387, 0.5000000000000001], + [0.8650793650793651, 0.8400259231507715, 0.47825397862131824], + [0.8968253968253967, 0.8119380057158566, 0.4562106573531631], + [0.9285714285714284, 0.7818314824680301, 0.43388373911755834], + [0.9603174603174602, 0.7497812029677344, 0.4112871031306117], + [0.9920634920634921, 0.7158668492597186, 0.3884347962746948], + [1.0, 0.6801727377709197, 0.3653410243663952], + [1.0, 0.6427876096865395, 0.3420201433256688], + [1.0, 0.6038044103254777, 0.31848665025168466], + [1.0, 0.5633200580636223, 0.2947551744109043], + [1.0, 0.5214352033794981, 0.27084046814300516], + [1.0, 0.47825397862131847, 0.24675739769029378], + [1.0, 0.43388373911755823, 0.22252093395631445], + [1.0, 0.3884347962746946, 0.19814614319939755], + [1.0, 0.3420201433256689, 0.17364817766693041], + [1.0, 0.2947551744109046, 0.14904226617617464], + [1.0, 0.24675739769029384, 0.12434370464748527], + [1.0, 0.1981461431993976, 0.09956784659581666], + [1.0, 0.14904226617617472, 0.07473009358642439], + [1.0, 0.09956784659581717, 0.04984588566069742], + [1.0, 0.04984588566069748, 0.024930691738073035], + [1.0, 1.2246467991473532e-16, 6.123233995736766e-17], + ] + + +def get_ocean_base() -> list[RGBColor]: + return [ + [0.0, 0.5, 0.0], + [0.0, 0.47619047619047616, 0.015873015873015872], + [0.0, 0.4523809523809524, 0.031746031746031744], + [0.0, 0.4285714285714286, 0.047619047619047616], + [0.0, 0.40476190476190477, 0.06349206349206349], + [0.0, 0.38095238095238093, 0.07936507936507936], + [0.0, 0.35714285714285715, 0.09523809523809523], + [0.0, 0.33333333333333337, 0.1111111111111111], + [0.0, 0.30952380952380953, 0.12698412698412698], + [0.0, 0.2857142857142857, 0.14285714285714285], + [0.0, 0.2619047619047619, 0.15873015873015872], + [0.0, 0.23809523809523814, 0.1746031746031746], + [0.0, 0.2142857142857143, 0.19047619047619047], + [0.0, 0.19047619047619047, 0.20634920634920634], + [0.0, 0.16666666666666669, 0.2222222222222222], + [0.0, 0.1428571428571429, 0.23809523809523808], + [0.0, 0.11904761904761907, 0.25396825396825395], + [0.0, 0.09523809523809523, 0.2698412698412698], + [0.0, 0.07142857142857145, 0.2857142857142857], + [0.0, 0.04761904761904767, 0.30158730158730157], + [0.0, 0.023809523809523836, 0.31746031746031744], + [0.0, 0.0, 0.3333333333333333], + [0.0, 0.023809523809523725, 0.3492063492063492], + [0.0, 0.04761904761904756, 0.36507936507936506], + [0.0, 0.0714285714285714, 0.38095238095238093], + [0.0, 0.09523809523809523, 0.3968253968253968], + [0.0, 0.11904761904761907, 0.4126984126984127], + [0.0, 0.1428571428571428, 0.42857142857142855], + [0.0, 0.16666666666666663, 0.4444444444444444], + [0.0, 0.19047619047619047, 0.4603174603174603], + [0.0, 0.2142857142857142, 0.47619047619047616], + [0.0, 0.23809523809523803, 0.49206349206349204], + [0.0, 0.26190476190476186, 0.5079365079365079], + [0.0, 0.2857142857142856, 0.5238095238095237], + [0.0, 0.30952380952380953, 0.5396825396825397], + [0.0, 0.33333333333333337, 0.5555555555555556], + [0.0, 0.3571428571428571, 0.5714285714285714], + [0.0, 0.3809523809523808, 0.5873015873015872], + [0.0, 0.40476190476190466, 0.6031746031746031], + [0.0, 0.4285714285714286, 0.6190476190476191], + [0.0, 0.45238095238095233, 0.6349206349206349], + [0.0, 0.47619047619047605, 0.6507936507936507], + [0.0, 0.5, 0.6666666666666666], + [0.04761904761904745, 0.5238095238095237, 0.6825396825396826], + [0.0952380952380949, 0.5476190476190474, 0.6984126984126984], + [0.14285714285714235, 0.5714285714285712, 0.7142857142857142], + [0.19047619047619024, 0.5952380952380951, 0.7301587301587301], + [0.23809523809523814, 0.6190476190476191, 0.746031746031746], + [0.2857142857142856, 0.6428571428571428, 0.7619047619047619], + [0.33333333333333304, 0.6666666666666665, 0.7777777777777777], + [0.38095238095238093, 0.6904761904761905, 0.7936507936507936], + [0.4285714285714288, 0.7142857142857144, 0.8095238095238095], + [0.4761904761904763, 0.7380952380952381, 0.8253968253968254], + [0.5238095238095237, 0.7619047619047619, 0.8412698412698412], + [0.5714285714285712, 0.7857142857142856, 0.8571428571428571], + [0.6190476190476191, 0.8095238095238095, 0.873015873015873], + [0.6666666666666665, 0.8333333333333333, 0.8888888888888888], + [0.714285714285714, 0.857142857142857, 0.9047619047619047], + [0.7619047619047619, 0.8809523809523809, 0.9206349206349206], + [0.8095238095238093, 0.9047619047619047, 0.9365079365079365], + [0.8571428571428568, 0.9285714285714284, 0.9523809523809523], + [0.9047619047619042, 0.9523809523809521, 0.9682539682539681], + [0.9523809523809521, 0.976190476190476, 0.9841269841269841], + [1.0, 1.0, 1.0], + ] + + +def get_summer_base() -> list[RGBColor]: + return [ + [0.0, 0.5, 0.4], + [0.015873015873015872, 0.5079365079365079, 0.4], + [0.031746031746031744, 0.5158730158730158, 0.4], + [0.047619047619047616, 0.5238095238095238, 0.4], + [0.06349206349206349, 0.5317460317460317, 0.4], + [0.07936507936507936, 0.5396825396825397, 0.4], + [0.09523809523809523, 0.5476190476190477, 0.4], + [0.1111111111111111, 0.5555555555555556, 0.4], + [0.12698412698412698, 0.5634920634920635, 0.4], + [0.14285714285714285, 0.5714285714285714, 0.4], + [0.15873015873015872, 0.5793650793650793, 0.4], + [0.1746031746031746, 0.5873015873015873, 0.4], + [0.19047619047619047, 0.5952380952380952, 0.4], + [0.20634920634920634, 0.6031746031746031, 0.4], + [0.2222222222222222, 0.6111111111111112, 0.4], + [0.23809523809523808, 0.6190476190476191, 0.4], + [0.25396825396825395, 0.626984126984127, 0.4], + [0.2698412698412698, 0.6349206349206349, 0.4], + [0.2857142857142857, 0.6428571428571428, 0.4], + [0.30158730158730157, 0.6507936507936508, 0.4], + [0.31746031746031744, 0.6587301587301587, 0.4], + [0.3333333333333333, 0.6666666666666666, 0.4], + [0.3492063492063492, 0.6746031746031746, 0.4], + [0.36507936507936506, 0.6825396825396826, 0.4], + [0.38095238095238093, 0.6904761904761905, 0.4], + [0.3968253968253968, 0.6984126984126984, 0.4], + [0.4126984126984127, 0.7063492063492063, 0.4], + [0.42857142857142855, 0.7142857142857143, 0.4], + [0.4444444444444444, 0.7222222222222222, 0.4], + [0.4603174603174603, 0.7301587301587301, 0.4], + [0.47619047619047616, 0.7380952380952381, 0.4], + [0.49206349206349204, 0.746031746031746, 0.4], + [0.5079365079365079, 0.753968253968254, 0.4], + [0.5238095238095237, 0.7619047619047619, 0.4], + [0.5396825396825397, 0.7698412698412698, 0.4], + [0.5555555555555556, 0.7777777777777778, 0.4], + [0.5714285714285714, 0.7857142857142857, 0.4], + [0.5873015873015872, 0.7936507936507936, 0.4], + [0.6031746031746031, 0.8015873015873016, 0.4], + [0.6190476190476191, 0.8095238095238095, 0.4], + [0.6349206349206349, 0.8174603174603174, 0.4], + [0.6507936507936507, 0.8253968253968254, 0.4], + [0.6666666666666666, 0.8333333333333333, 0.4], + [0.6825396825396826, 0.8412698412698413, 0.4], + [0.6984126984126984, 0.8492063492063492, 0.4], + [0.7142857142857142, 0.8571428571428571, 0.4], + [0.7301587301587301, 0.8650793650793651, 0.4], + [0.746031746031746, 0.873015873015873, 0.4], + [0.7619047619047619, 0.8809523809523809, 0.4], + [0.7777777777777777, 0.8888888888888888, 0.4], + [0.7936507936507936, 0.8968253968253967, 0.4], + [0.8095238095238095, 0.9047619047619048, 0.4], + [0.8253968253968254, 0.9126984126984127, 0.4], + [0.8412698412698412, 0.9206349206349206, 0.4], + [0.8571428571428571, 0.9285714285714286, 0.4], + [0.873015873015873, 0.9365079365079365, 0.4], + [0.8888888888888888, 0.9444444444444444, 0.4], + [0.9047619047619047, 0.9523809523809523, 0.4], + [0.9206349206349206, 0.9603174603174602, 0.4], + [0.9365079365079365, 0.9682539682539683, 0.4], + [0.9523809523809523, 0.9761904761904762, 0.4], + [0.9682539682539681, 0.9841269841269841, 0.4], + [0.9841269841269841, 0.9920634920634921, 0.4], + [1.0, 1.0, 0.4], + ] + + +def get_spring_base() -> list[RGBColor]: + return [ + [1.0, 0.0, 1.0], + [1.0, 0.015873015873015872, 0.9841269841269842], + [1.0, 0.031746031746031744, 0.9682539682539683], + [1.0, 0.047619047619047616, 0.9523809523809523], + [1.0, 0.06349206349206349, 0.9365079365079365], + [1.0, 0.07936507936507936, 0.9206349206349207], + [1.0, 0.09523809523809523, 0.9047619047619048], + [1.0, 0.1111111111111111, 0.8888888888888888], + [1.0, 0.12698412698412698, 0.873015873015873], + [1.0, 0.14285714285714285, 0.8571428571428572], + [1.0, 0.15873015873015872, 0.8412698412698413], + [1.0, 0.1746031746031746, 0.8253968253968254], + [1.0, 0.19047619047619047, 0.8095238095238095], + [1.0, 0.20634920634920634, 0.7936507936507937], + [1.0, 0.2222222222222222, 0.7777777777777778], + [1.0, 0.23809523809523808, 0.7619047619047619], + [1.0, 0.25396825396825395, 0.746031746031746], + [1.0, 0.2698412698412698, 0.7301587301587302], + [1.0, 0.2857142857142857, 0.7142857142857143], + [1.0, 0.30158730158730157, 0.6984126984126984], + [1.0, 0.31746031746031744, 0.6825396825396826], + [1.0, 0.3333333333333333, 0.6666666666666667], + [1.0, 0.3492063492063492, 0.6507936507936508], + [1.0, 0.36507936507936506, 0.6349206349206349], + [1.0, 0.38095238095238093, 0.6190476190476191], + [1.0, 0.3968253968253968, 0.6031746031746033], + [1.0, 0.4126984126984127, 0.5873015873015873], + [1.0, 0.42857142857142855, 0.5714285714285714], + [1.0, 0.4444444444444444, 0.5555555555555556], + [1.0, 0.4603174603174603, 0.5396825396825398], + [1.0, 0.47619047619047616, 0.5238095238095238], + [1.0, 0.49206349206349204, 0.5079365079365079], + [1.0, 0.5079365079365079, 0.4920634920634921], + [1.0, 0.5238095238095237, 0.4761904761904763], + [1.0, 0.5396825396825397, 0.46031746031746035], + [1.0, 0.5555555555555556, 0.4444444444444444], + [1.0, 0.5714285714285714, 0.4285714285714286], + [1.0, 0.5873015873015872, 0.4126984126984128], + [1.0, 0.6031746031746031, 0.39682539682539686], + [1.0, 0.6190476190476191, 0.38095238095238093], + [1.0, 0.6349206349206349, 0.3650793650793651], + [1.0, 0.6507936507936507, 0.3492063492063493], + [1.0, 0.6666666666666666, 0.33333333333333337], + [1.0, 0.6825396825396826, 0.31746031746031744], + [1.0, 0.6984126984126984, 0.3015873015873016], + [1.0, 0.7142857142857142, 0.2857142857142858], + [1.0, 0.7301587301587301, 0.2698412698412699], + [1.0, 0.746031746031746, 0.25396825396825395], + [1.0, 0.7619047619047619, 0.23809523809523814], + [1.0, 0.7777777777777777, 0.22222222222222232], + [1.0, 0.7936507936507936, 0.2063492063492064], + [1.0, 0.8095238095238095, 0.19047619047619047], + [1.0, 0.8253968253968254, 0.17460317460317465], + [1.0, 0.8412698412698412, 0.15873015873015883], + [1.0, 0.8571428571428571, 0.1428571428571429], + [1.0, 0.873015873015873, 0.12698412698412698], + [1.0, 0.8888888888888888, 0.11111111111111116], + [1.0, 0.9047619047619047, 0.09523809523809534], + [1.0, 0.9206349206349206, 0.07936507936507942], + [1.0, 0.9365079365079365, 0.06349206349206349], + [1.0, 0.9523809523809523, 0.04761904761904767], + [1.0, 0.9682539682539681, 0.031746031746031855], + [1.0, 0.9841269841269841, 0.015873015873015928], + [1.0, 1.0, 0.0], + ] + + +def get_cool_base() -> list[RGBColor]: + return [ + [0.0, 1.0, 1.0], + [0.015873015873015872, 0.9841269841269842, 1.0], + [0.031746031746031744, 0.9682539682539683, 1.0], + [0.047619047619047616, 0.9523809523809523, 1.0], + [0.06349206349206349, 0.9365079365079365, 1.0], + [0.07936507936507936, 0.9206349206349207, 1.0], + [0.09523809523809523, 0.9047619047619048, 1.0], + [0.1111111111111111, 0.8888888888888888, 1.0], + [0.12698412698412698, 0.873015873015873, 1.0], + [0.14285714285714285, 0.8571428571428572, 1.0], + [0.15873015873015872, 0.8412698412698413, 1.0], + [0.1746031746031746, 0.8253968253968254, 1.0], + [0.19047619047619047, 0.8095238095238095, 1.0], + [0.20634920634920634, 0.7936507936507937, 1.0], + [0.2222222222222222, 0.7777777777777778, 1.0], + [0.23809523809523808, 0.7619047619047619, 1.0], + [0.25396825396825395, 0.746031746031746, 1.0], + [0.2698412698412698, 0.7301587301587302, 1.0], + [0.2857142857142857, 0.7142857142857143, 1.0], + [0.30158730158730157, 0.6984126984126984, 1.0], + [0.31746031746031744, 0.6825396825396826, 1.0], + [0.3333333333333333, 0.6666666666666667, 1.0], + [0.3492063492063492, 0.6507936507936508, 1.0], + [0.36507936507936506, 0.6349206349206349, 1.0], + [0.38095238095238093, 0.6190476190476191, 1.0], + [0.3968253968253968, 0.6031746031746033, 1.0], + [0.4126984126984127, 0.5873015873015873, 1.0], + [0.42857142857142855, 0.5714285714285714, 1.0], + [0.4444444444444444, 0.5555555555555556, 1.0], + [0.4603174603174603, 0.5396825396825398, 1.0], + [0.47619047619047616, 0.5238095238095238, 1.0], + [0.49206349206349204, 0.5079365079365079, 1.0], + [0.5079365079365079, 0.4920634920634921, 1.0], + [0.5238095238095237, 0.4761904761904763, 1.0], + [0.5396825396825397, 0.46031746031746035, 1.0], + [0.5555555555555556, 0.4444444444444444, 1.0], + [0.5714285714285714, 0.4285714285714286, 1.0], + [0.5873015873015872, 0.4126984126984128, 1.0], + [0.6031746031746031, 0.39682539682539686, 1.0], + [0.6190476190476191, 0.38095238095238093, 1.0], + [0.6349206349206349, 0.3650793650793651, 1.0], + [0.6507936507936507, 0.3492063492063493, 1.0], + [0.6666666666666666, 0.33333333333333337, 1.0], + [0.6825396825396826, 0.31746031746031744, 1.0], + [0.6984126984126984, 0.3015873015873016, 1.0], + [0.7142857142857142, 0.2857142857142858, 1.0], + [0.7301587301587301, 0.2698412698412699, 1.0], + [0.746031746031746, 0.25396825396825395, 1.0], + [0.7619047619047619, 0.23809523809523814, 1.0], + [0.7777777777777777, 0.22222222222222232, 1.0], + [0.7936507936507936, 0.2063492063492064, 1.0], + [0.8095238095238095, 0.19047619047619047, 1.0], + [0.8253968253968254, 0.17460317460317465, 1.0], + [0.8412698412698412, 0.15873015873015883, 1.0], + [0.8571428571428571, 0.1428571428571429, 1.0], + [0.873015873015873, 0.12698412698412698, 1.0], + [0.8888888888888888, 0.11111111111111116, 1.0], + [0.9047619047619047, 0.09523809523809534, 1.0], + [0.9206349206349206, 0.07936507936507942, 1.0], + [0.9365079365079365, 0.06349206349206349, 1.0], + [0.9523809523809523, 0.04761904761904767, 1.0], + [0.9682539682539681, 0.031746031746031855, 1.0], + [0.9841269841269841, 0.015873015873015928, 1.0], + [1.0, 0.0, 1.0], + ] + + +def get_hsv_base() -> list[RGBColor]: + return [ + [1.0, 0.0, 0.0], + [1.0, 0.09375009375009374, 0.0], + [1.0, 0.1875001875001875, 0.0], + [1.0, 0.28125028125028123, 0.0], + [1.0, 0.375000375000375, 0.0], + [1.0, 0.4687504687504688, 0.0], + [1.0, 0.5625005625005625, 0.0], + [1.0, 0.6562506562506563, 0.0], + [1.0, 0.75000075000075, 0.0], + [1.0, 0.8437508437508436, 0.0], + [0.9999996874996875, 0.937500625000625, 0.0], + [0.9687489687489689, 1.0, 0.0], + [0.8749988749988751, 1.0, 0.0], + [0.7812487812487814, 1.0, 0.0], + [0.6874986874986876, 1.0, 0.0], + [0.5937485937485938, 1.0, 0.0], + [0.4999984999985, 1.0, 0.0], + [0.4062484062484063, 1.0, 0.0], + [0.3124983124983125, 1.0, 0.0], + [0.21874821874821881, 1.0, 0.0], + [0.12499812499812502, 1.0, 0.0], + [0.031249343749343742, 1.0, 1.3125013125182343e-06], + [0.0, 1.0, 0.0625020624890686], + [0.0, 1.0, 0.15625156561670206], + [0.0, 1.0, 0.2500010687443355], + [0.0, 1.0, 0.34375057187196895], + [0.0, 1.0, 0.43750007499960236], + [0.0, 1.0, 0.5312495781272358], + [0.0, 1.0, 0.6249990812548692], + [0.0, 1.0, 0.7187485843825027], + [0.0, 1.0, 0.8124980875101362], + [0.0, 1.0, 0.9062475906377696], + [0.0, 1.0, 0.999997093765403], + [0.0, 0.9062528125028132, 1.0], + [0.0, 0.8125027187527188, 1.0], + [0.0, 0.718752625002625, 1.0], + [0.0, 0.6250025312525314, 1.0], + [0.0, 0.5312524375024383, 1.0], + [0.0, 0.437502343752344, 1.0], + [0.0, 0.3437522500022503, 1.0], + [0.0, 0.2500021562521566, 1.0], + [0.0, 0.15625206250206347, 1.0], + [0.0, 0.06250196875196912, 1.0], + [0.031249374999375024, 1.2500012499527813e-06, 1.0], + [0.12499821874821886, 0.0, 1.0], + [0.21874831249831198, 0.0, 1.0], + [0.3124984062484064, 0.0, 1.0], + [0.4062484999985002, 0.0, 1.0], + [0.49999859374859396, 0.0, 1.0], + [0.593748687498687, 0.0, 1.0], + [0.6874987812487816, 0.0, 1.0], + [0.7812488749988753, 0.0, 1.0], + [0.8749989687489691, 0.0, 1.0], + [0.9687490624990622, 0.0, 1.0], + [0.9999997187497188, 0.0, 0.9375005625005625], + [1.0, 0.0, 0.8437507500007498], + [1.0, 0.0, 0.7500006562506562], + [1.0, 0.0, 0.656250562500563], + [1.0, 0.0, 0.5625004687504687], + [1.0, 0.0, 0.468750375000375], + [1.0, 0.0, 0.3750002812502812], + [1.0, 0.0, 0.28125018750018815], + [1.0, 0.0, 0.1875000937500937], + [1.0, 0.0, 0.09375], + ] + + +def get_bgr_base() -> list[RGBColor]: + return [ + [0.0, 0.0, 1.0], + [0.031746031746031744, 0.0, 0.9682539682539683], + [0.06349206349206349, 0.0, 0.9365079365079365], + [0.09523809523809523, 0.0, 0.9047619047619048], + [0.12698412698412698, 0.0, 0.873015873015873], + [0.15873015873015872, 0.0, 0.8412698412698413], + [0.19047619047619047, 0.0, 0.8095238095238095], + [0.2222222222222222, 0.0, 0.7777777777777778], + [0.25396825396825395, 0.0, 0.746031746031746], + [0.2857142857142857, 0.0, 0.7142857142857143], + [0.31746031746031744, 0.0, 0.6825396825396826], + [0.3492063492063492, 0.0, 0.6507936507936508], + [0.38095238095238093, 0.0, 0.6190476190476191], + [0.4126984126984127, 0.0, 0.5873015873015873], + [0.4444444444444444, 0.0, 0.5555555555555556], + [0.47619047619047616, 0.0, 0.5238095238095238], + [0.5079365079365079, 0.0, 0.4920634920634921], + [0.5396825396825397, 0.0, 0.46031746031746035], + [0.5714285714285714, 0.0, 0.4285714285714286], + [0.6031746031746031, 0.0, 0.39682539682539686], + [0.6349206349206349, 0.0, 0.3650793650793651], + [0.6666666666666666, 0.0, 0.33333333333333337], + [0.6984126984126984, 0.0, 0.3015873015873016], + [0.7301587301587301, 0.0, 0.2698412698412699], + [0.7619047619047619, 0.0, 0.23809523809523814], + [0.7936507936507936, 0.0, 0.2063492063492064], + [0.8253968253968254, 0.0, 0.17460317460317465], + [0.8571428571428571, 0.0, 0.1428571428571429], + [0.8888888888888888, 0.0, 0.11111111111111116], + [0.9206349206349206, 0.0, 0.07936507936507942], + [0.9523809523809523, 0.0, 0.04761904761904767], + [0.9841269841269841, 0.0, 0.015873015873015928], + [0.9841269841269842, 0.015873015873015872, 0.0], + [0.9523809523809526, 0.047619047619047394, 0.0], + [0.9206349206349207, 0.07936507936507936, 0.0], + [0.8888888888888888, 0.1111111111111111, 0.0], + [0.8571428571428572, 0.14285714285714285, 0.0], + [0.8253968253968256, 0.17460317460317437, 0.0], + [0.7936507936507937, 0.20634920634920634, 0.0], + [0.7619047619047619, 0.23809523809523808, 0.0], + [0.7301587301587302, 0.2698412698412698, 0.0], + [0.6984126984126986, 0.30158730158730135, 0.0], + [0.6666666666666667, 0.3333333333333333, 0.0], + [0.6349206349206349, 0.36507936507936506, 0.0], + [0.6031746031746033, 0.3968253968253968, 0.0], + [0.5714285714285716, 0.4285714285714283, 0.0], + [0.5396825396825398, 0.4603174603174603, 0.0], + [0.5079365079365079, 0.49206349206349204, 0.0], + [0.47619047619047616, 0.5238095238095238, 0.0], + [0.44444444444444464, 0.5555555555555554, 0.0], + [0.4126984126984127, 0.5873015873015873, 0.0], + [0.38095238095238093, 0.6190476190476191, 0.0], + [0.3492063492063492, 0.6507936507936508, 0.0], + [0.31746031746031766, 0.6825396825396823, 0.0], + [0.2857142857142857, 0.7142857142857143, 0.0], + [0.25396825396825395, 0.746031746031746, 0.0], + [0.2222222222222222, 0.7777777777777778, 0.0], + [0.1904761904761907, 0.8095238095238093, 0.0], + [0.15873015873015872, 0.8412698412698413, 0.0], + [0.12698412698412698, 0.873015873015873, 0.0], + [0.09523809523809523, 0.9047619047619048, 0.0], + [0.06349206349206371, 0.9365079365079363, 0.0], + [0.031746031746031744, 0.9682539682539683, 0.0], + [0.0, 1.0, 0.0], + ] + + +def get_pink_base() -> list[RGBColor]: + return [ + [0.1178, 0.0, 0.0], + [0.1958570548040548, 0.10286904261004261, 0.10286904261004261], + [0.2506610896140896, 0.1454790653900654, 0.1454790653900654], + [0.29546811656811656, 0.17817408269208268, 0.17817408269208268], + [0.33432413915213915, 0.20573809713609714, 0.20573809713609714], + [0.3691121589001589, 0.23002210977010978, 0.23002210977010978], + [0.40089217663417664, 0.25197612114012113, 0.25197612114012113], + [0.4303311928571929, 0.27216613153713154, 0.27216613153713154], + [0.4578822078802079, 0.2909571412001412, 0.2909571412001412], + [0.48386722192222187, 0.30860715023715024, 0.30860715023715024], + [0.5085252351702352, 0.3253001587801588, 0.3253001587801588], + [0.5320422477312478, 0.34117816687016683, 0.34117816687016683], + [0.5545632596922597, 0.3563481746121746, 0.3563481746121746], + [0.5762042711412712, 0.370899182013182, 0.370899182013182], + [0.5970612821282821, 0.38490018914018914, 0.38490018914018914], + [0.6172132927402928, 0.39841019599019595, 0.39841019599019595], + [0.6367293029443029, 0.4114762026082026, 0.4114762026082026], + [0.6556633128513129, 0.42413920904920904, 0.42413920904920904], + [0.6740663224523225, 0.4364362152622152, 0.4364362152622152], + [0.6919803317593318, 0.44839522133122134, 0.44839522133122134], + [0.7094413408403408, 0.4600442272202272, 0.4600442272202272], + [0.7264833496713496, 0.471405232953233, 0.471405232953233], + [0.7431343583143583, 0.48249823856823854, 0.48249823856823854], + [0.7594211595051595, 0.4933425567615568, 0.49334224405324406], + [0.766356164952165, 0.517549555000555, 0.5039532493842493], + [0.7732291703251704, 0.5406745543755543, 0.5143442546752547], + [0.7800421756041757, 0.5628495546845547, 0.5245312597662597], + [0.7867961807921808, 0.5841835557145558, 0.5345222648432649], + [0.7934921859201859, 0.6047655573125573, 0.5443312697802698], + [0.800132190994191, 0.6246695593815594, 0.5539662746302746], + [0.806718195960196, 0.6439585618705619, 0.5634362794202794], + [0.8132502008675463, 0.6626875645679968, 0.572750284066388], + [0.8197297991324537, 0.6808994354320032, 0.5819137159336122], + [0.8261598070998071, 0.6986374678594677, 0.5909367293097293], + [0.832538815008815, 0.7159364983284983, 0.5998237422767423], + [0.8388698227318228, 0.7328275270515271, 0.6085807548037548], + [0.8451538303318303, 0.7493375542295543, 0.617212766935767], + [0.8513918378118378, 0.7654925799695799, 0.6257267786357786], + [0.8575838451998452, 0.7813126044996045, 0.63412579002479], + [0.8637308524718524, 0.7968186278556278, 0.642415801039801], + [0.8698348596078597, 0.8120286501696502, 0.6505998117678118], + [0.8758968666358666, 0.8269596715176715, 0.6586818221958222], + [0.8819168735798735, 0.8416246920346919, 0.6666668323148323], + [0.8878958804198804, 0.8560397116997117, 0.6745558422198422], + [0.8938348871588873, 0.8702157306557308, 0.6823548518188519], + [0.8997348937998937, 0.8841637489357488, 0.6900658612018611], + [0.9055969003459003, 0.8978957665557666, 0.6976908703748703], + [0.9114209068159068, 0.9114207835997836, 0.7052338793118793], + [0.9172079131949132, 0.9172079131949132, 0.7271656710196709], + [0.9229579194999195, 0.9229579194999195, 0.7484547019537018], + [0.9286729257049257, 0.9286729257049257, 0.7691557308867307], + [0.9343529318399318, 0.9343529318399318, 0.7893137581037581], + [0.939998937893938, 0.939998937893938, 0.8089687837947839], + [0.9456109438799438, 0.9456109438799438, 0.828158808099808], + [0.9511899497889498, 0.9511899497889498, 0.8469128312138313], + [0.9567359556319557, 0.9567359556319557, 0.8652608532158531], + [0.9622499614019614, 0.9622499614019614, 0.8832288742238742], + [0.967732967101967, 0.967732967101967, 0.9008368943518943], + [0.9731849727399727, 0.9731849727399727, 0.9181089136399135], + [0.9786069783119783, 0.9786069783119783, 0.9350609321919323], + [0.9839989838239838, 0.9839989838239838, 0.95171095004995], + [0.9893609892759894, 0.9893609892759894, 0.9680749672719673], + [0.9946949946659946, 0.9946949946659946, 0.984166983907984], + [1.0, 1.0, 1.0], + ] + + +def get_hot_base() -> list[RGBColor]: + return [ + [0.0416, 0.0, 0.0], + [0.0832696068869982, 0.0, 0.0], + [0.12493921377399639, 0.0, 0.0], + [0.16660882066099458, 0.0, 0.0], + [0.20827842754799278, 0.0, 0.0], + [0.24994803443499097, 0.0, 0.0], + [0.2916176413219892, 0.0, 0.0], + [0.33328724820898736, 0.0, 0.0], + [0.37495685509598553, 0.0, 0.0], + [0.4166264619829838, 0.0, 0.0], + [0.458296068869982, 0.0, 0.0], + [0.49996567575698014, 0.0, 0.0], + [0.5416352826439783, 0.0, 0.0], + [0.5833048895309766, 0.0, 0.0], + [0.6249744964179746, 0.0, 0.0], + [0.6666441033049729, 0.0, 0.0], + [0.7083137101919711, 0.0, 0.0], + [0.7499833170789694, 0.0, 0.0], + [0.7916529239659675, 0.0, 0.0], + [0.8333225308529657, 0.0, 0.0], + [0.8749921377399639, 0.0, 0.0], + [0.916661744626962, 0.0, 0.0], + [0.9583313515139603, 0.0, 0.0], + [1.0, 9.583317761411433e-07, 0.0], + [1.0, 0.041667557290219495, 0.0], + [1.0, 0.08333415624866285, 0.0], + [1.0, 0.12500075520710618, 0.0], + [1.0, 0.16666735416554954, 0.0], + [1.0, 0.2083339531239929, 0.0], + [1.0, 0.25000055208243627, 0.0], + [1.0, 0.2916671510408796, 0.0], + [1.0, 0.33333374999932297, 0.0], + [1.0, 0.3750003489577663, 0.0], + [1.0, 0.41666694791620934, 0.0], + [1.0, 0.458333546874653, 0.0], + [1.0, 0.5000001458330964, 0.0], + [1.0, 0.5416667447915398, 0.0], + [1.0, 0.5833333437499828, 0.0], + [1.0, 0.6249999427084264, 0.0], + [1.0, 0.6666665416668698, 0.0], + [1.0, 0.7083331406253132, 0.0], + [1.0, 0.7499997395837562, 0.0], + [1.0, 0.7916663385421998, 0.0], + [1.0, 0.8333329375006432, 0.0], + [1.0, 0.8749995364590866, 0.0], + [1.0, 0.9166661354175296, 0.0], + [1.0, 0.9583327343759732, 0.0], + [1.0, 0.9999993333344166, 0.0], + [1.0, 1.0, 0.06249906249906237], + [1.0, 1.0, 0.12499912499912444], + [1.0, 1.0, 0.1874991874991874], + [1.0, 1.0, 0.24999924999924988], + [1.0, 1.0, 0.3124993124993124], + [1.0, 1.0, 0.37499937499937447], + [1.0, 1.0, 0.4374994374994374], + [1.0, 1.0, 0.49999949999949994], + [1.0, 1.0, 0.5624995624995625], + [1.0, 1.0, 0.6249996249996245], + [1.0, 1.0, 0.6874996874996875], + [1.0, 1.0, 0.7499997499997499], + [1.0, 1.0, 0.8124998124998125], + [1.0, 1.0, 0.8749998749998745], + [1.0, 1.0, 0.9374999374999375], + [1.0, 1.0, 1.0], + ] + + +def get_plasma_base() -> list[RGBColor]: + return [ + [0.050383, 0.029803, 0.527975], + [0.096379, 0.025165, 0.547103], + [0.132381, 0.022258, 0.56325], + [0.16407, 0.020171, 0.577478], + [0.193374, 0.018354, 0.59033], + [0.221197, 0.016497, 0.602083], + [0.248032, 0.014439, 0.612868], + [0.274191, 0.012109, 0.622722], + [0.299855, 0.009561, 0.631624], + [0.32515, 0.006915, 0.639512], + [0.35015, 0.004382, 0.646298], + [0.374897, 0.002245, 0.651876], + [0.399411, 0.000859, 0.656133], + [0.423689, 0.000646, 0.658956], + [0.447714, 0.00208, 0.66024], + [0.471457, 0.005678, 0.659897], + [0.500678, 0.014055, 0.657088], + [0.523633, 0.024532, 0.652901], + [0.546157, 0.038954, 0.64701], + [0.568201, 0.055778, 0.639477], + [0.589719, 0.072878, 0.630408], + [0.610667, 0.090204, 0.619951], + [0.631017, 0.107699, 0.608287], + [0.650746, 0.125309, 0.595617], + [0.669845, 0.142992, 0.582154], + [0.688318, 0.160709, 0.568103], + [0.706178, 0.178437, 0.553657], + [0.723444, 0.196158, 0.538981], + [0.740143, 0.213864, 0.524216], + [0.756304, 0.231555, 0.509468], + [0.771958, 0.249237, 0.494813], + [0.787133, 0.266922, 0.480307], + [0.805467, 0.289057, 0.462415], + [0.819651, 0.306812, 0.448306], + [0.833422, 0.324635, 0.434366], + [0.846788, 0.342551, 0.420579], + [0.85975, 0.360588, 0.406917], + [0.872303, 0.378774, 0.393355], + [0.884436, 0.397139, 0.37986], + [0.896131, 0.415712, 0.366407], + [0.907365, 0.434524, 0.35297], + [0.918109, 0.453603, 0.339529], + [0.928329, 0.472975, 0.326067], + [0.93799, 0.492667, 0.312575], + [0.947051, 0.512699, 0.299049], + [0.95547, 0.533093, 0.28549], + [0.963203, 0.553865, 0.271909], + [0.970205, 0.575028, 0.258325], + [0.977856, 0.602051, 0.241387], + [0.983041, 0.624131, 0.227937], + [0.987332, 0.646633, 0.214648], + [0.990681, 0.669558, 0.201642], + [0.993032, 0.692907, 0.189084], + [0.994324, 0.716681, 0.177208], + [0.994495, 0.74088, 0.166335], + [0.993482, 0.765499, 0.156891], + [0.991209, 0.790537, 0.149377], + [0.987621, 0.815978, 0.144363], + [0.982653, 0.841812, 0.142303], + [0.976265, 0.868016, 0.143351], + [0.968443, 0.894564, 0.147014], + [0.959276, 0.921407, 0.151566], + [0.949151, 0.948435, 0.152178], + [0.940015, 0.975158, 0.131326], + ] + + +def get_viridis_base() -> list[RGBColor]: + return [ + [0.267004, 0.004874, 0.329415], + [0.272594, 0.025563, 0.353093], + [0.277018, 0.050344, 0.375715], + [0.280267, 0.073417, 0.397163], + [0.282327, 0.094955, 0.417331], + [0.283197, 0.11568, 0.436115], + [0.282884, 0.13592, 0.453427], + [0.281412, 0.155834, 0.469201], + [0.278826, 0.17549, 0.483397], + [0.275191, 0.194905, 0.496005], + [0.270595, 0.214069, 0.507052], + [0.265145, 0.232956, 0.516599], + [0.258965, 0.251537, 0.524736], + [0.252194, 0.269783, 0.531579], + [0.244972, 0.287675, 0.53726], + [0.237441, 0.305202, 0.541921], + [0.227802, 0.326594, 0.546532], + [0.220057, 0.343307, 0.549413], + [0.212395, 0.359683, 0.55171], + [0.204903, 0.375746, 0.553533], + [0.197636, 0.391528, 0.554969], + [0.190631, 0.407061, 0.556089], + [0.183898, 0.422383, 0.556944], + [0.177423, 0.437527, 0.557565], + [0.171176, 0.45253, 0.557965], + [0.165117, 0.467423, 0.558141], + [0.159194, 0.482237, 0.558073], + [0.153364, 0.497, 0.557724], + [0.147607, 0.511733, 0.557049], + [0.141935, 0.526453, 0.555991], + [0.136408, 0.541173, 0.554483], + [0.131172, 0.555899, 0.552459], + [0.125394, 0.574318, 0.549086], + [0.121831, 0.589055, 0.545623], + [0.119738, 0.603785, 0.5414], + [0.119699, 0.61849, 0.536347], + [0.122312, 0.633153, 0.530398], + [0.128087, 0.647749, 0.523491], + [0.137339, 0.662252, 0.515571], + [0.150148, 0.676631, 0.506589], + [0.166383, 0.690856, 0.496502], + [0.185783, 0.704891, 0.485273], + [0.20803, 0.718701, 0.472873], + [0.232815, 0.732247, 0.459277], + [0.259857, 0.745492, 0.444467], + [0.288921, 0.758394, 0.428426], + [0.319809, 0.770914, 0.411152], + [0.35236, 0.783011, 0.392636], + [0.395174, 0.797475, 0.367757], + [0.430983, 0.808473, 0.346476], + [0.468053, 0.818921, 0.323998], + [0.506271, 0.828786, 0.300362], + [0.545524, 0.838039, 0.275626], + [0.585678, 0.846661, 0.249897], + [0.626579, 0.854645, 0.223353], + [0.668054, 0.861999, 0.196293], + [0.709898, 0.868751, 0.169257], + [0.751884, 0.874951, 0.143228], + [0.79376, 0.880678, 0.120005], + [0.83527, 0.886029, 0.102646], + [0.876168, 0.891125, 0.09525], + [0.916242, 0.896091, 0.100717], + [0.9553, 0.901065, 0.118128], + [0.993248, 0.906157, 0.143936], + ] + + +def get_cividis_base() -> list[RGBColor]: + return [ + [0.0, 0.135112, 0.304751], + [0.0, 0.146877, 0.330479], + [0.0, 0.157932, 0.357521], + [0.0, 0.168204, 0.385902], + [0.0, 0.178802, 0.414764], + [0.0, 0.188769, 0.439563], + [0.017852, 0.198528, 0.441248], + [0.068968, 0.209372, 0.438863], + [0.103401, 0.220406, 0.43579], + [0.130669, 0.231458, 0.43284], + [0.154261, 0.242475, 0.43012], + [0.17549, 0.253444, 0.42779], + [0.195057, 0.264372, 0.425924], + [0.213431, 0.275266, 0.42448], + [0.230871, 0.286134, 0.423498], + [0.247605, 0.296986, 0.422917], + [0.267693, 0.310542, 0.422821], + [0.28324, 0.32139, 0.423211], + [0.298421, 0.332247, 0.423973], + [0.313287, 0.34312, 0.42512], + [0.327875, 0.354016, 0.42667], + [0.342246, 0.364939, 0.428559], + [0.356418, 0.375896, 0.430823], + [0.37043, 0.38689, 0.433428], + [0.384268, 0.397928, 0.436475], + [0.397991, 0.409011, 0.439848], + [0.411607, 0.420145, 0.443577], + [0.42512, 0.431334, 0.447692], + [0.438504, 0.44258, 0.452341], + [0.451759, 0.453887, 0.457582], + [0.464947, 0.465241, 0.463395], + [0.478186, 0.476699, 0.468845], + [0.495913, 0.491076, 0.471751], + [0.51054, 0.502643, 0.47255], + [0.525348, 0.514285, 0.47266], + [0.540307, 0.526005, 0.472163], + [0.555393, 0.537807, 0.471147], + [0.570607, 0.549695, 0.469593], + [0.585916, 0.561674, 0.467618], + [0.601354, 0.573743, 0.465074], + [0.616852, 0.585913, 0.462237], + [0.632506, 0.59818, 0.458668], + [0.648222, 0.610553, 0.454801], + [0.664055, 0.623034, 0.450338], + [0.679979, 0.635626, 0.445424], + [0.695985, 0.648334, 0.440072], + [0.712105, 0.66116, 0.434117], + [0.728334, 0.674107, 0.427554], + [0.748772, 0.69047, 0.418448], + [0.765223, 0.703705, 0.410587], + [0.781795, 0.717074, 0.401966], + [0.79848, 0.73058, 0.392597], + [0.815274, 0.744226, 0.382504], + [0.832192, 0.758014, 0.371529], + [0.849223, 0.771947, 0.359729], + [0.866421, 0.786028, 0.346571], + [0.88372, 0.800258, 0.332599], + [0.901195, 0.814639, 0.317021], + [0.918828, 0.829168, 0.29996], + [0.93666, 0.843841, 0.280876], + [0.954725, 0.858646, 0.259365], + [0.973114, 0.87355, 0.234677], + [0.992218, 0.888385, 0.205468], + [0.995737, 0.909344, 0.217772], + ] + + +def get_twilight_base() -> list[RGBColor]: + return [ + [0.8857501584075443, 0.8500092494306783, 0.8879736506427196], + [0.8669601550533358, 0.8510896085314068, 0.8790976697717334], + [0.8403042080595778, 0.8425605950855084, 0.865250882410458], + [0.8046916442981458, 0.8270983878056066, 0.8484244929697402], + [0.7618690793798029, 0.8071194938764749, 0.830266486168872], + [0.7150040307625213, 0.7842648709158119, 0.8131111655994991], + [0.6672147980375281, 0.7595112803730375, 0.7986367465453776], + [0.6209919306481755, 0.733369347989232, 0.7871899462469658], + [0.5778116438998202, 0.7061310615715398, 0.7782534512102552], + [0.5383401557517628, 0.677970811290954, 0.7711273443905772], + [0.5028853540415561, 0.6489721673409526, 0.7652144671817491], + [0.46808490970531597, 0.6153702869579029, 0.7594078989109274], + [0.441912717666964, 0.5846144110017557, 0.7545183888441067], + [0.42037822361396326, 0.5529928715810817, 0.7494412559451894], + [0.40346600333905397, 0.5204784765384003, 0.7437421522356709], + [0.390920175719949, 0.4870552025122304, 0.7369797713388125], + [0.38222094988570376, 0.45272225034283325, 0.7287165614400378], + [0.37662448930806297, 0.41749553747893614, 0.7185245473617611], + [0.37324816143326384, 0.38140848555753937, 0.7059820097480604], + [0.37115856636209105, 0.3445191141023017, 0.6906525358646499], + [0.3694333459127623, 0.3069292355186234, 0.6720505284912062], + [0.3668108554010799, 0.26404857724525643, 0.6464799165290824], + [0.3628764356004103, 0.2258414929328703, 0.6188041741082302], + [0.35641381143126327, 0.18837119869242164, 0.5855320765920552], + [0.3463150717579309, 0.15312802296627787, 0.5457599924248665], + [0.33146154891145124, 0.12244245263391232, 0.4991827458843107], + [0.31129434479712365, 0.0986847719340522, 0.4471931371516162], + [0.28669151388283165, 0.08242987384848069, 0.39331182532243375], + [0.26003895187439957, 0.0722947810433622, 0.3417175669546984], + [0.23433055727563878, 0.0669355996616394, 0.295740099298676], + [0.21237411048541005, 0.06608502466175845, 0.2576516509356954], + [0.1960826032659409, 0.07032122724242362, 0.22874673279569585], + [0.19571853588267552, 0.07215778103960274, 0.21617499187076789], + [0.21625279838647882, 0.06761633027051639, 0.22324568672294431], + [0.2438114972024792, 0.06828985152901187, 0.23550427698321885], + [0.277471508091428, 0.07258296989925478, 0.2512349399594277], + [0.3153587000920581, 0.07942099731524319, 0.2681190407694196], + [0.35573889947341125, 0.08803897435024335, 0.2840695897279818], + [0.39719876876325577, 0.09830232096827862, 0.2974385647628578], + [0.4385637818793154, 0.11077667828375456, 0.30712600062348083], + [0.47877034239726296, 0.12642401401409453, 0.3127417273582196], + [0.5169084880054446, 0.14598463068714407, 0.3147540759228004], + [0.5566322903496934, 0.17269677158182117, 0.31423043195101424], + [0.5889486193554471, 0.19997020971775276, 0.31288922211590126], + [0.6185686525887719, 0.2297879924917992, 0.3120046383872893], + [0.6456734938264543, 0.2615520316161528, 0.31248673753935263], + [0.670399595476762, 0.29486126309253047, 0.3150381395687221], + [0.6928154484676493, 0.32945984647042675, 0.3202754315314667], + [0.7129346683977347, 0.36517570519856757, 0.3288027427038317], + [0.7307482075484517, 0.401868526884681, 0.3412449533046818], + [0.7462661578916344, 0.439392450449735, 0.3582343914230064], + [0.7595711110534006, 0.4775687122205708, 0.380360657784311], + [0.7708809196230247, 0.516168926434691, 0.40808667584648967], + [0.7817418047898184, 0.5597509805259486, 0.44624687186865436], + [0.7904577684772788, 0.5982813427706246, 0.48622257801969754], + [0.7991624518501963, 0.6362837937202919, 0.5313449594256143], + [0.8087321917008969, 0.6734621670219452, 0.5806983480557674], + [0.8198723065045529, 0.7095047497878748, 0.6330385704006711], + [0.8328327718577798, 0.7439727181745988, 0.6868223587464855], + [0.847111955079651, 0.7761188023604716, 0.7401227576280706], + [0.8610711702756552, 0.8047851790981223, 0.7903952966373629], + [0.8721533197845432, 0.8284805282370967, 0.8330486712880828], + [0.8811916987134195, 0.8442906671771735, 0.8635659516866988], + [0.8857115512284565, 0.8500218611585632, 0.8857253899008712], + ] + + +def get_turbo_base() -> list[RGBColor]: + return [ + [0.18995, 0.07176, 0.23217], + [0.2086, 0.11802, 0.34607], + [0.225, 0.16354, 0.45096], + [0.23915, 0.20833, 0.54686], + [0.25107, 0.25237, 0.63374], + [0.26074, 0.29568, 0.71162], + [0.26816, 0.33825, 0.7805], + [0.27334, 0.38008, 0.84037], + [0.27628, 0.42118, 0.89123], + [0.27698, 0.46153, 0.93309], + [0.27543, 0.50115, 0.96594], + [0.27106, 0.54015, 0.9893], + [0.25862, 0.57958, 0.99876], + [0.23874, 0.61931, 0.99485], + [0.21382, 0.65886, 0.97959], + [0.18625, 0.69775, 0.95498], + [0.15173, 0.74472, 0.91416], + [0.12698, 0.78037, 0.8759], + [0.10738, 0.81381, 0.83484], + [0.09532, 0.84455, 0.79299], + [0.0932, 0.87211, 0.75237], + [0.10342, 0.896, 0.715], + [0.12733, 0.91701, 0.67627], + [0.16319, 0.93609, 0.63137], + [0.20877, 0.95304, 0.58199], + [0.2618, 0.96765, 0.52981], + [0.32006, 0.97974, 0.47654], + [0.38127, 0.98909, 0.42386], + [0.44321, 0.99551, 0.37345], + [0.50362, 0.99879, 0.32701], + [0.56026, 0.99873, 0.28623], + [0.61088, 0.99514, 0.2528], + [0.66428, 0.98524, 0.2237], + [0.70553, 0.97255, 0.21032], + [0.74617, 0.95593, 0.20406], + [0.78563, 0.93579, 0.20336], + [0.82333, 0.91253, 0.20663], + [0.85868, 0.88655, 0.2123], + [0.89112, 0.85826, 0.2188], + [0.92004, 0.82806, 0.22456], + [0.94489, 0.79634, 0.228], + [0.96507, 0.76352, 0.22754], + [0.98, 0.73, 0.22161], + [0.98986, 0.69382, 0.21043], + [0.99535, 0.65341, 0.19577], + [0.99672, 0.60977, 0.17842], + [0.99419, 0.56386, 0.15918], + [0.98799, 0.51667, 0.13883], + [0.97545, 0.4574, 0.11305], + [0.96187, 0.41093, 0.0931], + [0.94538, 0.36638, 0.07461], + [0.92623, 0.32473, 0.05837], + [0.90463, 0.28696, 0.04516], + [0.88066, 0.25334, 0.03521], + [0.8538, 0.2217, 0.02677], + [0.82399, 0.19182, 0.01966], + [0.79125, 0.16368, 0.01387], + [0.75556, 0.13731, 0.00942], + [0.71692, 0.11268, 0.00629], + [0.67535, 0.0898, 0.00449], + [0.63082, 0.06868, 0.00401], + [0.58336, 0.04931, 0.00486], + [0.53295, 0.03169, 0.00705], + [0.4796, 0.01583, 0.01055], + ] + + +def get_seismic_base() -> list[RGBColor]: + return [ + [0.0, 0.0, 0.3], + [0.0, 0.0, 0.34444444444444444], + [0.0, 0.0, 0.38888888888888884], + [0.0, 0.0, 0.4333333333333333], + [0.0, 0.0, 0.47777777777777775], + [0.0, 0.0, 0.5222222222222221], + [0.0, 0.0, 0.5666666666666667], + [0.0, 0.0, 0.611111111111111], + [0.0, 0.0, 0.6555555555555554], + [0.0, 0.0, 0.7], + [0.0, 0.0, 0.7444444444444444], + [0.0, 0.0, 0.7888888888888888], + [0.0, 0.0, 0.8333333333333333], + [0.0, 0.0, 0.8777777777777778], + [0.0, 0.0, 0.922222222222222], + [0.0, 0.0, 0.9666666666666666], + [0.015873015873015872, 0.015873015873015872, 1.0], + [0.07936507936507936, 0.07936507936507936, 1.0], + [0.14285714285714285, 0.14285714285714285, 1.0], + [0.20634920634920634, 0.20634920634920634, 1.0], + [0.2698412698412698, 0.2698412698412698, 1.0], + [0.3333333333333333, 0.3333333333333333, 1.0], + [0.3968253968253968, 0.3968253968253968, 1.0], + [0.4603174603174603, 0.4603174603174603, 1.0], + [0.5238095238095238, 0.5238095238095238, 1.0], + [0.5873015873015873, 0.5873015873015873, 1.0], + [0.6507936507936508, 0.6507936507936508, 1.0], + [0.7142857142857143, 0.7142857142857143, 1.0], + [0.7777777777777778, 0.7777777777777778, 1.0], + [0.8412698412698413, 0.8412698412698413, 1.0], + [0.9047619047619048, 0.9047619047619048, 1.0], + [0.9682539682539683, 0.9682539682539683, 1.0], + [1.0, 0.9682539682539683, 0.9682539682539683], + [1.0, 0.9047619047619052, 0.9047619047619052], + [1.0, 0.8412698412698413, 0.8412698412698413], + [1.0, 0.7777777777777778, 0.7777777777777778], + [1.0, 0.7142857142857143, 0.7142857142857143], + [1.0, 0.6507936507936513, 0.6507936507936513], + [1.0, 0.5873015873015873, 0.5873015873015873], + [1.0, 0.5238095238095238, 0.5238095238095238], + [1.0, 0.46031746031746035, 0.46031746031746035], + [1.0, 0.3968253968253973, 0.3968253968253973], + [1.0, 0.33333333333333337, 0.33333333333333337], + [1.0, 0.2698412698412699, 0.2698412698412699], + [1.0, 0.2063492063492064, 0.2063492063492064], + [1.0, 0.14285714285714335, 0.14285714285714335], + [1.0, 0.07936507936507942, 0.07936507936507942], + [1.0, 0.015873015873015928, 0.015873015873015928], + [0.9761904761904762, 0.0, 0.0], + [0.9444444444444446, 0.0, 0.0], + [0.9126984126984127, 0.0, 0.0], + [0.8809523809523809, 0.0, 0.0], + [0.8492063492063492, 0.0, 0.0], + [0.8174603174603177, 0.0, 0.0], + [0.7857142857142857, 0.0, 0.0], + [0.753968253968254, 0.0, 0.0], + [0.7222222222222222, 0.0, 0.0], + [0.6904761904761907, 0.0, 0.0], + [0.6587301587301587, 0.0, 0.0], + [0.626984126984127, 0.0, 0.0], + [0.5952380952380952, 0.0, 0.0], + [0.5634920634920637, 0.0, 0.0], + [0.5317460317460317, 0.0, 0.0], + [0.5, 0.0, 0.0], + ] diff --git a/kornia/color/colormap.py b/kornia/color/colormap.py new file mode 100644 index 0000000..bebb6d0 --- /dev/null +++ b/kornia/color/colormap.py @@ -0,0 +1,314 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from enum import Enum +from typing import Optional, Union + +import torch +from torch import nn +from torch.nn.functional import interpolate + +import kornia.color._colormap_data as cm_data +from kornia.color._colormap_data import RGBColor +from kornia.core.check import KORNIA_CHECK + + +class ColorMapType(Enum): + r"""An enumeration for available colormaps. + + List of available colormaps: + + .. image:: _static/img/ColorMapType.png + """ + + autumn = 1 + bone = 2 + jet = 3 + winter = 4 + rainbow = 5 + ocean = 6 + summer = 7 + spring = 8 + cool = 9 + hsv = 10 + brg = 11 + pink = 12 + hot = 13 + plasma = 14 + viridis = 15 + cividis = 16 + twilight = 17 + turbo = 18 + seismic = 19 + + def _load_base(self) -> list[RGBColor]: + r"""Load the base colormap corresponding to the enumeration member. + + Returns: + The base colormap. + + """ + return { + "autumn": cm_data.get_autumn_base, + "bone": cm_data.get_bone_base, + "jet": cm_data.get_jet_base, + "winter": cm_data.get_winter_base, + "rainbow": cm_data.get_rainbow_base, + "ocean": cm_data.get_ocean_base, + "summer": cm_data.get_summer_base, + "spring": cm_data.get_spring_base, + "cool": cm_data.get_cool_base, + "hsv": cm_data.get_hsv_base, + "brg": cm_data.get_bgr_base, + "pink": cm_data.get_pink_base, + "hot": cm_data.get_hot_base, + "plasma": cm_data.get_plasma_base, + "viridis": cm_data.get_viridis_base, + "cividis": cm_data.get_cividis_base, + "twilight": cm_data.get_twilight_base, + "turbo": cm_data.get_turbo_base, + "seismic": cm_data.get_seismic_base, + }[self.name]() + + @classmethod + def list(cls) -> list[str]: + r"""Return a list of names of enumeration members. + + Returns: + A list containing the names of enumeration members. + + """ + return [c.name for c in cls] + + +class ColorMap: + r"""Class to represent a colour map. + + It can be created or selected from the built-in colour map. Please refer to + the `ColorMapType` enum class to view all available colormaps. + + Args: + base: A list of RGB colors to define a new custom colormap or the name of a built-in colormap as str or + using `ColorMapType` class. + num_colors: Number of colors in the colormap. + device: The device to put the generated colormap on. + dtype: The data type of the generated colormap. + + Returns: + An object of the colormap with the num_colors length. + + Examples: + >>> ColorMap(base='viridis', num_colors=8).colors + tensor([[0.2813, 0.2621, 0.2013, 0.1505, 0.1210, 0.2463, 0.5259, 0.8557], + [0.0842, 0.2422, 0.3836, 0.5044, 0.6258, 0.7389, 0.8334, 0.8886], + [0.4072, 0.5207, 0.5543, 0.5574, 0.5334, 0.4519, 0.2880, 0.0989]]) + + Create a color map from the first color (RGB with range[0-1]) to the last one with num_colors length. + >>> ColorMap(base=[[0., 0.5 , 1.0], [1., 0.5, 0.]], num_colors=8).colors + tensor([[0.0000, 0.0000, 0.1250, 0.3750, 0.6250, 0.8750, 1.0000, 1.0000], + [0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000], + [1.0000, 1.0000, 0.8750, 0.6250, 0.3750, 0.1250, 0.0000, 0.0000]]) + + """ + + def __init__( + self, + base: Union[list[RGBColor], str, ColorMapType], + num_colors: int = 64, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ) -> None: + super().__init__() + self._dtype = dtype + self._device = device + + if isinstance(base, str): + base = base.lower() + if base not in ColorMapType.list(): + raise ValueError(f"Unsupported colormap: {base}. Available colormaps are {ColorMapType.list()}") + base_colormap_data = ColorMapType[base]._load_base() + self.name = base + elif isinstance(base, ColorMapType): + base_colormap_data = base._load_base() + self.name = base.name + elif isinstance(base, list): + base_colormap_data = base + self.name = "CustomColorMapType" + else: + raise ValueError( + "Base should be one of the available `ColorMapType` or a base colormap data (list[RGBColor])" + ) + + self.colors = self._generate_color_map(base_colormap_data, num_colors) + + def _generate_color_map(self, base_colormap: list[RGBColor], num_colors: int) -> torch.Tensor: + r"""Generate a colormap torch.Tensor using interpolation. + + Args: + base_colormap: A list of RGB colors defining the colormap. + num_colors: Number of colors in the colormap. + + Returns: + A torch.Tensor representing the colormap. + + """ + tensor_colors = torch.tensor(list(base_colormap), dtype=self._dtype, device=self._device).T + return interpolate(tensor_colors[None, ...], size=num_colors, mode="linear")[0, ...] + + def __len__(self) -> int: + r"""Return the number of colors in the colormap. + + Returns: + Number of colors in the colormap. + + """ + return self.colors.shape[-1] + + +def apply_colormap(input_tensor: torch.Tensor, colormap: ColorMap) -> torch.Tensor: + r"""Apply to a gray torch.Tensor a colormap. + + .. image:: _static/img/apply_colormap.png + + Args: + input_tensor: the input torch.Tensor of image. + colormap: the colormap desired to be applied to the input torch.Tensor. + + Returns: + A RGB torch.Tensor with the applied color map into the input_tensor. + + Raises: + ValueError: If `colormap` is not a ColorMap object. + + .. note:: + The input torch.Tensor must be integer values in the range of [0-255] or float values in the range of [0-1]. + + Example: + >>> input_tensor = torch.tensor([[[0, 1, 2], [15, 25, 33], [128, 158, 188]]]) + >>> colormap = ColorMap(base=ColorMapType.autumn) + >>> apply_colormap(input_tensor, colormap) + tensor([[[[1.0000, 1.0000, 1.0000], + [1.0000, 1.0000, 1.0000], + [1.0000, 1.0000, 1.0000]], + + [[0.0000, 0.0159, 0.0159], + [0.0635, 0.1111, 0.1429], + [0.5079, 0.6190, 0.7302]], + + [[0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000]]]]) + + """ + KORNIA_CHECK( + isinstance(input_tensor, torch.Tensor), f"`input_tensor` must be a torch.Tensor. Got: {type(input_tensor)}" + ) + valid_types = [ + torch.bfloat16, + torch.half, + torch.float, + torch.double, + torch.uint8, + torch.int, + torch.long, + torch.short, + ] + KORNIA_CHECK( + input_tensor.dtype in valid_types, f"`input_tensor` must be a {valid_types}. Got: {input_tensor.dtype}" + ) + KORNIA_CHECK(len(input_tensor.shape) in (3, 4), "Wrong input torch.Tensor dimension.") + if len(input_tensor.shape) == 3: + input_tensor = input_tensor.unsqueeze_(0) + + B, C, H, W = input_tensor.shape + input_tensor = input_tensor.reshape(B, C, -1) + max_value = 1.0 if input_tensor.max() <= 1.0 else 255.0 + input_tensor = input_tensor.float().div_(max_value) + + colors = colormap.colors.permute(1, 0) + num_colors, channels_cmap = colors.shape + keys = torch.linspace(0.0, 1.0, num_colors - 1, device=input_tensor.device, dtype=input_tensor.dtype) + indices = torch.bucketize(input_tensor, keys).unsqueeze(-1).expand(-1, -1, -1, 3) + + output = torch.gather(colors.expand(B, C, -1, -1), 2, indices) + # (B, C, H*W, channels_cmap) -> (B, C*channels_cmap, H, W) + output = output.permute(0, 1, 3, 2).reshape(B, C * channels_cmap, H, W) + + return output + + +class ApplyColorMap(nn.Module): + r"""Class for applying a colormap to images. + + .. image:: _static/img/ApplyColorMap.png + + Args: + colormap: Either the name of a built-in colormap or a ColorMap object. + num_colors: Number of colors in the colormap. Default is 256. + device: The device to put the generated colormap on. + dtype: The data type of the generated colormap. + + Returns: + A RGB torch.Tensor with the applied color map into the input_tensor + + Raises: + ValueError: If `colormap` is not a ColorMap object. + + .. note:: + The input torch.Tensor must be integer values in the range of [0-255] or float values in the range of [0-1]. + + Example: + >>> input_tensor = torch.tensor([[[0, 1, 2], [15, 25, 33], [128, 158, 188]]]) + >>> colormap = ColorMap(base=ColorMapType.autumn) + >>> ApplyColorMap(colormap=colormap)(input_tensor) + tensor([[[[1.0000, 1.0000, 1.0000], + [1.0000, 1.0000, 1.0000], + [1.0000, 1.0000, 1.0000]], + + [[0.0000, 0.0159, 0.0159], + [0.0635, 0.1111, 0.1429], + [0.5079, 0.6190, 0.7302]], + + [[0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000]]]]) + + """ + + def __init__( + self, + colormap: ColorMap, + ) -> None: + super().__init__() + self.colormap = colormap + + def forward(self, input_tensor: torch.Tensor) -> torch.Tensor: + r"""Apply the colormap to the input torch.Tensor. + + Args: + input_tensor: The input torch.Tensor representing the grayscale image. + + .. note:: + The input torch.Tensor must be integer values in the range of [0-255] or float values in the range of [0-1]. + + Returns: + The output torch.Tensor representing the image with the applied colormap. + + """ + return apply_colormap(input_tensor, self.colormap) diff --git a/kornia/color/gray.py b/kornia/color/gray.py new file mode 100644 index 0000000..7850435 --- /dev/null +++ b/kornia/color/gray.py @@ -0,0 +1,248 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import ClassVar, Optional + +import torch +from torch import nn + +from kornia.color.rgb import bgr_to_rgb +from kornia.core.check import KORNIA_CHECK_IS_TENSOR + + +def grayscale_to_rgb(image: torch.Tensor) -> torch.Tensor: + r"""Convert a grayscale image to RGB version of image. + + .. image:: _static/img/grayscale_to_rgb.png + + The image data is assumed to be in the range of (0, 1). + + Args: + image: grayscale image torch.Tensor to be converted to RGB with shape :math:`(*,1,H,W)`. + + Returns: + RGB version of the image with shape :math:`(*,3,H,W)`. + + Example: + >>> input = torch.randn(2, 1, 4, 5) + >>> gray = grayscale_to_rgb(input) # 2x3x4x5 + + """ + KORNIA_CHECK_IS_TENSOR(image) + + if len(image.shape) < 3 or image.shape[-3] != 1: + raise ValueError(f"Input size must have a shape of (*, 1, H, W). Got {image.shape}.") + + shape = list(image.shape) + shape[-3] = 3 + # Use expand to create a view that repeats along channel dimension, no memory overhead. + return image.expand(*shape) + + +def rgb_to_grayscale(image: torch.Tensor, rgb_weights: Optional[torch.Tensor] = None) -> torch.Tensor: + r"""Convert a RGB image to grayscale version of image. + + .. image:: _static/img/rgb_to_grayscale.png + + The image data is assumed to be in the range of (0, 1). + + Args: + image: RGB image to be converted to grayscale with shape :math:`(*,3,H,W)`. + rgb_weights: Weights that will be applied on each channel (RGB). + The sum of the weights should add up to one. + + Returns: + grayscale version of the image with shape :math:`(*,1,H,W)`. + + .. note:: + See a working example `here `__. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> gray = rgb_to_grayscale(input) # 2x1x4x5 + + """ + KORNIA_CHECK_IS_TENSOR(image) + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W). Got {image.shape}") + + if rgb_weights is None: + # 8 bit images + if image.dtype == torch.uint8: + rgb_weights = torch.tensor([76, 150, 29], device=image.device, dtype=torch.uint8) + # floating point images + elif image.dtype in (torch.bfloat16, torch.float16, torch.float32, torch.float64): + rgb_weights = torch.tensor([0.299, 0.587, 0.114], device=image.device, dtype=image.dtype) + else: + raise TypeError(f"Unknown data type: {image.dtype}") + else: + # is torch.Tensor that we make sure is in the same device/dtype + rgb_weights = rgb_weights.to(image) + + # Unpack channels (View, don't copy) + r, g, b = image.unbind(dim=-3) + w_r, w_g, w_b = rgb_weights.unbind() + + # Accumulate results + out = r * w_r + out = torch.addcmul(out, g, w_g) + out = torch.addcmul(out, b, w_b) + + # Restore channel dim + return out.unsqueeze(-3) + + +def bgr_to_grayscale(image: torch.Tensor) -> torch.Tensor: + r"""Convert a BGR image to grayscale. + + The image data is assumed to be in the range of (0, 1). First flips to RGB, then converts. + + Args: + image: BGR image to be converted to grayscale with shape :math:`(*,3,H,W)`. + + Returns: + grayscale version of the image with shape :math:`(*,1,H,W)`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> gray = bgr_to_grayscale(input) # 2x1x4x5 + + """ + KORNIA_CHECK_IS_TENSOR(image) + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W). Got {image.shape}") + + image_rgb: torch.Tensor = bgr_to_rgb(image) + return rgb_to_grayscale(image_rgb) + + +class GrayscaleToRgb(nn.Module): + r"""nn.Module to convert a grayscale image to RGB version of image. + + The image data is assumed to be in the range of (0, 1). + + Shape: + - image: :math:`(*, 1, H, W)` + - output: :math:`(*, 3, H, W)` + + reference: + https://docs.opencv.org/4.0.1/de/d25/imgproc_color_conversions.html + + Example: + >>> input = torch.rand(2, 1, 4, 5) + >>> rgb = GrayscaleToRgb() + >>> output = rgb(input) # 2x3x4x5 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 1, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert a grayscale tensor to RGB. + + Args: + image: Input tensor with shape :math:`(*, 1, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``1`` is a single grayscale channel, and ``H``/``W`` are height and width. + + Returns: + RGB tensor with shape :math:`(*, 3, H, W)`. + """ + return grayscale_to_rgb(image) + + +class RgbToGrayscale(nn.Module): + r"""nn.Module to convert a RGB image to grayscale version of image. + + The image data is assumed to be in the range of (0, 1). + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 1, H, W)` + + reference: + https://docs.opencv.org/4.0.1/de/d25/imgproc_color_conversions.html + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> gray = RgbToGrayscale() + >>> output = gray(input) # 2x1x4x5 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 1, -1, -1] + + def __init__(self, rgb_weights: Optional[torch.Tensor] = None) -> None: + super().__init__() + if rgb_weights is None: + rgb_weights = torch.Tensor([0.299, 0.587, 0.114]) + self.rgb_weights = rgb_weights + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert an RGB tensor to grayscale. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to RGB channels, and ``H``/``W`` are height and width. + + Returns: + Grayscale tensor with shape :math:`(*, 1, H, W)`, computed with the module's RGB weights. + """ + return rgb_to_grayscale(image, rgb_weights=self.rgb_weights) + + +class BgrToGrayscale(nn.Module): + r"""nn.Module to convert a BGR image to grayscale version of image. + + The image data is assumed to be in the range of (0, 1). First flips to RGB, then converts. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 1, H, W)` + + reference: + https://docs.opencv.org/4.0.1/de/d25/imgproc_color_conversions.html + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> gray = BgrToGrayscale() + >>> output = gray(input) # 2x1x4x5 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 1, -1, -1] + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert a BGR tensor to grayscale. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to BGR channels, and ``H``/``W`` are height and width. + + Returns: + Grayscale tensor with shape :math:`(*, 1, H, W)`. + """ + return bgr_to_grayscale(image) diff --git a/kornia/color/hls.py b/kornia/color/hls.py new file mode 100644 index 0000000..f7febc6 --- /dev/null +++ b/kornia/color/hls.py @@ -0,0 +1,222 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import math +from typing import ClassVar + +import torch +from torch import nn + + +def rgb_to_hls(image: torch.Tensor, eps: float = 1e-8) -> torch.Tensor: + r"""Convert an RGB image to HLS. + + .. image:: _static/img/rgb_to_hls.png + + The image data is assumed to be in the range of (0, 1). + + NOTE: this method cannot be compiled with JIT in pytohrch < 1.7.0 + + Args: + image: RGB image to be converted to HLS with shape :math:`(*, 3, H, W)`. + eps: epsilon value to avoid div by zero. + + Returns: + HLS version of the image with shape :math:`(*, 3, H, W)`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = rgb_to_hls(input) # 2x3x4x5 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W). Got {image.shape}") + + _RGB2HSL_IDX = torch.tensor([[[0.0]], [[1.0]], [[2.0]]], device=image.device, dtype=image.dtype) # 3x1x1 + + _img_max: tuple[torch.Tensor, torch.Tensor] = image.max(-3) + maxc = _img_max[0] + imax = _img_max[1] + minc: torch.Tensor = image.min(-3)[0] + + if image.requires_grad: + l_ = maxc + minc + s = maxc - minc + # weird behaviour with undefined vars in JIT... + # scripting requires image_hls be defined even if it is not used :S + h = l_ # assign to any torch.Tensor... + image_hls = l_ # assign to any torch.Tensor... + else: + # define the resulting image to avoid the torch.stack([h, l, s]) + # so, h, l and s require inplace operations + # NOTE: torch.stack() increases in a 10% the cost in colab + image_hls = torch.empty_like(image) + h, l_, s = ( + image_hls[..., 0, :, :], + image_hls[..., 1, :, :], + image_hls[..., 2, :, :], + ) + torch.add(maxc, minc, out=l_) # l = max + min + torch.sub(maxc, minc, out=s) # s = max - min + + # precompute image / (max - min) + im = image / (s + eps).unsqueeze(-3) + + # epsilon cannot be inside the where to avoid precision issues + s /= torch.where(l_ < 1.0, l_, 2.0 - l_) + eps # saturation + l_ /= 2 # luminance + + # note that r,g and b were previously div by (max - min) + r, g, b = im[..., 0, :, :], im[..., 1, :, :], im[..., 2, :, :] + # h[imax == 0] = (((g - b) / (max - min)) % 6)[imax == 0] + # h[imax == 1] = (((b - r) / (max - min)) + 2)[imax == 1] + # h[imax == 2] = (((r - g) / (max - min)) + 4)[imax == 2] + cond = imax[..., None, :, :] == _RGB2HSL_IDX + if image.requires_grad: + h = ((g - b) % 6) * cond[..., 0, :, :] + else: + # replacing `torch.mul` with `out=h` with python * operator gives wrong results + torch.mul((g - b) % 6, cond[..., 0, :, :], out=h) + h += (b - r + 2) * cond[..., 1, :, :] + h += (r - g + 4) * cond[..., 2, :, :] + # h = 2.0 * math.pi * (60.0 * h) / 360.0 + h *= math.pi / 3.0 # hue [0, 2*pi] + + if image.requires_grad: + return torch.stack([h, l_, s], -3) + return image_hls + + +def hls_to_rgb(image: torch.Tensor) -> torch.Tensor: + r"""Convert a HLS image to RGB. + + The image data is assumed to be in the range of (0, 1). + + Args: + image: HLS image to be converted to RGB with shape :math:`(*, 3, H, W)`. + + Returns: + RGB version of the image with shape :math:`(*, 3, H, W)`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = hls_to_rgb(input) # 2x3x4x5 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W). Got {image.shape}") + + _HLS2RGB = torch.tensor([[[0.0]], [[8.0]], [[4.0]]], device=image.device, dtype=image.dtype) # 3x1x1 + + im: torch.Tensor = image.unsqueeze(-4) + h_ch: torch.Tensor = im[..., 0, :, :] + l_ch: torch.Tensor = im[..., 1, :, :] + s_ch: torch.Tensor = im[..., 2, :, :] + h_ch = h_ch * (6 / math.pi) # h * 360 / (2 * math.pi) / 30 + a = s_ch * torch.min(l_ch, 1.0 - l_ch) + + # kr = (0 + h) % 12 + # kg = (8 + h) % 12 + # kb = (4 + h) % 12 + k: torch.Tensor = (h_ch + _HLS2RGB) % 12 + + # l - a * max(min(min(k - 3.0, 9.0 - k), 1), -1) + mink = torch.min(k - 3.0, 9.0 - k) + return torch.addcmul(l_ch, a, mink.clamp_(min=-1.0, max=1.0), value=-1) + + +class RgbToHls(nn.Module): + r"""Convert an image from RGB to HLS. + + The image data is assumed to be in the range of (0, 1). + + Returns: + HLS version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Examples: + >>> input = torch.rand(2, 3, 4, 5) + >>> hls = RgbToHls() + >>> output = hls(input) # 2x3x4x5 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert an RGB tensor to HLS. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to RGB channels, and ``H``/``W`` are height and width. + + Returns: + HLS tensor with shape :math:`(*, 3, H, W)`. + """ + return rgb_to_hls(image) + + +class HlsToRgb(nn.Module): + r"""Convert an image from HLS to RGB. + + The image data is assumed to be in the range of (0, 1). + + Returns: + RGB version of the image. + + Shape: + - input: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Reference: + https://en.wikipedia.org/wiki/HSL_and_HSV + + Examples: + >>> input = torch.rand(2, 3, 4, 5) + >>> rgb = HlsToRgb() + >>> output = rgb(input) # 2x3x4x5 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert an HLS tensor to RGB. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to HLS channels, and ``H``/``W`` are height and width. + + Returns: + RGB tensor with shape :math:`(*, 3, H, W)`. + """ + return hls_to_rgb(image) diff --git a/kornia/color/hsv.py b/kornia/color/hsv.py new file mode 100644 index 0000000..8e05e73 --- /dev/null +++ b/kornia/color/hsv.py @@ -0,0 +1,194 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import math +from typing import ClassVar + +import torch +from torch import nn + + +def rgb_to_hsv(image: torch.Tensor, eps: float = 1e-8) -> torch.Tensor: + r"""Convert an image from RGB to HSV. + + .. image:: _static/img/rgb_to_hsv.png + + The image data is assumed to be in the range of (0, 1). + + Args: + image: RGB Image to be converted to HSV with shape of :math:`(*, 3, H, W)`. + eps: scalar to enforce numarical stability. + + Returns: + HSV version of the image with shape of :math:`(*, 3, H, W)`. + The H channel values are in the range 0..2pi. S and V are in the range 0..1. + + .. note:: + See a working example `here `__. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = rgb_to_hsv(input) # 2x3x4x5 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W). Got {image.shape}") + + max_rgb, argmax_rgb = image.max(-3) + min_rgb, _argmin_rgb = image.min(-3) + deltac = max_rgb - min_rgb + + v = max_rgb + s = deltac / (max_rgb + eps) + + deltac = torch.where(deltac == 0, torch.ones_like(deltac), deltac) + rc, gc, bc = torch.unbind((max_rgb.unsqueeze(-3) - image), dim=-3) + + h1 = bc - gc + h2 = (rc - bc) + 2.0 * deltac + h3 = (gc - rc) + 4.0 * deltac + + h = torch.stack((h1, h2, h3), dim=-3) / deltac.unsqueeze(-3) + h = torch.gather(h, dim=-3, index=argmax_rgb.unsqueeze(-3)).squeeze(-3) + h = (h / 6.0) % 1.0 + h = 2.0 * math.pi * h # we return 0/2pi output + + return torch.stack((h, s, v), dim=-3) + + +def hsv_to_rgb(image: torch.Tensor) -> torch.Tensor: + r"""Convert an image from HSV to RGB. + + The H channel values are assumed to be in the range 0..2pi. S and V are in the range 0..1. + + Args: + image: HSV Image to be converted to HSV with shape of :math:`(*, 3, H, W)`. + + Returns: + RGB version of the image with shape of :math:`(*, 3, H, W)`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = hsv_to_rgb(input) # 2x3x4x5 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W). Got {image.shape}") + + h: torch.Tensor = image[..., 0, :, :] / (2 * math.pi) + s: torch.Tensor = image[..., 1, :, :] + v: torch.Tensor = image[..., 2, :, :] + + hi: torch.Tensor = torch.floor(h * 6) % 6 + f: torch.Tensor = ((h * 6) % 6) - hi + p: torch.Tensor = v * (1.0 - s) + q: torch.Tensor = v * (1.0 - f * s) + t: torch.Tensor = v * (1.0 - (1.0 - f) * s) + + hi = hi.long().clamp_(0, 5) + indices: torch.Tensor = torch.stack([hi, hi + 6, hi + 12], dim=-3) + out = torch.stack((v, q, p, p, t, v, t, v, v, q, p, p, p, p, t, v, v, q), dim=-3) + out = torch.gather(out, -3, indices) + + return out + + +class RgbToHsv(nn.Module): + r"""Convert an image from RGB to HSV. + + The image data is assumed to be in the range of (0, 1). + + Args: + eps: scalar to enforce numarical stability. + + Returns: + HSV version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> hsv = RgbToHsv() + >>> output = hsv(input) # 2x3x4x5 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def __init__(self, eps: float = 1e-6) -> None: + super().__init__() + self.eps = eps + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert an RGB tensor to HSV. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` is the channel dimension, and ``H``/``W`` are height and width. + + Returns: + HSV tensor with shape :math:`(*, 3, H, W)`. + """ + return rgb_to_hsv(image, self.eps) + + +class HsvToRgb(nn.Module): + r"""Convert an image from HSV to RGB. + + H channel values are assumed to be in the range 0..2pi. S and V are in the range 0..1. + + Returns: + RGB version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> rgb = HsvToRgb() + >>> output = rgb(input) # 2x3x4x5 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert an HSV tensor to RGB. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` is the channel dimension, and ``H``/``W`` are height and width. + + Returns: + RGB tensor with shape :math:`(*, 3, H, W)`. + """ + return hsv_to_rgb(image) diff --git a/kornia/color/lab.py b/kornia/color/lab.py new file mode 100644 index 0000000..e42eee8 --- /dev/null +++ b/kornia/color/lab.py @@ -0,0 +1,232 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""The RGB to Lab color transformations were translated from scikit image's rgb2lab and lab2rgb. + +https://github.com/scikit-image/scikit-image/blob/a48bf6774718c64dade4548153ae16065b595ca9/skimage/color/colorconv.py +""" + +from __future__ import annotations + +from typing import ClassVar + +import torch +from torch import nn + +from .rgb import linear_rgb_to_rgb, rgb_to_linear_rgb +from .xyz import rgb_to_xyz, xyz_to_rgb + + +def rgb_to_lab(image: torch.Tensor) -> torch.Tensor: + r"""Convert a RGB image to Lab. + + .. image:: _static/img/rgb_to_lab.png + + The input RGB image is assumed to be in the range of :math:`[0, 1]`. Lab + color is computed using the D65 illuminant and Observer 2. + + Args: + image: RGB Image to be converted to Lab with shape :math:`(*, 3, H, W)`. + + Returns: + Lab version of the image with shape :math:`(*, 3, H, W)`. + The L channel values are in the range 0..100. a and b are in the range -128..127. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = rgb_to_lab(input) # 2x3x4x5 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W). Got {image.shape}") + + # Convert from sRGB to Linear RGB + lin_rgb = rgb_to_linear_rgb(image) + + xyz_im: torch.Tensor = rgb_to_xyz(lin_rgb) + + # F.normalize for D65 white point + xyz_ref_white = torch.tensor([0.95047, 1.0, 1.08883], device=xyz_im.device, dtype=xyz_im.dtype)[..., :, None, None] + xyz_normalized = torch.div(xyz_im, xyz_ref_white) + + threshold = 0.008856 + power = torch.pow(xyz_normalized.clamp(min=threshold), 1 / 3.0) + scale = 7.787 * xyz_normalized + 4.0 / 29.0 + xyz_int = torch.where(xyz_normalized > threshold, power, scale) + + x: torch.Tensor = xyz_int[..., 0, :, :] + y: torch.Tensor = xyz_int[..., 1, :, :] + z: torch.Tensor = xyz_int[..., 2, :, :] + + L: torch.Tensor = (116.0 * y) - 16.0 + a: torch.Tensor = 500.0 * (x - y) + _b: torch.Tensor = 200.0 * (y - z) + + out: torch.Tensor = torch.stack([L, a, _b], dim=-3) + + return out + + +def lab_to_rgb(image: torch.Tensor, clip: bool = True) -> torch.Tensor: + r"""Convert a Lab image to RGB. + + The L channel is assumed to be in the range of :math:`[0, 100]`. + a and b channels are in the range of :math:`[-128, 127]`. + + Args: + image: Lab image to be converted to RGB with shape :math:`(*, 3, H, W)`. + clip: Whether to apply clipping to insure output RGB values in range :math:`[0, 1]`. + + Returns: + Lab version of the image with shape :math:`(*, 3, H, W)`. + The output RGB image are in the range of :math:`[0, 1]`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = lab_to_rgb(input) # 2x3x4x5 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W). Got {image.shape}") + + L: torch.Tensor = image[..., 0, :, :] + a: torch.Tensor = image[..., 1, :, :] + _b: torch.Tensor = image[..., 2, :, :] + + fy = (L + 16.0) / 116.0 + fx = (a / 500.0) + fy + fz = fy - (_b / 200.0) + + # if color data out of range: Z < 0 + fz = fz.clamp(min=0.0) + + fxyz = torch.stack([fx, fy, fz], dim=-3) + + # Convert from Lab to XYZ + power = torch.pow(fxyz, 3.0) + scale = (fxyz - 4.0 / 29.0) / 7.787 + xyz = torch.where(fxyz > 0.2068966, power, scale) + + # For D65 white point + xyz_ref_white = torch.tensor([0.95047, 1.0, 1.08883], device=xyz.device, dtype=xyz.dtype)[..., :, None, None] + xyz_im = xyz * xyz_ref_white + + rgbs_im: torch.Tensor = xyz_to_rgb(xyz_im) + + # https://github.com/richzhang/colorization-pytorch/blob/66a1cb2e5258f7c8f374f582acc8b1ef99c13c27/util/util.py#L107 + # rgbs_im = torch.where(rgbs_im < 0, torch.zeros_like(rgbs_im), rgbs_im) + + # Convert from RGB Linear to sRGB + rgb_im = linear_rgb_to_rgb(rgbs_im) + + # Clip to 0,1 https://www.w3.org/Graphics/Color/srgb + if clip: + rgb_im = torch.clamp(rgb_im, min=0.0, max=1.0) + + return rgb_im + + +class RgbToLab(nn.Module): + r"""Convert an image from RGB to Lab. + + The image data is assumed to be in the range of :math:`[0, 1]`. Lab + color is computed using the D65 illuminant and Observer 2. + + Returns: + Lab version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Examples: + >>> input = torch.rand(2, 3, 4, 5) + >>> lab = RgbToLab() + >>> output = lab(input) # 2x3x4x5 + + Reference: + [1] https://docs.opencv.org/4.0.1/de/d25/imgproc_color_conversions.html + + [2] https://www.easyrgb.com/en/math.php + + [3] https://github.com/torch/image/blob/dc061b98fb7e946e00034a5fc73e883a299edc7f/generic/image.c#L1467 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert an RGB tensor to Lab. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to RGB channels, and ``H``/``W`` are height and width. + + Returns: + Lab tensor with shape :math:`(*, 3, H, W)`. + """ + return rgb_to_lab(image) + + +class LabToRgb(nn.Module): + r"""Convert an image from Lab to RGB. + + Returns: + RGB version of the image. Range may not be in :math:`[0, 1]`. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Examples: + >>> input = torch.rand(2, 3, 4, 5) + >>> rgb = LabToRgb() + >>> output = rgb(input) # 2x3x4x5 + + References: + [1] https://docs.opencv.org/4.0.1/de/d25/imgproc_color_conversions.html + + [2] https://www.easyrgb.com/en/math.php + + [3] https://github.com/torch/image/blob/dc061b98fb7e946e00034a5fc73e883a299edc7f/generic/image.c#L1518 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, image: torch.Tensor, clip: bool = True) -> torch.Tensor: + """Convert a Lab tensor to RGB. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to Lab channels, and ``H``/``W`` are height and width. + clip: If ``True``, clamp output values to :math:`[0, 1]`. + + Returns: + RGB tensor with shape :math:`(*, 3, H, W)`. + """ + return lab_to_rgb(image, clip) diff --git a/kornia/color/luv.py b/kornia/color/luv.py new file mode 100644 index 0000000..6b1f192 --- /dev/null +++ b/kornia/color/luv.py @@ -0,0 +1,224 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""The RGB to Luv color transformations were translated from scikit image's rgb2luv and luv2rgb. + +https://github.com/scikit-image/scikit-image/blob/a48bf6774718c64dade4548153ae16065b595ca9/skimage/color/colorconv.py +""" + +from __future__ import annotations + +from typing import ClassVar + +import torch +from torch import nn + +from .rgb import linear_rgb_to_rgb, rgb_to_linear_rgb +from .xyz import rgb_to_xyz, xyz_to_rgb + + +def rgb_to_luv(image: torch.Tensor, eps: float = 1e-12) -> torch.Tensor: + r"""Convert a RGB image to Luv. + + .. image:: _static/img/rgb_to_luv.png + + The image data is assumed to be in the range of :math:`[0, 1]`. Luv + color is computed using the D65 illuminant and Observer 2. + + Args: + image: RGB Image to be converted to Luv with shape :math:`(*, 3, H, W)`. + eps: for numerically stability when dividing. + + Returns: + Luv version of the image with shape :math:`(*, 3, H, W)`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = rgb_to_luv(input) # 2x3x4x5 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W). Got {image.shape}") + + # Convert from sRGB to Linear RGB + lin_rgb = rgb_to_linear_rgb(image) + + xyz_im: torch.Tensor = rgb_to_xyz(lin_rgb) + + x: torch.Tensor = xyz_im[..., 0, :, :] + y: torch.Tensor = xyz_im[..., 1, :, :] + z: torch.Tensor = xyz_im[..., 2, :, :] + + threshold = 0.008856 + L: torch.Tensor = torch.where( + y > threshold, + 116.0 * torch.pow(y.clamp(min=threshold), 1.0 / 3.0) - 16.0, + 903.3 * y, + ) + + # Compute reference white point + xyz_ref_white: tuple[float, float, float] = (0.95047, 1.0, 1.08883) + u_w: float = (4 * xyz_ref_white[0]) / (xyz_ref_white[0] + 15 * xyz_ref_white[1] + 3 * xyz_ref_white[2]) + v_w: float = (9 * xyz_ref_white[1]) / (xyz_ref_white[0] + 15 * xyz_ref_white[1] + 3 * xyz_ref_white[2]) + + u_p: torch.Tensor = (4 * x) / (x + 15 * y + 3 * z + eps) + v_p: torch.Tensor = (9 * y) / (x + 15 * y + 3 * z + eps) + + u: torch.Tensor = 13 * L * (u_p - u_w) + v: torch.Tensor = 13 * L * (v_p - v_w) + + out = torch.stack([L, u, v], dim=-3) + + return out + + +def luv_to_rgb(image: torch.Tensor, eps: float = 1e-12) -> torch.Tensor: + r"""Convert a Luv image to RGB. + + Args: + image: Luv image to be converted to RGB with shape :math:`(*, 3, H, W)`. + eps: for numerically stability when dividing. + + Returns: + Luv version of the image with shape :math:`(*, 3, H, W)`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = luv_to_rgb(input) # 2x3x4x5 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W). Got {image.shape}") + + L: torch.Tensor = image[..., 0, :, :] + u: torch.Tensor = image[..., 1, :, :] + v: torch.Tensor = image[..., 2, :, :] + + # Convert from Luv to XYZ + y: torch.Tensor = torch.where(L > 7.999625, torch.pow((L + 16) / 116, 3.0), L / 903.3) + + # Compute white point + xyz_ref_white: tuple[float, float, float] = (0.95047, 1.0, 1.08883) + u_w: float = (4 * xyz_ref_white[0]) / (xyz_ref_white[0] + 15 * xyz_ref_white[1] + 3 * xyz_ref_white[2]) + v_w: float = (9 * xyz_ref_white[1]) / (xyz_ref_white[0] + 15 * xyz_ref_white[1] + 3 * xyz_ref_white[2]) + + a: torch.Tensor = u_w + u / (13 * L + eps) + d: torch.Tensor = v_w + v / (13 * L + eps) + c: torch.Tensor = 3 * y * (5 * d - 3) + + z: torch.Tensor = ((a - 4) * c - 15 * a * d * y) / (12 * d + eps) + x: torch.Tensor = -(c / (d + eps) + 3.0 * z) + + xyz_im: torch.Tensor = torch.stack([x, y, z], -3) + + rgbs_im: torch.Tensor = xyz_to_rgb(xyz_im) + + # Convert from RGB Linear to sRGB + rgb_im = linear_rgb_to_rgb(rgbs_im) + + return rgb_im + + +class RgbToLuv(nn.Module): + r"""Convert an image from RGB to Luv. + + The image data is assumed to be in the range of :math:`[0, 1]`. Luv + color is computed using the D65 illuminant and Observer 2. + + Returns: + Luv version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Examples: + >>> input = torch.rand(2, 3, 4, 5) + >>> luv = RgbToLuv() + >>> output = luv(input) # 2x3x4x5 + + Reference: + [1] https://docs.opencv.org/4.0.1/de/d25/imgproc_color_conversions.html + + [2] https://www.easyrgb.com/en/math.php + + [3] http://www.poynton.com/ColorFAQ.html + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert an RGB tensor to Luv. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to RGB channels, and ``H``/``W`` are height and width. + + Returns: + Luv tensor with shape :math:`(*, 3, H, W)`. + """ + return rgb_to_luv(image) + + +class LuvToRgb(nn.Module): + r"""Convert an image from Luv to RGB. + + Returns: + RGB version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Examples: + >>> input = torch.rand(2, 3, 4, 5) + >>> rgb = LuvToRgb() + >>> output = rgb(input) # 2x3x4x5 + + References: + [1] https://docs.opencv.org/4.0.1/de/d25/imgproc_color_conversions.html + + [2] https://www.easyrgb.com/en/math.php + + [3] http://www.poynton.com/ColorFAQ.html + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert a Luv tensor to RGB. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to Luv channels, and ``H``/``W`` are height and width. + + Returns: + RGB tensor with shape :math:`(*, 3, H, W)`. + """ + return luv_to_rgb(image) diff --git a/kornia/color/raw.py b/kornia/color/raw.py new file mode 100644 index 0000000..2fba35d --- /dev/null +++ b/kornia/color/raw.py @@ -0,0 +1,405 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from enum import Enum +from typing import ClassVar + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SHAPE + + +class CFA(Enum): + r"""Define the configuration of the color filter array. + + So far only bayer images is supported and the enum sets the pixel order for bayer. Note that this can change due + to things like rotations and cropping of images. Take care if including the translations in pipeline. + This implementations is optimized to be reasonably fast, look better than simple nearest neighbour. + On top of this care is taken to make it reversible going raw -> rgb -> raw. the raw samples remain intact + during conversion and only unknown samples are interpolated. + + The names are based on the OpenCV convention where the BG indicates pixel 1,1 (counting from 0,0) is + blue and its neighbour to the right is green. In that case the top left pixel is red. Other options are GB, RG and + GR + + reference: + https://en.wikipedia.org/wiki/Color_filter_array + """ + + BG = 0 + GB = 1 + RG = 2 + GR = 3 + + +def raw_to_rgb(image: torch.Tensor, cfa: CFA) -> torch.Tensor: + r"""Convert a raw bayer image to RGB version of image. + + We are assuming a CFA with 2 green, 1 red, 1 blue. A bilinear interpolation is used for R/G and a fix convolution + for the green pixels. To simplify calculations we expect the Height Width to be evenly divisible by 2. + + The image data is assumed to be in the range of (0, 1). Image H/W is assumed to be evenly divisible by 2. + for simplicity reasons + + Args: + image: raw image to be converted to RGB with shape :math:`(*,1,H,W)`. + cfa: The configuration of the color filter. + + Returns: + RGB version of the image with shape :math:`(*,3,H,W)`. + + Example: + >>> rawinput = torch.randn(2, 1, 4, 6) + >>> rgb = raw_to_rgb(rawinput, CFA.RG) # 2x3x4x6 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if image.dim() < 3 or image.size(-3) != 1: + raise ValueError(f"Input size must have a shape of (*, 1, H, W). Got {image.shape}.") + + if len(image.shape) < 2 or image.shape[-2] % 2 == 1 or image.shape[-1] % 2 == 1: + raise ValueError(f"Input H&W must be evenly disible by 2. Got {image.shape}") + + imagesize = image.size() + + image = image.view(-1, 1, image.shape[-2], image.shape[-1]) + + # BG is defined as pel 1,1 being blue, that is the top left is actually green. This matches + # opencv naming so makes sense to keep + if cfa == CFA.BG: + r = image[..., :, ::2, ::2] + b = image[..., :, 1::2, 1::2] + rpad = (0, 1, 0, 1) + bpad = (1, 0, 1, 0) + elif cfa == CFA.GB: + r = image[..., :, ::2, 1::2] + b = image[..., :, 1::2, ::2] + rpad = (1, 0, 0, 1) + bpad = (0, 1, 1, 0) + elif cfa == CFA.RG: + r = image[..., :, 1::2, 1::2] + b = image[..., :, ::2, ::2] + rpad = (1, 0, 1, 0) + bpad = (0, 1, 0, 1) + elif cfa == CFA.GR: + r = image[..., :, 1::2, ::2] + b = image[..., :, ::2, 1::2] + rpad = (0, 1, 1, 0) + bpad = (1, 0, 0, 1) + else: + raise ValueError(f"Unsupported CFA Got {cfa}.") + + # upscaling r and b with bi-linear gives reasonable quality + # Note that depending on where these are sampled we need to F.pad appropriately + # the bilinear filter will pretty much be based on for example this layout (RG) + # (which needs to be padded bottom right) + # +-+-+ + # |B| | + # | | | + # +-+-+ + # While in this layout we need to F.pad with additional B samples top left to + # make sure we interpolate from the correct position + # +-+-+ + # | | | + # | |B| + # +-+-+ + # For an image like this (3x2 blue pixels) + # +------+ + # |B B B | + # | | + # |B B B | + # | | + # +------+ + # It needs to be expanded to this (4x3 pixels scaled to 7x5 for correct interpolation) + # +-------+ + # |B B B b| + # | | + # |B B B b| + # | | + # |b b b b| + # +-------+ + # and we crop the area afterwards. This is since the interpolation will be between first and last pixel + # evenly spaced between them while the B/R samples will be missing in the corners were they are assumed to exist + # Further we need to do align_corners to start the interpolation from the middle of the samples in the corners, that + # way we get to keep the known blue samples across the whole image + rpadded = F.pad(r, list(rpad), "replicate") + bpadded = F.pad(b, list(bpad), "replicate") + # use explicit padding instead of conv2d padding to be able to use reflect which mirror the correct colors + # for a 2x2 bayer filter + gpadded = F.pad(image, [1, 1, 1, 1], "reflect") + + r_up = F.interpolate(rpadded, size=(image.shape[-2] + 1, image.shape[-1] + 1), mode="bilinear", align_corners=True) + b_up = F.interpolate(bpadded, size=(image.shape[-2] + 1, image.shape[-1] + 1), mode="bilinear", align_corners=True) + + # remove the extra padding + r_up = F.pad(r_up, [-x for x in rpad]) + b_up = F.pad(b_up, [-x for x in bpad]) + + # all unknown pixels are the average of the nearby green samples + kernel = torch.tensor( + [[[[0.0, 0.25, 0.0], [0.25, 0.0, 0.25], [0.0, 0.25, 0.0]]]], dtype=image.dtype, device=image.device + ) + + # This is done on all samples but result for the known green samples is then overwritten by the input + g_up = F.conv2d(gpadded, kernel) + + # overwrite the already known samples which otherwise have values from r/b + # this depends on the CFA configuration + if cfa == CFA.BG: + g_up[:, :, ::2, 1::2] = image[:, :, ::2, 1::2] + g_up[:, :, 1::2, ::2] = image[:, :, 1::2, ::2] + elif cfa == CFA.GB: + g_up[:, :, ::2, ::2] = image[:, :, ::2, ::2] + g_up[:, :, 1::2, 1::2] = image[:, :, 1::2, 1::2] + elif cfa == CFA.RG: + g_up[:, :, 1::2, ::2] = image[:, :, 1::2, ::2] + g_up[:, :, ::2, 1::2] = image[:, :, ::2, 1::2] + elif cfa == CFA.GR: + g_up[:, :, 1::2, 1::2] = image[:, :, 1::2, 1::2] + g_up[:, :, ::2, ::2] = image[:, :, ::2, ::2] + else: + raise ValueError(f"Unsupported CFA Got {cfa}.") + + r_up = r_up.view(imagesize) + g_up = g_up.view(imagesize) + b_up = b_up.view(imagesize) + + rgb: torch.Tensor = torch.cat([r_up, g_up, b_up], dim=-3) + + return rgb + + +def rgb_to_raw(image: torch.Tensor, cfa: CFA) -> torch.Tensor: + r"""Convert a RGB image to RAW version of image with the specified color filter array. + + The image data is assumed to be in the range of (0, 1). + + Args: + image: RGB image to be converted to bayer raw with shape :math:`(*,3,H,W)`. + cfa: Which color filter array do we want the output to mimic. I.e. which pixels are red/green/blue. + + Returns: + raw version of the image with shape :math:`(*,1,H,W)`. + + Example: + >>> rgbinput = torch.rand(2, 3, 4, 6) + >>> raw = rgb_to_raw(rgbinput, CFA.BG) # 2x1x4x6 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W). Got {image.shape}") + + # pick the torch.Tensor with green pixels + # clone to make sure grad works + output: torch.Tensor = image[..., 1:2, :, :].clone() + + # overwrite the r/b positions (depending on the cfa configuration) with blue/red pixels + if cfa == CFA.BG: + output[..., :, ::2, ::2] = image[..., 0:1, ::2, ::2] # red + output[..., :, 1::2, 1::2] = image[..., 2:3, 1::2, 1::2] # blue + elif cfa == CFA.GB: + output[..., :, ::2, 1::2] = image[..., 0:1, ::2, 1::2] # red + output[..., :, 1::2, ::2] = image[..., 2:3, 1::2, ::2] # blue + elif cfa == CFA.RG: + output[..., :, 1::2, 1::2] = image[..., 0:1, 1::2, 1::2] # red + output[..., :, ::2, ::2] = image[..., 2:3, ::2, ::2] # blue + elif cfa == CFA.GR: + output[..., :, 1::2, ::2] = image[..., 0:1, 1::2, ::2] # red + output[..., :, ::2, 1::2] = image[..., 2:3, ::2, 1::2] # blue + + return output + + +def raw_to_rgb_2x2_downscaled(image: torch.Tensor, cfa: CFA) -> torch.Tensor: + r"""Convert the raw bayer image to RGB version of it and resize width and height by half. + + This is done efficiently by converting each superpixel of bayer image to the corresponding rgb triplet. + R and B channels of the raw image are left as are, while two G channels of raw image are averaged to obtain the + output G channel. + + We are assuming a CFA with 2 green, 1 red, 1 blue. + The image data is assumed to be in the range of (0, 1). Image H/W is assumed to be evenly divisible by 2 + for simplicity reasons. + + Args: + image: raw image to be converted to RGB and downscaled with shape :math:`(*,1,H,W)`. + cfa: The configuration of the color filter. + + Returns: + downscaled RGB version of the image with shape :math:`(*,3,\frac{H}{2},\frac{W}{2})`. + + Example: + >>> rawinput = torch.randn(2, 1, 4, 6) + >>> rgb = raw_to_rgb_2x2_downscaled(rawinput, CFA.RG) # 2x3x2x3 + + """ + KORNIA_CHECK(isinstance(image, torch.Tensor), "Input type is not a torch.Tensor") + + KORNIA_CHECK_SHAPE(image, ["*", "1", "H", "W"]) + + KORNIA_CHECK( + image.shape[-2] % 2 == 0 and image.shape[-1] % 2 == 0, + f"Input H&W must be evenly disible by 2. Got {image.shape}", + ) + + if cfa == CFA.BG: + r = image[..., :, ::2, ::2] + b = image[..., :, 1::2, 1::2] + g1 = image[..., :, ::2, 1::2] + g2 = image[..., :, 1::2, ::2] + elif cfa == CFA.GB: + r = image[..., :, ::2, 1::2] + b = image[..., :, 1::2, ::2] + g1 = image[..., :, ::2, ::2] + g2 = image[..., :, 1::2, 1::2] + elif cfa == CFA.RG: + r = image[..., :, 1::2, 1::2] + b = image[..., :, ::2, ::2] + g1 = image[..., :, 1::2, ::2] + g2 = image[..., :, ::2, 1::2] + elif cfa == CFA.GR: + r = image[..., :, 1::2, ::2] + b = image[..., :, ::2, 1::2] + g1 = image[..., :, 1::2, 1::2] + g2 = image[..., :, ::2, ::2] + else: + raise ValueError(f"Unsupported CFA Got {cfa}.") + + rgb: torch.Tensor = torch.cat([r, (g1 + g2) / 2, b], dim=-3) + + return rgb + + +class RawToRgb(nn.Module): + r"""nn.Module to convert a bayer raw image to RGB version of image. + + The image data is assumed to be in the range of (0, 1). + + Shape: + - image: :math:`(*, 1, H, W)` + - output: :math:`(*, 3, H, W)` + + Example: + >>> rawinput = torch.rand(2, 1, 4, 6) + >>> rgb = RawToRgb(CFA.RG) + >>> output = rgb(rawinput) # 2x3x4x6 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 1, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def __init__(self, cfa: CFA) -> None: + super().__init__() + self.cfa = cfa + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert a Bayer raw tensor to RGB. + + Args: + image: Input tensor with shape :math:`(*, 1, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``1`` is a single raw Bayer channel, and ``H``/``W`` are height and width. + + Returns: + RGB tensor with shape :math:`(*, 3, H, W)` using this module's CFA pattern. + """ + return raw_to_rgb(image, cfa=self.cfa) + + +class RgbToRaw(nn.Module): + r"""nn.Module to convert a RGB image to bayer raw version of image. + + The image data is assumed to be in the range of (0, 1). + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 1, H, W)` + + reference: + https://docs.opencv.org/4.0.1/de/d25/imgproc_color_conversions.html + + Example: + >>> rgbinput = torch.rand(2, 3, 4, 6) + >>> raw = RgbToRaw(CFA.GB) + >>> output = raw(rgbinput) # 2x1x4x6 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 1, -1, -1] + + def __init__(self, cfa: CFA) -> None: + super().__init__() + self.cfa = cfa + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert an RGB tensor to Bayer raw. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to RGB channels, and ``H``/``W`` are height and width. + + Returns: + Raw tensor with shape :math:`(*, 1, H, W)` using this module's CFA pattern. + """ + return rgb_to_raw(image, cfa=self.cfa) + + +class RawToRgb2x2Downscaled(nn.Module): + r"""nn.Module version of the :func:`raw_to_rgb_2x2_downscaled()` function. + + The image width and height have to be divisible by two. The image + data is assumed to be in the range of (0, 1). + + Shape: + - image: :math:`(*, 1, H, W)` + - output: :math:`(*, 3, \frac{H}{2}, \frac{W}{2})` + + Example: + >>> rawinput = torch.rand(2, 1, 4, 6) + >>> rgb_downscale = RawToRgb2x2Downscaled(CFA.RG) + >>> output = rgb_downscale(rawinput) # 2x3x2x3 + + """ + + def __init__(self, cfa: CFA) -> None: + super().__init__() + self.cfa = cfa + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert a Bayer raw tensor to downscaled RGB. + + Args: + image: Input tensor with shape :math:`(*, 1, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``1`` is a single raw Bayer channel, and ``H``/``W`` are height and width. + + Returns: + RGB tensor with shape :math:`(*, 3, H / 2, W / 2)`. + """ + return raw_to_rgb_2x2_downscaled(image, cfa=self.cfa) diff --git a/kornia/color/rgb.py b/kornia/color/rgb.py new file mode 100644 index 0000000..d3ec076 --- /dev/null +++ b/kornia/color/rgb.py @@ -0,0 +1,804 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import ClassVar, Optional, Union, cast + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK_IS_COLOR + + +def rgb_to_bgr(image: torch.Tensor) -> torch.Tensor: + r"""Convert a RGB image to BGR. + + .. image:: _static/img/rgb_to_bgr.png + + Args: + image: RGB Image to be converted to BGRof of shape :math:`(*,3,H,W)`. + + Returns: + BGR version of the image with shape of shape :math:`(*,3,H,W)`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = rgb_to_bgr(input) # 2x3x4x5 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W).Got {image.shape}") + + return bgr_to_rgb(image) + + +def bgr_to_rgb(image: torch.Tensor) -> torch.Tensor: + r"""Convert a BGR image to RGB. + + Args: + image: BGR Image to be converted to BGR of shape :math:`(*,3,H,W)`. + + Returns: + RGB version of the image with shape of shape :math:`(*,3,H,W)`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = bgr_to_rgb(input) # 2x3x4x5 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W).Got {image.shape}") + + # flip image channels + out: torch.Tensor = image.flip(-3) + return out + + +def rgb_to_rgba(image: torch.Tensor, alpha_val: Union[float, torch.Tensor]) -> torch.Tensor: + r"""Convert an image from RGB to RGBA. + + Args: + image: RGB Image to be converted to RGBA of shape :math:`(*,3,H,W)`. + alpha_val (float, torch.Tensor): A float number for the alpha value or a torch.tensor + of shape :math:`(*,1,H,W)`. + + Returns: + RGBA version of the image with shape :math:`(*,4,H,W)`. + + .. note:: The current functionality is NOT supported by Torchscript. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = rgb_to_rgba(input, 1.) # 2x4x4x5 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W).Got {image.shape}") + + if not isinstance(alpha_val, (float, torch.Tensor)): + raise TypeError(f"alpha_val type is not a float or torch.Tensor. Got {type(alpha_val)}") + + # add one channel + r, g, b = torch.chunk(image, image.shape[-3], dim=-3) + + a: torch.Tensor = cast(torch.Tensor, alpha_val) + + if isinstance(alpha_val, float): + a = torch.full_like(r, fill_value=float(alpha_val)) + + return torch.cat([r, g, b, a], dim=-3) + + +def bgr_to_rgba(image: torch.Tensor, alpha_val: Union[float, torch.Tensor]) -> torch.Tensor: + r"""Convert an image from BGR to RGBA. + + Args: + image: BGR Image to be converted to RGBA of shape :math:`(*,3,H,W)`. + alpha_val: A float number for the alpha value or a torch.Tensor + of shape :math:`(*,1,H,W)`. + + Returns: + RGBA version of the image with shape :math:`(*,4,H,W)`. + + .. note:: The current functionality is NOT supported by Torchscript. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = bgr_to_rgba(input, 1.) # 2x4x4x5 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W).Got {image.shape}") + + if not isinstance(alpha_val, (float, torch.Tensor)): + raise TypeError(f"alpha_val type is not a float or torch.Tensor. Got {type(alpha_val)}") + + # convert first to RGB, then add alpha channel + x_rgb: torch.Tensor = bgr_to_rgb(image) + return rgb_to_rgba(x_rgb, alpha_val) + + +def rgba_to_rgb(image: torch.Tensor, background_color: Optional[torch.Tensor] = None) -> torch.Tensor: + r"""Convert an image from RGBA to RGB using alpha compositing. + + The function composites the input RGBA image over a background color. If no + background color is provided, it defaults to a white background. + + Args: + image: The RGBA image to be converted, with shape :math:`(*,4,H,W)`. + background_color: An optional background color. It can be a *tuple or list* of 3 floats, + a torch.Tensor of shape :math:`(*,3,H,W)` for a per-pixel background, or a broadcastable + torch.Tensor (e.g., :math:`(*,3,1,1)`). If None, a white background is used. + + Returns: + The converted RGB image with shape :math:`(*,3,H,W)`. + + Example: + >>> rgba_image = torch.rand(2, 4, 32, 32) + >>> # Test with default white background + >>> rgb_image_default = rgba_to_rgb(rgba_image) # 2x3x32x32 + >>> # Test with a custom background color + >>> rgb_image_custom = rgba_to_rgb(rgba_image, background_color=(0.0, 0.0, 1.0)) # 2x3x32x32 + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 4: + raise ValueError(f"Input size must have a shape of (*, 4, H, W).Got {image.shape}") + + # unpack channels + image_rgb = image[..., :3, :, :] + alpha = image[..., 3:4, :, :] + + if background_color is None: + background_rgb = torch.ones_like(image_rgb) + elif isinstance(background_color, (tuple, list)): + if len(background_color) != 3: + raise ValueError("background_color as a list/tuple must have 3 elements (R, G, B).") + background_rgb = torch.as_tensor(background_color, device=image.device, dtype=image.dtype).view(-1, 3, 1, 1) + + elif isinstance(background_color, torch.Tensor): + if background_color.shape[-3] != 3: + raise ValueError( + f"background_color torch.Tensor must have 3 channels in dimension -3. " + f"Got shape {background_color.shape}" + ) + background_rgb = background_color + + else: + raise TypeError(f"Unsupported type for background_color: {type(background_color)}") + + blended_rgb = image_rgb * alpha + background_rgb * (1.0 - alpha) + + return blended_rgb + + +def rgba_to_bgr(image: torch.Tensor) -> torch.Tensor: + r"""Convert an image from RGBA to BGR. + + Args: + image: RGBA Image to be converted to BGR of shape :math:`(*,4,H,W)`. + + Returns: + RGB version of the image with shape :math:`(*,3,H,W)`. + + Example: + >>> input = torch.rand(2, 4, 4, 5) + >>> output = rgba_to_bgr(input) # 2x3x4x5 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 4: + raise ValueError(f"Input size must have a shape of (*, 4, H, W).Got {image.shape}") + + # convert to RGB first, then to BGR + x_rgb: torch.Tensor = rgba_to_rgb(image) + return rgb_to_bgr(x_rgb) + + +def rgb_to_linear_rgb(image: torch.Tensor) -> torch.Tensor: + r"""Convert an sRGB image to linear RGB. Used in colorspace conversions. + + .. image:: _static/img/rgb_to_linear_rgb.png + + Args: + image: sRGB Image to be converted to linear RGB of shape :math:`(*,3,H,W)`. + + Returns: + linear RGB version of the image with shape of :math:`(*,3,H,W)`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = rgb_to_linear_rgb(input) # 2x3x4x5 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W).Got {image.shape}") + + lin_rgb: torch.Tensor = torch.where(image > 0.04045, torch.pow(((image + 0.055) / 1.055), 2.4), image / 12.92) + + return lin_rgb + + +def linear_rgb_to_rgb(image: torch.Tensor) -> torch.Tensor: + r"""Convert a linear RGB image to sRGB. Used in colorspace conversions. + + Args: + image: linear RGB Image to be converted to sRGB of shape :math:`(*,3,H,W)`. + + Returns: + sRGB version of the image with shape of shape :math:`(*,3,H,W)`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = linear_rgb_to_rgb(input) # 2x3x4x5 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W).Got {image.shape}") + + threshold = 0.0031308 + rgb: torch.Tensor = torch.where( + image > threshold, 1.055 * torch.pow(image.clamp(min=threshold), 1 / 2.4) - 0.055, 12.92 * image + ) + + return rgb + + +def normals_to_rgb255(image: torch.Tensor) -> torch.Tensor: + r"""Convert surface normals to RGB [0, 255] for visualization purposes. + + Args: + image: surface normals to be converted to RGB with quantization of shape :math:`(*,3,H,W)`. + + Returns: + RGB version of the image with shape of shape :math:`(*,3,H,W)`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = normals_to_rgb255(input) # 2x3x4x5 + + """ + KORNIA_CHECK_IS_COLOR(image) + rgb255 = (0.5 * (image + 1.0)).clip(0.0, 1.0) * 255 + return rgb255 + + +def rgb_to_rgb255(image: torch.Tensor) -> torch.Tensor: + r"""Convert an image from RGB to RGB [0, 255] for visualization purposes. + + Args: + image: RGB Image to be converted to RGB [0, 255] of shape :math:`(*,3,H,W)`. + + Returns: + RGB version of the image with shape of shape :math:`(*,3,H,W)`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = rgb_to_rgb255(input) # 2x3x4x5 + + """ + KORNIA_CHECK_IS_COLOR(image) + rgb255 = (image * 255).clip(0.0, 255.0) + return rgb255 + + +def rgb255_to_rgb(image: torch.Tensor) -> torch.Tensor: + r"""Convert an image from RGB [0, 255] to RGB for visualization purposes. + + Args: + image: RGB Image to be converted to RGB of shape :math:`(*,3,H,W)`. + + Returns: + RGB version of the image with shape of shape :math:`(*,3,H,W)`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = rgb255_to_rgb(input) # 2x3x4x5 + + """ + KORNIA_CHECK_IS_COLOR(image) + rgb = image / 255.0 + return rgb + + +def rgb255_to_normals(image: torch.Tensor) -> torch.Tensor: + r"""Convert an image from RGB [0, 255] to surface normals for visualization purposes. + + Args: + image: RGB Image to be converted to surface normals of shape :math:`(*,3,H,W)`. + + Returns: + surface normals version of the image with shape of shape :math:`(*,3,H,W)`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = rgb255_to_normals(input) # 2x3x4x5 + + """ + KORNIA_CHECK_IS_COLOR(image) + normals = F.normalize((image / 255.0) * 2.0 - 1.0, dim=-3, p=2.0) + return normals + + +class BgrToRgb(nn.Module): + r"""Convert image from BGR to RGB. + + The image data is assumed to be in the range of (0, 1). + + Returns: + RGB version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> rgb = BgrToRgb() + >>> output = rgb(input) # 2x3x4x5 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert a BGR tensor to RGB. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to BGR channels, and ``H``/``W`` are height and width. + + Returns: + RGB tensor with shape :math:`(*, 3, H, W)`. + """ + return bgr_to_rgb(image) + + +class RgbToBgr(nn.Module): + r"""Convert an image from RGB to BGR. + + The image data is assumed to be in the range of (0, 1). + + Returns: + BGR version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> bgr = RgbToBgr() + >>> output = bgr(input) # 2x3x4x5 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert an RGB tensor to BGR. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to RGB channels, and ``H``/``W`` are height and width. + + Returns: + BGR tensor with shape :math:`(*, 3, H, W)`. + """ + return rgb_to_bgr(image) + + +class RgbToRgba(nn.Module): + r"""Convert an image from RGB to RGBA. + + Add an alpha channel to existing RGB image. + + Args: + alpha_val: A float number for the alpha value or a torch.Tensor + of shape :math:`(*,1,H,W)`. + + Returns: + torch.Tensor: RGBA version of the image with shape :math:`(*,4,H,W)`. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 4, H, W)` + + .. note:: The current functionality is NOT supported by Torchscript. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> rgba = RgbToRgba(1.) + >>> output = rgba(input) # 2x4x4x5 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 4, -1, -1] + + def __init__(self, alpha_val: Union[float, torch.Tensor]) -> None: + super().__init__() + self.alpha_val = alpha_val + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert an RGB tensor to RGBA. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to RGB channels, and ``H``/``W`` are height and width. + + Returns: + RGBA tensor with shape :math:`(*, 4, H, W)` using this module's alpha value. + """ + return rgb_to_rgba(image, self.alpha_val) + + +class BgrToRgba(nn.Module): + r"""Convert an image from BGR to RGBA. + + Add an alpha channel to existing RGB image. + + Args: + alpha_val: A float number for the alpha value or a torch.Tensor + of shape :math:`(*,1,H,W)`. + + Returns: + RGBA version of the image with shape :math:`(*,4,H,W)`. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 4, H, W)` + + .. note:: The current functionality is NOT supported by Torchscript. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> rgba = BgrToRgba(1.) + >>> output = rgba(input) # 2x4x4x5 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 4, -1, -1] + + def __init__(self, alpha_val: Union[float, torch.Tensor]) -> None: + super().__init__() + self.alpha_val = alpha_val + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Append this module's alpha channel to the input tensor. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` is the input channel count, and ``H``/``W`` are height and width. + + Returns: + RGBA tensor with shape :math:`(*, 4, H, W)`. + """ + return bgr_to_rgba(image, self.alpha_val) + + +class RgbaToRgb(nn.Module): + r"""Convert an image from RGBA to RGB. + + Remove an alpha channel from RGB image. + + Returns: + RGB version of the image. + + Shape: + - image: :math:`(*, 4, H, W)` + - output: :math:`(*, 3, H, W)` + + Example: + >>> input = torch.rand(2, 4, 4, 5) + >>> rgba = RgbaToRgb() + >>> output = rgba(input) # 2x3x4x5 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 4, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert an RGBA tensor to RGB. + + Args: + image: Input tensor with shape :math:`(*, 4, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``4`` corresponds to RGBA channels, and ``H``/``W`` are height and width. + + Returns: + RGB tensor with shape :math:`(*, 3, H, W)`. + """ + return rgba_to_rgb(image) + + +class RgbaToBgr(nn.Module): + r"""Convert an image from RGBA to BGR. + + Remove an alpha channel from BGR image. + + Returns: + BGR version of the image. + + Shape: + - image: :math:`(*, 4, H, W)` + - output: :math:`(*, 3, H, W)` + + Example: + >>> input = torch.rand(2, 4, 4, 5) + >>> rgba = RgbaToBgr() + >>> output = rgba(input) # 2x3x4x5 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 4, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert an RGBA tensor to BGR. + + Args: + image: Input tensor with shape :math:`(*, 4, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``4`` corresponds to RGBA channels, and ``H``/``W`` are height and width. + + Returns: + BGR tensor with shape :math:`(*, 3, H, W)`. + """ + return rgba_to_bgr(image) + + +class RgbToLinearRgb(nn.Module): + r"""Convert an image from sRGB to linear RGB. + + Reverses the gamma correction of sRGB to get linear RGB values for colorspace conversions. + The image data is assumed to be in the range of :math:`[0, 1]` + + Returns: + Linear RGB version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> rgb_lin = RgbToLinearRgb() + >>> output = rgb_lin(input) # 2x3x4x5 + + References: + [1] https://stackoverflow.com/questions/35952564/convert-rgb-to-srgb + + [2] https://www.cambridgeincolour.com/tutorials/gamma-correction.htm + + [3] https://en.wikipedia.org/wiki/SRGB + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert an sRGB tensor to linear RGB. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to RGB channels, and ``H``/``W`` are height and width. + + Returns: + Linear RGB tensor with shape :math:`(*, 3, H, W)`. + """ + return rgb_to_linear_rgb(image) + + +class LinearRgbToRgb(nn.Module): + r"""Convert a linear RGB image to sRGB. + + Applies gamma correction to linear RGB values, at the end of colorspace conversions, to get sRGB. + + Returns: + sRGB version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> srgb = LinearRgbToRgb() + >>> output = srgb(input) # 2x3x4x5 + + References: + [1] https://stackoverflow.com/questions/35952564/convert-rgb-to-srgb + + [2] https://www.cambridgeincolour.com/tutorials/gamma-correction.htm + + [3] https://en.wikipedia.org/wiki/SRGB + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert a linear RGB tensor to sRGB. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to RGB channels, and ``H``/``W`` are height and width. + + Returns: + sRGB tensor with shape :math:`(*, 3, H, W)`. + """ + return linear_rgb_to_rgb(image) + + +class NormalsToRgb255(nn.Module): + r"""Convert surface normals to RGB [0, 255] for visualization purposes. + + Returns: + RGB version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> rgb = NormalsToRgb255() + >>> output = rgb(input) # 2x3x4x5 + + """ + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert surface normals to RGB values in :math:`[0, 255]`. + + Args: + image: Input normal map tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to XYZ normal components, and ``H``/``W`` are height and width. + + Returns: + RGB tensor in :math:`[0, 255]` with shape :math:`(*, 3, H, W)`. + """ + return normals_to_rgb255(image) + + +class RgbToRgb255(nn.Module): + r"""Convert an image from RGB to RGB [0, 255] for visualization purposes. + + Returns: + RGB version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> rgb = RgbToRgb255() + >>> output = rgb(input) # 2x3x4x5 + + """ + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert RGB values from :math:`[0, 1]` to :math:`[0, 255]`. + + Args: + image: Input RGB tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to RGB channels, and ``H``/``W`` are height and width. + + Returns: + RGB tensor in :math:`[0, 255]` with shape :math:`(*, 3, H, W)`. + """ + return rgb_to_rgb255(image) + + +class Rgb255ToRgb(nn.Module): + r"""Convert an image from RGB [0, 255] to RGB for visualization purposes. + + Returns: + RGB version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> rgb = Rgb255ToRgb() + >>> output = rgb(input) # 2x3x4x5 + + """ + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert RGB values from :math:`[0, 255]` to :math:`[0, 1]`. + + Args: + image: Input RGB tensor in :math:`[0, 255]` with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to RGB channels, and ``H``/``W`` are height and width. + + Returns: + RGB tensor in :math:`[0, 1]` with shape :math:`(*, 3, H, W)`. + """ + return rgb255_to_rgb(image) + + +class Rgb255ToNormals(nn.Module): + r"""Convert an image from RGB [0, 255] to surface normals for visualization purposes. + + Returns: + surface normals version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> normals = Rgb255ToNormals() + >>> output = normals(input) # 2x3x4x5 + + """ + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert RGB values in :math:`[0, 255]` to surface normals. + + Args: + image: Input RGB tensor in :math:`[0, 255]` with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to RGB channels, and ``H``/``W`` are height and width. + + Returns: + Normal map tensor with shape :math:`(*, 3, H, W)`. + """ + return rgb255_to_normals(image) diff --git a/kornia/color/sepia.py b/kornia/color/sepia.py new file mode 100644 index 0000000..cfaf1ff --- /dev/null +++ b/kornia/color/sepia.py @@ -0,0 +1,107 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Sepia color filter for RGB image tensors.""" + +import torch +from torch import nn + + +def sepia_from_rgb(input: torch.Tensor, rescale: bool = True, eps: float = 1e-6) -> torch.Tensor: + r"""Apply the sepia filter to an RGB tensor. + + Args: + input: the input tensor with shape :math:`(*, C, H, W)`. + rescale: if True, rescale the output so the max channel value is 1. + eps: small constant added to the denominator for numerical stability. + + Returns: + torch.Tensor: sepia-filtered tensor with shape :math:`(*, C, H, W)`. + + Example: + >>> input = torch.ones(3, 1, 1) + >>> sepia_from_rgb(input, rescale=False) + tensor([[[1.3510]], + + [[1.2030]], + + [[0.9370]]]) + """ + if len(input.shape) < 3 or input.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W). Got {input.shape}") + + r = input[..., 0, :, :] + g = input[..., 1, :, :] + b = input[..., 2, :, :] + + r_out = 0.393 * r + 0.769 * g + 0.189 * b + g_out = 0.349 * r + 0.686 * g + 0.168 * b + b_out = 0.272 * r + 0.534 * g + 0.131 * b + + sepia_out = torch.stack([r_out, g_out, b_out], dim=-3) + + if rescale: + max_values = sepia_out.amax(dim=-1).amax(dim=-1) + sepia_out = sepia_out / (max_values[..., None, None] + eps) + + return sepia_out + + +class Sepia(nn.Module): + r"""Apply the sepia filter to image tensors. + + Args: + rescale: if True, rescale the output so the max channel value is 1. + eps: small constant added to the denominator for numerical stability. + + Returns: + torch.Tensor: sepia-filtered tensor with shape :math:`(*, C, H, W)`. + + Example: + >>> input = torch.ones(3, 1, 1) + >>> Sepia(rescale=False)(input) + tensor([[[1.3510]], + + [[1.2030]], + + [[0.9370]]]) + """ + + def __init__(self, rescale: bool = True, eps: float = 1e-6) -> None: + """Initialize Sepia. + + Args: + rescale: if True, rescale the output so the max channel value is 1. + eps: small constant added to the denominator for numerical stability. + """ + self.rescale = rescale + self.eps = eps + super().__init__() + + def __repr__(self) -> str: + """Return a string representation of this module.""" + return self.__class__.__name__ + f"(rescale={self.rescale}, eps={self.eps})" + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Apply the sepia filter. + + Args: + input: RGB image tensor with shape :math:`(*, 3, H, W)`. + + Returns: + Sepia-filtered tensor with shape :math:`(*, 3, H, W)`. + """ + return sepia_from_rgb(input, rescale=self.rescale, eps=self.eps) diff --git a/kornia/color/utils.py b/kornia/color/utils.py new file mode 100644 index 0000000..4376485 --- /dev/null +++ b/kornia/color/utils.py @@ -0,0 +1,69 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch +from torch.nn import functional as F + + +def _apply_linear_transformation( + image: torch.Tensor, kernel: torch.Tensor, bias: torch.Tensor | None = None +) -> torch.Tensor: + """Apply a 3x3 linear color transformation with device-aware optimization. + + Args: + image: Input image tensor with shape :math:`(*, 3, H, W)`. + kernel: Transformation matrix with shape :math:`(3, 3)` applied along the channel + dimension. + bias: Bias vector with shape :math:`(3,)` to be added to the output. + + Returns: + Tensor with the same shape as ``image`` containing the transformed values. + """ + # Handle Integer inputs by casting to float safely + if image.is_floating_point(): + image_compute = image + else: + image_compute = image.float() + + # Match kernel dtype to the image (propagates float64 if needed) + kernel_compute = kernel.to(dtype=image_compute.dtype, device=image_compute.device) + input_shape = image_compute.shape + + # Empirical benchmarks show that einsum is faster on CPU for this specific pattern, + # while conv2d offers significant speedups on GPU/CUDA. + # We branch to ensure optimal performance on both devices. + # BRANCH 1: CPU (Einsum) + if image.device.type == "cpu": + out = torch.einsum("oi, ...ihw -> ...ohw", kernel, image) + + if bias is not None: + out = out + bias.view(-1, 1, 1) + + return out.contiguous() + + # BRANCH 2: GPU/Accelerators (Conv2d) + else: + # Reshape for conv2d: (B*..., C, H, W) + input_flat = image_compute.reshape(-1, 3, input_shape[-2], input_shape[-1]) + + weight = kernel_compute.view(3, 3, 1, 1) + out_flat = F.conv2d(input_flat, weight, bias=bias) + + # Unflatten back to original shape + out = out_flat.reshape(input_shape) + + return out diff --git a/kornia/color/xyz.py b/kornia/color/xyz.py new file mode 100644 index 0000000..b4eaa0d --- /dev/null +++ b/kornia/color/xyz.py @@ -0,0 +1,215 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import ClassVar + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE + + +def rgb_to_xyz(image: torch.Tensor) -> torch.Tensor: + r"""Convert a RGB image to XYZ. + + .. image:: _static/img/rgb_to_xyz.png + + Args: + image: RGB Image to be converted to XYZ with shape :math:`(*, 3, H, W)`. + + Returns: + XYZ version of the image with shape :math:`(*, 3, H, W)`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = rgb_to_xyz(input) # 2x3x4x5 + + """ + KORNIA_CHECK_IS_TENSOR(image) + KORNIA_CHECK_SHAPE(image, ["*", "3", "H", "W"]) + + # CIE RGB to XYZ Matrix (D65 White Point) + kernel = torch.tensor( + [ + [0.412453, 0.357580, 0.180423], + [0.212671, 0.715160, 0.072169], + [0.019334, 0.119193, 0.950227], + ], + device=image.device, + dtype=torch.float32, + ) + + # Apply Optimized Linear Transformation + return _apply_linear_transformation(image, kernel) + + +def xyz_to_rgb(image: torch.Tensor) -> torch.Tensor: + r"""Convert a XYZ image to RGB. + + Args: + image: XYZ Image to be converted to RGB with shape :math:`(*, 3, H, W)`. + + Returns: + RGB version of the image with shape :math:`(*, 3, H, W)`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = xyz_to_rgb(input) # 2x3x4x5 + + """ + KORNIA_CHECK_IS_TENSOR(image) + KORNIA_CHECK_SHAPE(image, ["*", "3", "H", "W"]) + + # CIE XYZ to RGB Matrix (D65 White Point) + kernel = torch.tensor( + [ + [3.2404813432005266, -1.5371515162713185, -0.4985363261688878], + [-0.9692549499965682, 1.8759900014898907, 0.0415559265582928], + [0.0556466391351772, -0.2040413383665112, 1.0573110696453443], + ], + device=image.device, + dtype=torch.float32, + ) + + # Apply Optimized Linear Transformation + return _apply_linear_transformation(image, kernel) + + +def _apply_linear_transformation(image: torch.Tensor, kernel: torch.Tensor) -> torch.Tensor: + """Apply a 3x3 linear color transformation with device-aware optimization. + + Args: + image: Input image tensor with shape :math:`(*, 3, H, W)`. + kernel: Transformation matrix with shape :math:`(3, 3)` applied along the channel + dimension. + + Returns: + Tensor with the same shape as ``image`` containing the transformed values. + """ + # Handle Integer inputs by casting to float safely + # If it's already floating point (e.g. float64 from gradcheck), we preserve it + if image.is_floating_point(): + image_compute = image + else: + image_compute = image.float() + + # Match kernel dtype to the image (propagates float64 if needed) + kernel_compute = kernel.to(dtype=image_compute.dtype, device=image_compute.device) + + input_shape = image_compute.shape + + # Empirical benchmarks show that einsum is faster on CPU for this specific pattern, + # while conv2d offers significant speedups on GPU/CUDA. + # We branch to ensure optimal performance on both devices. + # BRANCH 1: CPU (Einsum) + if image_compute.device.type == "cpu": + out = torch.einsum("...chw,oc->...ohw", image_compute, kernel_compute) + out = out.contiguous() + + # BRANCH 2: GPU/Accelerators (Conv2d) + else: + # Reshape for conv2d: (B*..., C, H, W) + input_flat = image_compute.reshape(-1, 3, input_shape[-2], input_shape[-1]) + + # Reshape kernel: (3, 3) -> (3, 3, 1, 1) + weight = kernel_compute.view(3, 3, 1, 1) + + out_flat = F.conv2d(input_flat, weight) + + # Unflatten back to original shape + out = out_flat.reshape(input_shape) + + return out + + +class RgbToXyz(nn.Module): + r"""Convert an image from RGB to XYZ. + + The image data is assumed to be in the range of (0, 1). + + Returns: + XYZ version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Examples: + >>> input = torch.rand(2, 3, 4, 5) + >>> xyz = RgbToXyz() + >>> output = xyz(input) # 2x3x4x5 + + Reference: + [1] https://docs.opencv.org/4.0.1/de/d25/imgproc_color_conversions.html + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert an RGB tensor to XYZ. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to RGB channels, and ``H``/``W`` are height and width. + + Returns: + XYZ tensor with shape :math:`(*, 3, H, W)`. + """ + return rgb_to_xyz(image) + + +class XyzToRgb(nn.Module): + r"""Converts an image from XYZ to RGB. + + Returns: + RGB version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Examples: + >>> input = torch.rand(2, 3, 4, 5) + >>> rgb = XyzToRgb() + >>> output = rgb(input) # 2x3x4x5 + + Reference: + [1] https://docs.opencv.org/4.0.1/de/d25/imgproc_color_conversions.html + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert an XYZ tensor to RGB. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to XYZ channels, and ``H``/``W`` are height and width. + + Returns: + RGB tensor with shape :math:`(*, 3, H, W)`. + """ + return xyz_to_rgb(image) diff --git a/kornia/color/ycbcr.py b/kornia/color/ycbcr.py new file mode 100644 index 0000000..082bf0e --- /dev/null +++ b/kornia/color/ycbcr.py @@ -0,0 +1,197 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import ClassVar + +import torch +from torch import nn + + +def _rgb_to_y(r: torch.Tensor, g: torch.Tensor, b: torch.Tensor) -> torch.Tensor: + y: torch.Tensor = 0.299 * r + 0.587 * g + 0.114 * b + return y + + +def rgb_to_ycbcr(image: torch.Tensor) -> torch.Tensor: + r"""Convert an RGB image to YCbCr. + + .. image:: _static/img/rgb_to_ycbcr.png + + Args: + image: RGB Image to be converted to YCbCr with shape :math:`(*, 3, H, W)`. + + Returns: + YCbCr version of the image with shape :math:`(*, 3, H, W)`. + + Examples: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = rgb_to_ycbcr(input) # 2x3x4x5 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W). Got {image.shape}") + + r: torch.Tensor = image[..., 0, :, :] + g: torch.Tensor = image[..., 1, :, :] + b: torch.Tensor = image[..., 2, :, :] + + delta: float = 0.5 + y: torch.Tensor = _rgb_to_y(r, g, b) + cb: torch.Tensor = (b - y) * 0.564 + delta + cr: torch.Tensor = (r - y) * 0.713 + delta + return torch.stack([y, cb, cr], -3) + + +def rgb_to_y(image: torch.Tensor) -> torch.Tensor: + r"""Convert an RGB image to Y. + + Args: + image: RGB Image to be converted to Y with shape :math:`(*, 3, H, W)`. + + Returns: + Y version of the image with shape :math:`(*, 1, H, W)`. + + Examples: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = rgb_to_y(input) # 2x1x4x5 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W). Got {image.shape}") + + r: torch.Tensor = image[..., 0:1, :, :] + g: torch.Tensor = image[..., 1:2, :, :] + b: torch.Tensor = image[..., 2:3, :, :] + + y: torch.Tensor = _rgb_to_y(r, g, b) + return y + + +def ycbcr_to_rgb(image: torch.Tensor) -> torch.Tensor: + r"""Convert an YCbCr image to RGB. + + The image data is assumed to be in the range of (0, 1). + + Args: + image: YCbCr Image to be converted to RGB with shape :math:`(*, 3, H, W)`. + + Returns: + RGB version of the image with shape :math:`(*, 3, H, W)`. + + Examples: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = ycbcr_to_rgb(input) # 2x3x4x5 + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(image)}") + + if len(image.shape) < 3 or image.shape[-3] != 3: + raise ValueError(f"Input size must have a shape of (*, 3, H, W). Got {image.shape}") + + y: torch.Tensor = image[..., 0, :, :] + cb: torch.Tensor = image[..., 1, :, :] + cr: torch.Tensor = image[..., 2, :, :] + + delta: float = 0.5 + cb_shifted: torch.Tensor = cb - delta + cr_shifted: torch.Tensor = cr - delta + + r: torch.Tensor = y + 1.403 * cr_shifted + g: torch.Tensor = y - 0.714 * cr_shifted - 0.344 * cb_shifted + b: torch.Tensor = y + 1.773 * cb_shifted + return torch.stack([r, g, b], -3).clamp(0, 1) + + +class RgbToYcbcr(nn.Module): + r"""Convert an image from RGB to YCbCr. + + The image data is assumed to be in the range of (0, 1). + + Returns: + YCbCr version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Examples: + >>> input = torch.rand(2, 3, 4, 5) + >>> ycbcr = RgbToYcbcr() + >>> output = ycbcr(input) # 2x3x4x5 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert an RGB tensor to YCbCr. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to RGB channels, and ``H``/``W`` are height and width. + + Returns: + YCbCr tensor with shape :math:`(*, 3, H, W)`. + """ + return rgb_to_ycbcr(image) + + +class YcbcrToRgb(nn.Module): + r"""Convert an image from YCbCr to Rgb. + + The image data is assumed to be in the range of (0, 1). + + Returns: + RGB version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Examples: + >>> input = torch.rand(2, 3, 4, 5) + >>> rgb = YcbcrToRgb() + >>> output = rgb(input) # 2x3x4x5 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Convert a YCbCr tensor to RGB. + + Args: + image: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to YCbCr channels, and ``H``/``W`` are height and width. + + Returns: + RGB tensor with shape :math:`(*, 3, H, W)`. + """ + return ycbcr_to_rgb(image) diff --git a/kornia/color/yuv.py b/kornia/color/yuv.py new file mode 100644 index 0000000..c4c1106 --- /dev/null +++ b/kornia/color/yuv.py @@ -0,0 +1,536 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import ClassVar + +import torch +from torch import nn + +from kornia.color.utils import _apply_linear_transformation +from kornia.core.check import KORNIA_CHECK_SHAPE +from kornia.core.exceptions import ShapeError + + +def rgb_to_yuv(image: torch.Tensor) -> torch.Tensor: + r"""Convert an RGB image to YUV. + + .. image:: _static/img/rgb_to_yuv.png + + The image data is assumed to be in the range of :math:`(0, 1)`. The range of the output is of + :math:`(0, 1)` to luma and the ranges of U and V are :math:`(-0.436, 0.436)` and :math:`(-0.615, 0.615)`, + respectively. + + The YUV model adopted here follows M/PAL values (see + `BT.470-5 `_, Table 2, + items 2.5 and 2.6). + + Args: + image: RGB Image to be converted to YUV with shape :math:`(*, 3, H, W)`. + + Returns: + YUV version of the image with shape :math:`(*, 3, H, W)`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = rgb_to_yuv(input) # 2x3x4x5 + + """ + KORNIA_CHECK_SHAPE(image, ["*", "3", "H", "W"]) + + kernel = torch.tensor( + [ + [0.299, 0.587, 0.114], + [-0.147, -0.289, 0.436], + [0.615, -0.515, -0.100], + ], + device=image.device, + dtype=image.dtype, + ) + + return _apply_linear_transformation(image, kernel) + + +def rgb_to_yuv420(image: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + r"""Convert an RGB image to YUV 420 (subsampled). + + Input need to be padded to be evenly divisible by 2 horizontal and vertical. + + The image data is assumed to be in the range of :math:`(0, 1)`. The range of the output is of :math:`(0, 1)` to + luma and the ranges of U and V are :math:`(-0.436, 0.436)` and :math:`(-0.615, 0.615)`, respectively. + + The YUV model adopted here follows M/PAL values (see + `BT.470-5 `_, Table 2, + items 2.5 and 2.6). + + Args: + image: RGB Image to be converted to YUV with shape :math:`(*, 3, H, W)`. + + Returns: + A torch.Tensor containing the Y plane with shape :math:`(*, 1, H, W)` + A torch.Tensor containing the UV planes with shape :math:`(*, 2, H/2, W/2)` + + Example: + >>> input = torch.rand(2, 3, 4, 6) + >>> output = rgb_to_yuv420(input) # (2x1x4x6, 2x2x2x3) + + """ + KORNIA_CHECK_SHAPE(image, ["*", "3", "H", "W"]) + + if len(image.shape) < 2 or image.shape[-2] % 2 == 1 or image.shape[-1] % 2 == 1: + raise ShapeError(f"Input H&W must be evenly disible by 2. Got {image.shape}") + + yuvimage = rgb_to_yuv(image) + + return ( + yuvimage[..., :1, :, :], + yuvimage[..., 1:3, :, :].unfold(-2, 2, 2).unfold(-2, 2, 2).mean((-1, -2)), + ) + + +def rgb_to_yuv422(image: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + r"""Convert an RGB image to YUV 422 (subsampled). + + Input need to be padded to be evenly divisible by 2 vertical. + + The image data is assumed to be in the range of :math:`(0, 1)`. The range of the output is of + :math:`(0, 1)` to luma and the ranges of U and V are :math:`(-0.436, 0.436)` and :math:`(-0.615, 0.615)`, + respectively. + + The YUV model adopted here follows M/PAL values (see + `BT.470-5 `_, Table 2, + items 2.5 and 2.6). + + Args: + image: RGB Image to be converted to YUV with shape :math:`(*, 3, H, W)`. + + Returns: + A torch.Tensor containing the Y plane with shape :math:`(*, 1, H, W)` + A torch.Tensor containing the UV planes with shape :math:`(*, 2, H, W/2)` + + Example: + >>> input = torch.rand(2, 3, 4, 6) + >>> output = rgb_to_yuv420(input) # (2x1x4x6, 2x1x4x3) + + """ + KORNIA_CHECK_SHAPE(image, ["*", "3", "H", "W"]) + + if len(image.shape) < 2 or image.shape[-2] % 2 == 1 or image.shape[-1] % 2 == 1: + raise ShapeError(f"Input H&W must be evenly disible by 2. Got {image.shape}") + + yuvimage = rgb_to_yuv(image) + + return (yuvimage[..., :1, :, :], yuvimage[..., 1:3, :, :].unfold(-1, 2, 2).mean(-1)) + + +def yuv_to_rgb(image: torch.Tensor) -> torch.Tensor: + r"""Convert an YUV image to RGB. + + The image data is assumed to be in the range of :math:`(0, 1)` for luma (Y). The ranges of U and V are + :math:`(-0.436, 0.436)` and :math:`(-0.615, 0.615)`, respectively. + + YUV formula follows M/PAL values (see + `BT.470-5 `_, Table 2, + items 2.5 and 2.6). + + Args: + image: YUV Image to be converted to RGB with shape :math:`(*, 3, H, W)`. + + Returns: + RGB version of the image with shape :math:`(*, 3, H, W)`. + + Example: + >>> input = torch.rand(2, 3, 4, 5) + >>> output = yuv_to_rgb(input) # 2x3x4x5 + + """ + KORNIA_CHECK_SHAPE(image, ["*", "3", "H", "W"]) + + kernel = torch.tensor( + [ + [1.0, 0.0, 1.14], + [1.0, -0.396, -0.581], + [1.0, 2.029, 0.0], + ], + device=image.device, + dtype=image.dtype, + ) + + return _apply_linear_transformation(image, kernel) + + +def yuv420_to_rgb(imagey: torch.Tensor, imageuv: torch.Tensor) -> torch.Tensor: + r"""Convert an YUV420 image to RGB. + + Input need to be padded to be evenly divisible by 2 horizontal and vertical. + + The image data is assumed to be in the range of :math:`(0, 1)` for luma (Y). The ranges of U and V are + :math:`(-0.436, 0.436)` and :math:`(-0.615, 0.615)`, respectively. + + YUV formula follows M/PAL values (see + `BT.470-5 `_, Table 2, + items 2.5 and 2.6). + + Args: + imagey: Y (luma) Image plane to be converted to RGB with shape :math:`(*, 1, H, W)`. + imageuv: UV (chroma) Image planes to be converted to RGB with shape :math:`(*, 2, H/2, W/2)`. + + Returns: + RGB version of the image with shape :math:`(*, 3, H, W)`. + + Example: + >>> inputy = torch.rand(2, 1, 4, 6) + >>> inputuv = torch.rand(2, 2, 2, 3) + >>> output = yuv420_to_rgb(inputy, inputuv) # 2x3x4x6 + + """ + KORNIA_CHECK_SHAPE(imagey, ["*", "1", "H", "W"]) + KORNIA_CHECK_SHAPE(imageuv, ["*", "2", "H", "W"]) + + if len(imagey.shape) < 2 or imagey.shape[-2] % 2 == 1 or imagey.shape[-1] % 2 == 1: + raise ShapeError(f"Input H&W must be evenly disible by 2. Got {imagey.shape}") + + if ( + len(imageuv.shape) < 2 + or len(imagey.shape) < 2 + or imagey.shape[-2] / imageuv.shape[-2] != 2 + or imagey.shape[-1] / imageuv.shape[-1] != 2 + ): + raise ShapeError( + f"Input imageuv H&W must be half the size of the luma plane. Got {imagey.shape} and {imageuv.shape}" + ) + + # first upsample + yuv444image = torch.cat( + [imagey, imageuv.repeat_interleave(2, dim=-1).repeat_interleave(2, dim=-2)], + dim=-3, + ) + + return yuv_to_rgb(yuv444image) + + +def yuv422_to_rgb(imagey: torch.Tensor, imageuv: torch.Tensor) -> torch.Tensor: + r"""Convert an YUV422 image to RGB. + + Input need to be padded to be evenly divisible by 2 vertical. + + The image data is assumed to be in the range of :math:`(0, 1)` for luma (Y). The ranges of U and V are + :math:`(-0.436, 0.436)` and :math:`(-0.615, 0.615)`, respectively. + + YUV formula follows M/PAL values (see + `BT.470-5 `_, Table 2, + items 2.5 and 2.6). + + Args: + imagey: Y (luma) Image plane to be converted to RGB with shape :math:`(*, 1, H, W)`. + imageuv: UV (luma) Image planes to be converted to RGB with shape :math:`(*, 2, H, W/2)`. + + Returns: + RGB version of the image with shape :math:`(*, 3, H, W)`. + + Example: + >>> inputy = torch.rand(2, 1, 4, 6) + >>> inputuv = torch.rand(2, 2, 2, 3) + >>> output = yuv420_to_rgb(inputy, inputuv) # 2x3x4x5 + + """ + KORNIA_CHECK_SHAPE(imagey, ["*", "1", "H", "W"]) + KORNIA_CHECK_SHAPE(imageuv, ["*", "2", "H", "W"]) + + if len(imagey.shape) < 2 or imagey.shape[-2] % 2 == 1 or imagey.shape[-1] % 2 == 1: + raise ShapeError(f"Input H&W must be evenly disible by 2. Got {imagey.shape}") + + if len(imageuv.shape) < 2 or len(imagey.shape) < 2 or imagey.shape[-1] / imageuv.shape[-1] != 2: + raise ShapeError( + f"Input imageuv W must be half the size of the luma plane. Got {imagey.shape} and {imageuv.shape}" + ) + + # first upsample + yuv444image = torch.cat([imagey, imageuv.repeat_interleave(2, dim=-1)], dim=-3) + + return yuv_to_rgb(yuv444image) + + +class RgbToYuv(nn.Module): + r"""Convert an image from RGB to YUV. + + The image data is assumed to be in the range of :math:`(0, 1)`. + + YUV formula follows M/PAL values (see + `BT.470-5 `_, Table 2, + items 2.5 and 2.6). + + Returns: + YUV version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Examples: + >>> input = torch.rand(2, 3, 4, 5) + >>> yuv = RgbToYuv() + >>> output = yuv(input) # 2x3x4x5 + + Reference:: + [1] https://es.wikipedia.org/wiki/YUV#RGB_a_Y'UV + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Convert an RGB tensor to YUV. + + Args: + input: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to RGB channels, and ``H``/``W`` are height and width. + + Returns: + YUV tensor with shape :math:`(*, 3, H, W)`. + """ + return rgb_to_yuv(input) + + +class RgbToYuv420(nn.Module): + r"""Convert an image from RGB to YUV420. + + Width and Height evenly divisible by 2. + + The image data is assumed to be in the range of :math:`(0, 1)`. + + YUV formula follows M/PAL values (see + `BT.470-5 `_, Table 2, + items 2.5 and 2.6). + + Returns: + YUV420 version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 1, H, W)` and :math:`(*, 2, H/2, W/2)` + + Examples: + >>> yuvinput = torch.rand(2, 3, 4, 6) + >>> yuv = RgbToYuv420() + >>> output = yuv(yuvinput) # # (2x1x4x6, 2x1x2x3) + + Reference:: + [1] https://es.wikipedia.org/wiki/YUV#RGB_a_Y'UV + + """ + + # TODO: Handle multiple inputs and outputs models later + ONNX_EXPORTABLE = False + + def forward(self, yuvinput: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: # skipcq: PYL-R0201 + """Convert an RGB tensor to YUV420 planes. + + Args: + yuvinput: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to RGB channels, and ``H``/``W`` are height and width. + + Returns: + Tuple of ``(y, uv)`` where ``y`` has shape :math:`(*, 1, H, W)` and ``uv`` has + shape :math:`(*, 2, H / 2, W / 2)`. The ``1`` channel is luma and the ``2`` + channels are chroma. + """ + return rgb_to_yuv420(yuvinput) + + +class RgbToYuv422(nn.Module): + r"""Convert an image from RGB to YUV422. + + Width must be evenly disvisible by 2. + + The image data is assumed to be in the range of :math:`(0, 1)`. + + YUV formula follows M/PAL values (see + `BT.470-5 `_, Table 2, + items 2.5 and 2.6). + + Returns: + YUV422 version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 1, H, W)` and :math:`(*, 2, H, W/2)` + + Examples: + >>> yuvinput = torch.rand(2, 3, 4, 6) + >>> yuv = RgbToYuv422() + >>> output = yuv(yuvinput) # # (2x1x4x6, 2x2x4x3) + + Reference:: + [1] https://es.wikipedia.org/wiki/YUV#RGB_a_Y'UV + + """ + + # TODO: Handle multiple inputs and outputs models later + ONNX_EXPORTABLE = False + + def forward(self, yuvinput: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: # skipcq: PYL-R0201 + """Convert an RGB tensor to YUV422 planes. + + Args: + yuvinput: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to RGB channels, and ``H``/``W`` are height and width. + + Returns: + Tuple of ``(y, uv)`` where ``y`` has shape :math:`(*, 1, H, W)` and ``uv`` has + shape :math:`(*, 2, H, W / 2)`. The ``1`` channel is luma and the ``2`` channels + are chroma. + """ + return rgb_to_yuv422(yuvinput) + + +class YuvToRgb(nn.Module): + r"""Convert an image from YUV to RGB. + + The image data is assumed to be in the range of :math:`(0, 1)` for luma (Y). The ranges of U and V are + :math:`(-0.436, 0.436)` and :math:`(-0.615, 0.615)`, respectively. + + YUV formula follows M/PAL values (see + `BT.470-5 `_, Table 2, + items 2.5 and 2.6). + + Returns: + RGB version of the image. + + Shape: + - image: :math:`(*, 3, H, W)` + - output: :math:`(*, 3, H, W)` + + Examples: + >>> input = torch.rand(2, 3, 4, 5) + >>> rgb = YuvToRgb() + >>> output = rgb(input) # 2x3x4x5 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Convert a YUV tensor to RGB. + + Args: + input: Input tensor with shape :math:`(*, 3, H, W)`. + Here, ``*`` means any number of leading dimensions (for example, batch size), + ``3`` corresponds to YUV channels, and ``H``/``W`` are height and width. + + Returns: + RGB tensor with shape :math:`(*, 3, H, W)`. + """ + return yuv_to_rgb(input) + + +class Yuv420ToRgb(nn.Module): + r"""Convert an image from YUV to RGB. + + Width and Height must be evenly divisible by 2. + + The image data is assumed to be in the range of :math:`(0, 1)` for luma (Y). The ranges of U and V are + :math:`(-0.436, 0.436)` and :math:`(-0.615, 0.615)`, respectively. + + YUV formula follows M/PAL values (see + `BT.470-5 `_, Table 2, + items 2.5 and 2.6). + + Returns: + RGB version of the image. + + Shape: + - imagey: :math:`(*, 1, H, W)` + - imageuv: :math:`(*, 2, H/2, W/2)` + - output: :math:`(*, 3, H, W)` + + Examples: + >>> inputy = torch.rand(2, 1, 4, 6) + >>> inputuv = torch.rand(2, 2, 2, 3) + >>> rgb = Yuv420ToRgb() + >>> output = rgb(inputy, inputuv) # 2x3x4x6 + + """ + + # TODO: Handle multiple inputs and outputs models later + ONNX_EXPORTABLE = False + + def forward(self, inputy: torch.Tensor, inputuv: torch.Tensor) -> torch.Tensor: # skipcq: PYL-R0201 + """Convert YUV420 luma/chroma planes to RGB. + + Args: + inputy: Luma tensor with shape :math:`(*, 1, H, W)`. + inputuv: Chroma tensor with shape :math:`(*, 2, H / 2, W / 2)`. + For both tensors, ``*`` means any number of leading dimensions (for example, + batch size), and ``H``/``W`` are the full-resolution height and width. + + Returns: + RGB tensor with shape :math:`(*, 3, H, W)`. + """ + return yuv420_to_rgb(inputy, inputuv) + + +class Yuv422ToRgb(nn.Module): + r"""Convert an image from YUV to RGB. + + Width must be evenly divisible by 2. + + The image data is assumed to be in the range of :math:`(0, 1)` for luma (Y). The ranges of U and V are + :math:`(-0.436, 0.436)` and :math:`(-0.615, 0.615)`, respectively. + + YUV formula follows M/PAL values (see + `BT.470-5 `_, Table 2, + items 2.5 and 2.6). + + Returns: + RGB version of the image. + + Shape: + - imagey: :math:`(*, 1, H, W)` + - imageuv: :math:`(*, 2, H, W/2)` + - output: :math:`(*, 3, H, W)` + + Examples: + >>> inputy = torch.rand(2, 1, 4, 6) + >>> inputuv = torch.rand(2, 2, 4, 3) + >>> rgb = Yuv422ToRgb() + >>> output = rgb(inputy, inputuv) # 2x3x4x6 + + """ + + # TODO: Handle multiple inputs and outputs models later + ONNX_EXPORTABLE = False + + def forward(self, inputy: torch.Tensor, inputuv: torch.Tensor) -> torch.Tensor: # skipcq: PYL-R0201 + """Convert YUV422 luma/chroma planes to RGB. + + Args: + inputy: Luma tensor with shape :math:`(*, 1, H, W)`. + inputuv: Chroma tensor with shape :math:`(*, 2, H, W / 2)`. + For both tensors, ``*`` means any number of leading dimensions (for example, + batch size), and ``H``/``W`` are the full-resolution height and width. + + Returns: + RGB tensor with shape :math:`(*, 3, H, W)`. + """ + return yuv422_to_rgb(inputy, inputuv) diff --git a/kornia/config.py b/kornia/config.py new file mode 100644 index 0000000..32bd470 --- /dev/null +++ b/kornia/config.py @@ -0,0 +1,79 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from dataclasses import dataclass, field +from enum import StrEnum + +__all__ = ["InstallationMode", "kornia_config"] + + +class InstallationMode(StrEnum): + """Represent the installation mode for external dependencies.""" + + # Ask the user if to install the dependencies + ASK = "ASK" + # Install the dependencies + AUTO = "AUTO" + # Raise an error if the dependencies are not installed + RAISE = "RAISE" + + def __eq__(self, other: object) -> bool: + if isinstance(other, str): + return self.value.lower() == other.lower() # Case-insensitive comparison + return super().__eq__(other) + + +class LazyLoaderConfig: + """Configure lazy loading behavior for external dependencies.""" + + _installation_mode: InstallationMode = InstallationMode.ASK + + @property + def installation_mode(self) -> InstallationMode: + return self._installation_mode + + @installation_mode.setter + def installation_mode(self, value: str) -> None: + # Allow setting via string by converting to the Enum + if isinstance(value, str): + try: + self._installation_mode = InstallationMode(value.upper()) + except ValueError: + raise ValueError( + f"{value} is not a valid InstallationMode. Choose from: {list(InstallationMode)}" + ) from None + elif isinstance(value, InstallationMode): + self._installation_mode = value + else: + raise TypeError("installation_mode must be a string or InstallationMode Enum.") + + +@dataclass +class KorniaConfig: + """Configure Kornia's behavior.""" + + hub_models_dir: str + hub_onnx_dir: str + output_dir: str = "kornia_outputs" + hub_cache_dir: str = ".kornia_hub" + lazyloader: LazyLoaderConfig = field(default_factory=LazyLoaderConfig) + + +kornia_config = KorniaConfig( + hub_models_dir=os.path.join(".kornia_hub", "models"), hub_onnx_dir=os.path.join(".kornia_hub", "onnx_models") +) diff --git a/kornia/constants.py b/kornia/constants.py new file mode 100644 index 0000000..15aba0a --- /dev/null +++ b/kornia/constants.py @@ -0,0 +1,164 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from enum import Enum, EnumMeta +from typing import Iterator, Type, TypeVar, Union + +import torch + +__all__ = ["BorderType", "DType", "Resample", "SamplePadding", "TKEnum", "pi"] + +pi = torch.tensor(3.14159265358979323846) + + +T = TypeVar("T", bound=Enum) +TKEnum = Union[str, int, T] + + +class _KORNIA_EnumMeta(EnumMeta): + """Custom metaclass for enums to support string and integer comparisons.""" + + def __iter__(self) -> Iterator[Enum]: # type: ignore[override] + return super().__iter__() + + def __contains__(self, other: TKEnum[Enum]) -> bool: # type: ignore[override] + if isinstance(other, str): + return any(val.name.upper() == other.upper() for val in self) + + elif isinstance(other, int): + return any(val.value == other for val in self) + + return any(val == other for val in self) + + def __repr__(self) -> str: + return " | ".join(f"{self.__name__}.{val.name}" for val in self) + + +def _get(cls: Type[T], value: TKEnum[T]) -> T: + if isinstance(value, str): + return cls[value.upper()] + + elif isinstance(value, int): + return cls(value) + + elif isinstance(value, cls): + return value + + raise TypeError( + f"The `.get` method from `{cls}` expects a value with type `str`, `int` or `{cls}`. Gotcha {type(value)}" + ) + + +class Resample(Enum, metaclass=_KORNIA_EnumMeta): + """Represent the resampling mode for image interpolation.""" + + NEAREST = 0 + BILINEAR = 1 + BICUBIC = 2 + + @classmethod + def get(cls, value: TKEnum["Resample"]) -> "Resample": + return _get(cls, value) + + +class BorderType(Enum, metaclass=_KORNIA_EnumMeta): + """Represent the border padding mode for image operations.""" + + CONSTANT = 0 + REFLECT = 1 + REPLICATE = 2 + CIRCULAR = 3 + + @classmethod + def get(cls, value: TKEnum["BorderType"]) -> "BorderType": + return _get(cls, value) + + +class SamplePadding(Enum, metaclass=_KORNIA_EnumMeta): + """Represent the padding mode used during spatial sampling.""" + + ZEROS = 0 + BORDER = 1 + REFLECTION = 2 + FILL = 3 + + @classmethod + def get(cls, value: TKEnum["SamplePadding"]) -> "SamplePadding": + return _get(cls, value) + + +class DType(Enum, metaclass=_KORNIA_EnumMeta): + """Represent the internal data types used across the library.""" + + INT64 = 0 + FLOAT16 = 1 + FLOAT32 = 2 + FLOAT64 = 3 + + @classmethod + def get(cls, value: Union[str, int, torch.dtype, torch.Tensor, "DType"]) -> "DType": + if isinstance(value, torch.dtype): + return cls[str(value).upper()[6:]] + + elif isinstance(value, torch.Tensor): + return cls(int(value.item())) + + elif isinstance(value, str): + return cls[value.upper()] + + elif isinstance(value, int): + return cls(value) + + elif isinstance(value, cls): + return value + + raise TypeError(f"Invalid identifier {value} with type {type(value)}.") + + @classmethod + def to_torch(cls, value: TKEnum["DType"]) -> torch.dtype: + data = cls.get(value=value) + + if data == DType.INT64: + return torch.long + + elif data == DType.FLOAT16: + return torch.float16 + + elif data == DType.FLOAT32: + return torch.float32 + + elif data == DType.FLOAT64: + return torch.float64 + + raise ValueError + + +# TODO: (low-priority) add INPUT3D, MASK3D, BBOX3D, LAFs etc. +class DataKey(Enum, metaclass=_KORNIA_EnumMeta): + IMAGE = 0 + INPUT = 0 + MASK = 1 + BBOX = 2 + BBOX_XYXY = 3 + BBOX_XYWH = 4 + KEYPOINTS = 5 + LABEL = 6 + CLASS = 6 + + @classmethod + def get(cls, value: TKEnum["DataKey"]) -> "DataKey": + return _get(cls, value) diff --git a/kornia/contrib/__init__.py b/kornia/contrib/__init__.py new file mode 100644 index 0000000..83d0b0b --- /dev/null +++ b/kornia/contrib/__init__.py @@ -0,0 +1,69 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Contrib — Experimental and contributed modules for Kornia. + +This subpackage provides community-contributed and experimental features. +""" + +from kornia.models.tiny_vit import TinyViT + +from .boxmot_tracker import BoxMotTracker +from .connected_components import connected_components +from .diamond_square import diamond_square +from .distance_transform import DistanceTransform, distance_transform +from .edge_detection import EdgeDetector, EdgeDetectorBuilder +from .extract_patches import ( + CombineTensorPatches, + ExtractTensorPatches, + combine_tensor_patches, + compute_padding, + extract_tensor_patches, +) +from .face_detection import * +from .histogram_matching import histogram_matching, interp +from .image_stitching import ImageStitcher +from .kmeans import KMeans +from .lambda_module import Lambda +from .object_detection import ObjectDetector, RTDETRDetectorBuilder +from .super_resolution import RRDBNetBuilder, SmallSRBuilder, SuperResolution + +__all__ = [ + "BoxMotTracker", + "CombineTensorPatches", + "DistanceTransform", + "EdgeDetector", + "EdgeDetectorBuilder", + "ExtractTensorPatches", + "ImageStitcher", + "KMeans", + "Lambda", + "ObjectDetector", + "RRDBNetBuilder", + "RTDETRDetectorBuilder", + "SmallSRBuilder", + "SuperResolution", + "TinyViT", + "combine_tensor_patches", + "compute_padding", + "connected_components", + "diamond_square", + "distance_transform", + "extract_tensor_patches", + "histogram_matching", + "interp", +] diff --git a/kornia/contrib/boxmot_tracker.py b/kornia/contrib/boxmot_tracker.py new file mode 100644 index 0000000..a6a351d --- /dev/null +++ b/kornia/contrib/boxmot_tracker.py @@ -0,0 +1,182 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import datetime +import logging +import os +from pathlib import Path +from typing import Any, Optional, Union + +import torch + +from kornia.config import kornia_config +from kornia.contrib.object_detection import ObjectDetector, RTDETRDetectorBuilder +from kornia.core.external import boxmot +from kornia.core.external import numpy as np +from kornia.image.utils import tensor_to_image +from kornia.io import write_image + +__all__ = ["BoxMotTracker"] + +logger = logging.getLogger(__name__) + + +class BoxMotTracker: + """BoxMotTracker is a module that wraps a detector and a tracker model. + + This module uses BoxMot library for tracking. + + Args: + detector: The detector model. + tracker_model_name: The name of the tracker model. Valid options are: + - "BoTSORT" + - "DeepOCSORT" + - "OCSORT" + - "HybridSORT" + - "ByteTrack" + - "StrongSORT" + - "ImprAssoc" + tracker_model_weights: Path to the model weights for ReID (Re-Identification). + device: Union[str, torch.device, None] on which to run the model (e.g., 'cpu' or 'cuda'). + fp16: Whether to use half-precision (fp16) for faster inference on compatible devices. + per_class: Whether to perform per-class tracking. + track_high_thresh: High threshold for detection confidence. + Detections above this threshold are used in the first association round. + track_low_thresh: Low threshold for detection confidence. + Detections below this threshold are ignored. + new_track_thresh: Threshold for creating a new track. + Detections above this threshold will be considered as potential new tracks. + track_buffer: Number of frames to keep a track alive after it was last detected. + match_thresh: Threshold for the matching step in data association. + proximity_thresh: Threshold for IoU (Intersection over Union) distance in first-round association. + appearance_thresh: Threshold for appearance embedding distance in the ReID module. + cmc_method: Method for correcting camera motion. Options include "sof" (simple optical flow). + frame_rate: Frame rate of the video being processed. Used to scale the track buffer size. + fuse_first_associate: Whether to fuse appearance and motion information during the first association step. + with_reid: Whether to use ReID (Re-Identification) features for association. + + Note: + To use this tracker, load image tensors (shape: ``(1, C, H, W)``) and call ``model.update(image)`` + for at least 4 frames before calling ``model.save(image)``. + + .. note:: + At least 4 frames are needed to initialize the tracking position. + + """ + + name: str = "boxmot_tracker" + + def __init__( + self, + detector: Union[ObjectDetector, str] = "rtdetr_r18vd", + tracker_model_name: str = "DeepOCSORT", + tracker_model_weights: str = "osnet_x0_25_msmt17.pt", + device: str = "cpu", + fp16: bool = False, + **kwargs: Any, + ) -> None: + super().__init__() + if isinstance(detector, str): + if detector.startswith("rtdetr"): + detector = RTDETRDetectorBuilder.build(model_name=detector) + else: + raise ValueError( + f"Detector `{detector}` not available. You may pass an ObjectDetector instance instead." + ) + self.detector = detector + os.makedirs(f"{kornia_config.hub_models_dir}/boxmot", exist_ok=True) + self.tracker = getattr(boxmot, tracker_model_name)( + model_weights=Path(os.path.join(f"{kornia_config.hub_models_dir}/boxmot", tracker_model_weights)), + device=device, + fp16=fp16, + **kwargs, + ) + + def update(self, image: torch.Tensor) -> None: + """Update the tracker with a new image. + + Args: + image: The input image. + + """ + if not (image.ndim == 4 and image.shape[0] == 1) and not image.ndim == 3: + raise ValueError(f"Input torch.Tensor must be of shape (1, 3, H, W) or (3, H, W). Got {image.shape}") + + if image.ndim == 3: + image = image.unsqueeze(0) + + detections_raw: Union[torch.Tensor, list[torch.Tensor]] = self.detector(image) + + detections = detections_raw[0].cpu().numpy() # Batch size is 1 + + detections = np.array( # type: ignore + [ + detections[:, 2], + detections[:, 3], + detections[:, 2] + detections[:, 4], + detections[:, 3] + detections[:, 5], + detections[:, 1], + detections[:, 0], + ] + ).T + + if detections.shape[0] == 0: + # empty N X (x, y, x, y, conf, cls) + detections = np.empty((0, 6)) # type: ignore + + frame_raw = (tensor_to_image(image) * 255).astype(np.uint8) + # --> M X (x, y, x, y, id, conf, cls, ind) + return self.tracker.update(detections, frame_raw) + + def visualize(self, image: torch.Tensor, show_trajectories: bool = True) -> torch.Tensor: + """Visualize the results of the tracker. + + Args: + image: The input image. + show_trajectories: Whether to show the trajectories. + + Returns: + The image with the results of the tracker. + + """ + frame_raw = (tensor_to_image(image) * 255).astype(np.uint8) + self.tracker.plot_results(frame_raw, show_trajectories=show_trajectories) + + return torch.tensor(frame_raw).permute(2, 0, 1) + + def save(self, image: torch.Tensor, show_trajectories: bool = True, directory: Optional[str] = None) -> None: + """Save the model to ONNX format. + + Args: + image: The input image. + show_trajectories: Whether to visualize trajectories. + directory: Where to save the file(s). + + """ + if directory is None: + name = f"{self.name}_{datetime.datetime.now(tz=datetime.UTC).strftime('%Y%m%d%H%M%S')!s}" + directory = os.path.join("kornia_outputs", name) + output = self.visualize(image, show_trajectories=show_trajectories) + + os.makedirs(directory, exist_ok=True) + write_image( + os.path.join(directory, f"{str(0).zfill(6)}.jpg"), + output.byte(), + ) + logger.info(f"Outputs are saved in {directory}") diff --git a/kornia/contrib/connected_components.py b/kornia/contrib/connected_components.py new file mode 100644 index 0000000..1de842e --- /dev/null +++ b/kornia/contrib/connected_components.py @@ -0,0 +1,74 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch +import torch.nn.functional as F + + +def connected_components(image: torch.Tensor, num_iterations: int = 100) -> torch.Tensor: + r"""Compute the Connected-component labelling (CCL) algorithm. + + .. image:: https://github.com/kornia/data/raw/main/cells_segmented.png + + The implementation is an adaptation of the following repository: + + https://gist.github.com/efirdc/5d8bd66859e574c683a504a4690ae8bc + + .. warning:: + This is an experimental API subject to changes and optimization improvements. + + .. note:: + See a working example `here `__. + + Args: + image: the binarized input image with shape :math:`(*, 1, H, W)`. + The image must be in floating point with range [0, 1]. + num_iterations: the number of iterations to make the algorithm to converge. + + Return: + The labels image with the same shape of the input image. + + Example: + >>> img = torch.rand(2, 1, 4, 5) + >>> img_labels = connected_components(img, num_iterations=100) + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input imagetype is not a torch.Tensor. Got: {type(image)}") + + if not isinstance(num_iterations, int) or num_iterations < 1: + raise TypeError("Input num_iterations must be a positive integer.") + + if len(image.shape) < 3 or image.shape[-3] != 1: + raise ValueError(f"Input image shape must be (*,1,H,W). Got: {image.shape}") + + H, W = image.shape[-2:] + image_view = image.view(-1, 1, H, W) + + # precompute a mask with the valid values + mask = image_view == 1 + + # allocate the output tensors for labels + B, _, _, _ = image_view.shape + out = torch.arange(1, B * H * W + 1, device=image.device, dtype=image.dtype).view((-1, 1, H, W)) + out[~mask] = 0 + + for _ in range(num_iterations): + out = F.max_pool2d(out, kernel_size=3, stride=1, padding=1) + out = torch.mul(out, mask) # mask using element-wise multiplication + + return out.view_as(image) diff --git a/kornia/contrib/diamond_square.py b/kornia/contrib/diamond_square.py new file mode 100644 index 0000000..b95da88 --- /dev/null +++ b/kornia/contrib/diamond_square.py @@ -0,0 +1,233 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# based on: https://github.com/anguelos/tormentor/blob/e8050ac235b0c7ad3c7d931cfa47c308a305c486/diamond_square/diamond_square.py # noqa: E501 +import math +from typing import Callable, List, Optional, Tuple, Union + +import torch + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE +from kornia.enhance import normalize_min_max +from kornia.filters import filter2d + +# the default kernels for the diamond square +default_diamond_kernel: List[List[float]] = [[0.25, 0.0, 0.25], [0.0, 0.0, 0.0], [0.25, 0.0, 0.25]] +default_square_kernel: List[List[float]] = [[0.0, 0.25, 0.0], [0.25, 0.0, 0.25], [0.0, 0.25, 0.0]] + + +def _diamond_square_seed( + replicates: int, + width: int, + height: int, + random_fn: Callable[..., torch.Tensor], + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + """Generate the diamond square image seee. + + Args: + replicates: the num of batched replicas for the image. + width: the expected image width. + height: the expected image height. + random_fn: the random function to generate the image seed. + device: the torch device where to create the image seed. + dtype: the torch dtype where to create the image seed. + + Return: + the generated image seed of size Bx1xHxW. + + """ + KORNIA_CHECK(width == 3 or height == 3, "Height or Width must be equal to 3.") + # TODO(anguelos): can we avoid transposing and passing always fixed size. This will cause issues with onnx/jit + transpose: bool = False + if height == 3: + transpose = True + width, height = height, width + + # width is always 3 + KORNIA_CHECK(height % 2 == 1 and height > 2, "Height must be odd and height bigger than 2") + + res: torch.Tensor = random_fn([replicates, 1, width, height], device=device, dtype=dtype) + res[..., ::2, ::2] = random_fn([replicates, 1, 2, (height + 1) // 2], device=device, dtype=dtype) + + # Diamond step + res[..., 1, 1::2] = (res[..., ::2, :-2:2] + res[..., ::2, 2::2]).sum(dim=2) / 4.0 + + # Square step + if width > 3: + res[..., 1, 2:-3:2] = ( + res[..., 0, 2:-3:2] + res[..., 2, 2:-3:2] + res[..., 1, 0:-4:2] + res[..., 1, 2:-3:2] + ) / 4.0 + + tmp1 = res[..., 2, 0] + res[..., 1, 0] = res[..., 0, 0] + res[..., 1, 1] + tmp1 + res[..., 1, -1] = res[..., -1, -1] + res[..., 1, -2] + tmp1 + tmp2 = res[..., 1, 1::2] + res[..., 0, 1::2] = res[..., 0, 0:-2:2] + res[..., 0, 2::2] + tmp2 + res[..., 2, 1::2] = res[..., 2, 0:-2:2] + res[..., 2, 2::2] + tmp2 + res = res / 3.0 + + if transpose: + res = res.transpose(2, 3) + return res + + +def _one_diamond_one_square( + img: torch.Tensor, + random_scale: Union[float, torch.Tensor], + random_fn: Callable[..., torch.Tensor] = torch.rand, + diamond_kernel: Optional[torch.Tensor] = None, + square_kernel: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """Doubles the image resolution by applying a single diamond square steps. + + Recursive application of this method creates plasma fractals. + + Attention! The function is differentiable and gradients are computed as well. + + If this function is run in the usual sense, it is more efficient if it is run in a no_grad() + + Args: + img: a 4D torch.Tensor where dimensions are Batch, Channel, Width, Height. + Width and Height must both be 2^N+1 and + Batch and Channels should in the usual case be 1. + random_scale: a float number in [0,1] controlling the randomness created pixels get. I the usual case, it is + halved at every application of this function. + random_fn: the random function to generate the image seed. + diamond_kernel: the 3x3 kernel to perform the diamond step. + square_kernel: the 3x3 kernel to perform the square step. + + Return: + A torch.Tensor on the same device as img with the same channels as img and width, height of 2^(N+1)+1. + + """ + KORNIA_CHECK_SHAPE(img, ["B", "C", "H", "W"]) + # TODO (anguelos) test multi channel and batch size > 1 + + if diamond_kernel is None: + diamond_kernel = torch.Tensor([default_diamond_kernel]).to(img) # 1x3x3 + if square_kernel is None: + square_kernel = torch.Tensor([default_square_kernel]).to(img) # 1x3x3 + + batch_sz, _, height, width = img.shape + new_img: torch.Tensor = torch.zeros( + [batch_sz, 1, 2 * (height - 1) + 1, 2 * (width - 1) + 1], device=img.device, dtype=img.dtype + ) + new_img[:, :, ::2, ::2] = img + + factor: float = 1.0 / 0.75 + pad_compencate = torch.ones_like(new_img) + pad_compencate[:, :, :, 0] = factor + pad_compencate[:, :, :, -1] = factor + pad_compencate[:, :, 0, :] = factor + pad_compencate[:, :, -1, :] = factor + + random_img: torch.Tensor = random_fn(new_img.size(), device=img.device, dtype=img.dtype) * random_scale + + # TODO(edgar): use kornia.filter2d + # diamond + diamond_regions = filter2d(new_img, diamond_kernel) + diamond_centers = (diamond_regions > 0).to(img.dtype) + # TODO (anguelos) make sure diamond_regions*diamond_centers is needed + new_img = new_img + (1 - random_scale) * diamond_regions * diamond_centers + diamond_centers * random_img + + # square + square_regions = filter2d(new_img, square_kernel) * pad_compencate + square_centers = (square_regions > 0).to(img.dtype) + + # TODO (anguelos) make sure square_centers*square_regions is needed + new_img = new_img + square_centers * random_img + (1 - random_scale) * square_centers * square_regions + + return new_img + + +def diamond_square( + output_size: Tuple[int, int, int, int], + roughness: Union[float, torch.Tensor] = 0.5, + random_scale: Union[float, torch.Tensor] = 1.0, + random_fn: Callable[..., torch.Tensor] = torch.rand, + normalize_range: Optional[Tuple[float, float]] = None, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + """Generate Plasma Fractal Images using the diamond square algorithm. + + See: https://en.wikipedia.org/wiki/Diamond-square_algorithm + + Args: + output_size: a tuple of integers with the BxCxHxW of the image to be generated. + roughness: the scale value to apply at each recursion step. + random_scale: the initial value of the scale for recursion. + random_fn: the callable function to use to sample a random torch.Tensor. + normalize_range: whether to F.normalize using min-max the output map. In case of a + range is specified, min-max norm is applied between the provided range. + device: the torch device to place the output map. + dtype: the torch dtype to place the output map. + + Returns: + A torch.Tensor with shape :math:`(B,C,H,W)` containing the fractal image. + + """ + KORNIA_CHECK(len(output_size) == 4, "output_size must be (B,C,H,W)") + if not isinstance(random_scale, torch.Tensor): + random_scale = torch.Tensor([[[[random_scale]]]]).to(device, dtype) + random_scale = random_scale.expand([output_size[0] * output_size[1], 1, 1, 1]) + else: + KORNIA_CHECK_IS_TENSOR(random_scale) + random_scale = random_scale.view(-1, 1, 1, 1) + random_scale = random_scale.expand([output_size[0], output_size[1], 1, 1]) + random_scale = random_scale.reshape([-1, 1, 1, 1]) + + if not isinstance(roughness, torch.Tensor): + roughness = torch.Tensor([[[[roughness]]]]).to(device, dtype) + roughness = roughness.expand([output_size[0] * output_size[1], 1, 1, 1]) + else: + roughness = roughness.view(-1, 1, 1, 1) + roughness = roughness.expand([output_size[0], output_size[1], 1, 1]) + roughness = roughness.reshape([-1, 1, 1, 1]) + + width, height = output_size[-2:] + num_samples: int = 1 + for x in output_size[:-2]: + num_samples *= x + + # compute the image seed + p2_width: float = 2 ** math.ceil(math.log2(width - 1)) + 1 + p2_height: float = 2 ** math.ceil(math.log2(height - 1)) + 1 + recursion_depth: int = int(min(math.log2(p2_width - 1) - 1, math.log2(p2_height - 1) - 1)) + seed_width: int = (p2_width - 1) // 2**recursion_depth + 1 + seed_height: int = (p2_height - 1) // 2**recursion_depth + 1 + img: torch.Tensor = random_scale * _diamond_square_seed( + num_samples, seed_width, seed_height, random_fn, device, dtype + ) + + # perform recursion + scale = random_scale + for _ in range(recursion_depth): + scale = scale * roughness + img = _one_diamond_one_square(img, scale, random_fn) + + # slice to match with the output size + img = img[..., :width, :height] + img = img.view(output_size) + + # F.normalize the output in the range using min-max + if normalize_range is not None: + min_val, max_val = normalize_range + img = normalize_min_max(img.contiguous(), min_val, max_val) + return img diff --git a/kornia/contrib/distance_transform.py b/kornia/contrib/distance_transform.py new file mode 100644 index 0000000..8c1e5fd --- /dev/null +++ b/kornia/contrib/distance_transform.py @@ -0,0 +1,184 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import math + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE, KORNIA_CHECK_TYPE +from kornia.filters import filter2d, filter3d +from kornia.geometry.grid import create_meshgrid, create_meshgrid3d + + +def _distance_transform_2d_impl(image: torch.Tensor, kernel_size: int, h: float) -> torch.Tensor: + device = image.device + dtype = image.dtype + k_half = kernel_size // 2 + + n_iters = math.ceil(max(image.shape[2], image.shape[3]) / k_half) + grid = create_meshgrid(kernel_size, kernel_size, False, device, dtype) + grid = grid - k_half + + dist = torch.hypot(grid[0, ..., 0], grid[0, ..., 1]) + kernel = torch.exp(-dist / h).unsqueeze(0) + + out = torch.zeros_like(image) + boundary = image.clone() + signal_ones = torch.ones_like(boundary) + + for i in range(n_iters): + cdt = filter2d(boundary, kernel, border_type="replicate") + cdt = -h * torch.log(cdt) + cdt = torch.nan_to_num(cdt, nan=0.0, posinf=0.0, neginf=0.0) + + mask = cdt > 0 + if not mask.any(): + break + + offset: int = i * k_half + out = out + (offset + cdt) * mask.to(dtype=out.dtype) + boundary = torch.where(mask, signal_ones, boundary) + + return out + + +def _distance_transform_3d_impl(image: torch.Tensor, kernel_size: int, h: float) -> torch.Tensor: + device = image.device + dtype = image.dtype + k_half = kernel_size // 2 + + n_iters = math.ceil(max(image.shape[2:]) / k_half) + grid = create_meshgrid3d(kernel_size, kernel_size, kernel_size, False, device, dtype) + grid = grid - k_half + dist = torch.norm(grid[0], p=2, dim=-1) + kernel = torch.exp(-dist / h).unsqueeze(0) + + out = torch.zeros_like(image) + boundary = image.clone() + signal_ones = torch.ones_like(boundary) + + for i in range(n_iters): + cdt = filter3d(boundary, kernel, border_type="replicate") + cdt = -h * torch.log(cdt) + cdt = torch.nan_to_num(cdt, nan=0.0, posinf=0.0, neginf=0.0) + + mask = cdt > 0 + if not mask.any(): + break + + offset: int = i * k_half + out = out + (offset + cdt) * mask.to(dtype=out.dtype) + boundary = torch.where(mask, signal_ones, boundary) + + return out + + +def distance_transform(image: torch.Tensor, kernel_size: int = 3, h: float = 0.35) -> torch.Tensor: + r"""Approximates the Euclidean distance transform of images/volumes using cascaded convolution operations. + + The value at each pixel/voxel represents the distance to the nearest non-zero element. + It uses the method described in :cite:`pham2021dtlayer`. + The transformation is applied independently across the channel dimension. + + Args: + image: Image or volume with shape :math:`(B,C,H,W)` or :math:`(B,C,D,H,W)`. + kernel_size: size of the convolution kernel. Must be an odd number. + h: value that influence the approximation of the min function. + + Returns: + tensor with the same shape as input. + + Example: + >>> # 2D example: + >>> tensor = torch.zeros(1, 1, 5, 5) + >>> tensor[:,:, 1, 2] = 1 + >>> dt = distance_transform(tensor) + >>> # 3D example: + >>> volume = torch.zeros(1, 1, 5, 5, 5) + >>> volume[:, :, 2, 2, 2] = 1 + >>> dt = distance_transform(volume) + + """ + # Validation using KORNIA_CHECK API + KORNIA_CHECK_IS_TENSOR(image) + KORNIA_CHECK(image.is_floating_point(), "image must be a floating point tensor") + + KORNIA_CHECK(image.ndim in (4, 5), f"Invalid image shape, we expect BxCxHxW or BxCxDxHxW. Got: {image.shape}") + + if image.ndim == 4: + KORNIA_CHECK_SHAPE(image, ["B", "C", "H", "W"]) + else: + KORNIA_CHECK_SHAPE(image, ["B", "C", "D", "H", "W"]) + + # dtype / param checks + KORNIA_CHECK_TYPE(kernel_size, int, "kernel_size must be an int") + KORNIA_CHECK(kernel_size % 2 != 0 and kernel_size >= 3, "kernel_size must be an odd integer >= 3") + KORNIA_CHECK(h > 0, f"h must be a positive float, got {h}") + + if image.ndim == 4: + return _distance_transform_2d_impl(image, kernel_size, h) + + return _distance_transform_3d_impl(image, kernel_size, h) + + +class DistanceTransform(nn.Module): + r"""Module that approximates the Euclidean distance transform of images/volumes using convolutions. + + Args: + kernel_size: size of the convolution kernel. + h: value that influence the approximation of the min function. + + """ + + def __init__(self, kernel_size: int = 3, h: float = 0.35) -> None: + super().__init__() + self.kernel_size = kernel_size + self.h = h + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Compute the distance transform while preserving the original tensor layout. + + The underlying functional implementation expects a single-channel tensor. + For multi-channel inputs, channels are temporarily folded into the batch + dimension, processed independently, and then reshaped back. + + Args: + image: Input tensor with shape :math:`(B, C, H, W)` for 2D data or + :math:`(B, C, D, H, W)` for 3D data, where: + - ``B`` is batch size, + - ``C`` is the number of channels, + - ``D`` is depth (only for 3D volumes), + - ``H`` is height, + - ``W`` is width. + + Returns: + A distance-transform tensor with the same shape as ``image``. + Each channel is processed independently. + """ + # Reshape multi-channel inputs to batch dimension to ensure independent processing + if image.shape[1] > 1: + # Dynamically determine spatial dimensions (works for H,W or D,H,W) + spatial_dims = image.shape[2:] + # Use reshape to handle non-contiguous tensors safely + image_in = image.reshape(-1, 1, *spatial_dims) + else: + image_in = image + + return distance_transform(image_in, self.kernel_size, self.h).view_as(image) diff --git a/kornia/contrib/edge_detection.py b/kornia/contrib/edge_detection.py new file mode 100644 index 0000000..717b5bb --- /dev/null +++ b/kornia/contrib/edge_detection.py @@ -0,0 +1,249 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Any, Optional, Union + +import torch +from torch import nn + +from kornia.color.gray import grayscale_to_rgb +from kornia.core.external import PILImage as Image +from kornia.core.external import onnx +from kornia.core.mixin.onnx import ONNXExportMixin +from kornia.enhance.normalize import Normalize +from kornia.models.base import ModelBase +from kornia.models.dexined import DexiNed +from kornia.models.processors import ResizePostProcessor, ResizePreProcessor + +__all__ = ["EdgeDetector", "EdgeDetectorBuilder"] + + +class EdgeDetector(ModelBase, ONNXExportMixin): + """EdgeDetector is a module that wraps an edge detection model. + + This is a high-level API that wraps edge detection models like :py:class:`kornia.models.DexiNed`. + + Args: + model: The edge detection model. + pre_processor: Pre-processing module (e.g., ResizePreProcessor). + post_processor: Post-processing module (e.g., ResizePostProcessor). + name: Optional name for the detector. + + Example: + >>> from kornia.models.dexined import DexiNed + >>> from kornia.models.processors import ResizePreProcessor, ResizePostProcessor + >>> model = DexiNed(pretrained=True) + >>> detector = EdgeDetector(model, ResizePreProcessor(352, 352), ResizePostProcessor()) + >>> img = torch.rand(1, 3, 320, 320) + >>> out = detector(img) + + """ + + name: str = "edge_detection" + + def __init__( + self, + model: torch.nn.Module, + pre_processor: torch.nn.Module, + post_processor: torch.nn.Module, + name: Optional[str] = None, + ) -> None: + """Initialize EdgeDetector. + + Args: + model: The edge detection model. + pre_processor: Pre-processing module (e.g., ResizePreProcessor). + post_processor: Post-processing module (e.g., ResizePostProcessor). + name: Optional name for the detector. + + """ + super().__init__() + self.model = model.eval() + self.pre_processor = pre_processor + self.post_processor = post_processor + if name is not None: + self.name = name + + @staticmethod + def from_config(config: Any) -> EdgeDetector: + """Build EdgeDetector from config. + + This is a placeholder to satisfy the abstract method requirement. + Use EdgeDetectorBuilder.build() or instantiate EdgeDetector directly. + + Args: + config: Configuration object (not used, kept for interface compatibility). + + Returns: + EdgeDetector instance. + + """ + raise NotImplementedError( + "EdgeDetector.from_config() is not implemented. " + "Use EdgeDetectorBuilder.build() or instantiate EdgeDetector directly." + ) + + @torch.inference_mode() + def forward(self, images: Union[torch.Tensor, list[torch.Tensor]]) -> Union[torch.Tensor, list[torch.Tensor]]: + """Forward pass of the edge detection model. + + Args: + images: If list of RGB images. Each image is a torch.Tensor with shape :math:`(3, H, W)`. + If torch.Tensor, a torch.Tensor with shape :math:`(B, 3, H, W)`. + + Returns: + output torch.Tensor. + + """ + images, image_sizes = self.pre_processor(images) + out_images = self.model(images) + return self.post_processor(out_images, image_sizes) + + def visualize( + self, + images: Union[torch.Tensor, list[torch.Tensor]], + edge_maps: Optional[Union[torch.Tensor, list[torch.Tensor]]] = None, + output_type: str = "torch", + ) -> Union[torch.Tensor, list[torch.Tensor], list[Image.Image]]: # type: ignore + """Draw the edge detection results. + + Args: + images: input torch.Tensor. + edge_maps: detected edges. + output_type: type of the output. + + Returns: + output torch.Tensor. + + """ + if edge_maps is None: + edge_maps = self.forward(images) + output = [] + for edge_map in edge_maps: + output.append(grayscale_to_rgb(edge_map)[0]) + + return self._tensor_to_type(output, output_type, is_batch=isinstance(images, torch.Tensor)) + + def save( + self, + images: Union[torch.Tensor, list[torch.Tensor]], + edge_maps: Optional[Union[torch.Tensor, list[torch.Tensor]]] = None, + directory: Optional[str] = None, + output_type: str = "torch", + ) -> None: + """Save the edge detection results. + + Args: + images: input torch.Tensor. + edge_maps: detected edges. + output_type: type of the output. + directory: where to save outputs. + + Returns: + output torch.Tensor. + + """ + outputs = self.visualize(images, edge_maps, output_type) + self._save_outputs(images, directory, suffix="_src") + self._save_outputs(outputs, directory, suffix="_edge") + + def to_onnx( # type: ignore[override] + self, + onnx_name: Optional[str] = None, + image_size: Optional[int] = 352, + include_pre_and_post_processor: bool = True, + save: bool = True, + additional_metadata: Optional[list[tuple[str, str]]] = None, + **kwargs: Any, + ) -> onnx.ModelProto: # type: ignore + """Export the current edge detection model to an ONNX model file. + + Args: + onnx_name: + The name of the output ONNX file. If not provided, a default name in the + format "Kornia-.onnx" will be used. + image_size: + The size to which input images will be resized during preprocessing. + If None, image_size will be dynamic. For DexiNed, recommended scale is 352. + include_pre_and_post_processor: + Whether to include the pre-processor and post-processor in the exported model. + save: + If to save the model or load it. + additional_metadata: + Additional metadata to add to the ONNX model. + kwargs: Additional arguments to convert to onnx. + + """ + if onnx_name is None: + onnx_name = f"kornia_{self.name}_{image_size}.onnx" + + return ONNXExportMixin.to_onnx( + self, + onnx_name, + input_shape=[-1, 3, image_size or -1, image_size or -1], + output_shape=[-1, 1, image_size or -1, image_size or -1], + pseudo_shape=[1, 3, image_size or 352, image_size or 352], + model=self if include_pre_and_post_processor else self.model, + save=save, + additional_metadata=additional_metadata, + **kwargs, + ) + + +class EdgeDetectorBuilder: + """EdgeDetectorBuilder is a class that builds an edge detection model. + + This is a high-level API that builds edge detection models like :py:class:`kornia.models.DexiNed` + and wraps them with :py:class:`EdgeDetector`. + + Note: + To use this model, load image tensors and call ``model.save(images)``. + """ + + @staticmethod + def build(model_name: str = "dexined", pretrained: bool = True, image_size: int = 352) -> EdgeDetector: + """Build an edge detection model. + + Args: + model_name: Name of the model to build. Currently only "dexined" is supported. + pretrained: If True, loads pretrained weights. + image_size: Size to which input images will be resized during preprocessing. + + Returns: + EdgeDetector instance configured with the specified model. + + Example: + >>> detector = EdgeDetectorBuilder.build(pretrained=True, image_size=352) + >>> img = torch.rand(1, 3, 320, 320) + >>> out = detector(img) + + """ + if model_name.lower() == "dexined": + # Normalize then scale to [0, 255] + norm = Normalize(mean=torch.tensor([[0.485, 0.456, 0.406]]), std=torch.tensor([[1.0 / 255.0] * 3])) + model = nn.Sequential(norm, DexiNed(pretrained=pretrained), nn.Sigmoid()) + else: + raise ValueError(f"Model {model_name} not found. Please choose from 'dexined'.") + + return EdgeDetector( + model, + ResizePreProcessor(image_size, image_size), + ResizePostProcessor(), + name="dexined", + ) diff --git a/kornia/contrib/extract_patches.py b/kornia/contrib/extract_patches.py new file mode 100644 index 0000000..1ffd12d --- /dev/null +++ b/kornia/contrib/extract_patches.py @@ -0,0 +1,527 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from math import ceil +from typing import Optional, Tuple, Union, cast +from warnings import warn + +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn.modules.utils import _pair + +FullPadType = Tuple[int, int, int, int] +TuplePadType = Union[Tuple[int, int], FullPadType] +PadType = Union[int, TuplePadType] + + +def create_padding_tuple(padding: PadType, unpadding: bool = False) -> FullPadType: + """Create argument for padding op.""" + padding = cast(TuplePadType, _pair(padding)) + + if len(padding) not in [2, 4]: + raise AssertionError( + f"{'Unpadding' if unpadding else 'Padding'} must be either an int, tuple of two ints or tuple of four ints" + ) + + if len(padding) == 2: + pad_vert = _pair(padding[0]) + pad_horz = _pair(padding[1]) + else: + pad_vert = padding[:2] + pad_horz = padding[2:] + padding = cast(FullPadType, pad_horz + pad_vert) + + return padding + + +def compute_padding( + original_size: Union[int, Tuple[int, int]], + window_size: Union[int, Tuple[int, int]], + stride: Optional[Union[int, Tuple[int, int]]] = None, +) -> FullPadType: + r"""Compute required padding to ensure chaining of :func:`extract_tensor_patches` and + :func:`combine_tensor_patches` produces expected result. + + Args: + original_size: the size of the original torch.Tensor. + window_size: the size of the sliding window used while extracting patches. + stride: The stride of the sliding window. Optional: if not specified, window_size will be used. + + Return: + The required padding as a tuple of four ints: (top, bottom, left, right) + + Example: + >>> image = torch.arange(12).view(1, 1, 4, 3) + >>> padding = compute_padding((4,3), (3,3)) + >>> out = extract_tensor_patches(image, window_size=(3, 3), stride=(3, 3), padding=padding) + >>> combine_tensor_patches(out, original_size=(4, 3), window_size=(3, 3), stride=(3, 3), unpadding=padding) + tensor([[[[ 0, 1, 2], + [ 3, 4, 5], + [ 6, 7, 8], + [ 9, 10, 11]]]]) + + .. note:: + This function will be implicitly used in :func:`extract_tensor_patches` and :func:`combine_tensor_patches` if + `allow_auto_(un)padding` is set to True. + + """ # noqa: D205 + original_size = cast(Tuple[int, int], _pair(original_size)) + window_size = cast(Tuple[int, int], _pair(window_size)) + if stride is None: + stride = window_size + stride = cast(Tuple[int, int], _pair(stride)) + + remainder_vertical = (original_size[0] - window_size[0]) % stride[0] + remainder_horizontal = (original_size[1] - window_size[1]) % stride[1] + # it might be best to apply padding only to the far edges (right, bottom), so + # that fewer patches are affected by the padding. + # For now, just use the default padding + if remainder_vertical != 0: + vertical_padding = stride[0] - remainder_vertical + else: + vertical_padding = 0 + + if remainder_horizontal != 0: + horizontal_padding = stride[1] - remainder_horizontal + else: + horizontal_padding = 0 + + if vertical_padding % 2 == 0: + top_padding = bottom_padding = vertical_padding // 2 + else: + top_padding = vertical_padding // 2 + bottom_padding = ceil(vertical_padding / 2) + + if horizontal_padding % 2 == 0: + left_padding = right_padding = horizontal_padding // 2 + else: + left_padding = horizontal_padding // 2 + right_padding = ceil(horizontal_padding / 2) + # the new implementation with unfolding requires symmetric padding + padding = int(top_padding), int(bottom_padding), int(left_padding), int(right_padding) + return padding + + +class ExtractTensorPatches(nn.Module): + r"""nn.Module that extract patches from tensors and torch.stack them. + + In the simplest case, the output value of the operator with input size + :math:`(B, C, H, W)` is :math:`(B, N, C, H_{out}, W_{out})`. + + where + - :math:`B` is the batch size. + - :math:`N` denotes the total number of extracted patches stacked in + - :math:`C` denotes the number of input channels. + - :math:`H`, :math:`W` the input height and width of the input in pixels. + - :math:`H_{out}`, :math:`W_{out}` denote to denote to the patch size + defined in the function signature. + left-right and top-bottom order. + + * :attr:`window_size` is the size of the sliding window and controls the + shape of the output torch.Tensor and defines the shape of the output patch. + * :attr:`stride` controls the stride to apply to the sliding window and + regulates the overlapping between the extracted patches. + * :attr:`padding` controls the amount of implicit torch.zeros-paddings on both + sizes at each dimension. + * :attr:`allow_auto_padding` allows automatic calculation of the padding required + to fit the window and stride into the image. + + The parameters :attr:`window_size`, :attr:`stride` and :attr:`padding` can + be either: + + - a single ``int`` -- in which case the same value is used for the + height and width dimension. + - a ``tuple`` of two ints -- in which case, the first `int` is used for + the height dimension, and the second `int` for the width dimension. + + :attr:`padding` can also be a ``tuple`` of four ints -- in which case, the + first two ints are for the height dimension while the last two ints are for + the width dimension. + + Args: + input: torch.Tensor image where to extract the patches with shape :math:`(B, C, H, W)`. + window_size: the size of the sliding window and the output patch size. + stride: stride of the sliding window. + padding: Zero-padding added to both side of the input. + allow_auto_adding: whether to allow automatic padding if the window and stride do not fit into the image. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output: :math:`(B, N, C, H_{out}, W_{out})` + + Returns: + the torch.Tensor with the extracted patches. + + Examples: + >>> input = torch.arange(9.).view(1, 1, 3, 3) + >>> patches = extract_tensor_patches(input, (2, 3)) + >>> input + tensor([[[[0., 1., 2.], + [3., 4., 5.], + [6., 7., 8.]]]]) + >>> patches[:, -1] + tensor([[[[3., 4., 5.], + [6., 7., 8.]]]]) + + """ + + def __init__( + self, + window_size: Union[int, Tuple[int, int]], + stride: Union[int, Tuple[int, int]] = 1, + padding: PadType = 0, + allow_auto_padding: bool = False, + ) -> None: + super().__init__() + self.window_size: Union[int, Tuple[int, int]] = window_size + self.stride: Union[int, Tuple[int, int]] = stride + self.padding: PadType = padding + self.allow_auto_padding: bool = allow_auto_padding + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Extract sliding-window patches from a batched image tensor. + + Args: + input: Input tensor with shape :math:`(B, C, H, W)`, where: + - ``B`` is the batch size (number of images), + - ``C`` is the number of channels, + - ``H`` is height, + - ``W`` is width. + + Returns: + A tensor containing stacked patches with shape + :math:`(B, N, C, H_{out}, W_{out})`, where: + - ``N`` is the number of extracted windows per image, + - ``H_{out}`` and ``W_{out}`` are patch height and patch width. + """ + return extract_tensor_patches( + input, + self.window_size, + stride=self.stride, + padding=self.padding, + allow_auto_padding=self.allow_auto_padding, + ) + + +class CombineTensorPatches(nn.Module): + r"""nn.Module that combines patches back into full tensors. + + In the simplest case, the output value of the operator with input size + :math:`(B, N, C, H_{out}, W_{out})` is :math:`(B, C, H, W)`. + + where + - :math:`B` is the batch size. + - :math:`N` denotes the total number of extracted patches stacked in + - :math:`C` denotes the number of input channels. + - :math:`H`, :math:`W` the input height and width of the input in pixels. + - :math:`H_{out}`, :math:`W_{out}` denote to denote to the patch size + defined in the function signature. + left-right and top-bottom order. + + + * :attr:`original_size` is the size of the original image prior to + extracting torch.Tensor patches and defines the shape of the output patch. + * :attr:`window_size` is the size of the sliding window used while + extracting torch.Tensor patches. + * :attr:`stride` controls the stride to apply to the sliding window and + regulates the overlapping between the extracted patches. + * :attr:`unpadding` is the amount of padding to be removed. If specified, + this value must be the same as padding used while extracting torch.Tensor patches. + * :attr:`allow_auto_unpadding` allows automatic calculation of the padding required + to fit the window and stride into the image. This must be used if the + `allow_auto_padding` flag was used for extracting the patches. + + + The parameters :attr:`original_size`, :attr:`window_size`, :attr:`stride`, and :attr:`unpadding` can + be either: + + - a single ``int`` -- in which case the same value is used for the + height and width dimension. + - a ``tuple`` of two ints -- in which case, the first `int` is used for + the height dimension, and the second `int` for the width dimension. + + :attr:`unpadding` can also be a ``tuple`` of four ints -- in which case, the + first two ints are for the height dimension while the last two ints are for + the width dimension. + + Args: + patches: patched torch.Tensor with shape :math:`(B, N, C, H_{out}, W_{out})`. + original_size: the size of the original torch.Tensor and the output size. + window_size: the size of the sliding window used while extracting patches. + stride: stride of the sliding window. + unpadding: remove the padding added to both side of the input. + allow_auto_unpadding: whether to allow automatic unpadding of the input + if the window and stride do not fit into the original_size. + eps: small value used to prevent division by zero. + + Shape: + - Input: :math:`(B, N, C, H_{out}, W_{out})` + - Output: :math:`(B, C, H, W)` + + Example: + >>> out = extract_tensor_patches(torch.arange(16).view(1, 1, 4, 4), window_size=(2, 2), stride=(2, 2)) + >>> combine_tensor_patches(out, original_size=(4, 4), window_size=(2, 2), stride=(2, 2)) + tensor([[[[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11], + [12, 13, 14, 15]]]]) + + .. note:: + This function is supposed to be used in conjunction with :class:`ExtractTensorPatches`. + + """ + + def __init__( + self, + original_size: Tuple[int, int], + window_size: Union[int, Tuple[int, int]], + stride: Optional[Union[int, Tuple[int, int]]] = None, + unpadding: PadType = 0, + allow_auto_unpadding: bool = False, + ) -> None: + super().__init__() + self.original_size: Tuple[int, int] = original_size + self.window_size: Union[int, Tuple[int, int]] = window_size + self.stride: Union[int, Tuple[int, int]] = stride if stride is not None else window_size + self.unpadding: PadType = unpadding + self.allow_auto_unpadding: bool = allow_auto_unpadding + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Reconstruct full images from extracted patches. + + Args: + input: Patch tensor with shape :math:`(B, N, C, H_{out}, W_{out})`, where: + - ``B`` is the batch size, + - ``N`` is the number of patches, + - ``C`` is the number of channels, + - ``H_{out}`` and ``W_{out}`` are patch height and width. + + Returns: + A reconstructed tensor with shape :math:`(B, C, H, W)`, where ``H`` + and ``W`` correspond to ``original_size`` after optional unpadding. + """ + return combine_tensor_patches( + input, + self.original_size, + self.window_size, + stride=self.stride, + unpadding=self.unpadding, + allow_auto_unpadding=self.allow_auto_unpadding, + ) + + +def _check_patch_fit(original_size: Tuple[int, int], window_size: Tuple[int, int], stride: Tuple[int, int]) -> bool: + remainder_vertical = (original_size[0] - window_size[0]) % stride[0] + remainder_horizontal = (original_size[1] - window_size[1]) % stride[1] + # the remainder takes into account half a window on each side, + # the rest of the image is divided based on the stride, not the window + # size + if (remainder_horizontal != 0) or (remainder_vertical != 0): + # needs padding to fit + return False + # we can fit a full number of patches in, based on the stride + return True + + +def combine_tensor_patches( + patches: torch.Tensor, + original_size: Union[int, Tuple[int, int]], + window_size: Union[int, Tuple[int, int]], + stride: Union[int, Tuple[int, int]], + allow_auto_unpadding: bool = False, + unpadding: PadType = 0, + eps: float = 1e-8, +) -> torch.Tensor: + r"""Restore input from patches. + + See :class:`~kornia.contrib.CombineTensorPatches` for details. + + Args: + patches: patched torch.Tensor with shape :math:`(B, N, C, H_{out}, W_{out})`. + original_size: the size of the original torch.Tensor and the output size. + window_size: the size of the sliding window used while extracting patches. + stride: stride of the sliding window. + unpadding: remove the padding added to both side of the input. + allow_auto_unpadding: whether to allow automatic unpadding of the input + if the window and stride do not fit into the original_size. + eps: small value used to prevent division by zero. + + Return: + The combined patches in an image torch.Tensor with shape :math:`(B, C, H, W)`. + + Example: + >>> out = extract_tensor_patches(torch.arange(16).view(1, 1, 4, 4), window_size=(2, 2), stride=(2, 2)) + >>> combine_tensor_patches(out, original_size=(4, 4), window_size=(2, 2), stride=(2, 2)) + tensor([[[[ 0, 1, 2, 3], + [ 4, 5, 6, 7], + [ 8, 9, 10, 11], + [12, 13, 14, 15]]]]) + + .. note:: + This function is supposed to be used in conjunction with :func:`extract_tensor_patches`. + + """ + if patches.ndim != 5: + raise ValueError(f"Invalid input shape, we expect BxNxCxHxW. Got: {patches.shape}") + + original_size = cast(Tuple[int, int], _pair(original_size)) + window_size = cast(Tuple[int, int], _pair(window_size)) + stride = cast(Tuple[int, int], _pair(stride)) + + if (stride[0] > window_size[0]) | (stride[1] > window_size[1]): + raise AssertionError( + f"Stride={stride} should be less than or equal to Window size={window_size}, information is missing" + ) + + if not unpadding: + # if padding is specified, we leave it up to the user to ensure it fits + # otherwise we check here if it will fit and offer to calculate padding + if not _check_patch_fit(original_size, window_size, stride): + if not allow_auto_unpadding: + warn( + f"The window will not fit into the image. \nWindow size: {window_size}\nStride: {stride}\n" + f"Image size: {original_size}\n" + "This means we probably cannot correctly recombine patches. By enabling `allow_auto_unpadding`, " + "the input will be unpadded to fit the window and stride.\n" + "If the patches have been obtained through `extract_tensor_patches` with the correct padding or " + "the argument `allow_auto_padding`, this will result in a correct reconstruction.", + stacklevel=1, + ) + else: + unpadding = compute_padding(original_size=original_size, window_size=window_size, stride=stride) + # TODO: Can't we just do actual size minus original size to get padding? + + if unpadding: + unpadding = create_padding_tuple(unpadding) + + ones_tensor = torch.ones( + patches.shape[0], + patches.shape[2], + original_size[0], + original_size[1], + device=patches.device, + dtype=patches.dtype, + ) + + if unpadding: + ones_tensor = F.pad(ones_tensor, pad=unpadding) + restored_size = ones_tensor.shape[2:] + + patches = patches.permute(0, 2, 3, 4, 1) + patches = patches.reshape(patches.shape[0], -1, patches.shape[-1]) + int_flag = 0 + if not torch.is_floating_point(patches): + int_flag = 1 + dtype = patches.dtype + patches = patches.float() + ones_tensor = ones_tensor.float() + + # Calculate normalization map + unfold_ones = F.unfold(ones_tensor, kernel_size=window_size, stride=stride) + norm_map = F.fold(input=unfold_ones, output_size=restored_size, kernel_size=window_size, stride=stride) + if unpadding: + norm_map = F.pad(norm_map, [-i for i in unpadding]) + + # Restored torch.Tensor + saturated_restored_tensor = F.fold(input=patches, output_size=restored_size, kernel_size=window_size, stride=stride) + if unpadding: + saturated_restored_tensor = F.pad(saturated_restored_tensor, [-i for i in unpadding]) + + # Remove satuation effect due to multiple summations + restored_tensor = saturated_restored_tensor / (norm_map + eps) + if int_flag: + restored_tensor = restored_tensor.to(dtype) + return restored_tensor + + +def _extract_tensor_patchesnd( + input: torch.Tensor, window_sizes: Tuple[int, ...], strides: Tuple[int, ...] +) -> torch.Tensor: + batch_size, num_channels = input.size()[:2] + dims = range(2, input.dim()) + for dim, patch_size, stride in zip(dims, window_sizes, strides): + input = input.unfold(dim, patch_size, stride) + input = input.permute(0, *dims, 1, *(dim + len(dims) for dim in dims)).contiguous() + return input.view(batch_size, -1, num_channels, *window_sizes) + + +def extract_tensor_patches( + input: torch.Tensor, + window_size: Union[int, Tuple[int, int]], + stride: Union[int, Tuple[int, int]] = 1, + padding: PadType = 0, + allow_auto_padding: bool = False, +) -> torch.Tensor: + r"""Extract patches from tensors and stacks them. + + See :class:`~kornia.contrib.ExtractTensorPatches` for details. + + Args: + input: torch.Tensor image where to extract the patches with shape :math:`(B, C, H, W)`. + window_size: the size of the sliding window and the output patch size. + stride: stride of the sliding window. + padding: Zero-padding added to both side of the input. + allow_auto_padding: whether to allow automatic padding if the window and stride do not fit into the image. + + Returns: + the torch.Tensor with the extracted patches with shape :math:`(B, N, C, H_{out}, W_{out})`. + + Examples: + >>> input = torch.arange(9.).view(1, 1, 3, 3) + >>> patches = extract_tensor_patches(input, (2, 3)) + >>> input + tensor([[[[0., 1., 2.], + [3., 4., 5.], + [6., 7., 8.]]]]) + >>> patches[:, -1] + tensor([[[[3., 4., 5.], + [6., 7., 8.]]]]) + + """ + if not torch.is_tensor(input): + raise TypeError(f"Input input type is not a torch.Tensor. Got {type(input)}") + + if len(input.shape) != 4: + raise ValueError(f"Invalid input shape, we expect BxCxHxW. Got: {input.shape}") + + # check if the window sliding over the image will fit into the image + # torch's unfold drops the final patches that don't fit + window_size = cast(Tuple[int, int], _pair(window_size)) + stride = cast(Tuple[int, int], _pair(stride)) + original_size = (input.shape[-2], input.shape[-1]) + + if not padding: + # if padding is specified, we leave it up to the user to ensure it fits + # otherwise we check here if it will fit and offer to calculate padding + if not _check_patch_fit(original_size, window_size, stride): + if not allow_auto_padding: + warn( + f"The window will not fit into the image. \nWindow size: {window_size}\nStride: {stride}\n" + f"Image size: {original_size}\n" + "This means that the final incomplete patches will be dropped. By enabling `allow_auto_padding`, " + "the input will be padded to fit the window and stride.", + stacklevel=1, + ) + else: + padding = compute_padding(original_size=original_size, window_size=window_size, stride=stride) + + if padding: + padding = create_padding_tuple(padding) + input = F.pad(input, padding) + + return _extract_tensor_patchesnd(input, window_size, stride) diff --git a/kornia/contrib/face_detection.py b/kornia/contrib/face_detection.py new file mode 100644 index 0000000..83ed4e2 --- /dev/null +++ b/kornia/contrib/face_detection.py @@ -0,0 +1,273 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from enum import Enum +from typing import Dict, List, Optional + +import torch +from torch import nn + +from kornia.geometry.bbox import nms as nms_kornia +from kornia.models.yunet import YuNet +from kornia.models.yunet.processors import PriorBox +from kornia.models.yunet.processors import decode as _decode + +__all__ = ["FaceDetector", "FaceDetectorResult", "FaceKeypoint"] + + +class FaceKeypoint(Enum): + r"""Define the keypoints detected in a face. + + The left/right convention is based on the screen viewer. + """ + + EYE_LEFT = 0 + EYE_RIGHT = 1 + NOSE = 2 + MOUTH_LEFT = 3 + MOUTH_RIGHT = 4 + + +class FaceDetectorResult: + r"""Encapsulate the results obtained by the :py:class:`kornia.contrib.FaceDetector`. + + Args: + data: the encoded results coming from the feature detector with shape :math:`(14,)`. + + """ + + def __init__(self, data: torch.Tensor) -> None: + if len(data) < 15: + raise ValueError(f"Result must comes as vector of size(14). Got: {data.shape}.") + self._data = data + + def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> "FaceDetectorResult": + """Like :func:`torch.nn.Module.to()` method.""" + self._data = self._data.to(device=device, dtype=dtype) + return self + + @property + def xmin(self) -> torch.Tensor: + """The bounding box top-left x-coordinate.""" + return self._data[..., 0] + + @property + def ymin(self) -> torch.Tensor: + """The bounding box top-left y-coordinate.""" + return self._data[..., 1] + + @property + def xmax(self) -> torch.Tensor: + """The bounding box bottom-right x-coordinate.""" + return self._data[..., 2] + + @property + def ymax(self) -> torch.Tensor: + """The bounding box bottom-right y-coordinate.""" + return self._data[..., 3] + + def get_keypoint(self, keypoint: FaceKeypoint) -> torch.Tensor: + """Get the [x y] position of a given facial keypoint. + + Args: + keypoint: the keypoint type to return the position. + + """ + if keypoint == FaceKeypoint.EYE_LEFT: + out = self._data[..., (4, 5)] + elif keypoint == FaceKeypoint.EYE_RIGHT: + out = self._data[..., (6, 7)] + elif keypoint == FaceKeypoint.NOSE: + out = self._data[..., (8, 9)] + elif keypoint == FaceKeypoint.MOUTH_LEFT: + out = self._data[..., (10, 11)] + elif keypoint == FaceKeypoint.MOUTH_RIGHT: + out = self._data[..., (12, 13)] + else: + raise ValueError(f"Not valid keypoint type. Got: {keypoint}.") + return out + + @property + def score(self) -> torch.Tensor: + """The detection score.""" + return self._data[..., 14] + + @property + def width(self) -> torch.Tensor: + """The bounding box width.""" + return self.xmax - self.xmin + + @property + def height(self) -> torch.Tensor: + """The bounding box height.""" + return self.ymax - self.ymin + + @property + def top_left(self) -> torch.Tensor: + """The [x y] position of the top-left coordinate of the bounding box.""" + return self._data[..., (0, 1)] + + @property + def top_right(self) -> torch.Tensor: + """The [x y] position of the top-left coordinate of the bounding box.""" + out = self.top_left + out[..., 0] += self.width + return out + + @property + def bottom_right(self) -> torch.Tensor: + """The [x y] position of the bottom-right coordinate of the bounding box.""" + return self._data[..., (2, 3)] + + @property + def bottom_left(self) -> torch.Tensor: + """The [x y] position of the top-left coordinate of the bounding box.""" + out = self.top_left + out[..., 1] += self.height + return out + + +class FaceDetector(nn.Module): + r"""Detect faces in a given image using YuNet model. + + This is a high-level API that wraps the :py:class:`kornia.models.YuNet` model for face detection. + By default, it uses the method described in :cite:`facedetect-yu`. + + Args: + top_k: the maximum number of detections to return before the nms. + confidence_threshold: the threshold used to discard detections. + nms_threshold: the threshold used by the nms for iou. + keep_top_k: the maximum number of detections to return after the nms. + + Return: + A list of B tensors with shape :math:`(N,15)` to be used with :py:class:`kornia.contrib.FaceDetectorResult`. + + Example: + >>> img = torch.rand(1, 3, 320, 320) + >>> detect = FaceDetector() + >>> res = detect(img) + + """ + + def __init__( + self, top_k: int = 5000, confidence_threshold: float = 0.3, nms_threshold: float = 0.3, keep_top_k: int = 750 + ) -> None: + super().__init__() + self.top_k = top_k + self.confidence_threshold = confidence_threshold + self.nms_threshold = nms_threshold + self.keep_top_k = keep_top_k + self.config = { + "name": "YuNet", + "min_sizes": [[10, 16, 24], [32, 48], [64, 96], [128, 192, 256]], + "steps": [8, 16, 32, 64], + "variance": [0.1, 0.2], + "clip": False, + } + self.min_sizes = [[10, 16, 24], [32, 48], [64, 96], [128, 192, 256]] + self.steps = [8, 16, 32, 64] + self.variance = [0.1, 0.2] + self.clip = False + self.model = YuNet("test", pretrained=True) + self.nms = nms_kornia + + def preprocess(self, image: torch.Tensor) -> torch.Tensor: + """Preprocess input images before feeding them to YuNet. + + Args: + image: Batch of RGB images with shape :math:`(B, 3, H, W)`, where: + - ``B`` is batch size, + - ``3`` is the RGB channel dimension, + - ``H`` is image height, + - ``W`` is image width. + + Returns: + The preprocessed image tensor. The current implementation returns the + input unchanged and acts as an extension point for custom pipelines. + """ + return image + + def postprocess(self, data: Dict[str, torch.Tensor], height: int, width: int) -> List[torch.Tensor]: + """Decode model outputs into filtered face detections. + + Args: + data: Raw output dictionary from YuNet containing ``loc``, ``conf``, + and ``iou`` tensors. + height: Input image height used to scale decoded box coordinates. + width: Input image width used to scale decoded box coordinates. + + Returns: + A list with one tensor per batch element. Each tensor is shaped + :math:`(N, 15)`, where ``N`` is the number of detections kept after + confidence filtering and non-maximum suppression (NMS). + + The 15 values per detection are: + - 14 geometry values (bounding box + five keypoints), + - 1 confidence score. + """ + loc, conf, iou = data["loc"], data["conf"], data["iou"] + + scale = torch.tensor( + [width, height, width, height, width, height, width, height, width, height, width, height, width, height], + device=loc.device, + dtype=loc.dtype, + ) # 14 + + priors = PriorBox(self.min_sizes, self.steps, self.clip, image_size=(height, width)) + priors = priors.to(loc.device, loc.dtype) + + batched_dets: List[torch.Tensor] = [] + for batch_elem in range(loc.shape[0]): + boxes = _decode(loc[batch_elem], priors(), self.variance) # Nx14 + boxes = boxes * scale + + # clamp here for the compatibility for ONNX + cls_scores, iou_scores = conf[batch_elem, :, 1], iou[batch_elem, :, 0] + scores = (cls_scores * iou_scores.clamp(0.0, 1.0)).sqrt() + + # ignore low scores + inds = scores > self.confidence_threshold + boxes, scores = boxes[inds], scores[inds] + + # keep top-K before NMS + order = scores.sort(descending=True)[1][: self.top_k] + boxes, scores = boxes[order], scores[order] + + # performd NMS + # NOTE: nms need to be revise since does not export well to onnx + dets = torch.cat((boxes, scores[:, None]), dim=-1) # Nx15 + keep = self.nms(boxes[:, :4], scores, self.nms_threshold) + if len(keep) > 0: + dets = dets[keep, :] + + # keep top-K faster NMS + batched_dets.append(dets[: self.keep_top_k]) + return batched_dets + + def forward(self, image: torch.Tensor) -> List[torch.Tensor]: + r"""Detect faces in a given batch of images. + + Args: + image: batch of images :math:`(B,3,H,W)` + + Return: + List[torch.Tensor]: list with the boxes found on each image. :math:`Bx(N,15)`. + + """ + img = self.preprocess(image) + out = self.model(img) + return self.postprocess(out, img.shape[-2], img.shape[-1]) diff --git a/kornia/contrib/histogram_matching.py b/kornia/contrib/histogram_matching.py new file mode 100644 index 0000000..af4cd40 --- /dev/null +++ b/kornia/contrib/histogram_matching.py @@ -0,0 +1,85 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch + + +def histogram_matching(source: torch.Tensor, template: torch.Tensor) -> torch.Tensor: + """Adjust the pixel values of an image to match its histogram towards a target image. + + `Histogram matching `_ is the transformation + of an image so that its histogram matches a specified histogram. In this implementation, the + histogram is computed over the flattened image array. Code referred to + `here `_. + + Args: + source: Image to transform. + template: Template image. It can have different dimensions to source. + + Returns: + The transformed output image as the same shape as the source image. + + Note: + This function does not matches histograms element-wisely if input a batched tensor. + + """ + oldshape = source.shape + source = source.ravel() + template = template.ravel() + + # get the set of unique pixel values and their corresponding indices and counts. + _, bin_idx, s_counts = torch.unique(source, return_inverse=True, return_counts=True) + t_values, t_counts = torch.unique(template, return_counts=True) + + # take the cumsum of the counts and normalize by the number of pixels to + # get the empirical cumulative distribution functions for the source and + # template images (maps pixel value --> quantile) + + s_quantiles = torch.cumsum(s_counts, dim=0, dtype=source.dtype) + s_quantiles = s_quantiles / s_quantiles[-1] + t_quantiles = torch.cumsum(t_counts, dim=0, dtype=source.dtype) + t_quantiles = t_quantiles / t_quantiles[-1] + + # interpolate linearly to find the pixel values in the template image + # that correspond most closely to the quantiles in the source image + interp_t_values = interp(s_quantiles, t_quantiles, t_values) + + return interp_t_values[bin_idx].reshape(oldshape) + + +def interp(x: torch.Tensor, xp: torch.Tensor, fp: torch.Tensor) -> torch.Tensor: + """One-dimensional linear interpolation for monotonically increasing sample points. + + Returns the one-dimensional piecewise linear interpolant to a function with + given discrete data points :math:`(xp, fp)`, evaluated at :math:`x`. + + This is confirmed to be a correct implementation. + See https://github.com/pytorch/pytorch/issues/1552#issuecomment-979998307 + + Args: + x: the :math:`x`-coordinates at which to evaluate the interpolated + values. + xp: the :math:`x`-coordinates of the data points, must be increasing. + fp: the :math:`y`-coordinates of the data points, same length as `xp`. + + Returns: + the interpolated values, same size as `x`. + + """ + i = torch.clip(torch.searchsorted(xp, x, right=True), 1, len(xp) - 1) + + return (fp[i - 1] * (xp[i] - x) + fp[i] * (x - xp[i - 1])) / (xp[i] - xp[i - 1]) diff --git a/kornia/contrib/image_stitching.py b/kornia/contrib/image_stitching.py new file mode 100644 index 0000000..5c66123 --- /dev/null +++ b/kornia/contrib/image_stitching.py @@ -0,0 +1,209 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Optional, Tuple + +import torch +from torch import nn + +from kornia.color import rgb_to_grayscale +from kornia.feature import LocalFeatureMatcher, LoFTR +from kornia.geometry.homography import find_homography_dlt_iterated +from kornia.geometry.ransac import RANSAC +from kornia.geometry.transform import warp_perspective + + +class ImageStitcher(nn.Module): + """Stitch two images with overlapping fields of view. + + Args: + matcher: image feature matching module. + estimator: method to compute homography, either "vanilla" or "ransac". + "ransac" is slower with a better accuracy. + blending_method: method to blend two images together. + Only "naive" is currently supported. + + Note: + Current implementation requires strict image ordering from left to right. + + .. code-block:: python + + IS = ImageStitcher(KF.LoFTR(pretrained='outdoor'), estimator='ransac').cuda() + # Compute the stitched result with less GPU memory cost. + with torch.inference_mode(): + out = IS(img_left, img_right) + # Show the result + plt.imshow(K.tensor_to_image(out)) + + """ + + def __init__(self, matcher: nn.Module, estimator: str = "ransac", blending_method: str = "naive") -> None: + super().__init__() + self.matcher = matcher + self.estimator = estimator + self.blending_method = blending_method + if estimator not in ["ransac", "vanilla"]: + raise NotImplementedError(f"Unsupported estimator {estimator}. Use `ransac` or `vanilla` instead.") + if estimator == "ransac": + self.ransac = RANSAC("homography") + + def _estimate_homography(self, keypoints1: torch.Tensor, keypoints2: torch.Tensor) -> torch.Tensor: + """Estimate homography by the matched keypoints. + + Args: + keypoints1: matched keypoint set from an image, shaped as :math:`(N, 2)`. + keypoints2: matched keypoint set from the other image, shaped as :math:`(N, 2)`. + + """ + if self.estimator == "vanilla": + homo = find_homography_dlt_iterated( + keypoints2[None], keypoints1[None], torch.ones_like(keypoints1[None, :, 0]) + ) + elif self.estimator == "ransac": + homo, _ = self.ransac(keypoints2, keypoints1) + homo = homo[None] + else: + raise NotImplementedError(f"Unsupported estimator {self.estimator}. Use `ransac` or `vanilla` instead.") + return homo + + def estimate_transform(self, *args: torch.Tensor, **kwargs: torch.Tensor) -> torch.Tensor: + """Compute the corresponding homography.""" + kp1, kp2, idx = kwargs["keypoints0"], kwargs["keypoints1"], kwargs["batch_indexes"] + homos = [self._estimate_homography(kp1[idx == i], kp2[idx == i]) for i in range(len(idx.unique()))] + + if len(homos) == 0: + raise RuntimeError("Compute homography failed. No matched keypoints found.") + + return torch.cat(homos) + + def blend_image(self, src_img: torch.Tensor, dst_img: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + """Blend two images together.""" + out: torch.Tensor + if self.blending_method == "naive": + out = torch.where(mask == 1, src_img, dst_img) + else: + raise NotImplementedError(f"Unsupported blending method {self.blending_method}. Use `naive`.") + return out + + def preprocess(self, image_1: torch.Tensor, image_2: torch.Tensor) -> Dict[str, torch.Tensor]: + """Preprocess input to the required format.""" + # TODO: probably perform histogram matching here. + if isinstance(self.matcher, (LoFTR, LocalFeatureMatcher)): + input_dict = { # LofTR works on grayscale images only + "image0": rgb_to_grayscale(image_1), + "image1": rgb_to_grayscale(image_2), + } + else: + raise NotImplementedError(f"The preprocessor for {self.matcher} has not been implemented.") + return input_dict + + def postprocess(self, image: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + """Crop redundant empty columns from a stitched panorama. + + Args: + image: Stitched image tensor, typically shaped + :math:`(B, C, H, W_{panorama})`, where ``W_{panorama}`` is the + panorama width after concatenation/warping. + mask: Validity mask aligned with ``image`` where non-zero values mark + pixels covered by at least one input view. + + Returns: + The stitched image cropped to remove trailing invalid columns + (columns that contain no valid pixels in the mask). + If no redundant area is found, the original ``image`` is returned. + """ + # NOTE: assumes no batch mode. This method keeps all valid regions after stitching. + mask_ = mask.sum((0, 1)) + index = int(mask_.bool().any(0).long().argmin().item()) + if index == 0: # If no redundant space + return image + return image[..., :index] + + def on_matcher(self, data: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """Run the configured feature matcher on preprocessed inputs. + + Args: + data: Matcher input dictionary, usually containing ``image0`` and + ``image1`` tensors. + + Returns: + A correspondence dictionary produced by ``self.matcher``. + Typical entries are matched keypoints and batch indices used later + for homography estimation. + """ + return self.matcher(data) + + def stitch_pair( + self, + images_left: torch.Tensor, + images_right: torch.Tensor, + mask_left: Optional[torch.Tensor] = None, + mask_right: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Stitch two images into a shared panorama frame. + + Args: + images_left: Reference image tensor, shaped :math:`(B, C, H, W_l)`. + images_right: Image tensor to be warped onto the reference frame, + shaped :math:`(B, C, H, W_r)`. + mask_left: Optional validity mask for ``images_left`` with the same + shape as ``images_left``. + mask_right: Optional validity mask for ``images_right`` with the same + shape as ``images_right``. + + Returns: + A tuple ``(stitched_image, stitched_mask)``: + - ``stitched_image`` is the blended panorama tensor with width + ``W_l + W_r`` before final cropping. + - ``stitched_mask`` is a mask tensor indicating which panorama + pixels are valid after warping and blending. + """ + # Compute the transformed images + input_dict = self.preprocess(images_left, images_right) + out_shape = (images_left.shape[-2], images_left.shape[-1] + images_right.shape[-1]) + correspondences = self.on_matcher(input_dict) + homo = self.estimate_transform(**correspondences) + src_img = warp_perspective(images_right, homo, out_shape) + dst_img = torch.cat([images_left, torch.zeros_like(images_right)], -1) + + # Compute the transformed masks + if mask_left is None: + mask_left = torch.ones_like(images_left) + if mask_right is None: + mask_right = torch.ones_like(images_right) + # 'nearest' to ensure no floating points in the mask + src_mask = warp_perspective(mask_right, homo, out_shape, mode="nearest") + dst_mask = torch.cat([mask_left, torch.zeros_like(mask_right)], -1) + return self.blend_image(src_img, dst_img, src_mask), (dst_mask + src_mask).bool().to(src_mask.dtype) + + def forward(self, *imgs: torch.Tensor) -> torch.Tensor: + """Iteratively stitch a sequence of overlapping images. + + Args: + *imgs: Ordered image tensors from left to right. Each tensor is expected + to share a common height and overlap with the next image. + Each tensor typically follows :math:`(B, C, H, W)`. + + Returns: + A single panorama tensor obtained by stitching all provided images + in order and cropping trailing empty space. + """ + img_out = imgs[0] + mask_left = torch.ones_like(img_out) + for i in range(len(imgs) - 1): + img_out, mask_left = self.stitch_pair(img_out, imgs[i + 1], mask_left) + return self.postprocess(img_out, mask_left) diff --git a/kornia/contrib/kmeans.py b/kornia/contrib/kmeans.py new file mode 100644 index 0000000..51175c5 --- /dev/null +++ b/kornia/contrib/kmeans.py @@ -0,0 +1,222 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# based on https://github.com/subhadarship/kmeans_pytorch + +from __future__ import annotations + +import torch + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SHAPE +from kornia.geometry.linalg import euclidean_distance + + +class KMeans: + """Implements the kmeans clustering algorithm with euclidean distance as similarity measure. + + Args: + num_clusters: number of clusters the data has to be assigned to + cluster_centers: torch.Tensor of starting cluster centres can be passed instead of num_clusters + tolerance: float value. the algorithm terminates if the shift in centers is less than tolerance + max_iterations: number of iterations to run the algorithm for + seed: number to set torch manual seed for reproducibility + + Example: + >>> kmeans = kornia.contrib.KMeans(3, None, 10e-4, 100, 0) + >>> kmeans.fit(torch.rand((1000, 5))) + >>> predictions = kmeans.predict(torch.rand((10, 5))) + + """ + + def __init__( + self, + num_clusters: int, + cluster_centers: torch.Tensor | None, + tolerance: float = 10e-4, + max_iterations: int = 0, + seed: int | None = None, + ) -> None: + KORNIA_CHECK(num_clusters != 0, "num_clusters can't be 0") + + # cluster_centers should have only 2 dimensions + if cluster_centers is not None: + KORNIA_CHECK_SHAPE(cluster_centers, ["C", "D"]) + + self.num_clusters = num_clusters + self._cluster_centers = cluster_centers + self.tolerance = tolerance + self.max_iterations = max_iterations + + self._final_cluster_assignments: None | torch.Tensor = None + self._final_cluster_centers: None | torch.Tensor = None + + if seed is not None: + torch.manual_seed(seed) + + @property + def cluster_centers(self) -> torch.Tensor: + """Return the current cluster centers. + + Returns: + A tensor with shape :math:`(C, D)`: + - ``C`` is the number of clusters. + - ``D`` is the feature dimension of each sample. + + If :meth:`fit` has already been called, this returns the learned + final centers. Otherwise, it returns the initialization provided + during construction. + + Raises: + TypeError: If no initial centers were provided and ``fit`` has not been run. + """ + if isinstance(self._final_cluster_centers, torch.Tensor): + return self._final_cluster_centers + if isinstance(self._cluster_centers, torch.Tensor): + return self._cluster_centers + else: + raise TypeError("Model has not been fit to a dataset") + + @property + def cluster_assignments(self) -> torch.Tensor: + """Return cluster labels assigned during the most recent ``fit`` call. + + Returns: + A 1D tensor with shape :math:`(N,)`, where ``N`` is the number of + samples given to :meth:`fit`. Each value is the cluster index + assigned to the corresponding sample. + + Raises: + TypeError: If ``fit`` has not been run yet. + """ + if isinstance(self._final_cluster_assignments, torch.Tensor): + return self._final_cluster_assignments + else: + raise TypeError("Model has not been fit to a dataset") + + def _initialise_cluster_centers(self, X: torch.Tensor, num_clusters: int) -> torch.Tensor: + """Chooses num_cluster points from X as the initial cluster centers. + + Args: + X: 2D input torch.Tensor to be clustered + num_clusters: number of desired cluster centers + + Returns: + 2D torch.Tensor with num_cluster rows + + """ + num_samples: int = len(X) + perm = torch.randperm(num_samples, device=X.device) + idx = perm[:num_clusters] + initial_state = X[idx] + return initial_state + + def _pairwise_euclidean_distance(self, data1: torch.Tensor, data2: torch.Tensor) -> torch.Tensor: + """Compute pairwise squared distance between 2 sets of vectors. + + Args: + data1: 2D torch.Tensor of shape N, D + data2: 2D torch.Tensor of shape C, D + + Returns: + 2D torch.Tensor of shape N, C + + """ + # N*1*D + A = data1[:, None, ...] + # 1*C*D + B = data2[None, ...] + distance = euclidean_distance(A, B) + return distance + + def fit(self, X: torch.Tensor) -> None: + """Fit iterative KMeans clustering till a threshold for shift in cluster centers or a maximum no of iterations + have reached. + + Args: + X: 2D input torch.Tensor to be clustered + + """ # noqa: D205 + # X should have only 2 dimensions + KORNIA_CHECK_SHAPE(X, ["N", "D"]) + + if self._cluster_centers is None: + self._cluster_centers = self._initialise_cluster_centers(X, self.num_clusters) + else: + # X and cluster_centers should have same number of columns + KORNIA_CHECK( + X.shape[1] == self._cluster_centers.shape[1], + f"Dimensions at position 1 of X and cluster_centers do not match. \ + {X.shape[1]} != {self._cluster_centers.shape[1]}", + ) + + # X = X.to(self.device) + current_centers = self._cluster_centers + + previous_centers: torch.Tensor | None = None + iteration: int = 0 + + while True: + # find distance between X and current_centers + distance: torch.Tensor = self._pairwise_euclidean_distance(X, current_centers) + + cluster_assignment = distance.argmin(-1) + + previous_centers = current_centers.clone() + + for index in range(self.num_clusters): + selected = torch.nonzero(cluster_assignment == index).squeeze() + selected = torch.index_select(X, 0, selected) + # edge case when a certain cluster centre has no points assigned to it + # just choose a random point as it's update + if selected.shape[0] == 0: + selected = X[torch.randint(len(X), (1,), device=X.device)] + current_centers[index] = selected.mean(dim=0) + + # sum of distance of how much the newly computed clusters have moved from their previous positions + center_shift = torch.sum(torch.sqrt(torch.sum((current_centers - previous_centers) ** 2, dim=1))) + + iteration = iteration + 1 + + if self.tolerance is not None and center_shift**2 < self.tolerance: + break + + if self.max_iterations != 0 and iteration >= self.max_iterations: + break + + self._final_cluster_assignments = cluster_assignment + self._final_cluster_centers = current_centers + + def predict(self, x: torch.Tensor) -> torch.Tensor: + """Find the cluster center closest to each point in x. + + Args: + x: 2D torch.Tensor + + Returns: + 1D torch.Tensor containing cluster id assigned to each data point in x + + """ + # x and cluster_centers should have same number of columns + KORNIA_CHECK( + x.shape[1] == self.cluster_centers.shape[1], + f"Dimensions at position 1 of x and cluster_centers do not match. \ + {x.shape[1]} != {self.cluster_centers.shape[1]}", + ) + + distance = self._pairwise_euclidean_distance(x, self.cluster_centers) + cluster_assignment = distance.argmin(-1) + return cluster_assignment diff --git a/kornia/contrib/lambda_module.py b/kornia/contrib/lambda_module.py new file mode 100644 index 0000000..ab09458 --- /dev/null +++ b/kornia/contrib/lambda_module.py @@ -0,0 +1,67 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Callable + +import torch +from torch import nn + + +class Lambda(nn.Module): + """Applies user-defined lambda as a transform. + + Args: + func: Callable function. + + Returns: + The output of the user-defined lambda. + + Example: + >>> import kornia + >>> x = torch.rand(1, 3, 5, 5) + >>> f = Lambda(lambda x: kornia.color.rgb_to_grayscale(x)) + >>> f(x).shape + torch.Size([1, 1, 5, 5]) + + """ + + def __init__(self, func: Callable[..., torch.Tensor]) -> None: + super().__init__() + if not callable(func): + raise TypeError(f"Argument lambd should be callable, got {type(func).__name__!r}") + + self.func = func + + def forward(self, img: torch.Tensor, *args: Any, **kwargs: Any) -> torch.Tensor: + """Run the user-provided transform function. + + This method simply forwards all arguments to ``self.func``. + It is useful when you want to insert a custom Python callable inside a + ``torch.nn.Module`` pipeline. + + Args: + img: Primary input tensor passed as the first positional argument. + For image tasks this is commonly shaped ``(B, C, H, W)``, where + ``B`` = batch size, ``C`` = number of channels, + ``H`` = height, and ``W`` = width. + *args: Additional positional arguments forwarded to ``self.func``. + **kwargs: Additional keyword arguments forwarded to ``self.func``. + + Returns: + The output tensor returned by ``self.func``. + """ + return self.func(img, *args, **kwargs) diff --git a/kornia/contrib/object_detection.py b/kornia/contrib/object_detection.py new file mode 100644 index 0000000..9a5f3dd --- /dev/null +++ b/kornia/contrib/object_detection.py @@ -0,0 +1,479 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Any, ClassVar, List, Optional, Tuple, Union + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK_SHAPE +from kornia.core.external import PILImage as Image +from kornia.core.external import onnx +from kornia.core.mixin.onnx import ONNXExportMixin +from kornia.image.draw import draw_rectangle +from kornia.models.base import ModelBase +from kornia.models.processors import ResizePreProcessor + +__all__ = [ + "BoundingBox", + "BoundingBoxDataFormat", + "BoxFiltering", + "ObjectDetector", + "ObjectDetectorResult", + "RTDETRDetectorBuilder", + "results_from_detections", +] + + +class BoundingBoxDataFormat(Enum): + """Enum class that maps bounding box data format.""" + + XYWH = 0 + XYXY = 1 + CXCYWH = 2 + CENTER_XYWH = 2 + + +# NOTE: probably we should use a more generic name like BoundingBox2D +# and add a BoundingBox3D class for 3D bounding boxes. Also for serialization +# we should have an explicit class for each format to make it more production ready +# specially to serialize to protobuf and not saturate at a high rates. + + +@dataclass(frozen=True) +class BoundingBox: + """Bounding box data class. + + Useful for representing bounding boxes in different formats for object detection. + + Args: + data: tuple of bounding box data. The length of the tuple depends on the data format. + data_format: bounding box data format. + + """ + + data: tuple[float, float, float, float] + data_format: BoundingBoxDataFormat + + +@dataclass(frozen=True) +class ObjectDetectorResult: + """Object detection result. + + Args: + class_id: class id of the detected object. + confidence: confidence score of the detected object. + bbox: bounding box of the detected object in xywh format. + + """ + + class_id: int + confidence: float + bbox: BoundingBox + + +def results_from_detections( + detections: torch.Tensor, format: str | BoundingBoxDataFormat +) -> list[ObjectDetectorResult]: + """Convert a detection torch.Tensor to a list of :py:class:`ObjectDetectorResult`. + + Args: + detections: torch.Tensor with shape :math:`(D, 6)`, where :math:`D` is the number of + detections in the given image, + :math:`6` represents class id, score, and `xywh` bounding box. + format: detection format. + + Returns: + list of :py:class:`ObjectDetectorResult`. + + """ + KORNIA_CHECK_SHAPE(detections, ["D", "6"]) + + if isinstance(format, str): + format = BoundingBoxDataFormat[format.upper()] + + results: list[ObjectDetectorResult] = [] + for det in detections: + det = det.squeeze().tolist() + if len(det) != 6: + continue + results.append( + ObjectDetectorResult( + class_id=int(det[0]), + confidence=det[1], + bbox=BoundingBox(data=(det[2], det[3], det[4], det[5]), data_format=format), + ) + ) + return results + + +class ObjectDetector(ModelBase, ONNXExportMixin): + """Wrap an object detection model and perform pre-processing and post-processing.""" + + name: str = "detection" + + def __init__(self, model: nn.Module, pre_processor: nn.Module, post_processor: nn.Module) -> None: + """Initialize ObjectDetector. + + Args: + model: The object detection model. + pre_processor: Pre-processing module (e.g., ResizePreProcessor). + post_processor: Post-processing module (e.g., DETRPostProcessor). + """ + super().__init__() + self.model = model.eval() + self.pre_processor = pre_processor + self.post_processor = post_processor + + @staticmethod + def from_config(config: Any) -> ObjectDetector: + """Build ObjectDetector from config. + + This is a placeholder to satisfy the abstract method requirement. + Use kornia.contrib.object_detection.RTDETRDetectorBuilder.build() or instantiate ObjectDetector directly. + + Args: + config: Configuration object (not used, kept for interface compatibility). + + Returns: + ObjectDetector instance. + + """ + raise NotImplementedError( + "ObjectDetector.from_config() is not implemented. " + "Use kornia.contrib.object_detection.RtdetrBuilder.build() or instantiate ObjectDetector directly." + ) + + @torch.inference_mode() + def forward(self, images: Union[torch.Tensor, list[torch.Tensor]]) -> Union[torch.Tensor, list[torch.Tensor]]: + """Detect objects in a given list of images. + + Args: + images: If list of RGB images. Each image is a torch.Tensor with shape :math:`(3, H, W)`. + If torch.Tensor, a torch.Tensor with shape :math:`(B, 3, H, W)`. + + Returns: + list of detections found in each image. For item in a batch, shape is :math:`(D, 6)`, + where :math:`D` is the + number of detections in the given image, :math:`6` represents class id, score, and `xywh` bounding box. + + """ + images, images_sizes = self.pre_processor(images) + logits, boxes = self.model(images) + detections = self.post_processor(logits, boxes, images_sizes) + return detections + + def visualize( + self, + images: Union[torch.Tensor, list[torch.Tensor]], + detections: Optional[torch.Tensor] = None, + output_type: str = "torch", + ) -> Union[torch.Tensor, list[torch.Tensor], list[Image.Image]]: # type: ignore + """Very simple drawing. + + Needs to be more fancy later. + """ + dets = detections or self.forward(images) + output = [] + for image, detection in zip(images, dets): + out_img = image[None].clone() + for out in detection: + out_img = draw_rectangle( + out_img, + torch.Tensor([[[out[-4], out[-3], out[-4] + out[-2], out[-3] + out[-1]]]]), + ) + output.append(out_img[0]) + + return self._tensor_to_type(output, output_type, is_batch=isinstance(images, torch.Tensor)) + + def save( + self, + images: Union[torch.Tensor, list[torch.Tensor]], + detections: Optional[torch.Tensor] = None, + directory: Optional[str] = None, + ) -> None: + """Save the output image(s) to a directory. + + Args: + images: input torch.Tensor. + detections: detection torch.Tensor. + directory: directory to save the images. + + """ + outputs = self.visualize(images, detections) + self._save_outputs(outputs, directory) + + def to_onnx( # type: ignore[override] + self, + onnx_name: Optional[str] = None, + image_size: Optional[int] = 640, + include_pre_and_post_processor: bool = True, + save: bool = True, + additional_metadata: Optional[list[tuple[str, str]]] = None, + **kwargs: Any, + ) -> onnx.ModelProto: # type: ignore + """Export an RT-DETR object detection model to ONNX format. + + Either `model_name` or `config` must be provided. If neither is provided, + a default pretrained model (`rtdetr_r18vd`) will be built. + + Args: + onnx_name: + The name of the output ONNX file. If not provided, a default name in the + format "Kornia-.onnx" will be used. + image_size: + The size to which input images will be resized during preprocessing. + If None, image_size will be dynamic. + For RTDETR, recommended scales include [480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800]. + include_pre_and_post_processor: + Whether to include the pre-processor and post-processor in the exported model. + save: + If to save the model or load it. + additional_metadata: + Additional metadata to add to the ONNX model. + kwargs: Additional arguments to convert to onnx. + + """ + if onnx_name is None: + onnx_name = f"kornia_{self.name}_{image_size}.onnx" + + return ONNXExportMixin.to_onnx( + self, + onnx_name, + input_shape=[-1, 3, image_size or -1, image_size or -1], + output_shape=[-1, -1, 6], + pseudo_shape=[1, 3, image_size or 352, image_size or 352], + model=self if include_pre_and_post_processor else self.model, + save=save, + additional_metadata=additional_metadata, + **kwargs, + ) + + def compile( + self, + *, + fullgraph: bool = False, + dynamic: bool = False, + backend: str = "inductor", + mode: Optional[str] = None, + options: Optional[dict[str, str | int | bool]] = None, + disable: bool = False, + ) -> None: + """Compile the internal object detection model with :py:func:`torch.compile()`.""" + self.model = torch.compile( # type: ignore + self.model, + fullgraph=fullgraph, + dynamic=dynamic, + backend=backend, + mode=mode, + options=options, + disable=disable, + ) + + +class BoxFiltering(nn.Module, ONNXExportMixin): + """Filter boxes according to the desired threshold. + + Args: + confidence_threshold: an 0-d scalar that represents the desired threshold. + classes_to_keep: a 1-d list of classes to keep. If None, keep all classes. + filter_as_zero: whether to filter boxes as zero. + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[List[int]] = [-1, -1, 6] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[List[int]] = [-1, -1, 6] + ONNX_EXPORT_PSEUDO_SHAPE: ClassVar[List[int]] = [5, 20, 6] + + def __init__( + self, + confidence_threshold: Optional[Union[torch.Tensor, float]] = None, + classes_to_keep: Optional[Union[torch.Tensor, List[int]]] = None, + filter_as_zero: bool = False, + ) -> None: + super().__init__() + self.filter_as_zero = filter_as_zero + self.classes_to_keep = None + self.confidence_threshold = None + if classes_to_keep is not None: + self.classes_to_keep = ( + classes_to_keep if isinstance(classes_to_keep, torch.Tensor) else torch.tensor(classes_to_keep) + ) + if confidence_threshold is not None: + self.confidence_threshold = ( + confidence_threshold or confidence_threshold + if isinstance(confidence_threshold, torch.Tensor) + else torch.tensor(confidence_threshold) + ) + + def forward( + self, + boxes: torch.Tensor, + confidence_threshold: Optional[torch.Tensor] = None, + classes_to_keep: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, List[torch.Tensor]]: + """Filter boxes according to the desired threshold. + + To be ONNX-friendly, the inputs for direct forwarding need to be all tensors. + + Args: + boxes: [B, D, 6], where B is the batchsize, D is the number of detections in the image, + 6 represent (class_id, confidence_score, x, y, w, h). + confidence_threshold: an 0-d scalar that represents the desired threshold. + classes_to_keep: a 1-d torch.Tensor of classes to keep. If None, keep all classes. + + Returns: + Union[torch.Tensor, List[torch.Tensor]] + If `filter_as_zero` is True, return a torch.Tensor of shape [D, 6], where D is the total number of + detections as input. + If `filter_as_zero` is False, return a list of tensors of shape [D, 6], where D is the number of + valid detections for each element in the batch. + + """ + # Apply confidence filtering + zero_tensor = torch.tensor(0.0, device=boxes.device, dtype=boxes.dtype) + confidence_threshold = ( + confidence_threshold or self.confidence_threshold or zero_tensor + ) # If None, use 0 as threshold + confidence_mask = boxes[:, :, 1] > confidence_threshold # [B, D] + + # Apply class filtering + classes_to_keep = classes_to_keep or self.classes_to_keep + if classes_to_keep is not None: + class_ids = boxes[:, :, 0:1] # [B, D, 1] + classes_to_keep = classes_to_keep.view(1, 1, -1) # [1, 1, C] for broadcasting + class_mask = (class_ids == classes_to_keep).any(dim=-1) # [B, D] + else: + # If no class filtering is needed, just use a mask of all `True` + class_mask = (confidence_mask * 0 + 1).bool() + + # Combine the confidence and class masks + combined_mask = confidence_mask & class_mask # [B, D] + + if self.filter_as_zero: + filtered_boxes = boxes * combined_mask[:, :, None] + return filtered_boxes + + filtered_boxes_list = [] + for i in range(boxes.shape[0]): + box = boxes[i] + mask = combined_mask[i] # [D] + valid_boxes = box[mask] + filtered_boxes_list.append(valid_boxes) + + return filtered_boxes_list + + def _create_dummy_input( + self, input_shape: List[int], pseudo_shape: Optional[List[int]] = None + ) -> Union[Tuple[Any, ...], torch.Tensor]: + pseudo_input = torch.rand( + *[ + ((self.ONNX_EXPORT_PSEUDO_SHAPE[i] if pseudo_shape is None else pseudo_shape[i]) if dim == -1 else dim) + for i, dim in enumerate(input_shape) + ] + ) + if self.confidence_threshold is None: + return pseudo_input, 0.1 + return pseudo_input + + +class RTDETRDetectorBuilder: + """A builder class for constructing RT-DETR object detection models. + + This class provides static methods to: + - Build an object detection model from a model name or configuration. + - Export the model to ONNX format for inference. + + Note: + To use this model, load image tensors and call ``model.save(images)``. + """ + + @staticmethod + def build( + model_name: Optional[str] = None, + config: Optional[Any] = None, + pretrained: bool = True, + image_size: Optional[int] = None, + confidence_threshold: Optional[float] = None, + confidence_filtering: Optional[bool] = None, + ) -> ObjectDetector: + """Build and returns an RT-DETR object detector model. + + Either `model_name` or `config` must be provided. If neither is provided, + a default pretrained model (`rtdetr_r18vd`) will be built. + + Args: + model_name: + Name of the RT-DETR model to load. Can be one of the available pretrained models. + Including 'rtdetr_r18vd', 'rtdetr_r34vd', 'rtdetr_r50vd_m', 'rtdetr_r50vd', 'rtdetr_r101vd'. + config: + A custom configuration object for building the RT-DETR model. + pretrained: + Whether to load a pretrained version of the model (applies when `model_name` is provided). + image_size: + The size to which input images will be resized during preprocessing. + If None, no resizing will be inferred from config file. Recommended scales include + [480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800]. + confidence_threshold: Threshold to filter results based on confidence scores. + confidence_filtering: Whether to filter results based on confidence scores. + + + Returns: + ObjectDetector + An object detector instance initialized with the specified model, preprocessor, and post-processor. + + """ + import warnings + + from kornia.models.rt_detr import DETRPostProcessor + from kornia.models.rt_detr.model import RTDETR, RTDETRConfig + + if model_name is not None and config is not None: + raise ValueError("Either `model_name` or `config` should be `None`.") + + if config is not None: + model = RTDETR.from_config(config) + image_size = image_size or config.input_size + elif model_name is not None: + if pretrained: + model = RTDETR.from_pretrained(model_name) + image_size = RTDETRConfig.from_name(model_name).input_size + else: + model = RTDETR.from_name(model_name) + image_size = RTDETRConfig.from_name(model_name).input_size + else: + warnings.warn("No `model_name` or `config` found. Will build pretrained `rtdetr_r18vd`.", stacklevel=1) + model = RTDETR.from_pretrained("rtdetr_r18vd") + image_size = RTDETRConfig.from_name("rtdetr_r18vd").input_size + + if confidence_threshold is None: + confidence_threshold = config.confidence_threshold if config is not None else 0.3 + + return ObjectDetector( + model, + ResizePreProcessor(image_size, image_size), + DETRPostProcessor( + confidence_threshold=confidence_threshold, + confidence_filtering=confidence_filtering or not torch.onnx.is_in_onnx_export(), + num_classes=model.decoder.num_classes, + num_top_queries=model.decoder.num_queries, + ), + ) diff --git a/kornia/contrib/super_resolution.py b/kornia/contrib/super_resolution.py new file mode 100644 index 0000000..6e71e8a --- /dev/null +++ b/kornia/contrib/super_resolution.py @@ -0,0 +1,274 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, List, Optional, Tuple, Union + +import torch +from torch import nn + +from kornia.config import kornia_config +from kornia.core.external import PILImage as Image +from kornia.core.external import basicsr, onnx +from kornia.core.mixin.onnx import ONNXExportMixin +from kornia.models.base import ModelBase +from kornia.models.processors import OutputRangePostProcessor, ResizePreProcessor +from kornia.models.small_sr import SmallSRNetWrapper +from kornia.onnx.download import CachedDownloader + +__all__ = ["RRDBNetBuilder", "SmallSRBuilder", "SuperResolution"] + +URLs = { + "RealESRGAN_x4plus": "https://github.com/xinntao/Real-ESRGAN/releases/download/v0.1.0/RealESRGAN_x4plus.pth", + "RealESRNet_x4plus": "https://github.com/xinntao/Real-ESRGAN/releases/download/v0.1.1/RealESRNet_x4plus.pth", + "RealESRGAN_x4plus_anime_6B": "https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.2.4/RealESRGAN_x4plus_anime_6B.pth", + "RealESRGAN_x2plus": "https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.1/RealESRGAN_x2plus.pth", +} + + +# TODO: support patching -> SR -> unpatching pipeline +class SuperResolution(ModelBase, ONNXExportMixin): + """SuperResolution is a module that wraps an super resolution model.""" + + name: str = "super_resolution" + input_image_size: Optional[int] + output_image_size: Optional[int] + pseudo_image_size: Optional[int] + + @torch.inference_mode() + def forward(self, images: Union[torch.Tensor, List[torch.Tensor]]) -> Union[torch.Tensor, List[torch.Tensor]]: + """Forward pass of the super resolution model. + + Args: + images: If list of RGB images. Each image is a torch.Tensor with shape :math:`(3, H, W)`. + If torch.Tensor, a torch.Tensor with shape :math:`(B, 3, H, W)`. + + Returns: + output torch.Tensor. + + """ + output = self.pre_processor(images) + if isinstance(output, list | tuple): + images = output[0] + else: + images = output + if isinstance(images, list): + out_images = [self.model(image[None])[0] for image in images] + else: + out_images = self.model(images) + return self.post_processor(out_images) + + def visualize( + self, + images: Union[torch.Tensor, List[torch.Tensor]], + edge_maps: Optional[Union[torch.Tensor, List[torch.Tensor]]] = None, + output_type: str = "torch", + ) -> Union[torch.Tensor, List[torch.Tensor], List["Image.Image"]]: # type: ignore + """Draw the super resolution results. + + Args: + images: input torch.Tensor. + edge_maps: detected edges. + output_type: type of the output. + + Returns: + output torch.Tensor. + + """ + if edge_maps is None: + edge_maps = self.forward(images) + output = [] + for edge_map in edge_maps: + output.append(edge_map) + + return self._tensor_to_type(output, output_type, is_batch=isinstance(images, torch.Tensor)) + + def save( + self, + images: Union[torch.Tensor, List[torch.Tensor]], + edge_maps: Optional[Union[torch.Tensor, List[torch.Tensor]]] = None, + directory: Optional[str] = None, + output_type: str = "torch", + ) -> None: + """Save the super resolution results. + + Args: + images: input torch.Tensor. + edge_maps: detected edges. + output_type: type of the output. + directory: where to save outputs. + output_type: backend used to generate outputs. + + Returns: + output torch.Tensor. + + """ + outputs = self.visualize(images, edge_maps, output_type) + self._save_outputs(images, directory, suffix="_src") + self._save_outputs(outputs, directory, suffix="_sr") + + def to_onnx( # type: ignore[override] + self, + onnx_name: Optional[str] = None, + include_pre_and_post_processor: bool = True, + save: bool = True, + additional_metadata: Optional[List[Tuple[str, str]]] = None, + **kwargs: Any, + ) -> "onnx.ModelProto": # type: ignore + """Export the current super resolution model to an ONNX model file. + + Args: + onnx_name: + The name of the output ONNX file. If not provided, a default name in the + format "Kornia-.onnx" will be used. + image_size: + The size to which input images will be resized during preprocessing. + If None, image_size will be dynamic. For DexiNed, recommended scale is 352. + include_pre_and_post_processor: + Whether to include the pre-processor and post-processor in the exported model. + save: + If to save the model or load it. + additional_metadata: + Additional metadata to add to the ONNX model. + kwargs: Additional arguments for converting to onnx. + + """ + if onnx_name is None: + onnx_name = f"kornia_{self.name}.onnx" + + return ONNXExportMixin.to_onnx( + self, + onnx_name, + input_shape=[-1, 3, self.input_image_size or -1, self.input_image_size or -1], + output_shape=[-1, 3, self.output_image_size or -1, self.output_image_size or -1], + pseudo_shape=[1, 3, self.pseudo_image_size or 352, self.pseudo_image_size or 352], + model=self if include_pre_and_post_processor else self.model, + save=save, + additional_metadata=additional_metadata, + **kwargs, + ) + + +class RRDBNetBuilder: + """Provide a static factory to build Residual-in-Residual Dense Block (RRDB) networks.""" + + @staticmethod + def build(model_name: str = "RealESRNet_x4plus", pretrained: bool = True) -> SuperResolution: + """Build a preconfigured RRDB (Residual-in-Residual Dense Block) model. + + The returned object is a :class:`SuperResolution` wrapper configured for + inference. Internally, this factory selects architecture hyperparameters + based on ``model_name``, and optionally loads pretrained weights. + + Args: + model_name: Name of the RRDB variant to construct. Supported options are: + ``"RealESRGAN_x4plus"``, ``"RealESRNet_x4plus"``, + ``"RealESRGAN_x4plus_anime_6B"``, and ``"RealESRGAN_x2plus"``. + pretrained: If ``True``, download and load pretrained weights for the + selected model variant. + + Returns: + A :class:`SuperResolution` wrapper ready for inference. + + Raises: + ValueError: If ``model_name`` is not one of the supported variants. + """ + if model_name == "RealESRGAN_x4plus": + model = basicsr.archs.rrdbnet_arch.RRDBNet( # type: ignore + num_in_ch=3, num_out_ch=3, num_feat=64, num_block=23, num_grow_ch=32, scale=4 + ) + elif model_name == "RealESRNet_x4plus": + model = basicsr.archs.rrdbnet_arch.RRDBNet( # type: ignore + num_in_ch=3, num_out_ch=3, num_feat=64, num_block=23, num_grow_ch=32, scale=4 + ) + elif model_name == "RealESRGAN_x4plus_anime_6B": + model = basicsr.archs.rrdbnet_arch.RRDBNet( # type: ignore + num_in_ch=3, num_out_ch=3, num_feat=64, num_block=6, num_grow_ch=32, scale=4 + ) + elif model_name == "RealESRGAN_x2plus": + model = basicsr.archs.rrdbnet_arch.RRDBNet( # type: ignore + num_in_ch=3, num_out_ch=3, num_feat=64, num_block=23, num_grow_ch=32, scale=2 + ) + else: + raise ValueError( + f"Model {model_name} not found. Please choose from " + "'RealESRGAN_x4plus', 'RealESRNet_x4plus', 'RealESRGAN_x4plus_anime_6B', 'RealESRGAN_x2plus'." + ) + + model_path = None + if pretrained: + url = URLs[model_name] + model_path = CachedDownloader.download_to_cache( + url, model_name, download=True, suffix=".pth", cache_dir=kornia_config.hub_onnx_dir + ) + model.load_state_dict(torch.load(model_path, map_location=torch.device("cpu"))["params_ema"], strict=True) + model.eval() + + return SuperResolution( + model, + pre_processor=nn.Identity(), + post_processor=OutputRangePostProcessor(min_val=0.0, max_val=1.0), + name=model_name, + ) + + +class SmallSRBuilder: + """Provide a static factory to build lightweight Super-Resolution models.""" + + @staticmethod + def build( + model_name: str = "small_sr", pretrained: bool = True, upscale_factor: int = 3, image_size: Optional[int] = None + ) -> SuperResolution: + """Build a lightweight super-resolution (SR) model wrapper. + + SR stands for *super-resolution*, which means generating a higher + resolution output from a lower resolution input. + + Args: + model_name: Name of the model variant to build. Currently only + ``"small_sr"`` is supported. + pretrained: If ``True``, load pretrained weights for the selected model. + upscale_factor: Integer scale used by ``SmallSRNetWrapper``. + For example, ``3`` maps an input image of size ``H x W`` to an + output image of approximately ``3H x 3W``. + image_size: Optional fixed input size. When provided, output metadata is + configured as ``image_size * upscale_factor``. + + Returns: + A :class:`SuperResolution` instance configured with resizing pre-processing + and identity post-processing. + + Raises: + ValueError: If ``model_name`` is not supported. + """ + if model_name.lower() == "small_sr": + model = SmallSRNetWrapper(upscale_factor, pretrained=pretrained) + else: + raise ValueError(f"Model {model_name} not found. Please choose from 'small_sr'.") + + sr = SuperResolution( + model, + pre_processor=ResizePreProcessor(224, 224), + post_processor=nn.Identity(), + name=model_name, + ) + if image_size is None: + sr.pseudo_image_size = 224 + else: + sr.input_image_size = image_size + sr.output_image_size = image_size * 3 + sr.pseudo_image_size = image_size + return sr diff --git a/kornia/contrib/visual_prompter.py b/kornia/contrib/visual_prompter.py new file mode 100644 index 0000000..77688ff --- /dev/null +++ b/kornia/contrib/visual_prompter.py @@ -0,0 +1,434 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Any, Optional + +import torch +import torch.nn.functional as F + +from kornia.augmentation import LongestMaxSize +from kornia.augmentation.container.augment import AugmentationSequential +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE +from kornia.enhance import normalize as kornia_normalize +from kornia.geometry.boxes import Boxes +from kornia.geometry.keypoints import Keypoints +from kornia.models.sam import Sam, SamConfig +from kornia.models.structures import Prompts, SegmentationResults + + +class VisualPrompter: + r"""Allow the user to run multiple query with multiple prompts for a model. + + At the moment, we just support the SAM model. The model is loaded based on the given config. + + For default the images are transformed to have their long side with size of the `image_encoder.img_size`. This + Prompter class ensure to transform the images and the prompts before prediction. Also, the image is passed + automatically for the method `preprocess_image`, which is responsible for F.normalize the image and F.pad it to have + the right size for the SAM model :math:`(\text{image_encoder.img_size}, \text{image_encoder.img_size})`. For + default the image is normalized by the mean and standard deviation of the SAM dataset values. + + Args: + config: A model config to generate the model. Now just the SAM model is supported. + device: The desired device to use the model. + dtype: The desired dtype to use the model. + + Example: + >>> # prompter = VisualPrompter() # Will load the vit h for default + >>> # You can load a custom SAM type for modifying the config + >>> prompter = VisualPrompter(SamConfig('vit_b')) + >>> image = torch.rand(3, 25, 30) + >>> prompter.set_image(image) + >>> boxes = Boxes( + ... torch.tensor( + ... [[[[0, 0], [0, 10], [10, 0], [10, 10]]]], + ... device=prompter.device, + ... dtype=torch.float32 + ... ), + ... mode='xyxy' + ... ) + >>> prediction = prompter.predict(boxes=boxes) + >>> prediction.logits.shape + torch.Size([1, 3, 256, 256]) + + """ + + def __init__( + self, + config: Optional[SamConfig] = None, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ) -> None: + super().__init__() + if config is None: + config = SamConfig(model_type="vit_h", pretrained=True) + + if isinstance(config, SamConfig): + self.model = Sam.from_config(config) + transforms = (LongestMaxSize(self.model.image_encoder.img_size, p=1.0),) + self.pixel_mean: Optional[torch.Tensor] = ( + torch.tensor([123.675, 116.28, 103.53], device=device, dtype=dtype) / 255 + ) + self.pixel_std: Optional[torch.Tensor] = ( + torch.tensor([58.395, 57.12, 57.375], device=device, dtype=dtype) / 255 + ) + else: + raise NotImplementedError + + self.model = self.model.to(device=device, dtype=dtype) + self.transforms = AugmentationSequential(*transforms, same_on_batch=True) + + self.device = device + self.dtype = dtype + self._original_image_size: None | tuple[int, int] = None + self._input_image_size: None | tuple[int, int] = None + self._input_encoder_size: None | tuple[int, int] = None + self.reset_image() + + def preprocess_image( + self, x: torch.Tensor, mean: Optional[torch.Tensor] = None, std: Optional[torch.Tensor] = None + ) -> torch.Tensor: + """Normalize and F.pad a torch.Tensor. + + For F.normalize the tensor: will prioritize the `mean` and `std` passed as argument, + if None will use the default + Sam Dataset values. + + For F.pad the tensor: Will F.pad the torch.Tensor into the right and bottom to match with the size of + `self.model.image_encoder.img_size` + + Args: + x: The image to be preprocessed + mean: Mean for each channel. + std: Standard deviations for each channel. + + Returns: + The image preprocessed (normalized if has mean and str available and padded to encoder size) + + """ + if isinstance(mean, torch.Tensor) and isinstance(std, torch.Tensor): + x = kornia_normalize(x, mean, std) + elif isinstance(self.pixel_mean, torch.Tensor) and isinstance(self.pixel_std, torch.Tensor): + x = kornia_normalize(x, self.pixel_mean, self.pixel_std) + + encoder_im_size = self.model.image_encoder.img_size + pad_h = encoder_im_size - x.shape[-2] + pad_w = encoder_im_size - x.shape[-1] + x = F.pad(x, (0, pad_w, 0, pad_h)) + + return x + + @torch.no_grad() + def set_image( + self, image: torch.Tensor, mean: Optional[torch.Tensor] = None, std: Optional[torch.Tensor] = None + ) -> None: + """Set the embeddings from the given image with `image_decoder` of the model. + + Prepare the given image with the selected transforms and the preprocess method. + + Args: + image: RGB image. Normally images with range of [0-1], the model preprocess F.normalize the + pixel values with the mean and std defined in its initialization. Expected to be into a float32 + dtype. Shape :math:`(3, H, W)`. + mean: mean value of dataset for normalization. + std: standard deviation of dataset for normalization. + + """ + if image.ndim == 3: + KORNIA_CHECK_SHAPE(image, ["3", "H", "W"]) + image = image.unsqueeze(0) + else: + KORNIA_CHECK_SHAPE(image, ["B", "3", "H", "W"]) + + self.reset_image() + + self._original_image_size = (image.shape[-2], image.shape[-1]) + + image = self.transforms(image, data_keys=["input"]) + self._tfs_params = self.transforms._params + self._input_image_size = (image.shape[-2], image.shape[-1]) + + image = self.preprocess_image(image, mean, std) + + self._input_encoder_size = (image.shape[-2], image.shape[-1]) + + self.image_embeddings = self.model.image_encoder(image) + self.is_image_set = True + + def _valid_keypoints(self, keypoints: Keypoints | torch.Tensor, labels: torch.Tensor) -> Keypoints: + """Validate the keypoints shape and ensure to be a Keypoints.""" + KORNIA_CHECK_SHAPE(keypoints.data, ["K", "N", "2"]) + KORNIA_CHECK_SHAPE(labels.data, ["K", "N"]) + KORNIA_CHECK(keypoints.shape[0] == labels.shape[0], "The keypoints and labels should have the same batch size") + + if isinstance(keypoints, torch.Tensor): + keypoints = Keypoints.from_tensor(keypoints) + + return keypoints + + def _valid_boxes(self, boxes: Boxes | torch.Tensor) -> Boxes: + """Validate the boxes shape and ensure to be a Boxes into xyxy mode.""" + if isinstance(boxes, torch.Tensor): + if boxes.ndim == 2: + KORNIA_CHECK_SHAPE(boxes.data, ["K", "4"]) + else: + KORNIA_CHECK_SHAPE(boxes.data, ["B", "K", "4"]) + + boxes = Boxes.from_tensor(boxes, mode="xyxy") + + if boxes.mode == "xyxy": + boxes_xyxy = boxes + else: + boxes_xyxy = Boxes.from_tensor(boxes.to_tensor(mode="xyxy"), mode="xyxy") + + return boxes_xyxy + + def _valid_masks(self, masks: torch.Tensor) -> torch.Tensor: + """Validate the input masks shape.""" + KORNIA_CHECK_SHAPE(masks, ["K", "1", "256", "256"]) + return masks + + def _transform_prompts( + self, *prompts: torch.Tensor | Boxes | Keypoints, data_keys: Optional[list[str]] = None + ) -> dict[str, torch.Tensor | Boxes | Keypoints]: + transformed_prompts = self.transforms(*prompts, data_keys=data_keys, params=self._tfs_params) + if data_keys is None: + data_keys = [] + + # prevent unpacking torch.Tensor when creating the output dict (issue #2627) + if not isinstance(transformed_prompts, (list, tuple)): + transformed_prompts = [transformed_prompts] + return {key: transformed_prompts[idx] for idx, key in enumerate(data_keys)} + + def preprocess_prompts( + self, + keypoints: Optional[Keypoints | torch.Tensor] = None, + keypoints_labels: Optional[torch.Tensor] = None, + boxes: Optional[Boxes | torch.Tensor] = None, + masks: Optional[torch.Tensor] = None, + ) -> Prompts: + """Validate and preprocess the given prompts to be aligned with the input image.""" + data_keys = [] + to_transform: list[Keypoints | Boxes | torch.Tensor] = [] + + if isinstance(keypoints, (Keypoints, torch.Tensor)) and isinstance(keypoints_labels, torch.Tensor): + keypoints = self._valid_keypoints(keypoints, keypoints_labels) + data_keys.append("keypoints") + to_transform.append(keypoints) + + if isinstance(boxes, (Boxes, torch.Tensor)): + self._valid_boxes(boxes) + data_keys.append("bbox_xyxy") + to_transform.append(boxes) + + if isinstance(masks, torch.Tensor): + self._valid_masks(masks) + + data = self._transform_prompts(*to_transform, data_keys=data_keys) + + if "keypoints" in data and isinstance(data["keypoints"], Keypoints): + kpts_tensor = data["keypoints"].to_tensor() + if KORNIA_CHECK_IS_TENSOR(kpts_tensor) and KORNIA_CHECK_IS_TENSOR(keypoints_labels): + points = (kpts_tensor, keypoints_labels) + else: + points = None + + if "bbox_xyxy" in data and isinstance(data["bbox_xyxy"], Boxes): + _bbox = data["bbox_xyxy"].to_tensor(mode="xyxy") + if KORNIA_CHECK_IS_TENSOR(_bbox): + bbox = _bbox + else: + bbox = None + + return Prompts(points=points, boxes=bbox, masks=masks) + + @torch.no_grad() + def predict( + self, + keypoints: Optional[Keypoints | torch.Tensor] = None, + keypoints_labels: Optional[torch.Tensor] = None, + boxes: Optional[Boxes | torch.Tensor] = None, + masks: Optional[torch.Tensor] = None, + multimask_output: bool = True, + output_original_size: bool = True, + ) -> SegmentationResults: + """Predict masks for the given image based on the input prompts. + + Args: + keypoints: Point prompts to the model. Each point is in (X,Y) in pixels. Shape :math:`(K, N, 2)`. Where + `N` is the number of points and `K` the number of prompts. + keypoints_labels: Labels for the point prompts. 1 indicates a foreground point and 0 indicates a background + point. Shape :math:`(K, N)`. Where `N` is the number of points, and `K` the number of + prompts. + boxes: A box prompt to the model. If a torch.Tensor, should be in a xyxy mode. Shape :math:`(K, 4)` + masks: A low resolution mask input to the model, typically coming from a previous prediction + iteration. Has shape :math:`(K, 1, H, W)`, where for SAM, H=W=256. + multimask_output: If true, the model will return three masks. For ambiguous input prompts (such as a + single click), this will often produce better masks than a single prediction. If only + a single mask is needed, the model's predicted quality score can be used to select the + best mask. For non-ambiguous prompts, such as multiple input prompts, + multimask_output=False can give better results. + output_original_size: If true, the logits of `SegmentationResults` will be post-process to match the + original input image size. + + Returns: + A prediction with the logits and scores (IoU of each predicted mask) + + """ + KORNIA_CHECK(self.is_image_set, "An image must be set with `self.set_image(...)` before `predict` be called!") + + prompts = self.preprocess_prompts(keypoints, keypoints_labels, boxes, masks) + + # Embed prompts + sparse_embeddings, dense_embeddings = self.model.prompt_encoder( + points=prompts.points, boxes=prompts.boxes, masks=prompts.masks + ) + del prompts + + # Predict masks + logits, scores = self.model.mask_decoder( + image_embeddings=self.image_embeddings, + image_pe=self.model.prompt_encoder.get_dense_pe(), + sparse_prompt_embeddings=sparse_embeddings, + dense_prompt_embeddings=dense_embeddings, + multimask_output=multimask_output, + ) + results = SegmentationResults(logits, scores) + if ( + output_original_size + and isinstance(self._input_image_size, tuple) + and isinstance(self._original_image_size, tuple) + ): + results.original_res_logits(self._input_image_size, self._original_image_size, self._input_encoder_size) + + # results = results.squeeze(0) + + return results + + def reset_image(self) -> None: + """Clear cached image state and prompt-transform metadata. + + This method invalidates previously computed image embeddings and resets all + size/transform bookkeeping so a new call to :meth:`set_image` starts from a + clean state. + + In practice, this resets: + - transformed-image parameters, + - original/input/encoder spatial sizes, + - cached image embeddings, + - ``is_image_set`` status flag. + """ + self._tfs_params = None + self._original_image_size = None + self._input_image_size = None + self._input_encoder_size = None + + if hasattr(self, "image_embeddings"): + del self.image_embeddings + + self.image_embeddings = None + self.is_image_set = False + + def compile( + self, + *, + fullgraph: bool = False, + dynamic: bool = False, + backend: str = "inductor", + mode: Optional[str] = None, + options: Optional[dict[Any, Any]] = None, + disable: bool = False, + ) -> None: + """Apply `torch.compile(...)`/dynamo API into the VisualPrompter API. + + .. note:: For more information about the dynamo API check the official docs + https://pytorch.org/docs/stable/generated/torch.compile.html + + Args: + fullgraph: Whether it is ok to break model into several subgraphs + dynamic: Use dynamic shape tracing + backend: backend to be used + mode: Can be either “default”, “reduce-overhead” or “max-autotune” + options: A dictionary of options to pass to the backend. + disable: Turn torch.compile() into a no-op for testing + + Example: + >>> # prompter = VisualPrompter() + >>> # prompter.compile() # You should have torch >= 2.0.0 installed + >>> # Use the prompter methods ... + + """ + # self.set_image = torch.compile( # type: ignore[method-assign] + # self.set_image, + # fullgraph=fullgraph, + # dynamic=dynamic, + # backend=backend, + # mode=mode, + # options=options, + # disable=disable, + # ) + # FIXME: compile set image will try to compile AugmentationSequential which fails + self.model.image_encoder = torch.compile( # type: ignore + self.model.image_encoder, + fullgraph=fullgraph, + dynamic=dynamic, + backend=backend, + mode=mode, + options=options, + disable=disable, + ) + + # self.preprocess_image = torch.compile( # type: ignore[method-assign] + # self.preprocess_image, + # fullgraph=fullgraph, + # dynamic=dynamic, + # backend=backend, + # mode=mode, + # options=options, + # disable=disable, + # ) + + # FIXME: compile predict will try to compile Preproc prompts, which need to compileAugmentationSequential + # which fails + # self.predict = torch.compile( # type: ignore[method-assign] + # self.predict, + # fullgraph=fullgraph, + # dynamic=dynamic, + # backend=backend, + # mode=mode, + # options=options, + # disable=disable, + # ) + self.model.mask_decoder = torch.compile( # type: ignore + self.model.mask_decoder, + fullgraph=fullgraph, + dynamic=dynamic, + backend=backend, + mode=mode, + options=options, + disable=disable, + ) + self.model.prompt_encoder = torch.compile( # type: ignore + self.model.prompt_encoder, + fullgraph=fullgraph, + dynamic=dynamic, + backend=backend, + mode=mode, + options=options, + disable=disable, + ) diff --git a/kornia/core/__init__.py b/kornia/core/__init__.py new file mode 100644 index 0000000..bd40fef --- /dev/null +++ b/kornia/core/__init__.py @@ -0,0 +1,44 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Core — Core tensor operations, backends, and utilities for Kornia. + +This subpackage provides core functionality, device management, and backend support. +""" + +from . import external +from .mixin import ( + ImageModuleMixIn, + ONNXExportMixin, + ONNXMixin, + ONNXRuntimeMixin, +) +from .module import ImageModule, ImageSequential +from .tensor_wrapper import TensorWrapper + +__all__ = [ + "ImageModule", + "ImageModuleMixIn", + "ImageSequential", + "ONNXExportMixin", + "ONNXMixin", + "ONNXRuntimeMixin", + "TensorWrapper", + "external", + "eye_like", + "vec_like", +] diff --git a/kornia/core/_compat.py b/kornia/core/_compat.py new file mode 100644 index 0000000..fa5af28 --- /dev/null +++ b/kornia/core/_compat.py @@ -0,0 +1,207 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import warnings +from functools import wraps +from inspect import isclass +from typing import Any, Callable, Optional + +import torch +from packaging import version + + +def torch_version() -> str: + """Parse the PyTorch version string and remove build metadata. + + Extracts the clean version number from `torch.__version__` by removing any + build suffixes like "+cu118" (CUDA builds) or "+cpu" (CPU builds). + + Returns: + The clean version string (e.g., "2.0.1" from "2.0.1+cu118"). + """ + return torch.__version__.partition("+")[0] + + +def torch_version_lt(major: int, minor: int, patch: int) -> bool: + """Check if the current PyTorch version is less than the specified version. + + Args: + major: Major version number (e.g., 2 for version 2.x.x). + minor: Minor version number (e.g., 0 for version x.0.x). + patch: Patch version number (e.g., 1 for version x.x.1). + + Returns: + True if the current PyTorch version is strictly less than the specified version, + False otherwise. + + """ + _version = version.parse(torch_version()) + return _version < version.parse(f"{major}.{minor}.{patch}") + + +def torch_version_le(major: int, minor: int, patch: int) -> bool: + """Check if the current PyTorch version is less than or equal to the specified version. + + Args: + major: Major version number (e.g., 2 for version 2.x.x). + minor: Minor version number (e.g., 0 for version x.0.x). + patch: Patch version number (e.g., 1 for version x.x.1). + + Returns: + True if the current PyTorch version is less than or equal to the specified version, + False otherwise. + + """ + _version = version.parse(torch_version()) + return _version <= version.parse(f"{major}.{minor}.{patch}") + + +def torch_version_ge(major: int, minor: int, patch: Optional[int] = None) -> bool: + """Check if the current PyTorch version is greater than or equal to the specified version. + + This function is useful for implementing version-specific behavior or compatibility checks. + When `patch` is None, only major and minor versions are compared. + + Args: + major: Major version number (e.g., 2 for version 2.x.x). + minor: Minor version number (e.g., 0 for version x.0.x). + patch: Optional patch version number (e.g., 1 for version x.x.1). + If None, only major.minor version is compared. + + Returns: + True if the current PyTorch version is greater than or equal to the specified version, + False otherwise. + + Example: + >>> torch_version_ge(2, 0, 0) # True if PyTorch >= 2.0.0 + True + >>> torch_version_ge(2, 4) # True if PyTorch >= 2.4.0 (any patch) + True + >>> torch_version_ge(3, 0, 0) # True if PyTorch >= 3.0.0 + False + """ + _version = version.parse(torch_version()) + if patch is None: + return _version >= version.parse(f"{major}.{minor}") + else: + return _version >= version.parse(f"{major}.{minor}.{patch}") + + +def _emit_deprecation_warning( + name: str, replace_with: Optional[str], version: Optional[str], extra_reason: Optional[str] +) -> None: + """Emit a deprecation warning with the given parameters.""" + beginning = f"Since kornia {version} the " if version is not None else "" + extra = f" {extra_reason}" if extra_reason else "" + + warnings.simplefilter("always", DeprecationWarning) + try: + if replace_with is not None: + msg = f"{beginning}`{name}` is deprecated in favor of `{replace_with}`.{extra}" + else: + msg = f"{beginning}`{name}` is deprecated and will be removed in the future versions.{extra}" + warnings.warn( + msg, + category=DeprecationWarning, + stacklevel=3, + ) + finally: + warnings.simplefilter("default", DeprecationWarning) + + +def deprecated( + replace_with: Optional[str] = None, version: Optional[str] = None, extra_reason: Optional[str] = None +) -> Any: + """Mark functions or classes as deprecated with a warning. + + This decorator emits a :class:`DeprecationWarning` when the decorated function or class is called. + It provides information about when the deprecation was introduced and what should be used instead. + + Args: + replace_with: The name of the replacement function/class that should be used instead. + If provided, the warning message will suggest using this replacement. + version: The kornia version when the deprecation was introduced (e.g., "0.8.3"). + If provided, the warning message will include "Since kornia {version}". + extra_reason: Additional context or reason for the deprecation. This will be appended + to the warning message. + + Returns: + A decorator that wraps the function or class with deprecation warnings. + + Example: + Basic usage without replacement: + ```python + @deprecated(version="0.8.3") + def old_function(): + pass + ``` + + With replacement suggestion: + ```python + @deprecated(replace_with="new_function", version="0.8.3") + def old_function(): + pass + ``` + + With additional context: + ```python + @deprecated( + replace_with="new_function", + version="0.8.3", + extra_reason=" The old implementation has performance issues." + ) + def old_function(): + pass + ``` + + Works with classes too: + ```python + @deprecated(replace_with="NewClass", version="0.8.3") + class OldClass: + pass + ``` + """ + + def _deprecated(func: Callable[..., Any]) -> Any: + # Get the name - use __name__ for both functions and classes + name = getattr(func, "__name__", "unknown") + + if isclass(func): + # For classes, wrap in a function that emits warning and instantiates the class + # We manually preserve important class attributes since @wraps doesn't work on classes + def class_wrapper(*args: Any, **kwargs: Any) -> Any: + _emit_deprecation_warning(name, replace_with, version, extra_reason) + return func(*args, **kwargs) + + # Preserve important class attributes + class_wrapper.__name__ = name + class_wrapper.__qualname__ = getattr(func, "__qualname__", name) + class_wrapper.__module__ = getattr(func, "__module__", None) + class_wrapper.__doc__ = func.__doc__ + class_wrapper.__annotations__ = getattr(func, "__annotations__", {}) + return class_wrapper + else: + # For functions, use @wraps normally + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + _emit_deprecation_warning(name, replace_with, version, extra_reason) + return func(*args, **kwargs) + + return wrapper + + return _deprecated diff --git a/kornia/core/check.py b/kornia/core/check.py new file mode 100644 index 0000000..1fc5e82 --- /dev/null +++ b/kornia/core/check.py @@ -0,0 +1,804 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""The testing package contains testing-specific utilities.""" + +from __future__ import annotations + +import os +from typing import Any, Optional, Sequence, TypeVar, cast + +import torch +from torch import float16, float32, float64 + +from kornia.core.exceptions import ( + BaseError, + DeviceError, + ImageError, + ShapeError, + TypeCheckError, + ValueCheckError, +) + +__all__ = [ + "KORNIA_CHECK", + "KORNIA_CHECK_DM_DESC", + "KORNIA_CHECK_IS_COLOR", + "KORNIA_CHECK_IS_GRAY", + "KORNIA_CHECK_IS_IMAGE", + "KORNIA_CHECK_IS_LIST_OF_TENSOR", + "KORNIA_CHECK_IS_TENSOR", + "KORNIA_CHECK_LAF", + "KORNIA_CHECK_SAME_DEVICE", + "KORNIA_CHECK_SAME_DEVICES", + "KORNIA_CHECK_SHAPE", + "KORNIA_CHECK_TYPE", + "KORNIA_UNWRAP", + "BaseError", + "DeviceError", + "ImageError", + "ShapeError", + "TypeCheckError", + "ValueCheckError", + "are_checks_enabled", + "disable_checks", + "enable_checks", +] + + +def _should_enable_checks() -> bool: + """Determine if checks should be enabled. + + Returns: + True if checks should be enabled, False otherwise. + + Checks are enabled by default in debug mode (normal Python execution). + Checks are disabled when: + - Running with `python -O` (optimized mode) + - Environment variable KORNIA_CHECKS=0 is set + """ + env_var = os.getenv("KORNIA_CHECKS", None) + if env_var is not None: + # Explicit override via environment variable + return env_var.lower() in ("1", "true", "yes", "on") + # Default: enabled in debug mode, disabled in optimized mode + return __debug__ + + +# Module-level flag - evaluated once at import time, but can be changed at runtime +_KORNIA_CHECKS_ENABLED: bool = _should_enable_checks() + + +def are_checks_enabled() -> bool: + """Check if validation is currently enabled. + + Returns: + True if checks are enabled, False otherwise. + + Example: + >>> are_checks_enabled() + True + """ + return _KORNIA_CHECKS_ENABLED + + +def disable_checks() -> None: + """Disable all Kornia validation checks for production. + + Note: + This function has no effect if checks were disabled at import time + (via `python -O` or KORNIA_CHECKS=0). The module-level flag is + evaluated once at import time for performance. + + Example: + >>> disable_checks() + >>> are_checks_enabled() + False + """ + global _KORNIA_CHECKS_ENABLED # noqa: PLW0603 + _KORNIA_CHECKS_ENABLED = False + + +def enable_checks() -> None: + """Enable all Kornia validation checks. + + Example: + >>> enable_checks() + >>> are_checks_enabled() + True + """ + global _KORNIA_CHECKS_ENABLED # noqa: PLW0603 + _KORNIA_CHECKS_ENABLED = True + + +# Logger api + + +def KORNIA_CHECK_SHAPE(x: torch.Tensor, shape: list[str], msg: Optional[str] = None, raises: bool = True) -> bool: + """Check whether a tensor has a specified shape. + + The shape can be specified with a implicit or explicit list of strings. + The guard also check whether the variable is a type `Tensor`. + + Args: + x: the tensor to evaluate. + shape: a list with strings with the expected shape. + msg: optional custom message to append to error. + raises: bool indicating whether an exception should be raised upon failure. + + Raises: + ShapeError: if the input tensor does not have the expected shape and raises is True. + + Note: + Checks can be disabled in Python mode using `disable_checks()` or the KORNIA_CHECKS + environment variable. In TorchScript-compiled code, checks always run (TorchScript + cannot access module-level globals, but the validation logic is fast). When running + with `python -O`, Python's optimizer may eliminate some checks. + + Example: + >>> x = torch.rand(2, 3, 4, 4) + >>> KORNIA_CHECK_SHAPE(x, ["B", "C", "H", "W"]) # implicit + True + + >>> x = torch.rand(2, 3, 4, 4) + >>> KORNIA_CHECK_SHAPE(x, ["2", "3", "H", "W"]) # explicit + True + + """ + if not torch.jit.is_scripting(): + if not _KORNIA_CHECKS_ENABLED: + return True + + if "*" == shape[0]: + shape_to_check = shape[1:] + x_shape_to_check = x.shape[-len(shape) + 1 :] + elif "*" == shape[-1]: + shape_to_check = shape[:-1] + x_shape_to_check = x.shape[: len(shape) - 1] + else: + shape_to_check = shape + x_shape_to_check = x.shape + + if len(x_shape_to_check) != len(shape_to_check): + if raises: + expected_dims = len(shape_to_check) + actual_dims = len(x_shape_to_check) + error_msg = f"Shape dimension mismatch: expected {expected_dims} dimensions, got {actual_dims}.\n" + error_msg += f" Expected shape: {shape}\n" + x_shape_list = list(x.shape) + error_msg += f" Actual shape: {x_shape_list}" + if msg is not None: + error_msg += f"\n {msg}" + raise ShapeError( + error_msg, + actual_shape=x_shape_list, + expected_shape=shape, + ) + else: + return False + + for i in range(len(x_shape_to_check)): + # The voodoo below is because torchscript does not like + # that dim can be both int and str + dim_: str = shape_to_check[i] + if not dim_.isnumeric(): + continue + dim = int(dim_) + if x_shape_to_check[i] != dim: + if raises: + error_msg = f"Shape mismatch at dimension {i}: expected {dim}, got {x_shape_to_check[i]}.\n" + error_msg += f" Expected shape: {shape}\n" + x_shape_list = list(x.shape) + error_msg += f" Actual shape: {x_shape_list}" + if msg is not None: + error_msg += f"\n {msg}" + raise ShapeError( + error_msg, + actual_shape=x_shape_list, + expected_shape=shape, + ) + else: + return False + return True + + +def KORNIA_CHECK(condition: bool, msg: Optional[str] = None, raises: bool = True) -> bool: + """Check any arbitrary boolean condition. + + Args: + condition: the condition to evaluate. + msg: message to show in the exception. + raises: bool indicating whether an exception should be raised upon failure. + + Raises: + BaseError: if the condition is not met and raises is True. + + Note: + Checks can be disabled in Python mode using `disable_checks()` or the KORNIA_CHECKS + environment variable. In TorchScript-compiled code, checks always run (TorchScript + cannot access module-level globals, but the validation logic is fast). When running + with `python -O`, Python's optimizer may eliminate some checks. + + Example: + >>> x = torch.rand(2, 3, 3) + >>> KORNIA_CHECK(x.shape[-2:] == (3, 3), "Invalid homography") + True + + """ + if not torch.jit.is_scripting(): + if not _KORNIA_CHECKS_ENABLED: + return True + + if not condition: + if raises: + if msg is None: + error_msg = "Validation condition failed" + else: + error_msg = msg + raise BaseError(error_msg) + return False + return True + + +def KORNIA_UNWRAP(maybe_obj: Any, typ: Any) -> Any: + """Unwrap an optional contained value that may or not be present. + + Args: + maybe_obj: the object to unwrap. + typ: expected type after unwrap. + + """ + # TODO: this function will change after kornia/pr#1987 + return cast(typ, maybe_obj) + + +T = TypeVar("T", bound=type) + + +def KORNIA_CHECK_TYPE(x: Any, typ: T | tuple[T, ...], msg: Optional[str] = None, raises: bool = True) -> bool: + """Check the type of an aribratry variable. + + Args: + x: any input variable. + typ: the expected type of the variable. + msg: message to show in the exception. + raises: bool indicating whether an exception should be raised upon failure. + + Raises: + TypeCheckError: if the input variable does not match with the expected and raises is True. + + Note: + Checks can be disabled in Python mode using `disable_checks()` or the KORNIA_CHECKS + environment variable. In TorchScript-compiled code, checks always run (TorchScript + cannot access module-level globals, but the validation logic is fast). When running + with `python -O`, Python's optimizer may eliminate some checks. + + Example: + >>> KORNIA_CHECK_TYPE("foo", str, "Invalid string") + True + + """ + if not torch.jit.is_scripting(): + if not _KORNIA_CHECKS_ENABLED: + return True + + if not isinstance(x, typ): + if raises: + # JIT doesn't support try-except or type introspection, so use simple message + if torch.jit.is_scripting(): + error_msg = "Type mismatch." + if msg is not None: + error_msg += f"\n {msg}" + raise TypeCheckError(error_msg) + else: + # In Python mode, we can safely use type introspection + expected_type_str = typ.__name__ if not isinstance(typ, tuple) else " | ".join(t.__name__ for t in typ) + type_name = str(type(x)) + error_msg = f"Type mismatch: expected {expected_type_str}, got {type_name}." + if msg is not None: + error_msg += f"\n {msg}" + raise TypeCheckError( + error_msg, + actual_type=type(x), + expected_type=typ, + ) + return False + return True + + +def KORNIA_CHECK_IS_TENSOR(x: Any, msg: Optional[str] = None, raises: bool = True) -> bool: + """Check the input variable is a Tensor. + + Args: + x: any input variable. + msg: message to show in the exception. + raises: bool indicating whether an exception should be raised upon failure. + + Raises: + TypeCheckError: if the input variable does not match with the expected and raises is True. + + Note: + Checks can be disabled in Python mode using `disable_checks()` or the KORNIA_CHECKS + environment variable. In TorchScript-compiled code, checks always run (TorchScript + cannot access module-level globals, but the validation logic is fast). When running + with `python -O`, Python's optimizer may eliminate some checks. + + Example: + >>> x = torch.rand(2, 3, 3) + >>> KORNIA_CHECK_IS_TENSOR(x, "Invalid tensor") + True + + """ + if not torch.jit.is_scripting(): + if not _KORNIA_CHECKS_ENABLED: + return True + + if not isinstance(x, torch.Tensor): + if raises: + # JIT doesn't support try-except or type introspection, so use simple message + if torch.jit.is_scripting(): + error_msg = "Type mismatch: expected Tensor." + if msg is not None: + error_msg += f"\n {msg}" + raise TypeCheckError(error_msg) + else: + # In Python mode, we can safely use type introspection + type_name = str(type(x)) + error_msg = f"Type mismatch: expected Tensor, got {type_name}." + if msg is not None: + error_msg += f"\n {msg}" + raise TypeCheckError( + error_msg, + actual_type=type(x), + expected_type=torch.Tensor, + ) + return False + return True + + +def KORNIA_CHECK_IS_LIST_OF_TENSOR(x: Optional[Sequence[Any]], raises: bool = True) -> bool: + """Check the input variable is a List of Tensors. + + Args: + x: Any sequence of objects + raises: bool indicating whether an exception should be raised upon failure. + + Raises: + TypeCheckError: if the input variable does not match with the expected and raises is True. + + Return: + True if the input is a list of Tensors, otherwise return False. + + Note: + Checks can be disabled in Python mode using `disable_checks()` or the KORNIA_CHECKS + environment variable. In TorchScript-compiled code, checks always run (TorchScript + cannot access module-level globals, but the validation logic is fast). When running + with `python -O`, Python's optimizer may eliminate some checks. + + Example: + >>> x = torch.rand(2, 3, 3) + >>> KORNIA_CHECK_IS_LIST_OF_TENSOR(x, raises=False) + False + >>> KORNIA_CHECK_IS_LIST_OF_TENSOR([x]) + True + + """ + if not torch.jit.is_scripting(): + if not _KORNIA_CHECKS_ENABLED: + return True + + are_tensors = isinstance(x, list) and all(isinstance(d, torch.Tensor) for d in x) + if not are_tensors: + if raises: + error_msg = f"Type mismatch: expected list[Tensor], got {type(x).__name__}." + raise TypeCheckError( + error_msg, + actual_type=type(x), + expected_type=list[torch.Tensor], + ) + return False + return True + + +def KORNIA_CHECK_SAME_DEVICE(x: torch.Tensor, y: torch.Tensor, raises: bool = True) -> bool: + """Check whether two tensor in the same device. + + Args: + x: first tensor to evaluate. + y: second tensor to evaluate. + raises: bool indicating whether an exception should be raised upon failure. + + Raises: + DeviceError: if the two tensors are not in the same device and raises is True. + + Note: + Checks can be disabled in Python mode using `disable_checks()` or the KORNIA_CHECKS + environment variable. In TorchScript-compiled code, checks always run (TorchScript + cannot access module-level globals, but the validation logic is fast). When running + with `python -O`, Python's optimizer may eliminate some checks. + + Example: + >>> x1 = torch.rand(2, 3, 3) + >>> x2 = torch.rand(1, 3, 1) + >>> KORNIA_CHECK_SAME_DEVICE(x1, x2) + True + + """ + if not torch.jit.is_scripting(): + if not _KORNIA_CHECKS_ENABLED: + return True + + if x.device != y.device: + if raises: + error_msg = "Device mismatch: tensors must be on the same device.\n" + error_msg += f" First tensor device: {x.device}\n" + error_msg += f" Second tensor device: {y.device}" + raise DeviceError( + error_msg, + actual_devices=[x.device, y.device], + expected_device=x.device, + ) + return False + return True + + +def KORNIA_CHECK_SAME_DEVICES(tensors: list[torch.Tensor], msg: Optional[str] = None, raises: bool = True) -> bool: + """Check whether a list provided tensors live in the same device. + + Args: + tensors: a list of tensors. + msg: message to show in the exception. + raises: bool indicating whether an exception should be raised upon failure. + + Raises: + DeviceError: if all the tensors are not in the same device and raises is True. + + Note: + Checks can be disabled in Python mode using `disable_checks()` or the KORNIA_CHECKS + environment variable. In TorchScript-compiled code, checks always run (TorchScript + cannot access module-level globals, but the validation logic is fast). When running + with `python -O`, Python's optimizer may eliminate some checks. + + Example: + >>> x1 = torch.rand(2, 3, 3) + >>> x2 = torch.rand(1, 3, 1) + >>> KORNIA_CHECK_SAME_DEVICES([x1, x2], "Tensors not in the same device") + True + + """ + if not torch.jit.is_scripting(): + if not _KORNIA_CHECKS_ENABLED: + return True + + KORNIA_CHECK(isinstance(tensors, list) and len(tensors) >= 1, "Expected a list with at least one element", raises) + if not all(tensors[0].device == x.device for x in tensors): + if raises: + devices = [x.device for x in tensors] + error_msg = "Device mismatch: all tensors must be on the same device.\n" + error_msg += f" Expected device: {tensors[0].device}\n" + error_msg += f" Actual devices: {devices}" + if msg is not None: + error_msg += f"\n {msg}" + raise DeviceError( + error_msg, + actual_devices=devices, + expected_device=tensors[0].device, + ) + return False + return True + + +def KORNIA_CHECK_SAME_SHAPE(x: torch.Tensor, y: torch.Tensor, raises: bool = True) -> bool: + """Check whether two tensor have the same shape. + + Args: + x: first tensor to evaluate. + y: second tensor to evaluate. + raises: bool indicating whether an exception should be raised upon failure. + + Raises: + ShapeError: if the two tensors have not the same shape and raises is True. + + Note: + Checks can be disabled in Python mode using `disable_checks()` or the KORNIA_CHECKS + environment variable. In TorchScript-compiled code, checks always run (TorchScript + cannot access module-level globals, but the validation logic is fast). When running + with `python -O`, Python's optimizer may eliminate some checks. + + Example: + >>> x1 = torch.rand(2, 3, 3) + >>> x2 = torch.rand(2, 3, 3) + >>> KORNIA_CHECK_SAME_SHAPE(x1, x2) + True + + """ + if not torch.jit.is_scripting(): + if not _KORNIA_CHECKS_ENABLED: + return True + + if x.shape != y.shape: + if raises: + error_msg = "Shape mismatch: tensors must have the same shape.\n" + x_shape_list = list(x.shape) + y_shape_list = list(y.shape) + error_msg += f" First tensor shape: {x_shape_list}\n" + error_msg += f" Second tensor shape: {y_shape_list}" + raise ShapeError( + error_msg, + actual_shape=x_shape_list, + expected_shape=y_shape_list, + ) + return False + return True + + +def KORNIA_CHECK_IS_COLOR(x: torch.Tensor, msg: Optional[str] = None, raises: bool = True) -> bool: + """Check whether an image tensor is a color images. + + Args: + x: image tensor to evaluate. + msg: message to show in the exception. + raises: bool indicating whether an exception should be raised upon failure. + + Raises: + ImageError: if the input tensor does not have a shape :math:`(3,H,W)` and raises is True. + + Note: + Checks can be disabled in Python mode using `disable_checks()` or the KORNIA_CHECKS + environment variable. In TorchScript-compiled code, checks always run (TorchScript + cannot access module-level globals, but the validation logic is fast). When running + with `python -O`, Python's optimizer may eliminate some checks. + + Example: + >>> img = torch.rand(2, 3, 4, 4) + >>> KORNIA_CHECK_IS_COLOR(img, "Image is not color") + True + + """ + if not torch.jit.is_scripting(): + if not _KORNIA_CHECKS_ENABLED: + return True + + if len(x.shape) < 3 or x.shape[-3] != 3: + if raises: + error_msg = f"Not a color tensor. Got: {type(x)}." + if msg is not None: + error_msg += f"\n{msg}" + raise ImageError(error_msg) + return False + return True + + +def KORNIA_CHECK_IS_GRAY(x: torch.Tensor, msg: Optional[str] = None, raises: bool = True) -> bool: + """Check whether an image tensor is grayscale. + + Args: + x: image tensor to evaluate. + msg: message to show in the exception. + raises: bool indicating whether an exception should be raised upon failure. + + Raises: + ImageError: if the tensor does not have a shape :math:`(1,H,W)` or :math:`(H,W)` and raises is True. + + Note: + Checks can be disabled in Python mode using `disable_checks()` or the KORNIA_CHECKS + environment variable. In TorchScript-compiled code, checks always run (TorchScript + cannot access module-level globals, but the validation logic is fast). When running + with `python -O`, Python's optimizer may eliminate some checks. + + Example: + >>> img = torch.rand(2, 1, 4, 4) + >>> KORNIA_CHECK_IS_GRAY(img, "Image is not grayscale") + True + + """ + if not torch.jit.is_scripting(): + if not _KORNIA_CHECKS_ENABLED: + return True + + if len(x.shape) < 2 or (len(x.shape) >= 3 and x.shape[-3] != 1): + if raises: + error_msg = f"Not a gray tensor. Got: {type(x)}." + if msg is not None: + error_msg += f"\n{msg}" + raise ImageError(error_msg) + return False + return True + + +def KORNIA_CHECK_IS_COLOR_OR_GRAY(x: torch.Tensor, msg: Optional[str] = None, raises: bool = True) -> bool: + """Check whether an image tensor is grayscale or color. + + Args: + x: image tensor to evaluate. + msg: message to show in the exception. + raises: bool indicating whether an exception should be raised upon failure. + + Raises: + ImageError: if the tensor does not have a shape :math:`(1,H,W)` or :math:`(3,H,W)` and raises is True. + + Note: + Checks can be disabled in Python mode using `disable_checks()` or the KORNIA_CHECKS + environment variable. In TorchScript-compiled code, checks always run (TorchScript + cannot access module-level globals, but the validation logic is fast). When running + with `python -O`, Python's optimizer may eliminate some checks. + + Example: + >>> img = torch.rand(2, 3, 4, 4) + >>> KORNIA_CHECK_IS_COLOR_OR_GRAY(img, "Image is not color or grayscale") + True + + """ + if not torch.jit.is_scripting(): + if not _KORNIA_CHECKS_ENABLED: + return True + + if len(x.shape) < 3 or x.shape[-3] not in [1, 3]: + if raises: + error_msg = f"Not a color or gray tensor. Got: {type(x)}." + if msg is not None: + error_msg += f"\n{msg}" + raise ImageError(error_msg) + return False + return True + + +def KORNIA_CHECK_IS_IMAGE(x: torch.Tensor, msg: Optional[str] = None, raises: bool = True, bits: int = 8) -> bool: + """Check whether an image tensor is ranged properly [0, 1] for float or [0, 2 ** bits] for int. + + Args: + x: image tensor to evaluate. + msg: message to show in the exception. + raises: bool indicating whether an exception should be raised upon failure. + bits: the image bits. The default checks if given integer input image is an + 8-bit image (0-255) or not. + + Raises: + ImageError: if the tensor shape is invalid and raises is True. + ValueCheckError: if the tensor value range is invalid and raises is True. + + Note: + Checks can be disabled in Python mode using `disable_checks()` or the KORNIA_CHECKS + environment variable. In TorchScript-compiled code, checks always run (TorchScript + cannot access module-level globals, but the validation logic is fast). When running + with `python -O`, Python's optimizer may eliminate some checks. + + Example: + >>> img = torch.rand(2, 3, 4, 4) + >>> KORNIA_CHECK_IS_IMAGE(img, "It is not an image") + True + + """ + if not torch.jit.is_scripting(): + if not _KORNIA_CHECKS_ENABLED: + return True + + # Combine the color or gray check with the range check + if not raises and not KORNIA_CHECK_IS_COLOR_OR_GRAY(x, msg, raises): + return False + + amin, amax = torch.aminmax(x) + + if x.dtype in (torch.bfloat16, float16, float32, float64): + invalid = (amin < 0) | (amax > 1) + else: + max_int_value = (1 << bits) - 1 + invalid = (amin < 0) | (amax > max_int_value) + + if invalid.item(): + return _handle_invalid_range(msg, raises, amin, amax) + + return True + + +def KORNIA_CHECK_DM_DESC(desc1: torch.Tensor, desc2: torch.Tensor, dm: torch.Tensor, raises: bool = True) -> bool: + """Check whether the provided descriptors match with a distance matrix. + + Args: + desc1: first descriptor tensor to evaluate. + desc2: second descriptor tensor to evaluate. + dm: distance matrix tensor to evaluate. + raises: bool indicating whether an exception should be raised upon failure. + + Raises: + ShapeError: if the descriptors shape do not match with the distance matrix and raises is True. + + Note: + Checks can be disabled in Python mode using `disable_checks()` or the KORNIA_CHECKS + environment variable. In TorchScript-compiled code, checks always run (TorchScript + cannot access module-level globals, but the validation logic is fast). When running + with `python -O`, Python's optimizer may eliminate some checks. + + Example: + >>> desc1 = torch.rand(4) + >>> desc2 = torch.rand(8) + >>> dm = torch.rand(4, 8) + >>> KORNIA_CHECK_DM_DESC(desc1, desc2, dm) + True + + """ + if not torch.jit.is_scripting(): + if not _KORNIA_CHECKS_ENABLED: + return True + + if not ((dm.size(0) == desc1.size(0)) and (dm.size(1) == desc2.size(0))): + if raises: + expected_shape = (desc1.size(0), desc2.size(0)) + desc1_size = desc1.size(0) + desc2_size = desc2.size(0) + error_msg = "Distance matrix shape mismatch.\n" + error_msg += ( + f" Expected shape: {expected_shape} (from desc1.shape[0]={desc1_size}, desc2.shape[0]={desc2_size})\n" + ) + dm_shape_list = list(dm.shape) + desc1_shape_list = list(desc1.shape) + desc2_shape_list = list(desc2.shape) + error_msg += f" Actual shape: {dm_shape_list}\n" + error_msg += f" desc1 shape: {desc1_shape_list}\n" + error_msg += f" desc2 shape: {desc2_shape_list}" + raise ShapeError( + error_msg, + actual_shape=dm_shape_list, + expected_shape=expected_shape, + ) + return False + return True + + +def KORNIA_CHECK_LAF(laf: torch.Tensor, raises: bool = True) -> bool: + """Check whether a Local Affine Frame (laf) has a valid shape. + + Args: + laf: local affine frame tensor to evaluate. + raises: bool indicating whether an exception should be raised upon failure. + + Raises: + ShapeError: if the input laf does not have a shape :math:`(B,N,2,3)` and raises is True. + + Note: + Checks can be disabled in Python mode using `disable_checks()` or the KORNIA_CHECKS + environment variable. In TorchScript-compiled code, checks always run (TorchScript + cannot access module-level globals, but the validation logic is fast). When running + with `python -O`, Python's optimizer may eliminate some checks. + + Example: + >>> lafs = torch.rand(2, 10, 2, 3) + >>> KORNIA_CHECK_LAF(lafs) + True + + """ + return KORNIA_CHECK_SHAPE(laf, ["B", "N", "2", "3"], raises=raises) + + +def _handle_invalid_range( + msg: Optional[str], raises: bool, min_val: float | torch.Tensor, max_val: float | torch.Tensor +) -> bool: + """Handle invalid range cases.""" + # Extract scalar values if tensors + min_scalar = min_val.item() if isinstance(min_val, torch.Tensor) else min_val + max_scalar = max_val.item() if isinstance(max_val, torch.Tensor) else max_val + + err_msg = f"Value range mismatch: expected [0, 1], got [{min_scalar}, {max_scalar}]." + if msg is not None: + err_msg += f"\n {msg}" + if raises: + raise ValueCheckError( + err_msg, + actual_value=(min_scalar, max_scalar), + expected_range=(0.0, 1.0), + ) + return False diff --git a/kornia/core/exceptions.py b/kornia/core/exceptions.py new file mode 100644 index 0000000..c7a1dbe --- /dev/null +++ b/kornia/core/exceptions.py @@ -0,0 +1,123 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Custom exception classes for Kornia validation checks.""" + +from __future__ import annotations + +from typing import Any, Optional + +__all__ = [ + "BaseError", + "DeviceError", + "ImageError", + "ShapeError", + "TypeCheckError", + "ValueCheckError", +] + + +class BaseError(Exception): + """Base exception class for all Kornia errors.""" + + pass + + +class ShapeError(BaseError): + """Raised when tensor shape validation fails. + + Attributes: + actual_shape: The actual shape of the tensor that failed validation. + expected_shape: The expected shape specification. + """ + + def __init__( + self, + message: str, + *, + actual_shape: Optional[tuple[int, ...] | list[int]] = None, + expected_shape: Optional[list[str] | tuple[int, ...]] = None, + ): + super().__init__(message) + self.actual_shape = actual_shape + self.expected_shape = expected_shape + + +class TypeCheckError(BaseError): + """Raised when type validation fails. + + Attributes: + actual_type: The actual type that failed validation. + expected_type: The expected type. + """ + + def __init__( + self, + message: str, + *, + actual_type: Optional[type] = None, + expected_type: Optional[type | tuple[type, ...]] = None, + ): + super().__init__(message) + self.actual_type = actual_type + self.expected_type = expected_type + + +class ValueCheckError(BaseError): + """Raised when value/range validation fails. + + Attributes: + actual_value: The actual value that failed validation. + expected_range: The expected value range (min, max). + """ + + def __init__( + self, + message: str, + *, + actual_value: Optional[Any] = None, + expected_range: Optional[tuple[Any, Any]] = None, + ): + super().__init__(message) + self.actual_value = actual_value + self.expected_range = expected_range + + +class DeviceError(BaseError): + """Raised when device mismatch validation fails. + + Attributes: + actual_devices: The actual device(s) that failed validation. + expected_device: The expected device. + """ + + def __init__( + self, + message: str, + *, + actual_devices: Optional[list] = None, + expected_device: Optional[Any] = None, + ): + super().__init__(message) + self.actual_devices = actual_devices + self.expected_device = expected_device + + +class ImageError(BaseError): + """Raised when image-specific validation fails.""" + + pass diff --git a/kornia/core/external.py b/kornia/core/external.py new file mode 100644 index 0000000..11aed16 --- /dev/null +++ b/kornia/core/external.py @@ -0,0 +1,162 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import importlib +import logging +import subprocess +import sys +from types import ModuleType +from typing import List, Optional + +from kornia.config import InstallationMode, kornia_config + +logger = logging.getLogger(__name__) + + +class LazyLoader: + """A class that implements lazy loading for Python modules. + + This class defers the import of a module until an attribute of the module is accessed. + It helps in reducing the initial load time and memory usage of a script, especially when + dealing with large or optional dependencies that might not be used in every execution. + + Attributes: + module_name: The name of the module to be lazily loaded. + module: The actual module object, initialized to None and loaded upon first access. + + """ + + auto_install: bool = False + + def __init__(self, module_name: str, dev_dependency: bool = False) -> None: + """Initialize the LazyLoader with the name of the module. + + Args: + module_name: The name of the module to be lazily loaded. + dev_dependency: If the dependency is required in the dev environment. + If True, the module will be loaded in the dev environment. + If False, the module will not be loaded in the dev environment. + + """ + self.module_name = module_name + self.module: Optional[ModuleType] = None + self.dev_dependency = dev_dependency + + def _install_package(self, module_name: str) -> None: + logger.info(f"Installing `{module_name}` ...") + subprocess.run([sys.executable, "-m", "pip", "install", "-U", module_name], shell=False, check=False) # noqa: S603 + + def _load(self) -> None: + """Load the module if it hasn't been loaded yet. + + This method is called internally when an attribute of the module is accessed for the first time. It attempts to + import the module and raises an ImportError with a custom message if the module is not installed. + """ + if not self.dev_dependency: + if "--doctest-modules" in sys.argv: + logger.info(f"Doctest detected, skipping loading of '{self.module_name}'") + return + try: + if __sphinx_build__: # type:ignore + logger.info(f"Sphinx detected, skipping loading of '{self.module_name}'") + return + except NameError: + pass + + if self.module is None: + try: + self.module = importlib.import_module(self.module_name) + except ImportError as e: + if kornia_config.lazyloader.installation_mode == InstallationMode.AUTO or self.auto_install: + self._install_package(self.module_name) + elif kornia_config.lazyloader.installation_mode == InstallationMode.ASK: + to_ask = True + if_install = input( + f"Optional dependency '{self.module_name}' is not installed. " + "You may silent this prompt by `kornia_config.lazyloader.installation_mode = 'auto'`. " + "Do you wish to install the dependency? [Y]es, [N]o, [A]ll." + ) + while to_ask: + if if_install.lower() == "y" or if_install.lower() == "yes": + self._install_package(self.module_name) + self.module = importlib.import_module(self.module_name) + to_ask = False + elif if_install.lower() == "a" or if_install.lower() == "all": + self.auto_install = True + self._install_package(self.module_name) + self.module = importlib.import_module(self.module_name) + to_ask = False + elif if_install.lower() == "n" or if_install.lower() == "no": + raise ImportError( + f"Optional dependency '{self.module_name}' is not installed. " + f"Please install it to use this functionality." + ) from e + else: + if_install = input("Invalid input. Please enter 'Y', 'N', or 'A'.") + + elif kornia_config.lazyloader.installation_mode == InstallationMode.RAISE: + raise ImportError( + f"Optional dependency '{self.module_name}' is not installed. " + f"Please install it to use this functionality." + ) from e + self.module = importlib.import_module(self.module_name) + + def __getattr__(self, item: str) -> object: + """Load the module (if not already loaded) and returns the requested attribute. + + This method is called when an attribute of the LazyLoader instance is accessed. + It ensures that the module is loaded and then returns the requested attribute. + + Args: + item: The name of the attribute to be accessed. + + Returns: + The requested attribute of the loaded module. + + """ + self._load() + return getattr(self.module, item) + + def __dir__(self) -> List[str]: + """Load the module (if not already loaded) and returns the list of attributes of the module. + + This method is called when the built-in dir() function is used on the LazyLoader instance. + It ensures that the module is loaded and then returns the list of attributes of the module. + + Returns: + list: The list of attributes of the loaded module. + + """ + self._load() + return dir(self.module) + + +# NOTE: This section is used for lazy loading of external modules. However, sphinx +# would also try to support lazy loading of external modules. To avoid that, we +# may set the module name to `autodoc_mock_imports` in conf.py to avoid undesired +# installation of external modules. +numpy = LazyLoader("numpy", dev_dependency=True) +PILImage = LazyLoader("PIL.Image", dev_dependency=True) +onnx = LazyLoader("onnx", dev_dependency=True) +diffusers = LazyLoader("diffusers") +transformers = LazyLoader("transformers") +onnxruntime = LazyLoader("onnxruntime") +boxmot = LazyLoader("boxmot") +segmentation_models_pytorch = LazyLoader("segmentation_models_pytorch") +basicsr = LazyLoader("basicsr") +requests = LazyLoader("requests") +ivy = LazyLoader("ivy") diff --git a/kornia/core/mixin/__init__.py b/kornia/core/mixin/__init__.py new file mode 100644 index 0000000..2be7a65 --- /dev/null +++ b/kornia/core/mixin/__init__.py @@ -0,0 +1,24 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Core Mixin — Mixins for image modules and ONNX export in Kornia. + +This subpackage provides mixin classes for extending core functionality. +""" + +from .image_module import ImageModuleMixIn +from .onnx import ONNXExportMixin, ONNXMixin, ONNXRuntimeMixin diff --git a/kornia/core/mixin/image_module.py b/kornia/core/mixin/image_module.py new file mode 100644 index 0000000..0629af2 --- /dev/null +++ b/kornia/core/mixin/image_module.py @@ -0,0 +1,251 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import datetime +import math +import os +from functools import wraps +from typing import Any, Callable, List, Literal, Optional, Tuple, Union + +import torch + +from kornia.core.external import PILImage as Image +from kornia.core.external import numpy as np + + +class ImageModuleMixIn: + """A MixIn that handles image-based operations. + + This modules accepts multiple input and output data types, provides end-to-end visualization, file saving features. + Note that this MixIn fits the classes that return one image tensor only. + """ + + _output_image: Any + + def convert_input_output( + self, + input_names_to_handle: Optional[List[Any]] = None, + output_type: Literal["pt", "numpy", "pil"] = "pt", + ) -> Callable[[Any], Any]: + """Convert input and output types for a function. + + Args: + input_names_to_handle: List of input names to convert, if None, handle all inputs. + output_type: Desired output type ('pt', 'numpy', or 'pil'). + + Returns: + Callable: Decorated function with converted input and output types. + + """ + # Validate output_type at the start + if output_type not in ("pt", "numpy", "pil"): + raise ValueError(f"Invalid output_type '{output_type}'. Must be one of 'pt', 'numpy', or 'pil'.") + + def decorator(func: Callable[[Any], Any]) -> Callable[[Any], Any]: + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Union[Any, List[Any]]: + # If input_names_to_handle is None, handle all inputs + if input_names_to_handle is None: + # Convert all args to tensors + args = tuple(self.to_tensor(arg) if self._is_valid_arg(arg) else arg for arg in args) + # Convert all kwargs to tensors + kwargs = {k: self.to_tensor(v) if self._is_valid_arg(v) else v for k, v in kwargs.items()} + else: + # Convert specified args to tensors + args = list(args) # type:ignore + for i, (arg, name) in enumerate(zip(args, func.__code__.co_varnames)): # ty: ignore[unresolved-attribute] + if name in input_names_to_handle: + args[i] = self.to_tensor(arg) # type:ignore + # Convert specified kwargs to tensors + for name, value in kwargs.items(): + if name in input_names_to_handle: + kwargs[name] = self.to_tensor(value) + + # Call the actual forward method + tensor_outputs = func(*args, **kwargs) + + if not isinstance(tensor_outputs, tuple): + tensor_outputs = (tensor_outputs,) + + # Convert outputs to the desired type + outputs = [] + for output in tensor_outputs: + if output_type == "pt": + outputs.append(output) + elif output_type == "numpy": + outputs.append(self.to_numpy(output)) + elif output_type == "pil": + outputs.append(self.to_pil(output)) + else: + raise ValueError("Output type not supported. Choose from 'pt', 'numpy', or 'pil'.") + + return outputs if len(outputs) > 1 else outputs[0] + + return wrapper + + return decorator + + def _is_valid_arg(self, arg: Any) -> bool: + """Check if the argument is a valid type for conversion. + + Args: + arg: The argument to check. + + Returns: + bool: True if valid, False otherwise. + + """ + if isinstance(arg, str) and os.path.exists(arg): + return True + if isinstance(arg, torch.Tensor): + return True + # Make sure that the numpy and PIL are not necessarily needed to be imported. + if isinstance(arg, np.ndarray): # type: ignore + return True + if isinstance(arg, (Image.Image)): # type: ignore + return True + return False + + def to_tensor(self, x: Any) -> torch.Tensor: + """Convert input to tensor. + + Supports image path, numpy array, PIL image, and raw tensor. + + Args: + x: The input to convert. + + Returns: + Tensor: The converted tensor. + + """ + if isinstance(x, str): + from kornia.io import ImageLoadType, load_image # pylint: disable=C0415 + + return load_image(x, ImageLoadType.UNCHANGED) / 255 + if isinstance(x, torch.Tensor): + return x + if isinstance(x, np.ndarray): # type: ignore + from kornia.image.utils import image_to_tensor # pylint: disable=C0415 + + return image_to_tensor(x) / 255 + if isinstance(x, Image.Image): # type: ignore + return torch.from_numpy(np.array(x)).permute(2, 0, 1).float() / 255 # type: ignore + raise TypeError("Input type not supported") + + def to_numpy(self, x: Any) -> "np.array": # type: ignore + """Convert input to numpy array. + + Args: + x: The input to convert. + + Returns: + np.array: The converted numpy array. + + """ + if isinstance(x, torch.Tensor): + return x.cpu().detach().numpy() + if isinstance(x, np.ndarray): # type: ignore + return x + if isinstance(x, Image.Image): # type: ignore + return np.array(x) # type: ignore + raise TypeError("Input type not supported") + + def to_pil(self, x: Any) -> "Image.Image": # type: ignore + """Convert input to PIL image. + + Args: + x: The input to convert. + + Returns: + Image.Image: The converted PIL image. + + """ + if isinstance(x, torch.Tensor): + x = x.cpu().detach() * 255 + if x.dim() == 3: + x = x.permute(1, 2, 0) + return Image.fromarray(x.byte().numpy()) # type: ignore + elif x.dim() == 4: + x = x.permute(0, 2, 3, 1) + return [Image.fromarray(_x.byte().numpy()) for _x in x] # type: ignore + else: + raise NotImplementedError + if isinstance(x, np.ndarray): # type: ignore + raise NotImplementedError + if isinstance(x, Image.Image): # type: ignore + return x + raise TypeError("Input type not supported") + + def _detach_tensor_to_cpu( + self, output_image: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]] + ) -> Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]]: + if isinstance(output_image, torch.Tensor): + return output_image.detach().cpu() + if isinstance(output_image, list | tuple): + return type(output_image)([self._detach_tensor_to_cpu(out) for out in output_image]) # type: ignore + raise RuntimeError(f"Unexpected object {output_image} with a type of `{type(output_image)}`") + + def show(self, n_row: Optional[int] = None, backend: str = "pil", display: bool = True) -> Optional[Any]: + """Return PIL images. + + Args: + n_row: Number of images displayed in each row of the grid. + backend: visualization backend. Only PIL is supported now. + display: Whether or not to show the image. + + """ + if self._output_image is None: + raise ValueError("No pre-computed images found. Needs to execute first.") + + if len(self._output_image.shape) == 3: + out_image = self._output_image + elif len(self._output_image.shape) == 4: + from kornia.image.utils import make_grid # pylint: disable=C0415 + + if n_row is None: + n_row = math.ceil(self._output_image.shape[0] ** 0.5) + out_image = make_grid(self._output_image, n_row, padding=2) + else: + raise ValueError + + if backend == "pil" and display: + Image.fromarray((out_image.permute(1, 2, 0).squeeze().numpy() * 255).astype(np.uint8)).show() # type: ignore + return None + if backend == "pil": + return Image.fromarray((out_image.permute(1, 2, 0).squeeze().numpy() * 255).astype(np.uint8)) # type: ignore + raise ValueError(f"Unsupported backend `{backend}`.") + + def save(self, name: Optional[str] = None, n_row: Optional[int] = None) -> None: + """Save the output image(s) to a directory. + + Args: + name: Directory to save the images. + n_row: Number of images displayed in each row of the grid. + + """ + from kornia.image.utils import make_grid # pylint: disable=C0415 + from kornia.io import write_image # pylint: disable=C0415 + + if name is None: + name = f"Kornia-{datetime.datetime.now(tz=datetime.UTC).strftime('%Y%m%d%H%M%S')!s}.jpg" + if len(self._output_image.shape) == 3: + out_image = self._output_image + if len(self._output_image.shape) == 4: + if n_row is None: + n_row = math.ceil(self._output_image.shape[0] ** 0.5) + out_image = make_grid(self._output_image, n_row, padding=2) + write_image(name, out_image.mul(255.0).byte()) diff --git a/kornia/core/mixin/onnx.py b/kornia/core/mixin/onnx.py new file mode 100644 index 0000000..0a05524 --- /dev/null +++ b/kornia/core/mixin/onnx.py @@ -0,0 +1,434 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import copy +import io +from typing import ( + Any, + ClassVar, + Optional, + Union, +) + +import torch +from torch import nn + +import kornia +from kornia.core.external import numpy as np +from kornia.core.external import onnx +from kornia.core.external import onnxruntime as ort + + +class ONNXExportMixin: + """Mixin class that provides ONNX export functionality for objects that support it. + + Attributes: + ONNX_EXPORTABLE: + A flag indicating whether the object can be exported to ONNX. Default is True. + ONNX_DEFAULT_INPUTSHAPE: + Default input shape for the ONNX export. A list of integers where `-1` indicates + dynamic dimensions. Default is [-1, -1, -1, -1]. + ONNX_DEFAULT_OUTPUTSHAPE: + Default output shape for the ONNX export. A list of integers where `-1` indicates + dynamic dimensions. Default is [-1, -1, -1, -1]. + ONNX_EXPORT_PSEUDO_SHAPE: + This is used to create a dummy input tensor for the ONNX export. Default is [1, 3, 256, 256]. + It dimension shall match the ONNX_DEFAULT_INPUTSHAPE and ONNX_DEFAULT_OUTPUTSHAPE. + Non-image dimensions are allowed. + + Note: + - If `ONNX_EXPORTABLE` is False, indicating that the object cannot be exported to ONNX. + + """ + + ONNX_EXPORTABLE: bool = True + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, -1, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, -1, -1, -1] + ONNX_EXPORT_PSEUDO_SHAPE: ClassVar[list[int]] = [1, 3, 256, 256] + ADDITIONAL_METADATA: ClassVar[list[tuple[str, str]]] = [] + + def to_onnx( + self, + onnx_name: Optional[str] = None, + input_shape: Optional[list[int]] = None, + output_shape: Optional[list[int]] = None, + pseudo_shape: Optional[list[int]] = None, + model: Optional[nn.Module] = None, + save: bool = True, + additional_metadata: Optional[list[tuple[str, str]]] = None, + **kwargs: Any, + ) -> onnx.ModelProto: # type: ignore + """Export the current object to an ONNX model file. + + Args: + onnx_name: + The name of the output ONNX file. If not provided, a default name in the + format "Kornia-.onnx" will be used. + input_shape: + The input shape for the model as a list of integers. If None, + `ONNX_DEFAULT_INPUTSHAPE` will be used. Dynamic dimensions can be indicated by `-1`. + output_shape: + The output shape for the model as a list of integers. If None, + `ONNX_DEFAULT_OUTPUTSHAPE` will be used. Dynamic dimensions can be indicated by `-1`. + pseudo_shape: + The pseudo shape for the model as a list of integers. If None, + `ONNX_EXPORT_PSEUDO_SHAPE` will be used. + model: + The model to export. If not provided, the current object will be used. + save: + If to save the model or load it. + additional_metadata: + Additional metadata to add to the ONNX model. + **kwargs: + Additional keyword arguments to pass to the `torch.onnx.export` function. + + Notes: + - A dummy input tensor is created based on the provided or default input shape. + - Dynamic axes for input and output tensors are configured where dimensions are marked `-1`. + - The model is exported with `torch.onnx.export`, with constant folding enabled and opset version set to 17. + + """ + if not self.ONNX_EXPORTABLE: + raise RuntimeError("This object cannot be exported to ONNX.") + + if input_shape is None: + input_shape = self.ONNX_DEFAULT_INPUTSHAPE + if output_shape is None: + output_shape = self.ONNX_DEFAULT_OUTPUTSHAPE + + if onnx_name is None: + onnx_name = f"Kornia-{self.__class__.__name__}.onnx" + + dummy_input = self._create_dummy_input(input_shape, pseudo_shape) + dynamic_axes = self._create_dynamic_axes(input_shape, output_shape) + + default_args: dict[str, Any] = { + "export_params": True, + "opset_version": 17, + "do_constant_folding": True, + "input_names": ["input"], + "output_names": ["output"], + "dynamic_axes": dynamic_axes, + } + default_args.update(kwargs) + + onnx_buffer = io.BytesIO() + torch.onnx.export( + model or self, # type: ignore[arg-type] + dummy_input, # type: ignore[arg-type] + onnx_buffer, # type: ignore[arg-type] + **default_args, + ) + onnx_buffer.seek(0) + onnx_model = onnx.load(onnx_buffer) # type: ignore + + if additional_metadata is None: + additional_metadata = [] + additional_metadata = copy.deepcopy(additional_metadata) + additional_metadata.extend(self.ADDITIONAL_METADATA) + onnx_model = kornia.onnx.utils.add_metadata(onnx_model, additional_metadata) + if save: + onnx.save(onnx_model, onnx_name) # type: ignore + return onnx_model + + def _create_dummy_input( + self, input_shape: list[int], pseudo_shape: Optional[list[int]] = None + ) -> Union[tuple[Any, ...], torch.Tensor]: + return torch.rand( + *[ + ((self.ONNX_EXPORT_PSEUDO_SHAPE[i] if pseudo_shape is None else pseudo_shape[i]) if dim == -1 else dim) + for i, dim in enumerate(input_shape) + ] + ) + + def _create_dynamic_axes(self, input_shape: list[int], output_shape: list[int]) -> dict[str, dict[int, str]]: + return { + "input": {i: "dim_" + str(i) for i, dim in enumerate(input_shape) if dim == -1}, + "output": {i: "dim_" + str(i) for i, dim in enumerate(output_shape) if dim == -1}, + } + + +class ONNXRuntimeMixin: + """Provide methods to manage ONNX Runtime inference sessions.""" + + def _create_session( + self, + op: onnx.ModelProto, # type:ignore + providers: Optional[list[str]] = None, + session_options: Optional[ort.InferenceSession] = None, # type:ignore + ) -> ort.InferenceSession: # type:ignore + """Create an optimized ONNXRuntime InferenceSession for the combined model. + + Args: + op: Onnx operation. + providers: + Execution providers for ONNXRuntime (e.g., ['CUDAExecutionProvider', 'CPUExecutionProvider']). + session_options: + Optional ONNXRuntime session options for session configuration and optimizations. + + Returns: + ort.InferenceSession: The ONNXRuntime session optimized for inference. + + """ + if session_options is None: + sess_options = ort.SessionOptions() # type:ignore + sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_EXTENDED # type:ignore + session = ort.InferenceSession( # type:ignore + op.SerializeToString(), + sess_options=sess_options, + providers=providers or ["CPUExecutionProvider"], + ) + return session + + def set_session(self, session: ort.InferenceSession) -> None: # type: ignore + """Set a custom ONNXRuntime InferenceSession. + + Args: + session: ort.InferenceSession + The custom ONNXRuntime session to be set for inference. + + """ + self._session = session + + def get_session(self) -> ort.InferenceSession: # type: ignore + """Get the current ONNXRuntime InferenceSession. + + Returns: + ort.InferenceSession: The current ONNXRuntime session. + + """ + return self._session + + def as_cpu(self, **kwargs: Any) -> None: + """Set the session to run on CPU.""" + self._session.set_providers(["CPUExecutionProvider"], provider_options=[{**kwargs}]) + + def as_cuda(self, device_id: int = 0, **kwargs: Any) -> None: + """Set the session to run on CUDA. + + We set the ONNX runtime session to use CUDAExecutionProvider. + For other CUDAExecutionProvider configurations, or CUDA/cuDNN/ONNX version issues, + you may refer to https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html. + + Note: + For using CUDA ONNXRuntime, you need to install `onnxruntime-gpu`. + For handling different CUDA version, you may refer to + https://github.com/microsoft/onnxruntime/issues/21769#issuecomment-2295342211. + + Args: + device_id: Select GPU to execute. + kwargs: Additional arguments for cuda. + + """ + self._session.set_providers(["CUDAExecutionProvider"], provider_options=[{"device_id": device_id, **kwargs}]) + + def as_tensorrt(self, device_id: int = 0, **kwargs: Any) -> None: + """Set the session to run on TensorRT. + + We set the ONNX runtime session to use TensorrtExecutionProvider. + For other TensorrtExecutionProvider configurations, or CUDA/cuDNN/ONNX/TensorRT version issues, + you may refer to https://onnxruntime.ai/docs/execution-providers/TensorRT-ExecutionProvider.html. + + Args: + device_id: select GPU to execute. + kwargs: additional arguments from TensorRT. + + """ + self._session.set_providers( + ["TensorrtExecutionProvider"], provider_options=[{"device_id": device_id, **kwargs}] + ) + + def as_openvino(self, device_type: str = "GPU", **kwargs: Any) -> None: + """Set the session to run on OpenVINO. + + We set the ONNX runtime session to use OpenVINOExecutionProvider. + For other OpenVINOExecutionProvider configurations, or CUDA/cuDNN/ONNX/TensorRT version issues, + you may refer to https://onnxruntime.ai/docs/execution-providers/OpenVINO-ExecutionProvider.html. + + Args: + device_type: CPU, NPU, GPU, GPU.0, GPU.1 based on the available GPUs, NPU, Any valid Hetero combination, + Any valid Multi or Auto devices combination. + kwargs: Additional arguments for OpenVINO. + + """ + self._session.set_providers( + ["OpenVINOExecutionProvider"], provider_options=[{"device_type": device_type, **kwargs}] + ) + + def __call__(self, *inputs: np.ndarray) -> list[np.ndarray]: # type:ignore + """Perform inference using the combined ONNX model. + + Args: + *inputs: Inputs to the ONNX model. The number of inputs must match the expected inputs of the session. + + Returns: + list: The outputs from the ONNX model inference. + + """ + ort_inputs = self._session.get_inputs() + ort_input_values = {ort_inputs[i].name: inputs[i] for i in range(len(ort_inputs))} + outputs = self._session.run(None, ort_input_values) + + return outputs + + +class ONNXMixin: + """Provide functionality for exporting and loading models in ONNX format.""" + + def _load_op( + self, + arg: Union[onnx.ModelProto, str], # type:ignore + cache_dir: Optional[str] = None, + ) -> onnx.ModelProto: # type:ignore + """Load an ONNX model, either from a file path or use the provided ONNX ModelProto. + + Args: + arg: Either an ONNX ModelProto object or a file path to an ONNX model. + cache_dir: Where to read onnx objects from if stored on disk. + + Returns: + onnx.ModelProto: The loaded ONNX model. + + """ + if isinstance(arg, str): + return kornia.onnx.utils.ONNXLoader.load_model(arg, cache_dir=cache_dir) + if isinstance(arg, onnx.ModelProto): # type:ignore + return arg + raise ValueError(f"Invalid argument type. Got {type(arg)}") + + def _load_ops( + self, + *args: Union[onnx.ModelProto, str], # type:ignore + cache_dir: Optional[str] = None, + ) -> list[onnx.ModelProto]: # type:ignore + """Load multiple ONNX models or operators and returns them as a list. + + Args: + *args: A variable number of ONNX models (either ONNX ModelProto objects or file paths). + For Hugging Face-hosted models, use the format 'hf://model_name'. Valid `model_name` can be found on + https://huggingface.co/kornia/ONNX_models. Or a URL to the ONNX model. + cache_dir: Where to read operations from if stored on disk. + + Returns: + list[onnx.ModelProto]: The loaded ONNX models as a list of ONNX graphs. + + """ + op_list = [] + for arg in args: + op_list.append(self._load_op(arg, cache_dir=cache_dir)) + return op_list + + def _combine( + self, + *args: list[onnx.ModelProto], # type:ignore + io_maps: Optional[list[tuple[str, str]]] = None, + ) -> onnx.ModelProto: # type:ignore + """Combine the provided ONNX models into a single ONNX graph. + + Optionally, map inputs and outputs between operators using the `io_map`. + + Args: + args: list of onnx operations. + io_maps: + A list of list of tuples representing input-output mappings for combining the models. + Example: [[(model1_output_name, model2_input_name)], [(model2_output_name, model3_input_name)]]. + + Returns: + onnx.ModelProto: The combined ONNX model as a single ONNX graph. + + """ + if len(args) == 0: + raise ValueError("No operators found.") + + combined_op = args[0] + combined_op = onnx.compose.add_prefix(combined_op, prefix=f"K{str(0).zfill(2)}-") # type:ignore + + for i, op in enumerate(args[1:]): + next_op = onnx.compose.add_prefix(op, prefix=f"K{str(i + 1).zfill(2)}-") # type:ignore + if io_maps is None: + io_map = [(f"K{str(i).zfill(2)}-output", f"K{str(i + 1).zfill(2)}-input")] + else: + io_map = [(f"K{str(i).zfill(2)}-{it[0]}", f"K{str(i + 1).zfill(2)}-{it[1]}") for it in io_maps[i]] + combined_op = onnx.compose.merge_models(combined_op, next_op, io_map=io_map) # type:ignore + + return combined_op + + def _export( + self, + op: onnx.ModelProto, # type:ignore + file_path: str, + **kwargs: Any, + ) -> None: + """Export the combined ONNX model to a file. + + Args: + op: onnx operation. + file_path: + The file path to export the combined ONNX model. + kwargs: Additional arguments to save onnx model. + + """ + onnx.save(op, file_path, **kwargs) # type:ignore + + def _add_metadata( + self, + op: onnx.ModelProto, # type:ignore + additional_metadata: Optional[list[tuple[str, str]]] = None, + ) -> onnx.ModelProto: # type:ignore + """Add metadata to the combined ONNX model. + + Args: + op: onnx operation. + additional_metadata: + A list of tuples representing additional metadata to add to the combined ONNX model. + Example: [("version", 0.1)], [("date", 20240909)]. + + """ + op = kornia.onnx.utils.add_metadata(op, additional_metadata) + return op + + def _onnx_version_conversion( + self, + op: onnx.ModelProto, # type:ignore + target_ir_version: Optional[int] = None, + target_opset_version: Optional[int] = None, + ) -> onnx.ModelProto: # type:ignore + """Automatic conversion of the model's IR/OPSET version to the given target version. + + Args: + op: onnx operation. + target_ir_version: The target IR version to convert to. + target_opset_version: The target OPSET version to convert to. + + """ + if op.ir_version != target_ir_version or op.opset_import[0].version != target_opset_version: + # Check if all ops are supported in the current IR version + model_bytes = io.BytesIO() + onnx.save_model(op, model_bytes) # type:ignore + loaded_model = onnx.load_model_from_string(model_bytes.getvalue()) # type:ignore + if target_opset_version is not None: + loaded_model = onnx.version_converter.convert_version( # type:ignore + loaded_model, target_opset_version + ) + onnx.checker.check_model(loaded_model) # type:ignore + # Set the IR version if it passed the checking + if target_ir_version is not None: + loaded_model.ir_version = target_ir_version + op = loaded_model + return op diff --git a/kornia/core/module.py b/kornia/core/module.py new file mode 100644 index 0000000..fc6ac4d --- /dev/null +++ b/kornia/core/module.py @@ -0,0 +1,179 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Any, Literal, Optional + +from torch import nn + +from .mixin.image_module import ImageModuleMixIn +from .mixin.onnx import ONNXExportMixin + + +class ImageModule(nn.Module, ImageModuleMixIn, ONNXExportMixin): + """Handles image-based operations. + + This modules accepts multiple input and output data types, provides end-to-end + visualization, file saving features. Note that this module fits the classes that + return one image tensor only. + + Note: + The additional add-on features increase the use of memories. To restore the + original behaviour, you may set `disable_features = True`. + + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._disable_features: bool = False + + @property + def disable_features(self) -> bool: + """Whether convenience I/O (input/output) helper features are disabled. + + This flag controls the extra behavior provided by :class:`ImageModuleMixIn`, + such as automatic input conversion (for example, PIL or NumPy inputs), + automatic output conversion, and output caching for visualization helpers. + + Returns: + ``True`` if the helper features are bypassed and the module behaves like + a plain :class:`torch.nn.Module` call. ``False`` if helper features are enabled. + """ + return self._disable_features + + @disable_features.setter + def disable_features(self, value: bool = True) -> None: + """Enable or disable convenience I/O (input/output) handling features. + + Args: + value: Feature toggle. + - ``True``: disable automatic type conversion and output caching, + so ``__call__`` behaves closer to a raw PyTorch module call. + - ``False``: keep helper features active. + """ + self._disable_features = value + + def __call__( + self, + *inputs: Any, + input_names_to_handle: Optional[list[Any]] = None, + output_type: Literal["pt", "numpy", "pil"] = "pt", + **kwargs: Any, + ) -> Any: + """Overwrite the __call__ function to handle various inputs. + + Args: + inputs: Inputs to operate on. + input_names_to_handle: List of input names to convert, if None, handle all inputs. + output_type: Desired output type ('pt', 'numpy', or 'pil'). + kwargs: Additional arguments. + + Returns: + Callable: Decorated function with converted input and output types. + + """ + # Wrap the forward method with the decorator + if not self._disable_features: + decorated_forward = self.convert_input_output( + input_names_to_handle=input_names_to_handle, output_type=output_type + )(super().__call__) + _output_image = decorated_forward(*inputs, **kwargs) + if output_type == "pt": + self._output_image = self._detach_tensor_to_cpu(_output_image) + else: + self._output_image = _output_image + else: + _output_image = super().__call__(*inputs, **kwargs) + return _output_image + + +class ImageSequential(nn.Sequential, ImageModuleMixIn, ONNXExportMixin): + """Handles image-based operations as a sequential module. + + This modules accepts multiple input and output data types, provides end-to-end + visualization, file saving features. Note that this module fits the classes that + return one image tensor only. + + Note: + The additional add-on features increase the use of memories. To restore the + original behaviour, you may set `disable_features = True`. + + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._disable_features: bool = False + + @property + def disable_features(self) -> bool: + """Whether convenience I/O (input/output) helper features are disabled. + + This flag controls the extra behavior provided by :class:`ImageModuleMixIn`, + such as automatic input conversion (for example, PIL or NumPy inputs), + automatic output conversion, and output caching for visualization helpers. + + Returns: + ``True`` if the helper features are bypassed and the module behaves like + a plain :class:`torch.nn.Sequential` call. ``False`` if helper features are enabled. + """ + return self._disable_features + + @disable_features.setter + def disable_features(self, value: bool = True) -> None: + """Enable or disable convenience I/O (input/output) handling features. + + Args: + value: Feature toggle. + - ``True``: disable automatic type conversion and output caching, + so ``__call__`` behaves closer to a raw PyTorch sequential call. + - ``False``: keep helper features active. + """ + self._disable_features = value + + def __call__( + self, + *inputs: Any, + input_names_to_handle: Optional[list[Any]] = None, + output_type: Literal["pt", "numpy", "pil"] = "pt", + **kwargs: Any, + ) -> Any: + """Overwrite the __call__ function to handle various inputs. + + Args: + inputs: Inputs to operate on. + input_names_to_handle: List of input names to convert, if None, handle all inputs. + output_type: Desired output type ('pt', 'numpy', or 'pil'). + kwargs: Additional arguments. + + Returns: + Callable: Decorated function with converted input and output types. + + """ + # Wrap the forward method with the decorator + if not self._disable_features: + decorated_forward = self.convert_input_output( + input_names_to_handle=input_names_to_handle, output_type=output_type + )(super().__call__) + _output_image = decorated_forward(*inputs, **kwargs) + if output_type == "pt": + self._output_image = self._detach_tensor_to_cpu(_output_image) + else: + self._output_image = _output_image + else: + _output_image = super().__call__(*inputs, **kwargs) + return _output_image diff --git a/kornia/core/ops.py b/kornia/core/ops.py new file mode 100644 index 0000000..82f3d59 --- /dev/null +++ b/kornia/core/ops.py @@ -0,0 +1,72 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch + +from kornia.core.check import KORNIA_CHECK + + +def eye_like(n: int, input: torch.Tensor, shared_memory: bool = False) -> torch.Tensor: + r"""Return a 2-D tensor with ones on the diagonal and zeros elsewhere with the same batch size as the input. + + Args: + n: the number of rows :math:`(N)`. + input: image tensor that will determine the batch size of the output matrix. + The expected shape is :math:`(B, *)`. + shared_memory: when set, all samples in the batch will share the same memory. + + Returns: + The identity matrix with the same batch size as the input :math:`(B, N, N)`. + + Notes: + When the dimension to expand is of size 1, using torch.expand(...) yields the same tensor as torch.repeat(...) + without using extra memory. Thus, when the tensor obtained by this method will be later assigned - + use this method with shared_memory=False, otherwise, prefer using it with shared_memory=True. + + """ + KORNIA_CHECK(n > 0, f"n must be positive. Got: {n}") + KORNIA_CHECK(len(input.shape) >= 1, f"input must have at least 1 dimension. Got shape: {input.shape}") + + # Use torch.eye with dtype parameter directly (available since PyTorch 2.0+) + identity = torch.eye(n, device=input.device, dtype=input.dtype) + + return identity[None].expand(input.shape[0], n, n) if shared_memory else identity[None].repeat(input.shape[0], 1, 1) + + +def vec_like(n: int, tensor: torch.Tensor, shared_memory: bool = False) -> torch.Tensor: + r"""Return a 2-D tensor with a vector containing zeros with the same batch size as the input. + + Args: + n: the number of rows :math:`(N)`. + tensor: image tensor that will determine the batch size of the output matrix. + The expected shape is :math:`(B, *)`. + shared_memory: when set, all samples in the batch will share the same memory. + + Returns: + The vector with the same batch size as the input :math:`(B, N, 1)`. + + Notes: + When the dimension to expand is of size 1, using torch.expand(...) yields the same tensor as torch.repeat(...) + without using extra memory. Thus, when the tensor obtained by this method will be later assigned - + use this method with shared_memory=False, otherwise, prefer using it with shared_memory=True. + + """ + KORNIA_CHECK(n > 0, f"n must be positive. Got: {n}") + KORNIA_CHECK(len(tensor.shape) >= 1, f"tensor must have at least 1 dimension. Got shape: {tensor.shape}") + + vec = torch.zeros(n, 1, device=tensor.device, dtype=tensor.dtype) + return vec[None].expand(tensor.shape[0], n, 1) if shared_memory else vec[None].repeat(tensor.shape[0], 1, 1) diff --git a/kornia/core/tensor_wrapper.py b/kornia/core/tensor_wrapper.py new file mode 100644 index 0000000..b21e0cb --- /dev/null +++ b/kornia/core/tensor_wrapper.py @@ -0,0 +1,282 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import collections.abc +from typing import Any, Optional + +import torch +from torch import Tensor + +from kornia.core.check import KORNIA_CHECK_IS_TENSOR + + +def _wrap(v: Any, cls: type[TensorWrapper]) -> Any: + """Wrap type. + + Args: + v: Value to wrap (tensor, list, tuple, or other). + cls: TensorWrapper class to use for wrapping. + + Returns: + Wrapped value if tensor, otherwise original value or wrapped collection. + """ + # wrap inputs if necessary + if type(v) in {tuple, list}: + return type(v)(_wrap(vi, cls) for vi in v) + + return cls(v) if isinstance(v, Tensor) else v + + +def _unwrap(v: Any) -> Any: + """Unwrap nested type. + + Args: + v: Value to unwrap (TensorWrapper, list, tuple, or other). + + Returns: + Unwrapped value (underlying tensor or original value). + """ + if type(v) in {tuple, list}: + return type(v)(_unwrap(vi) for vi in v) + + return v._data if isinstance(v, TensorWrapper) else v + + +class TensorWrapper: + """Wrapper around PyTorch tensors that tracks attribute and function usage. + + This class provides a transparent wrapper around PyTorch tensors while + tracking which attributes and functions are accessed. Useful for debugging + and understanding tensor usage patterns. + + Attributes: + _data: The underlying PyTorch tensor. + used_attrs: Set of attribute names that have been accessed. + used_calls: Set of functions that have been called. + """ + + __slots__ = ("_data", "used_attrs", "used_calls") + + def __init__(self, data: Tensor) -> None: + """Initialize TensorWrapper with a PyTorch tensor. + + Args: + data: The PyTorch tensor to wrap. If data is already a TensorWrapper, + its underlying tensor will be extracted. + + Raises: + TypeCheckError: If data is not a PyTorch tensor or TensorWrapper. + """ + # Handle case where data is already a TensorWrapper (e.g., Scalar wrapping another Scalar) + if isinstance(data, TensorWrapper): + data = data._data + KORNIA_CHECK_IS_TENSOR(data, "Expected Tensor for TensorWrapper") + object.__setattr__(self, "_data", data) + object.__setattr__(self, "used_attrs", set()) + object.__setattr__(self, "used_calls", set()) + + def unwrap(self) -> Tensor: + """Return the underlying PyTorch tensor.""" + return _unwrap(self) + + @property + def data(self) -> Tensor: + """Access the underlying tensor.""" + return self._data + + def __getstate__(self) -> dict[str, Any]: + """Support for pickle serialization.""" + return { + "_data": self._data, + "used_attrs": self.used_attrs, + "used_calls": self.used_calls, + } + + def __setstate__(self, state: dict[str, Any]) -> None: + """Support for pickle deserialization.""" + object.__setattr__(self, "_data", state["_data"]) + object.__setattr__(self, "used_attrs", state.get("used_attrs", set())) + object.__setattr__(self, "used_calls", state.get("used_calls", set())) + + def __repr__(self) -> str: + """Return string representation.""" + return f"{self.__class__.__name__}({self._data})" + + def __getattr__(self, name: str) -> Any: + """Get attribute from underlying tensor.""" + # Handle special 'data' property + if name == "data": + return self._data + + # Track attribute usage + self.used_attrs.add(name) + + # Get value from underlying tensor + val = getattr(self._data, name) + + # Wrap the result if it's a tensor + return _wrap(val, type(self)) + + def __setattr__(self, name: str, value: Any) -> None: + """Set attribute on underlying tensor.""" + # Only track non-internal attributes + if name not in self.__slots__: + self.used_attrs.add(name) + setattr(self._data, name, value) + else: + # Use object.__setattr__ for internal attributes to avoid recursion + object.__setattr__(self, name, value) + + def __setitem__(self, key: Any, value: Any) -> None: + """Set item on underlying tensor.""" + self._data[key] = value + + def __getitem__(self, key: Any) -> TensorWrapper: + """Get item from underlying tensor.""" + return _wrap(self._data[key], type(self)) + + @classmethod + def __torch_function__( + cls, + func: Any, + types: tuple[type, ...], + args: tuple[Any, ...] = (), + kwargs: Optional[dict[str, Any]] = None, + ) -> Any: + """Intercept PyTorch function calls.""" + if kwargs is None: + kwargs = {} + + # Find instances of this class in the arguments + args_of_this_cls: list[TensorWrapper] = [] + for a in args: + if isinstance(a, cls): + args_of_this_cls.append(a) + elif isinstance(a, collections.abc.Sequence) and not isinstance(a, str | bytes): + args_of_this_cls.extend(el for el in a if isinstance(el, cls)) + + # Track function usage + for a in args_of_this_cls: + a.used_calls.add(func) + + # Unwrap arguments and call the function + unwrapped_args = _unwrap(args) + unwrapped_kwargs = {k: _unwrap(v) for k, v in kwargs.items()} + + return _wrap(func(*unwrapped_args, **unwrapped_kwargs), cls) + + def __add__(self, other: Any) -> TensorWrapper: + """Add operation.""" + return self.__binary_op__(torch.add, other) + + def __radd__(self, other: Any) -> TensorWrapper: + """Right-side add operation.""" + return self.__binary_op__(torch.add, other, swap=True) + + def __mul__(self, other: Any) -> TensorWrapper: + """Multiply operation.""" + return self.__binary_op__(torch.mul, other) + + def __rmul__(self, other: Any) -> TensorWrapper: + """Right-side multiply operation.""" + return self.__binary_op__(torch.mul, other, swap=True) + + def __sub__(self, other: Any) -> TensorWrapper: + """Subtract operation.""" + return self.__binary_op__(torch.sub, other) + + def __rsub__(self, other: Any) -> TensorWrapper: + """Right-side subtract operation.""" + return self.__binary_op__(torch.sub, other, swap=True) + + def __truediv__(self, other: Any) -> TensorWrapper: + """True division operation.""" + return self.__binary_op__(torch.true_divide, other) + + def __floordiv__(self, other: Any) -> TensorWrapper: + """Floor division operation.""" + return self.__binary_op__(torch.floor_divide, other) + + def __ge__(self, other: Any) -> TensorWrapper: + """Greater than or equal comparison.""" + return self.__binary_op__(torch.ge, other) + + def __gt__(self, other: Any) -> TensorWrapper: + """Greater than comparison.""" + return self.__binary_op__(torch.gt, other) + + def __lt__(self, other: Any) -> TensorWrapper: + """Less than comparison.""" + return self.__binary_op__(torch.lt, other) + + def __le__(self, other: Any) -> TensorWrapper: + """Less than or equal comparison.""" + return self.__binary_op__(torch.le, other) + + def __eq__(self, other: object) -> TensorWrapper: + """Equality comparison.""" + return self.__binary_op__(torch.eq, other) + + def __ne__(self, other: object) -> TensorWrapper: + """Inequality comparison.""" + return self.__binary_op__(torch.ne, other) + + def __bool__(self) -> bool: + """Convert to boolean (unwrapped).""" + return bool(self._data) + + def __int__(self) -> int: + """Convert to integer (unwrapped).""" + return int(self._data) + + def __neg__(self) -> TensorWrapper: + """Negation operation.""" + return self.__unary_op__(torch.neg) + + def __len__(self) -> int: + """Return length of tensor.""" + return len(self._data) + + def __binary_op__(self, func: Any, other: Any, swap: bool = False) -> TensorWrapper: + """Helper for binary operations. + + Args: + func: The PyTorch function to call. + other: The other operand. + swap: If True, swap the order of operands (for right-side operations). + + Returns: + Wrapped result of the operation. + """ + if swap: + args = (other, self) + else: + args = (self, other) + return self.__torch_function__(func, (type(self),), args) + + def __unary_op__(self, func: Any) -> TensorWrapper: + """Helper for unary operations. + + Args: + func: The PyTorch function to call. + + Returns: + Wrapped result of the operation. + """ + return self.__torch_function__(func, (type(self),), (self,)) diff --git a/kornia/core/utils.py b/kornia/core/utils.py new file mode 100644 index 0000000..b75923e --- /dev/null +++ b/kornia/core/utils.py @@ -0,0 +1,366 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import importlib.util +import platform +import sys +from dataclasses import asdict, fields, is_dataclass +from typing import Any, Dict, List, Optional, Tuple, Type, TypeVar, Union + +import torch +from torch.linalg import inv_ex + +from kornia.core._compat import torch_version_ge +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_TYPE +from kornia.core.exceptions import DeviceError + + +def xla_is_available() -> bool: + """Return whether `torch_xla` is available in the system.""" + if importlib.util.find_spec("torch_xla") is not None: + return True + return False + + +def is_mps_tensor_safe(x: torch.Tensor) -> bool: + """Return whether tensor is on MPS device.""" + return "mps" in str(x.device) + + +def get_cuda_device_if_available(index: int = 0) -> torch.device: + """Try to get cuda device, if fail, return cpu. + + Args: + index: cuda device index + + Returns: + torch.device + + """ + if torch.cuda.is_available(): + return torch.device(f"cuda:{index}") + + return torch.device("cpu") + + +def get_mps_device_if_available() -> torch.device: + """Try to get mps device, if fail, return cpu. + + Returns: + torch.device + + """ + dev = "cpu" + if hasattr(torch.backends, "mps"): + if torch.backends.mps.is_available(): + dev = "mps" + return torch.device(dev) + + +def get_cuda_or_mps_device_if_available() -> torch.device: + """Check OS and platform and run get_cuda_device_if_available or get_mps_device_if_available. + + Returns: + torch.device + + """ + if sys.platform == "darwin" and platform.machine() == "arm64": + return get_mps_device_if_available() + else: + return get_cuda_device_if_available() + + +def _extract_device_dtype(tensor_list: List[Optional[Any]]) -> Tuple[torch.device, torch.dtype]: + """Check if all the input are in the same device (only if when they are torch.Tensor). + + If so, it would return a tuple of (device, dtype). + Default: (``torch.get_default_device()``, ``torch.get_default_dtype()``). + + Returns: + [torch.device, torch.dtype] + + """ + device, dtype = None, None + for tensor in tensor_list: + if tensor is not None: + if not isinstance(tensor, torch.Tensor): + continue + _device = tensor.device + _dtype = tensor.dtype + if device is None and dtype is None: + device = _device + dtype = _dtype + elif device != _device or dtype != _dtype: + raise DeviceError( + f"Passed values are not in the same device and dtype. " + f"Got ({device}, {dtype}) and ({_device}, {_dtype}).", + actual_devices=[device, _device], + expected_device=device, + ) + if device is None: + device = torch.get_default_device() + if dtype is None: + dtype = torch.get_default_dtype() + return (device, dtype) + + +def _normalize_to_float32_or_float64(dtype: torch.dtype) -> torch.dtype: + """Normalize dtype to float32 or float64 for operations that require full precision. + + Args: + dtype: The input dtype to normalize. + + Returns: + torch.float32 if dtype is not float32 or float64, otherwise returns the original dtype. + """ + return dtype if dtype in (torch.float32, torch.float64) else torch.float32 + + +def _inverse_3x3_closed_form(input: torch.Tensor) -> torch.Tensor: + """Closed-form inverse for batched 3x3 matrices. + + Used as an ONNX-traceable fallback to ``torch.linalg.inv``: the legacy ONNX + exporter does not lower ``aten::linalg_inv`` (as of opset 17). Computed via + the adjugate / determinant formula, which is composed entirely of basic + arithmetic ops that all standard ONNX opsets support. + + Args: + input: Tensor of shape ``(..., 3, 3)``. + + Returns: + Tensor of shape ``(..., 3, 3)`` containing the matrix inverse for each + leading-dim slice. Numerically equivalent to ``torch.linalg.inv`` for + well-conditioned matrices; behavior on singular matrices is undefined + (no explicit check, same as ``torch.linalg.inv`` itself). + """ + a = input[..., 0, 0] + b = input[..., 0, 1] + c = input[..., 0, 2] + d = input[..., 1, 0] + e = input[..., 1, 1] + f = input[..., 1, 2] + g = input[..., 2, 0] + h = input[..., 2, 1] + i = input[..., 2, 2] + + # Cofactors (signed minors). + c00 = e * i - f * h + c01 = -(d * i - f * g) + c02 = d * h - e * g + c10 = -(b * i - c * h) + c11 = a * i - c * g + c12 = -(a * h - b * g) + c20 = b * f - c * e + c21 = -(a * f - c * d) + c22 = a * e - b * d + + det = a * c00 + b * c01 + c * c02 + + # Adjugate is the transpose of the cofactor matrix. + row0 = torch.stack([c00, c10, c20], dim=-1) + row1 = torch.stack([c01, c11, c21], dim=-1) + row2 = torch.stack([c02, c12, c22], dim=-1) + adj = torch.stack([row0, row1, row2], dim=-2) + + return adj / det[..., None, None] + + +def _torch_inverse_cast(input: torch.Tensor) -> torch.Tensor: + """Make torch.inverse work with other than fp32/64. + + The function torch.inverse is only implemented for fp32/64 which makes impossible to be used by fp16 or others. What + this function does, is cast input data type to fp32, apply torch.inverse, and cast back to the input dtype. + + During tracing (``torch.jit.trace`` and legacy ``torch.onnx.export``) on 3x3 + matrices, falls back to a closed-form inverse so the resulting graph does not + include ``aten::linalg_inv`` (unsupported by the legacy ONNX exporter at + opset 20). ``torch.jit.is_tracing()`` is JIT-script-safe (unlike + ``torch.onnx.is_in_onnx_export``, which contains an ``import`` statement). + """ + KORNIA_CHECK_IS_TENSOR(input, "Input must be torch.Tensor") + dtype = _normalize_to_float32_or_float64(input.dtype) + if torch.jit.is_tracing() and input.shape[-2:] == (3, 3): + return _inverse_3x3_closed_form(input.to(dtype)).to(input.dtype) + return torch.linalg.inv(input.to(dtype)).to(input.dtype) + + +def _torch_histc_cast(input: torch.Tensor, bins: int, min: Union[float, bool], max: Union[float, bool]) -> torch.Tensor: + """Make torch.histc work with other than fp32/64. + + The function torch.histc is only implemented for fp32/64 which makes impossible to be used by fp16 or others. What + this function does, is cast input data type to fp32, apply torch.inverse, and cast back to the input dtype. + """ + KORNIA_CHECK_IS_TENSOR(input, "Input must be torch.Tensor") + dtype = _normalize_to_float32_or_float64(input.dtype) + return torch.histc(input.to(dtype), bins, min, max).to(input.dtype) + + +def _torch_svd_cast(input: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Make torch.svd work with other than fp32/64. + + The function torch.svd is only implemented for fp32/64 which makes + impossible to be used by fp16 or others. What this function does, is cast + input data type to fp32, apply torch.svd, and cast back to the input dtype. + + NOTE: in torch 1.8.1 this function is recommended to use as torch.linalg.svd + + For numerical stability, fp32 inputs are promoted to fp64 (except on MPS where fp64 is unsupported). + """ + if is_mps_tensor_safe(input): + dtype = torch.float32 + elif input.dtype == torch.float32: + dtype = torch.float64 + else: + dtype = _normalize_to_float32_or_float64(input.dtype) + + out1, out2, out3H = torch.linalg.svd(input.to(dtype)) + # Since kornia requires torch>=2.0.0, we can always use .mH + out3 = out3H.mH + return (out1.to(input.dtype), out2.to(input.dtype), out3.to(input.dtype)) + + +def _torch_linalg_svdvals(input: torch.Tensor) -> torch.Tensor: + """Make torch.linalg.svdvals work with other than fp32/64. + + The function torch.svd is only implemented for fp32/64 which makes + impossible to be used by fp16 or others. What this function does, is cast + input data type to fp32, apply torch.svd, and cast back to the input dtype. + + NOTE: in torch 1.8.1 this function is recommended to use as torch.linalg.svd + """ + KORNIA_CHECK_IS_TENSOR(input, "Input must be torch.Tensor") + dtype = _normalize_to_float32_or_float64(input.dtype) + + # Since kornia requires torch>=2.0.0, we can always use torch.linalg.svdvals + out = torch.linalg.svdvals(input.to(dtype)) + return out.to(input.dtype) + + +def _torch_solve_cast(A: torch.Tensor, B: torch.Tensor) -> torch.Tensor: + """Make torch.solve work with other than fp32/64. + + For stable operation, the input matrices should be cast to fp64, and the output will + be cast back to the input dtype. However, fp64 is not yet supported on MPS. + + This function is actively used in: + - kornia.geometry.transform.imgwarp + - kornia.geometry.transform.thin_plate_spline + - kornia.geometry.epipolar.essential + """ + if is_mps_tensor_safe(A): + dtype = torch.float32 + else: + dtype = torch.float64 + + out = torch.linalg.solve(A.to(dtype), B.to(dtype)) + + # cast back to the input dtype + return out.to(A.dtype) + + +def safe_solve_with_mask(B: torch.Tensor, A: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + r"""Solves the system of equations. + + Avoids crashing because of singular matrix input and outputs the mask of valid solution. + """ + # Based on https://github.com/pytorch/pytorch/issues/31546#issuecomment-694135622 + KORNIA_CHECK_IS_TENSOR(B, "B must be torch.Tensor") + dtype: torch.dtype = B.dtype + if dtype not in (torch.float32, torch.float64): + dtype = torch.float32 + + # Since kornia requires torch>=2.0.0, we can always use torch.linalg.lu_factor_ex and torch.linalg.lu_solve + A_LU, pivots, info = torch.linalg.lu_factor_ex(A.to(dtype)) + + valid_mask: torch.Tensor = info == 0 + n_dim_B = len(B.shape) + n_dim_A = len(A.shape) + if n_dim_A - n_dim_B == 1: + B = B.unsqueeze(-1) + + X = torch.linalg.lu_solve(A_LU, pivots, B.to(dtype)) + + return X.to(B.dtype), A_LU.to(A.dtype), valid_mask + + +def safe_inverse_with_mask(A: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Perform inverse. + + Avoids crashing because of non-invertable matrix input and outputs the mask of valid solution. + """ + KORNIA_CHECK_IS_TENSOR(A, "A must be torch.Tensor") + + dtype_original = A.dtype + dtype = _normalize_to_float32_or_float64(dtype_original) + + inverse, info = inv_ex(A.to(dtype)) + mask = info == 0 + return inverse.to(dtype_original), mask + + +def is_autocast_enabled(both: bool = True) -> bool: + """Check if torch autocast is enabled. + + Args: + both: if True will consider autocast region for both types of devices + + Returns: + Return a Bool, + will always return False for a torch without support, otherwise will be: if both is True + `torch.is_autocast_enabled() or torch.is_autocast_enabled('cpu')`. If both is False will return just + `torch.is_autocast_enabled()`. + + """ + # Since kornia requires torch>=2.0.0, autocast is always available + if both: + if torch_version_ge(2, 4): + return torch.is_autocast_enabled() or torch.is_autocast_enabled("cpu") + else: + return torch.is_autocast_enabled() or torch.is_autocast_cpu_enabled() + + return torch.is_autocast_enabled() + + +def dataclass_to_dict(obj: Any) -> Any: + """Recursively convert dataclass instances to dictionaries.""" + if is_dataclass(obj) and not isinstance(obj, type): + return {key: dataclass_to_dict(value) for key, value in asdict(obj).items()} + elif isinstance(obj, list | tuple): + return type(obj)(dataclass_to_dict(item) for item in obj) + elif isinstance(obj, dict): + return {key: dataclass_to_dict(value) for key, value in obj.items()} + else: + return obj + + +T = TypeVar("T") + + +def dict_to_dataclass(dict_obj: Dict[str, Any], dataclass_type: Type[T]) -> T: + """Recursively convert dictionaries to dataclass instances.""" + KORNIA_CHECK_TYPE(dict_obj, dict, "Input conf must be dict") + KORNIA_CHECK(is_dataclass(dataclass_type), "dataclass_type must be a dataclass") + field_types: dict[str, Any] = {f.name: f.type for f in fields(dataclass_type)} + constructor_args = {} + for key, value in dict_obj.items(): + if key in field_types and is_dataclass(field_types[key]): + constructor_args[key] = dict_to_dataclass(value, field_types[key]) + else: + constructor_args[key] = value + # TODO: remove type ignore when https://github.com/python/mypy/issues/14941 be andressed + return dataclass_type(**constructor_args) diff --git a/kornia/enhance/__init__.py b/kornia/enhance/__init__.py new file mode 100644 index 0000000..a249277 --- /dev/null +++ b/kornia/enhance/__init__.py @@ -0,0 +1,122 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Enhance — Image enhancement operations for Kornia. + +This subpackage provides modules for brightness, contrast, and other image enhancements. +""" + +from .adjust import ( + AdjustBrightness, + AdjustBrightnessAccumulative, + AdjustContrast, + AdjustContrastWithMeanSubtraction, + AdjustGamma, + AdjustHue, + AdjustLog, + AdjustSaturation, + AdjustSaturationWithGraySubtraction, + AdjustSigmoid, + Invert, + adjust_brightness, + adjust_brightness_accumulative, + adjust_contrast, + adjust_contrast_with_mean_subtraction, + adjust_gamma, + adjust_hue, + adjust_hue_raw, + adjust_log, + adjust_saturation, + adjust_saturation_raw, + adjust_saturation_with_gray_subtraction, + adjust_sigmoid, + equalize, + equalize3d, + invert, + posterize, + sharpness, + solarize, +) +from .core import AddWeighted, add_weighted +from .equalization import equalize_clahe +from .histogram import histogram, histogram2d, image_histogram2d +from .integral import IntegralImage, IntegralTensor, integral_image, integral_tensor +from .jpeg import JPEGCodecDifferentiable, jpeg_codec_differentiable +from .normalize import Denormalize, Normalize, denormalize, normalize, normalize_min_max +from .rescale import Rescale +from .shift_rgb import shift_rgb +from .threshold import Threshold, ThresholdType, threshold +from .zca import ZCAWhitening, linear_transform, zca_mean, zca_whiten + +__all__ = [ + "AddWeighted", + "AdjustBrightness", + "AdjustBrightnessAccumulative", + "AdjustContrast", + "AdjustContrastWithMeanSubtraction", + "AdjustGamma", + "AdjustHue", + "AdjustLog", + "AdjustSaturation", + "AdjustSaturationWithGraySubtraction", + "AdjustSigmoid", + "Denormalize", + "IntegralImage", + "IntegralTensor", + "Invert", + "JPEGCodecDifferentiable", + "JPEGCodecDifferentiable", + "Normalize", + "Rescale", + "Threshold", + "ThresholdType", + "ZCAWhitening", + "add_weighted", + "adjust_brightness", + "adjust_brightness_accumulative", + "adjust_contrast", + "adjust_contrast_with_mean_subtraction", + "adjust_gamma", + "adjust_hue", + "adjust_hue_raw", + "adjust_log", + "adjust_saturation", + "adjust_saturation_raw", + "adjust_saturation_with_gray_subtraction", + "adjust_sigmoid", + "denormalize", + "equalize", + "equalize3d", + "equalize_clahe", + "histogram", + "histogram2d", + "image_histogram2d", + "integral_image", + "integral_tensor", + "invert", + "jpeg_codec_differentiable", + "linear_transform", + "normalize", + "normalize_min_max", + "posterize", + "sharpness", + "shift_rgb", + "solarize", + "threshold", + "zca_mean", + "zca_whiten", +] diff --git a/kornia/enhance/adjust.py b/kornia/enhance/adjust.py new file mode 100644 index 0000000..8d68023 --- /dev/null +++ b/kornia/enhance/adjust.py @@ -0,0 +1,1598 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from math import pi +from typing import ClassVar, Optional, Union + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.color import hsv_to_rgb, rgb_to_grayscale, rgb_to_hsv +from kornia.core.check import ( + KORNIA_CHECK, + KORNIA_CHECK_IS_COLOR_OR_GRAY, + KORNIA_CHECK_IS_TENSOR, +) +from kornia.core.utils import _torch_histc_cast +from kornia.image.utils import perform_keep_shape_image, perform_keep_shape_video + + +def adjust_saturation_raw(image: torch.Tensor, factor: Union[float, torch.Tensor]) -> torch.Tensor: + r"""Adjust color saturation of an image. + + Expecting image to be in hsv format already. + """ + KORNIA_CHECK_IS_TENSOR(image, "Expected shape (*, H, W)") + KORNIA_CHECK(isinstance(factor, (float, torch.Tensor)), "Factor should be float or torch.Tensor.") + + if isinstance(factor, float): + # TODO: figure out how to create later a torch.Tensor without importing torch + factor = torch.as_tensor(factor, device=image.device, dtype=image.dtype) + elif isinstance(factor, torch.Tensor): + factor = factor.to(image.device, image.dtype) + + # make factor broadcastable + while len(factor.shape) != len(image.shape): + factor = factor[..., None] + + # unpack the hsv values + h, s, v = torch.chunk(image, chunks=3, dim=-3) + + # transform the hue value and appl module + s_out: torch.Tensor = torch.clamp(s * factor, min=0, max=1) + + # pack back back the corrected hue + out: torch.Tensor = torch.cat([h, s_out, v], dim=-3) + + return out + + +def adjust_saturation_with_gray_subtraction(image: torch.Tensor, factor: Union[float, torch.Tensor]) -> torch.Tensor: + r"""Adjust color saturation of an image by blending the image with its grayscaled version. + + The image is expected to be an RGB image or a gray image in the range of [0, 1]. + If it is an RGB image, returns blending of the image with its grayscaled version. + If it is a gray image, returns the image. + + .. note:: + this is just a convenience function to have compatibility with Pil + + Args: + image: Image/torch.Tensor to be adjusted in the shape of :math:`(*, 3, H, W)`. + factor: How much to adjust the saturation. 0 will give a black + and white image, 1 will give the original image while 2 will enhance the saturation by a factor of 2. + + Return: + Adjusted image in the shape of :math:`(*, 3, H, W)`. + + Example: + >>> x = torch.ones(1, 3, 3, 3) + >>> adjust_saturation_with_gray_subtraction(x, 2.).shape + torch.Size([1, 3, 3, 3]) + + >>> x = torch.ones(2, 3, 3, 3) + >>> y = torch.tensor([1., 2.]) + >>> adjust_saturation_with_gray_subtraction(x, y).shape + torch.Size([2, 3, 3, 3]) + + """ + KORNIA_CHECK_IS_TENSOR(image, "Expected shape (*, H, W)") + KORNIA_CHECK(isinstance(factor, (float, torch.Tensor)), "Factor should be float or torch.Tensor.") + KORNIA_CHECK_IS_COLOR_OR_GRAY(image, "Image should be an RGB or gray image") + + if image.shape[-3] == 1: + return image + + if isinstance(factor, float): + # TODO: figure out how to create later a torch.Tensor without importing torch + factor = torch.as_tensor(factor, device=image.device, dtype=image.dtype) + elif isinstance(factor, torch.Tensor): + factor = factor.to(image.device, image.dtype) + + # make factor broadcastable + while len(factor.shape) != len(image.shape): + factor = factor[..., None] + + x_other: torch.Tensor = rgb_to_grayscale(image) + + # blend the image with the grayscaled image + x_adjusted: torch.Tensor = (1 - factor) * x_other + factor * image + + # clamp the output + out: torch.Tensor = torch.clamp(x_adjusted, 0.0, 1.0) + + return out + + +def adjust_saturation(image: torch.Tensor, factor: Union[float, torch.Tensor]) -> torch.Tensor: + r"""Adjust color saturation of an image. + + .. image:: _static/img/adjust_saturation.png + + The image is expected to be an RGB image in the range of [0, 1]. + + Args: + image: Image/torch.Tensor to be adjusted in the shape of :math:`(*, 3, H, W)`. + factor: How much to adjust the saturation. 0 will give a black + and white image, 1 will give the original image while 2 will enhance the saturation by a factor of 2. + saturation_mode: The mode to adjust saturation. + + Return: + Adjusted image in the shape of :math:`(*, 3, H, W)`. + + .. note:: + See a working example `here `__. + + Example: + >>> x = torch.ones(1, 3, 3, 3) + >>> adjust_saturation(x, 2.).shape + torch.Size([1, 3, 3, 3]) + + >>> x = torch.ones(2, 3, 3, 3) + >>> y = torch.tensor([1., 2.]) + >>> adjust_saturation(x, y).shape + torch.Size([2, 3, 3, 3]) + + """ + # convert the rgb image to hsv + x_hsv: torch.Tensor = rgb_to_hsv(image) + + # perform the conversion + x_adjusted: torch.Tensor = adjust_saturation_raw(x_hsv, factor) + + # convert back to rgb + out: torch.Tensor = hsv_to_rgb(x_adjusted) + + return out + + +def adjust_hue_raw(image: torch.Tensor, factor: Union[float, torch.Tensor]) -> torch.Tensor: + r"""Adjust hue of an image. + + Expecting image to be in hsv format already. + """ + KORNIA_CHECK_IS_TENSOR(image, "Expected shape (*, H, W)") + KORNIA_CHECK( + isinstance(factor, (float, torch.Tensor)), + f"The factor should be a float number or torch.Tensor in the range between [-PI, PI]. Got {type(factor)}", + ) + + if isinstance(factor, float): + factor = torch.as_tensor(factor) + + factor = factor.to(image.device, image.dtype) + + # make factor broadcastable + while len(factor.shape) != len(image.shape): + factor = factor[..., None] + + # unpack the hsv values + h, s, v = torch.chunk(image, chunks=3, dim=-3) + + # transform the hue value and appl module + divisor: float = 2 * pi + h_out: torch.Tensor = torch.fmod(h + factor, divisor) + + # pack back back the corrected hue + out: torch.Tensor = torch.cat([h_out, s, v], dim=-3) + + return out + + +def adjust_hue(image: torch.Tensor, factor: Union[float, torch.Tensor]) -> torch.Tensor: + r"""Adjust hue of an image. + + .. image:: _static/img/adjust_hue.png + + The image is expected to be an RGB image in the range of [0, 1]. + + Args: + image: Image to be adjusted in the shape of :math:`(*, 3, H, W)`. + factor: How much to shift the hue channel. Should be in [-PI, PI]. PI + and -PI give complete reversal of hue channel in HSV space in positive and negative + direction respectively. 0 means no shift. Therefore, both -PI and PI will give an + image with complementary colors while 0 gives the original image. + + Return: + Adjusted image in the shape of :math:`(*, 3, H, W)`. + + .. note:: + See a working example `here `__. + + Example: + >>> x = torch.ones(1, 3, 2, 2) + >>> adjust_hue(x, 3.141516).shape + torch.Size([1, 3, 2, 2]) + + >>> x = torch.ones(2, 3, 3, 3) + >>> y = torch.ones(2) * 3.141516 + >>> adjust_hue(x, y).shape + torch.Size([2, 3, 3, 3]) + + """ + # convert the rgb image to hsv + x_hsv: torch.Tensor = rgb_to_hsv(image) + + # perform the conversion + x_adjusted: torch.Tensor = adjust_hue_raw(x_hsv, factor) + + # convert back to rgb + out: torch.Tensor = hsv_to_rgb(x_adjusted) + + return out + + +def adjust_gamma( + input: torch.Tensor, gamma: Union[float, torch.Tensor], gain: Union[float, torch.Tensor] = 1.0 +) -> torch.Tensor: + r"""Perform gamma correction on an image. + + .. image:: _static/img/adjust_contrast.png + + The input image is expected to be in the range of [0, 1]. + + Args: + input: Image to be adjusted in the shape of :math:`(*, H, W)`. + gamma: Non negative real number, same as y\gammay in the equation. + gamma larger than 1 make the shadows darker, while gamma smaller than 1 make + dark regions lighter. + gain: The constant multiplier. + + Return: + Adjusted image in the shape of :math:`(*, H, W)`. + + .. note:: + See a working example `here `__. + + Example: + >>> x = torch.ones(1, 1, 2, 2) + >>> adjust_gamma(x, 1.0, 2.0) + tensor([[[[1., 1.], + [1., 1.]]]]) + + >>> x = torch.ones(2, 5, 3, 3) + >>> y1 = torch.ones(2) * 1.0 + >>> y2 = torch.ones(2) * 2.0 + >>> adjust_gamma(x, y1, y2).shape + torch.Size([2, 5, 3, 3]) + + """ + if not isinstance(input, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(input)}") + + if not isinstance(gamma, (float, torch.Tensor)): + raise TypeError(f"The gamma should be a positive float or torch.Tensor. Got {type(gamma)}") + + if not isinstance(gain, (float, torch.Tensor)): + raise TypeError(f"The gain should be a positive float or torch.Tensor. Got {type(gain)}") + + if isinstance(gamma, float): + gamma = torch.Tensor([gamma]) + + if isinstance(gain, float): + gain = torch.Tensor([gain]) + + gamma = gamma.to(input.device).to(input.dtype) + gain = gain.to(input.device).to(input.dtype) + + if (gamma < 0.0).any(): + raise ValueError(f"Gamma must be non-negative. Got {gamma}") + + if (gain < 0.0).any(): + raise ValueError(f"Gain must be non-negative. Got {gain}") + + for _ in range(len(input.shape) - len(gamma.shape)): + gamma = torch.unsqueeze(gamma, dim=-1) + + for _ in range(len(input.shape) - len(gain.shape)): + gain = torch.unsqueeze(gain, dim=-1) + + # Apply the gamma correction + x_adjust: torch.Tensor = gain * torch.pow(input, gamma) + + # Truncate between pixel values + out: torch.Tensor = torch.clamp(x_adjust, 0.0, 1.0) + + return out + + +def adjust_contrast(image: torch.Tensor, factor: Union[float, torch.Tensor], clip_output: bool = True) -> torch.Tensor: + r"""Adjust the contrast of an image torch.Tensor. + + .. image:: _static/img/adjust_contrast.png + + This implementation follows Szeliski's book convention, where contrast is defined as + a `multiplicative` operation directly to raw pixel values. Beware that other frameworks + might use different conventions which can be difficult to reproduce exact results. + + The input image and factor is expected to be in the range of [0, 1]. + + .. tip:: + This is not the preferred way to adjust the contrast of an image. Ideally one must + implement :func:`kornia.enhance.adjust_gamma`. More details in the following link: + https://scikit-image.org/docs/dev/auto_examples/color_exposure/plot_log_gamma.html#sphx-glr-auto-examples-color-exposure-plot-log-gamma-py + + Args: + image: Image to be adjusted in the shape of :math:`(*, H, W)`. + factor: Contrast adjust factor per element + in the batch. 0 generates a completely black image, 1 does not modify + the input image while any other non-negative number modify the + brightness by this factor. + clip_output: whether to clip the output image with range of [0, 1]. + + Return: + Adjusted image in the shape of :math:`(*, H, W)`. + + .. note:: + See a working example `here `__. + + Example: + >>> import torch + >>> x = torch.ones(1, 1, 2, 2) + >>> adjust_contrast(x, 0.5) + tensor([[[[0.5000, 0.5000], + [0.5000, 0.5000]]]]) + + >>> x = torch.ones(2, 5, 3, 3) + >>> y = torch.tensor([0.65, 0.50]) + >>> adjust_contrast(x, y).shape + torch.Size([2, 5, 3, 3]) + + """ + KORNIA_CHECK_IS_TENSOR(image, "Expected shape (*, H, W)") + KORNIA_CHECK(isinstance(factor, (float, torch.Tensor)), "Factor should be float or torch.Tensor.") + + if isinstance(factor, float): + # TODO: figure out how to create later a torch.Tensor without importing torch + factor = torch.as_tensor(factor, device=image.device, dtype=image.dtype) + elif isinstance(factor, torch.Tensor): + factor = factor.to(image.device, image.dtype) + + # make factor broadcastable + while len(factor.shape) != len(image.shape): + factor = factor[..., None] + + KORNIA_CHECK(any(factor >= 0), "Contrast factor must be positive.") + + # Apply contrast factor to each channel + img_adjust: torch.Tensor = image * factor + + # Truncate between pixel values + if clip_output: + img_adjust = img_adjust.clamp(min=0.0, max=1.0) + + return img_adjust + + +def adjust_contrast_with_mean_subtraction(image: torch.Tensor, factor: Union[float, torch.Tensor]) -> torch.Tensor: + r"""Adjust the contrast of an image torch.Tensor by subtracting the mean over channels. + + .. note:: + this is just a convenience function to have compatibility with Pil. For exact + definition of image contrast adjustment consider using :func:`kornia.enhance.adjust_gamma`. + + Args: + image: Image to be adjusted in the shape of :math:`(*, H, W)`. + factor: Contrast adjust factor per element + in the batch. 0 generates a completely black image, 1 does not modify + the input image while any other non-negative number modify the + brightness by this factor. + + Return: + Adjusted image in the shape of :math:`(*, H, W)`. + + Example: + >>> import torch + >>> x = torch.ones(1, 1, 2, 2) + >>> adjust_contrast_with_mean_subtraction(x, 0.5) + tensor([[[[1., 1.], + [1., 1.]]]]) + + >>> x = torch.ones(2, 5, 3, 3) + >>> y = torch.tensor([0.65, 0.50]) + >>> adjust_contrast_with_mean_subtraction(x, y).shape + torch.Size([2, 5, 3, 3]) + + """ + KORNIA_CHECK_IS_TENSOR(image, "Expected shape (*, H, W)") + KORNIA_CHECK(isinstance(factor, (float, torch.Tensor)), "Factor should be float or torch.Tensor.") + + if isinstance(factor, float): + # TODO: figure out how to create later a torch.Tensor without importing torch + factor = torch.as_tensor(factor, device=image.device, dtype=image.dtype) + elif isinstance(factor, torch.Tensor): + factor = factor.to(image.device, image.dtype) + + # make factor broadcastable + while len(factor.shape) != len(image.shape): + factor = factor[..., None] + + # KORNIA_CHECK(any(factor >= 0), "Contrast factor must be positive.") + + if image.shape[-3] == 3: + img_mean = rgb_to_grayscale(image).mean((-2, -1), True) + else: + img_mean = image.mean() + + # Apply contrast factor subtracting the mean + img_adjust: torch.Tensor = image * factor + img_mean * (1 - factor) + + img_adjust = img_adjust.clamp(min=0.0, max=1.0) + + return img_adjust + + +def adjust_brightness( + image: torch.Tensor, factor: Union[float, torch.Tensor], clip_output: bool = True +) -> torch.Tensor: + r"""Adjust the brightness of an image torch.Tensor. + + .. image:: _static/img/adjust_brightness.png + + This implementation follows Szeliski's book convention, where brightness is defined as + an `additive` operation directly to raw pixel and shift its values according the applied + factor and range of the image values. Beware that other framework might use different + conventions which can be difficult to reproduce exact results. + + The input image and factor is expected to be in the range of [0, 1]. + + .. tip:: + By applying a large factor might prouce clipping or loss of image detail. We recommenda to + apply small factors to avoid the mentioned issues. Ideally one must implement the adjustment + of image intensity with other techniques suchs as :func:`kornia.enhance.adjust_gamma`. More + details in the following link: + https://scikit-image.org/docs/dev/auto_examples/color_exposure/plot_log_gamma.html#sphx-glr-auto-examples-color-exposure-plot-log-gamma-py + + Args: + image: Image to be adjusted in the shape of :math:`(*, H, W)`. + factor: Brightness adjust factor per element in the batch. It's recommended to + bound the factor by [0, 1]. 0 does not modify the input image while any other + number modify the brightness. + clip_output: Whether to clip output to be in [0,1]. + + Return: + Adjusted torch.Tensor in the shape of :math:`(*, H, W)`. + + .. note:: + See a working example `here `__. + + Example: + >>> x = torch.ones(1, 1, 2, 2) + >>> adjust_brightness(x, 1.) + tensor([[[[1., 1.], + [1., 1.]]]]) + + >>> x = torch.ones(2, 5, 3, 3) + >>> y = torch.tensor([0.25, 0.50]) + >>> adjust_brightness(x, y).shape + torch.Size([2, 5, 3, 3]) + + """ + KORNIA_CHECK_IS_TENSOR(image, "Expected shape (*, H, W)") + KORNIA_CHECK(isinstance(factor, (float, torch.Tensor)), "Factor should be float or torch.Tensor.") + + # convert factor to a torch.Tensor + if isinstance(factor, float): + # TODO: figure out how to create later a torch.Tensor without importing torch + factor = torch.as_tensor(factor, device=image.device, dtype=image.dtype) + elif isinstance(factor, torch.Tensor): + factor = factor.to(image.device, image.dtype) + + # make factor broadcastable + while len(factor.shape) != len(image.shape): + factor = factor[..., None] + + # shift pixel values + img_adjust: torch.Tensor = image + factor + + # truncate between pixel values + if clip_output: + img_adjust = img_adjust.clamp(min=0.0, max=1.0) + + return img_adjust + + +def adjust_brightness_accumulative( + image: torch.Tensor, factor: Union[float, torch.Tensor], clip_output: bool = True +) -> torch.Tensor: + r"""Adjust the brightness accumulatively of an image torch.Tensor. + + This implementation follows PIL convention. + + The input image and factor is expected to be in the range of [0, 1]. + + Args: + image: Image to be adjusted in the shape of :math:`(*, H, W)`. + factor: Brightness adjust factor per element in the batch. It's recommended to + bound the factor by [0, 1]. 0 does not modify the input image while any other + number modify the brightness. + clip_output: Whether to clip output to be in [0,1]. + + Return: + Adjusted torch.Tensor in the shape of :math:`(*, H, W)`. + + Example: + >>> x = torch.ones(1, 1, 2, 2) + >>> adjust_brightness_accumulative(x, 1.) + tensor([[[[1., 1.], + [1., 1.]]]]) + + >>> x = torch.ones(2, 5, 3, 3) + >>> y = torch.tensor([0.25, 0.50]) + >>> adjust_brightness_accumulative(x, y).shape + torch.Size([2, 5, 3, 3]) + + """ + KORNIA_CHECK_IS_TENSOR(image, "Expected shape (*, H, W)") + KORNIA_CHECK(isinstance(factor, (float, torch.Tensor)), "Factor should be float or torch.Tensor.") + + # convert factor to a torch.Tensor + if isinstance(factor, float): + # TODO: figure out how to create later a torch.Tensor without importing torch + factor = torch.as_tensor(factor, device=image.device, dtype=image.dtype) + elif isinstance(factor, torch.Tensor): + factor = factor.to(image.device, image.dtype) + + # make factor broadcastable + while len(factor.shape) != len(image.shape): + factor = factor[..., None] + + # shift pixel values + img_adjust: torch.Tensor = image * factor + + # truncate between pixel values + if clip_output: + img_adjust = img_adjust.clamp(min=0.0, max=1.0) + + return img_adjust + + +def adjust_sigmoid(image: torch.Tensor, cutoff: float = 0.5, gain: float = 10, inv: bool = False) -> torch.Tensor: + """Adjust sigmoid correction on the input image torch.Tensor. + + The input image is expected to be in the range of [0, 1]. + + Reference: + [1]: Gustav J. Braun, "Image Lightness Rescaling Using Sigmoidal Contrast Enhancement Functions", + http://markfairchild.org/PDFs/PAP07.pdf + + Args: + image: Image to be adjusted in the shape of :math:`(*, H, W)`. + cutoff: The cutoff of sigmoid function. + gain: The multiplier of sigmoid function. + inv: If is set to True the function will return the inverse sigmoid correction. + + Returns: + Adjusted torch.Tensor in the shape of :math:`(*, H, W)`. + + Example: + >>> x = torch.ones(1, 1, 2, 2) + >>> adjust_sigmoid(x, gain=0) + tensor([[[[0.5000, 0.5000], + [0.5000, 0.5000]]]]) + + """ + KORNIA_CHECK_IS_TENSOR(image, "Expected shape (*, H, W)") + + if inv: + img_adjust = 1 - 1 / (1 + (gain * (cutoff - image)).exp()) + else: + img_adjust = 1 / (1 + (gain * (cutoff - image)).exp()) + return img_adjust + + +def adjust_log(image: torch.Tensor, gain: float = 1, inv: bool = False, clip_output: bool = True) -> torch.Tensor: + """Adjust log correction on the input image torch.Tensor. + + The input image is expected to be in the range of [0, 1]. + + Reference: + [1]: http://www.ece.ucsb.edu/Faculty/Manjunath/courses/ece178W03/EnhancePart1.pdf + + Args: + image: Image to be adjusted in the shape of :math:`(*, H, W)`. + gain: The multiplier of logarithmic function. + inv: If is set to True the function will return the inverse logarithmic correction. + clip_output: Whether to clip the output image with range of [0, 1]. + + Returns: + Adjusted torch.Tensor in the shape of :math:`(*, H, W)`. + + Example: + >>> x = torch.zeros(1, 1, 2, 2) + >>> adjust_log(x, inv=True) + tensor([[[[0., 0.], + [0., 0.]]]]) + + """ + KORNIA_CHECK_IS_TENSOR(image, "Expected shape (*, H, W)") + + if inv: + img_adjust = (2**image - 1) * gain + else: + img_adjust = (1 + image).log2() * gain + + # truncate between pixel values + if clip_output: + img_adjust = img_adjust.clamp(min=0.0, max=1.0) + + return img_adjust + + +def _solarize(input: torch.Tensor, thresholds: Union[float, torch.Tensor] = 0.5) -> torch.Tensor: + r"""For each pixel in the image, select the pixel if the value is less than the threshold. + + Otherwise, subtract 1.0 from the pixel. + + Args: + input: image or batched images to solarize. + thresholds: solarize thresholds. + If int or one element torch.Tensor, input will be solarized across the whole batch. + If 1-d torch.Tensor, input will be solarized element-wise, len(thresholds) == len(input). + + Returns: + Solarized images. + + """ + if not isinstance(input, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(input)}") + + if not isinstance(thresholds, (float, torch.Tensor)): + raise TypeError(f"The factor should be either a float or torch.Tensor. Got {type(thresholds)}") + + if isinstance(thresholds, torch.Tensor) and len(thresholds.shape) != 0: + if not (input.size(0) == len(thresholds) and len(thresholds.shape) == 1): + raise AssertionError(f"thresholds must be a 1-d vector of shape ({input.size(0)},). Got {thresholds}") + # TODO: I am not happy about this line, but no easy to do batch-wise operation + thresholds = thresholds.to(input.device).to(input.dtype) + thresholds = torch.stack([x.expand(*input.shape[-3:]) for x in thresholds]) + + return torch.where(input < thresholds, input, 1.0 - input) + + +def solarize( + input: torch.Tensor, + thresholds: Union[float, torch.Tensor] = 0.5, + additions: Optional[Union[float, torch.Tensor]] = None, +) -> torch.Tensor: + r"""For each pixel in the image less than threshold. + + .. image:: _static/img/solarize.png + + We add 'addition' amount to it and then clip the pixel value to be between 0 and 1.0. + The value of 'addition' is between -0.5 and 0.5. + + Args: + input: image torch.Tensor with shapes like :math:`(*, C, H, W)` to solarize. + thresholds: solarize thresholds. + If int or one element torch.Tensor, input will be solarized across the whole batch. + If 1-d torch.Tensor, input will be solarized element-wise, len(thresholds) == len(input). + additions: between -0.5 and 0.5. + If None, no addition will be performed. + If int or one element torch.Tensor, same addition will be added across the whole batch. + If 1-d torch.Tensor, additions will be added element-wisely, len(additions) == len(input). + + Returns: + The solarized images with shape :math:`(*, C, H, W)`. + + Example: + >>> x = torch.rand(1, 4, 3, 3) + >>> out = solarize(x, thresholds=0.5, additions=0.) + >>> out.shape + torch.Size([1, 4, 3, 3]) + + >>> x = torch.rand(2, 4, 3, 3) + >>> thresholds = torch.tensor([0.8, 0.5]) + >>> additions = torch.tensor([-0.25, 0.25]) + >>> solarize(x, thresholds, additions).shape + torch.Size([2, 4, 3, 3]) + + """ + if not isinstance(input, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(input)}") + + if not isinstance(thresholds, (float, torch.Tensor)): + raise TypeError(f"The factor should be either a float or torch.Tensor. Got {type(thresholds)}") + + if isinstance(thresholds, float): + thresholds = torch.as_tensor(thresholds) + + if additions is not None: + if not isinstance(additions, (float, torch.Tensor)): + raise TypeError(f"The factor should be either a float or torch.Tensor. Got {type(additions)}") + + if isinstance(additions, float): + additions = torch.as_tensor(additions) + + if not torch.all((additions < 0.5) * (additions > -0.5)): + raise AssertionError(f"The value of 'addition' is between -0.5 and 0.5. Got {additions}.") + + if isinstance(additions, torch.Tensor) and len(additions.shape) != 0: + if not (input.size(0) == len(additions) and len(additions.shape) == 1): + raise AssertionError(f"additions must be a 1-d vector of shape ({input.size(0)},). Got {additions}") + # TODO: I am not happy about this line, but no easy to do batch-wise operation + additions = additions.to(input.device).to(input.dtype) + additions = torch.stack([x.expand(*input.shape[-3:]) for x in additions]) + input = input + additions + input = input.clamp(0.0, 1.0) + + return _solarize(input, thresholds) + + +@perform_keep_shape_image +def posterize(input: torch.Tensor, bits: Union[int, torch.Tensor]) -> torch.Tensor: + r"""Reduce the number of bits for each color channel. + + .. image:: _static/img/posterize.png + + Non-differentiable function, ``torch.uint8`` involved. + + Args: + input: image torch.Tensor with shape :math:`(*, C, H, W)` to posterize. + bits: number of high bits. Must be in range [0, 8]. + If int or one element torch.Tensor, input will be posterized by this bits. + If 1-d torch.Tensor, input will be posterized element-wisely, len(bits) == input.shape[-3]. + If n-d torch.Tensor, input will be posterized element-channel-wisely, + bits.shape == input.shape[:len(bits.shape)] + + Returns: + Image with reduced color channels with shape :math:`(*, C, H, W)`. + + Example: + >>> x = torch.rand(1, 6, 3, 3) + >>> out = posterize(x, bits=8) + >>> torch.testing.assert_close(x, out) + + >>> x = torch.rand(2, 6, 3, 3) + >>> bits = torch.tensor([4, 2]) + >>> posterize(x, bits).shape + torch.Size([2, 6, 3, 3]) + + """ + if not isinstance(input, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(input)}") + + if not isinstance(bits, (int, torch.Tensor)): + raise TypeError(f"bits type is not an int or torch.Tensor. Got {type(bits)}") + + if isinstance(bits, int): + bits = torch.as_tensor(bits) + + # TODO: find a better way to check boundaries on tensors + # if not torch.all((bits >= 0) * (bits <= 8)) and bits.dtype == torch.int: + # raise ValueError(f"bits must be integers within range [0, 8]. Got {bits}.") + + # TODO: Make a differentiable version + # Current version: + # Ref: https://github.com/open-mmlab/mmcv/pull/132/files#diff-309c9320c7f71bedffe89a70ccff7f3bR19 + # Ref: https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/autoaugment.py#L222 + # Potential approach: implementing kornia.LUT with floating points + # https://github.com/albumentations-team/albumentations/blob/master/albumentations/augmentations/functional.py#L472 + def _left_shift(input: torch.Tensor, shift: torch.Tensor) -> torch.Tensor: + return ((input * 255).to(torch.uint8) * (2**shift)).to(input.dtype) / 255.0 + + def _right_shift(input: torch.Tensor, shift: torch.Tensor) -> torch.Tensor: + return (input * 255).to(torch.uint8) / (2**shift).to(input.dtype) / 255.0 + + def _posterize_one(input: torch.Tensor, bits: torch.Tensor) -> torch.Tensor: + # Single bits value condition + if bits == 0: + return torch.zeros_like(input) + if bits == 8: + return input.clone() + bits = 8 - bits + return _left_shift(_right_shift(input, bits), bits) + + if len(bits.shape) == 0 or (len(bits.shape) == 1 and len(bits) == 1): + return _posterize_one(input, bits) + + res = [] + if len(bits.shape) == 1: + if bits.shape[0] != input.shape[0]: + raise AssertionError( + f"Batch size must be equal between bits and input. Got {bits.shape[0]}, {input.shape[0]}." + ) + + for i in range(input.shape[0]): + res.append(_posterize_one(input[i], bits[i])) + return torch.stack(res, dim=0) + + if bits.shape != input.shape[: len(bits.shape)]: + raise AssertionError( + "Batch and channel must be equal between bits and input. " + f"Got {bits.shape}, {input.shape[: len(bits.shape)]}." + ) + _input = input.view(-1, *input.shape[len(bits.shape) :]) + _bits = bits.flatten() + for i in range(input.shape[0]): + res.append(_posterize_one(_input[i], _bits[i])) + return torch.stack(res, dim=0).reshape(*input.shape) + + +@perform_keep_shape_image +def sharpness(input: torch.Tensor, factor: Union[float, torch.Tensor]) -> torch.Tensor: + r"""Apply sharpness to the input torch.Tensor. + + .. image:: _static/img/sharpness.png + + Implemented Sharpness function from PIL using torch ops. This implementation refers to: + https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/autoaugment.py#L326 + + Args: + input: image torch.Tensor with shape :math:`(*, C, H, W)` to sharpen. + factor: factor of sharpness strength. Must be above 0. + If float or one element torch.Tensor, input will be sharpened by the same factor across the whole batch. + If 1-d torch.Tensor, input will be sharpened element-wisely, len(factor) == len(input). + + Returns: + Sharpened image or images with shape :math:`(*, C, H, W)`. + + Example: + >>> x = torch.rand(1, 1, 5, 5) + >>> sharpness(x, 0.5).shape + torch.Size([1, 1, 5, 5]) + + """ + if not isinstance(factor, torch.Tensor): + factor = torch.as_tensor(factor, device=input.device, dtype=input.dtype) + + if len(factor.size()) != 0 and factor.shape != torch.Size([input.size(0)]): + raise AssertionError( + "Input batch size shall match with factor size if factor is not a 0-dim torch.Tensor. " + f"Got {input.size(0)} and {factor.shape}" + ) + + kernel = ( + torch.as_tensor([[1, 1, 1], [1, 5, 1], [1, 1, 1]], dtype=input.dtype, device=input.device) + .view(1, 1, 3, 3) + .repeat(input.size(1), 1, 1, 1) + / 13 + ) + + # This shall be equivalent to depthwise conv2d: + # Ref: https://discuss.pytorch.org/t/depthwise-and-separable-convolutions-in-pytorch/7315/2 + degenerate = torch.nn.functional.conv2d(input, kernel, bias=None, stride=1, groups=input.size(1)) + degenerate = torch.clamp(degenerate, 0.0, 1.0) + + # For the borders of the resulting image, fill in the values of the original image. + mask = torch.ones_like(degenerate) + padded_mask = F.pad(mask, [1, 1, 1, 1]) + padded_degenerate = F.pad(degenerate, [1, 1, 1, 1]) + result = torch.where(padded_mask == 1, padded_degenerate, input) + + if len(factor.size()) == 0: + return _blend_one(result, input, factor) + return torch.stack([_blend_one(result[i], input[i], factor[i]) for i in range(len(factor))]) + + +def _blend_one(input1: torch.Tensor, input2: torch.Tensor, factor: torch.Tensor) -> torch.Tensor: + r"""Blend two images into one. + + Args: + input1: image torch.Tensor with shapes like :math:`(H, W)` or :math:`(D, H, W)`. + input2: image torch.Tensor with shapes like :math:`(H, W)` or :math:`(D, H, W)`. + factor: factor 0-dim torch.Tensor. + + Returns: + : image torch.Tensor with the batch in the zero position. + + """ + if not isinstance(input1, torch.Tensor): + raise AssertionError(f"`input1` must be a torch.Tensor. Got {input1}.") + if not isinstance(input2, torch.Tensor): + raise AssertionError(f"`input1` must be a torch.Tensor. Got {input2}.") + + if isinstance(factor, torch.Tensor) and len(factor.size()) != 0: + raise AssertionError(f"Factor shall be a float or single element torch.Tensor. Got {factor}.") + if factor == 0.0: + return input1 + if factor == 1.0: + return input2 + diff = (input2 - input1) * factor + res = input1 + diff + if factor > 0.0 and factor < 1.0: + return res + return torch.clamp(res, 0, 1) + + +def _build_lut(histo: torch.Tensor, step: torch.Tensor) -> torch.Tensor: + # Compute the cumulative sum, shifting by step // 2 + # and then normalization by step. + step_trunc = torch.div(step, 2, rounding_mode="trunc") + lut = torch.div(torch.cumsum(histo, 0) + step_trunc, step, rounding_mode="trunc") + # Shift lut, prepending with 0. + lut = torch.cat([torch.zeros(1, device=lut.device, dtype=lut.dtype), lut[:-1]]) + # Clip the counts to be in range. This is done + # in the C code for image.point. + return torch.clamp(lut, 0, 255) + + +# Code taken from: https://github.com/pytorch/vision/pull/796 +def _scale_channel(im: torch.Tensor) -> torch.Tensor: + r"""Scale the data in the channel to implement equalize. + + Args: + im: image torch.Tensor with shapes like :math:`(H, W)` or :math:`(D, H, W)`. + + Returns: + image torch.Tensor with the batch in the zero position. + + """ + min_ = im.min() + max_ = im.max() + + if min_.item() < 0.0 and not torch.isclose(min_, torch.as_tensor(0.0, dtype=min_.dtype)): + raise ValueError(f"Values in the input torch.Tensor must greater or equal to 0.0. Found {min_.item()}.") + + if max_.item() > 1.0 and not torch.isclose(max_, torch.as_tensor(1.0, dtype=max_.dtype)): + raise ValueError(f"Values in the input torch.Tensor must lower or equal to 1.0. Found {max_.item()}.") + + ndims = len(im.shape) + if ndims not in (2, 3): + raise TypeError(f"Input torch.Tensor must have 2 or 3 dimensions. Found {ndims}.") + + im = im * 255.0 + # Compute the histogram of the image channel. + histo = _torch_histc_cast(im, bins=256, min=0, max=255) + # For the purposes of computing the step, filter out the nonzeros. + nonzero_histo = torch.reshape(histo[histo != 0], [-1]) + step = torch.div(torch.sum(nonzero_histo) - nonzero_histo[-1], 255, rounding_mode="trunc") + + # If step is zero, return the original image. Otherwise, build + # lut from the full histogram and step and then index from it. + if step == 0: + result = im + else: + # can't index using 2d index. Have to flatten and then reshape + result = torch.gather(_build_lut(histo, step), 0, im.flatten().long()) + result = result.reshape_as(im) + + return result / 255.0 + + +@perform_keep_shape_image +def equalize(input: torch.Tensor) -> torch.Tensor: + r"""Apply equalize on the input torch.Tensor. + + .. image:: _static/img/equalize.png + + Implements Equalize function from PIL using PyTorch ops based on uint8 format: + https://github.com/tensorflow/tpu/blob/5f71c12a020403f863434e96982a840578fdd127/models/official/efficientnet/autoaugment.py#L355 + + Args: + input: image torch.Tensor to equalize with shape :math:`(*, C, H, W)`. + + Returns: + Equalized image torch.Tensor with shape :math:`(*, C, H, W)`. + + Example: + >>> x = torch.rand(1, 2, 3, 3) + >>> equalize(x).shape + torch.Size([1, 2, 3, 3]) + + """ + res = [] + for image in input: + # Assumes RGB for now. Scales each channel independently + # and then stacks the result. + scaled_image = torch.stack([_scale_channel(image[i, :, :]) for i in range(len(image))]) + res.append(scaled_image) + return torch.stack(res) + + +@perform_keep_shape_video +def equalize3d(input: torch.Tensor) -> torch.Tensor: + r"""Equalize the values for a 3D volumetric torch.Tensor. + + Implements Equalize function for a sequence of images using PyTorch ops based on uint8 format: + https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/autoaugment.py#L352 + + Args: + input: image torch.Tensor with shape :math:`(*, C, D, H, W)` to equalize. + + Returns: + Equalized volume with shape :math:`(B, C, D, H, W)`. + + """ + res = [] + for volume in input: + # Assumes RGB for now. Scales each channel independently + # and then stacks the result. + scaled_input = torch.stack([_scale_channel(volume[i, :, :, :]) for i in range(len(volume))]) + res.append(scaled_input) + + return torch.stack(res) + + +def invert(image: torch.Tensor, max_val: Optional[torch.Tensor] = None) -> torch.Tensor: + r"""Invert the values of an input image torch.Tensor by its maximum value. + + .. image:: _static/img/invert.png + + Args: + image: The input torch.Tensor to invert with an arbitatry shape. + max_val: The expected maximum value in the input torch.Tensor. The shape has to + according to the input torch.Tensor shape, or at least has to work with broadcasting. + + Example: + >>> img = torch.rand(1, 2, 4, 4) + >>> invert(img).shape + torch.Size([1, 2, 4, 4]) + + >>> img = 255. * torch.rand(1, 2, 3, 4, 4) + >>> invert(img, torch.as_tensor(255.)).shape + torch.Size([1, 2, 3, 4, 4]) + + >>> img = torch.rand(1, 3, 4, 4) + >>> invert(img, torch.as_tensor([[[[1.]]]])).shape + torch.Size([1, 3, 4, 4]) + + """ + if not isinstance(image, torch.Tensor): + raise AssertionError(f"Input is not a torch.Tensor. Got: {type(input)}") + + if max_val is None: + _max_val = torch.tensor([1.0]) + else: + _max_val = max_val + if not isinstance(_max_val, torch.Tensor): + raise AssertionError(f"max_val is not a torch.Tensor. Got: {type(_max_val)}") + + return _max_val.to(image) - image + + +class AdjustSaturation(nn.Module): + r"""Adjust color saturation of an image. + + The input image is expected to be an RGB image in the range of [0, 1]. + + Args: + saturation_factor: How much to adjust the saturation. 0 will give a black + and white image, 1 will give the original image while 2 will enhance the saturation by a factor of 2. + saturation_mode: The mode to adjust saturation. + + Shape: + - Input: Image/torch.Tensor to be adjusted in the shape of :math:`(*, 3, H, W)`. + - Output: Adjusted image in the shape of :math:`(*, 3, H, W)`. + + Example: + >>> x = torch.ones(1, 3, 3, 3) + >>> AdjustSaturation(2.)(x) + tensor([[[[1., 1., 1.], + [1., 1., 1.], + [1., 1., 1.]], + + [[1., 1., 1.], + [1., 1., 1.], + [1., 1., 1.]], + + [[1., 1., 1.], + [1., 1., 1.], + [1., 1., 1.]]]]) + + >>> x = torch.ones(2, 3, 3, 3) + >>> y = torch.ones(2) + >>> out = AdjustSaturation(y)(x) + >>> torch.nn.functional.mse_loss(x, out) + tensor(0.) + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def __init__(self, saturation_factor: Union[float, torch.Tensor]) -> None: + super().__init__() + self.saturation_factor: Union[float, torch.Tensor] = saturation_factor + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Adjust RGB saturation using the module's configured factor. + + Args: + input: Input RGB tensor, typically shaped :math:`(*, 3, H, W)`. + + Returns: + Saturation-adjusted tensor with the same shape as ``input``. + """ + return adjust_saturation(input, self.saturation_factor) + + +class AdjustSaturationWithGraySubtraction(nn.Module): + r"""Adjust color saturation of an image. + + This implementation aligns PIL. Hence, the output is close to TorchVision. + The input image is expected to be in the range of [0, 1]. + + The input image is expected to be an RGB or gray image in the range of [0, 1]. + + Args: + saturation_factor: How much to adjust the saturation. 0 will give a black + and white image, 1 will give the original image while 2 will enhance the saturation by a factor of 2. + saturation_mode: The mode to adjust saturation. + + Shape: + - Input: Image/torch.Tensor to be adjusted in the shape of :math:`(*, 3, H, W)`. + - Output: Adjusted image in the shape of :math:`(*, 3, H, W)`. + + Example: + >>> x = torch.ones(1, 3, 3, 3) + >>> AdjustSaturationWithGraySubtraction(2.)(x) + tensor([[[[1., 1., 1.], + [1., 1., 1.], + [1., 1., 1.]], + + [[1., 1., 1.], + [1., 1., 1.], + [1., 1., 1.]], + + [[1., 1., 1.], + [1., 1., 1.], + [1., 1., 1.]]]]) + + >>> x = torch.ones(2, 3, 3, 3) + >>> y = torch.ones(2) + >>> out = AdjustSaturationWithGraySubtraction(y)(x) + >>> torch.nn.functional.mse_loss(x, out) + tensor(0.) + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def __init__(self, saturation_factor: Union[float, torch.Tensor]) -> None: + super().__init__() + self.saturation_factor: Union[float, torch.Tensor] = saturation_factor + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Adjust saturation using gray-subtraction interpolation behavior. + + This forwards to :func:`adjust_saturation_with_gray_subtraction`, + which follows PIL-like saturation behavior. + + Args: + input: Input image tensor, typically shaped :math:`(*, 3, H, W)`. + + Returns: + Saturation-adjusted tensor with the same shape as ``input``. + """ + return adjust_saturation_with_gray_subtraction(input, self.saturation_factor) + + +class AdjustHue(nn.Module): + r"""Adjust hue of an image. + + This implementation aligns PIL. Hence, the output is close to TorchVision. + The input image is expected to be in the range of [0, 1]. + + The input image is expected to be an RGB image in the range of [0, 1]. + + Args: + hue_factor: How much to shift the hue channel. Should be in [-PI, PI]. PI + and -PI give complete reversal of hue channel in HSV space in positive and negative + direction respectively. 0 means no shift. Therefore, both -PI and PI will give an + image with complementary colors while 0 gives the original image. + + Shape: + - Input: Image/torch.Tensor to be adjusted in the shape of :math:`(*, 3, H, W)`. + - Output: Adjusted image in the shape of :math:`(*, 3, H, W)`. + + Example: + >>> x = torch.ones(1, 3, 3, 3) + >>> AdjustHue(3.141516)(x) + tensor([[[[1., 1., 1.], + [1., 1., 1.], + [1., 1., 1.]], + + [[1., 1., 1.], + [1., 1., 1.], + [1., 1., 1.]], + + [[1., 1., 1.], + [1., 1., 1.], + [1., 1., 1.]]]]) + + >>> x = torch.ones(2, 3, 3, 3) + >>> y = torch.ones(2) * 3.141516 + >>> AdjustHue(y)(x).shape + torch.Size([2, 3, 3, 3]) + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + + def __init__(self, hue_factor: Union[float, torch.Tensor]) -> None: + super().__init__() + self.hue_factor: Union[float, torch.Tensor] = hue_factor + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Shift image hue by the module's configured hue factor. + + Args: + input: Input RGB tensor in the expected image range, usually shaped + :math:`(*, 3, H, W)`. + + Returns: + Hue-adjusted tensor with the same shape as ``input``. + """ + return adjust_hue(input, self.hue_factor) + + +class AdjustGamma(nn.Module): + r"""Perform gamma correction on an image. + + The input image is expected to be in the range of [0, 1]. + + Args: + gamma: Non negative real number, same as y\gammay in the equation. + gamma larger than 1 make the shadows darker, while gamma smaller than 1 make + dark regions lighter. + gain: The constant multiplier. + + Shape: + - Input: Image to be adjusted in the shape of :math:`(*, N)`. + - Output: Adjusted image in the shape of :math:`(*, N)`. + + Example: + >>> x = torch.ones(1, 1, 3, 3) + >>> AdjustGamma(1.0, 2.0)(x) + tensor([[[[1., 1., 1.], + [1., 1., 1.], + [1., 1., 1.]]]]) + + >>> x = torch.ones(2, 5, 3, 3) + >>> y1 = torch.ones(2) * 1.0 + >>> y2 = torch.ones(2) * 2.0 + >>> AdjustGamma(y1, y2)(x).shape + torch.Size([2, 5, 3, 3]) + + """ + + def __init__(self, gamma: Union[float, torch.Tensor], gain: Union[float, torch.Tensor] = 1.0) -> None: + super().__init__() + self.gamma: Union[float, torch.Tensor] = gamma + self.gain: Union[float, torch.Tensor] = gain + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Apply gamma correction using this module's ``gamma`` and ``gain``. + + Args: + input: Input tensor to correct, commonly image-shaped + :math:`(*, C, H, W)`. + + Returns: + Gamma-corrected tensor with the same shape as ``input``. + """ + return adjust_gamma(input, self.gamma, self.gain) + + +class AdjustContrast(nn.Module): + r"""Adjust Contrast of an image. + + This implementation aligns OpenCV, not PIL. Hence, the output differs from TorchVision. + The input image is expected to be in the range of [0, 1]. + + Args: + contrast_factor: Contrast adjust factor per element + in the batch. 0 generates a completely black image, 1 does not modify + the input image while any other non-negative number modify the + brightness by this factor. + + Shape: + - Input: Image/Input to be adjusted in the shape of :math:`(*, N)`. + - Output: Adjusted image in the shape of :math:`(*, N)`. + + Example: + >>> x = torch.ones(1, 1, 3, 3) + >>> AdjustContrast(0.5)(x) + tensor([[[[0.5000, 0.5000, 0.5000], + [0.5000, 0.5000, 0.5000], + [0.5000, 0.5000, 0.5000]]]]) + + >>> x = torch.ones(2, 5, 3, 3) + >>> y = torch.ones(2) + >>> AdjustContrast(y)(x).shape + torch.Size([2, 5, 3, 3]) + + """ + + def __init__(self, contrast_factor: Union[float, torch.Tensor]) -> None: + super().__init__() + self.contrast_factor: Union[float, torch.Tensor] = contrast_factor + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Adjust contrast using the configured contrast factor. + + Args: + input: Input tensor to adjust, typically shaped :math:`(*, C, H, W)`. + + Returns: + Contrast-adjusted tensor with the same shape as ``input``. + """ + return adjust_contrast(input, self.contrast_factor) + + +class AdjustContrastWithMeanSubtraction(nn.Module): + r"""Adjust Contrast of an image. + + This implementation aligns PIL. Hence, the output is close to TorchVision. + The input image is expected to be in the range of [0, 1]. + + Args: + contrast_factor: Contrast adjust factor per element + in the batch by subtracting its mean grayscaled version. + 0 generates a completely black image, 1 does not modify + the input image while any other non-negative number modify the + brightness by this factor. + + Shape: + - Input: Image/Input to be adjusted in the shape of :math:`(*, N)`. + - Output: Adjusted image in the shape of :math:`(*, N)`. + + Example: + >>> x = torch.ones(1, 1, 3, 3) + >>> AdjustContrastWithMeanSubtraction(0.5)(x) + tensor([[[[1., 1., 1.], + [1., 1., 1.], + [1., 1., 1.]]]]) + + >>> x = torch.ones(2, 5, 3, 3) + >>> y = torch.ones(2) + >>> AdjustContrastWithMeanSubtraction(y)(x).shape + torch.Size([2, 5, 3, 3]) + + """ + + def __init__(self, contrast_factor: Union[float, torch.Tensor]) -> None: + super().__init__() + self.contrast_factor: Union[float, torch.Tensor] = contrast_factor + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Adjust contrast with mean-subtraction based interpolation. + + This forwards to :func:`adjust_contrast_with_mean_subtraction`, which + applies PIL-like contrast behavior. + + Args: + input: Input tensor to adjust, typically shaped :math:`(*, C, H, W)`. + + Returns: + Contrast-adjusted tensor with the same shape as ``input``. + """ + return adjust_contrast_with_mean_subtraction(input, self.contrast_factor) + + +class AdjustBrightness(nn.Module): + r"""Adjust Brightness of an image. + + This implementation aligns OpenCV, not PIL. Hence, the output differs from TorchVision. + The input image is expected to be in the range of [0, 1]. + + Args: + brightness_factor: Brightness adjust factor per element + in the batch. 0 does not modify the input image while any other number modify the + brightness. + + Shape: + - Input: Image/Input to be adjusted in the shape of :math:`(*, N)`. + - Output: Adjusted image in the shape of :math:`(*, N)`. + + Example: + >>> x = torch.ones(1, 1, 3, 3) + >>> AdjustBrightness(1.)(x) + tensor([[[[1., 1., 1.], + [1., 1., 1.], + [1., 1., 1.]]]]) + + >>> x = torch.ones(2, 5, 3, 3) + >>> y = torch.ones(2) + >>> AdjustBrightness(y)(x).shape + torch.Size([2, 5, 3, 3]) + + """ + + def __init__(self, brightness_factor: Union[float, torch.Tensor]) -> None: + super().__init__() + self.brightness_factor: Union[float, torch.Tensor] = brightness_factor + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Adjust image brightness using the configured brightness factor. + + Args: + input: Input tensor to adjust, commonly shaped :math:`(*, C, H, W)`. + + Returns: + Brightness-adjusted tensor with the same shape as ``input``. + """ + return adjust_brightness(input, self.brightness_factor) + + +class AdjustSigmoid(nn.Module): + r"""Adjust the contrast of an image torch.Tensor or performs sigmoid correction on the input image torch.Tensor. + + The input image is expected to be in the range of [0, 1]. + + Reference: + [1]: Gustav J. Braun, "Image Lightness Rescaling Using Sigmoidal Contrast Enhancement Functions", + http://markfairchild.org/PDFs/PAP07.pdf + + Args: + image: Image to be adjusted in the shape of :math:`(*, H, W)`. + cutoff: The cutoff of sigmoid function. + gain: The multiplier of sigmoid function. + inv: If is set to True the function will return the negative sigmoid correction. + + Example: + >>> x = torch.ones(1, 1, 2, 2) + >>> AdjustSigmoid(gain=0)(x) + tensor([[[[0.5000, 0.5000], + [0.5000, 0.5000]]]]) + + """ + + def __init__(self, cutoff: float = 0.5, gain: float = 10, inv: bool = False) -> None: + super().__init__() + self.cutoff: float = cutoff + self.gain: float = gain + self.inv: bool = inv + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Apply sigmoid-based intensity remapping to the input image. + + Args: + image: Input image tensor to remap, typically shaped + :math:`(*, C, H, W)` or :math:`(*, H, W)`. + + Returns: + Tensor with sigmoid correction applied, preserving input shape. + """ + return adjust_sigmoid(image, cutoff=self.cutoff, gain=self.gain, inv=self.inv) + + +class AdjustLog(nn.Module): + """Adjust log correction on the input image torch.Tensor. + + The input image is expected to be in the range of [0, 1]. + + Reference: + [1]: http://www.ece.ucsb.edu/Faculty/Manjunath/courses/ece178W03/EnhancePart1.pdf + + Args: + image: Image to be adjusted in the shape of :math:`(*, H, W)`. + gain: The multiplier of logarithmic function. + inv: If is set to True the function will return the inverse logarithmic correction. + clip_output: Whether to clip the output image with range of [0, 1]. + + Example: + >>> x = torch.zeros(1, 1, 2, 2) + >>> AdjustLog(inv=True)(x) + tensor([[[[0., 0.], + [0., 0.]]]]) + + """ + + def __init__(self, gain: float = 1, inv: bool = False, clip_output: bool = True) -> None: + super().__init__() + self.gain: float = gain + self.inv: bool = inv + self.clip_output: bool = clip_output + + def forward(self, image: torch.Tensor) -> torch.Tensor: + """Apply logarithmic or inverse-logarithmic intensity correction. + + Args: + image: Input image tensor to transform, typically shaped + :math:`(*, C, H, W)` or :math:`(*, H, W)`. + + Returns: + Log-corrected tensor with the same shape as ``image``. + """ + return adjust_log(image, gain=self.gain, inv=self.inv, clip_output=self.clip_output) + + +class AdjustBrightnessAccumulative(nn.Module): + r"""Adjust Brightness of an image accumulatively. + + This implementation aligns PIL. Hence, the output is close to TorchVision. + The input image is expected to be in the range of [0, 1]. + + Args: + brightness_factor: Brightness adjust factor per element + in the batch. 0 does not modify the input image while any other number modify the + brightness. + + Shape: + - Input: Image/Input to be adjusted in the shape of :math:`(*, N)`. + - Output: Adjusted image in the shape of :math:`(*, N)`. + + Example: + >>> x = torch.ones(1, 1, 3, 3) + >>> AdjustBrightnessAccumulative(1.)(x) + tensor([[[[1., 1., 1.], + [1., 1., 1.], + [1., 1., 1.]]]]) + + >>> x = torch.ones(2, 5, 3, 3) + >>> y = torch.ones(2) + >>> AdjustBrightnessAccumulative(y)(x).shape + torch.Size([2, 5, 3, 3]) + + """ + + def __init__(self, brightness_factor: Union[float, torch.Tensor]) -> None: + super().__init__() + self.brightness_factor: Union[float, torch.Tensor] = brightness_factor + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Adjust brightness with accumulative (PIL-like) behavior. + + Args: + input: Input tensor to adjust, commonly shaped :math:`(*, C, H, W)`. + + Returns: + Brightness-adjusted tensor with the same shape as ``input``. + """ + return adjust_brightness_accumulative(input, self.brightness_factor) + + +class Invert(nn.Module): + r"""Invert the values of an input torch.Tensor by its maximum value. + + Args: + input: The input torch.Tensor to invert with an arbitatry shape. + max_val: The expected maximum value in the input torch.Tensor. The shape has to + according to the input torch.Tensor shape, or at least has to work with broadcasting. Default: 1.0. + + Example: + >>> img = torch.rand(1, 2, 4, 4) + >>> Invert()(img).shape + torch.Size([1, 2, 4, 4]) + + >>> img = 255. * torch.rand(1, 2, 3, 4, 4) + >>> Invert(torch.as_tensor(255.))(img).shape + torch.Size([1, 2, 3, 4, 4]) + + >>> img = torch.rand(1, 3, 4, 4) + >>> Invert(torch.as_tensor([[[[1.]]]]))(img).shape + torch.Size([1, 3, 4, 4]) + + """ + + def __init__(self, max_val: Optional[torch.Tensor] = None) -> None: + super().__init__() + if max_val is None: + max_val = torch.tensor(1.0) + if not isinstance(max_val, nn.Parameter): + self.register_buffer("max_val", max_val) + else: + self.max_val = max_val + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Invert values using the module's configured maximum reference. + + Args: + input: Tensor to invert. Any shape is accepted as long as it is + broadcast-compatible with ``self.max_val``. + + Returns: + Inverted tensor with the same shape as ``input``. + """ + return invert(input, self.max_val) diff --git a/kornia/enhance/core.py b/kornia/enhance/core.py new file mode 100644 index 0000000..77c2a59 --- /dev/null +++ b/kornia/enhance/core.py @@ -0,0 +1,138 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Union + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR + + +def add_weighted( + src1: torch.Tensor, + alpha: Union[float, torch.Tensor], + src2: torch.Tensor, + beta: Union[float, torch.Tensor], + gamma: Union[float, torch.Tensor], +) -> torch.Tensor: + r"""Calculate the weighted sum of two Tensors. + + .. image:: _static/img/add_weighted.png + + The function calculates the weighted sum of two Tensors as follows: + + .. math:: + out = src1 * alpha + src2 * beta + gamma + + Args: + src1: torch.Tensor with an arbitrary shape, equal to shape of src2. + alpha: weight of the src1 elements as Union[float, torch.Tensor]. + src2: torch.Tensor with an arbitrary shape, equal to shape of src1. + beta: weight of the src2 elements as Union[float, torch.Tensor]. + gamma: scalar added to each sum as Union[float, torch.Tensor]. + + Returns: + Weighted torch.Tensor with shape equal to src1 and src2 shapes. + + Example: + >>> input1 = torch.rand(1, 1, 5, 5) + >>> input2 = torch.rand(1, 1, 5, 5) + >>> output = add_weighted(input1, 0.5, input2, 0.5, 1.0) + >>> output.shape + torch.Size([1, 1, 5, 5]) + + Notes: + torch.Tensor alpha/beta/gamma have to be with shape broadcastable to src1 and src2 shapes. + + """ + KORNIA_CHECK_IS_TENSOR(src1) + KORNIA_CHECK_IS_TENSOR(src2) + KORNIA_CHECK(src1.shape == src2.shape, f"src1 and src2 have different shapes. Got {src1.shape} and {src2.shape}") + + if isinstance(alpha, torch.Tensor): + KORNIA_CHECK(src1.shape == alpha.shape, "alpha has a different shape than src.") + else: + alpha = torch.tensor(alpha, dtype=src1.dtype, device=src1.device) + + if isinstance(beta, torch.Tensor): + KORNIA_CHECK(src1.shape == beta.shape, "beta has a different shape than src.") + else: + beta = torch.tensor(beta, dtype=src1.dtype, device=src1.device) + + if isinstance(gamma, torch.Tensor): + KORNIA_CHECK(src1.shape == gamma.shape, "gamma has a different shape than src.") + else: + gamma = torch.tensor(gamma, dtype=src1.dtype, device=src1.device) + + return src1 * alpha + src2 * beta + gamma + + +class AddWeighted(nn.Module): + r"""Calculate the weighted sum of two Tensors. + + The function calculates the weighted sum of two Tensors as follows: + + .. math:: + out = src1 * alpha + src2 * beta + gamma + + Args: + alpha: weight of the src1 elements as Union[float, torch.Tensor]. + beta: weight of the src2 elements as Union[float, torch.Tensor]. + gamma: scalar added to each sum as Union[float, torch.Tensor]. + + Shape: + - Input1: torch.Tensor with an arbitrary shape, equal to shape of Input2. + - Input2: torch.Tensor with an arbitrary shape, equal to shape of Input1. + - Output: Weighted torch.Tensor with shape equal to src1 and src2 shapes. + + Example: + >>> input1 = torch.rand(1, 1, 5, 5) + >>> input2 = torch.rand(1, 1, 5, 5) + >>> output = AddWeighted(0.5, 0.5, 1.0)(input1, input2) + >>> output.shape + torch.Size([1, 1, 5, 5]) + + Notes: + torch.Tensor alpha/beta/gamma have to be with shape broadcastable to src1 and src2 shapes. + + """ + + def __init__( + self, alpha: Union[float, torch.Tensor], beta: Union[float, torch.Tensor], gamma: Union[float, torch.Tensor] + ) -> None: + super().__init__() + self.alpha = alpha + self.beta = beta + self.gamma = gamma + + def forward(self, src1: torch.Tensor, src2: torch.Tensor) -> torch.Tensor: + """Compute a weighted sum of two tensors with a scalar bias term. + + This method is a module wrapper around :func:`add_weighted`, using the + ``alpha``, ``beta``, and ``gamma`` values defined at initialization. + + Args: + src1: First input tensor. + src2: Second input tensor. It must have the same shape as ``src1``. + + Returns: + A tensor with the same shape as the inputs, computed as + ``src1 * alpha + src2 * beta + gamma`` with broadcasting applied + where supported by :func:`add_weighted`. + """ + return add_weighted(src1, self.alpha, src2, self.beta, self.gamma) diff --git a/kornia/enhance/equalization.py b/kornia/enhance/equalization.py new file mode 100644 index 0000000..ed85583 --- /dev/null +++ b/kornia/enhance/equalization.py @@ -0,0 +1,403 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""In this module several equalization methods are exposed: he, ahe, clahe.""" + +import math +from typing import Tuple + +import torch +import torch.nn.functional as F + +from kornia.core.utils import _torch_histc_cast +from kornia.image.utils import perform_keep_shape_image + +from .histogram import histogram + + +def _compute_tiles( + imgs: torch.Tensor, grid_size: Tuple[int, int], even_tile_size: bool = False +) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Compute tiles on an image according to a grid size. + + Note that padding can be added to the image in order to crop properly the image. + So, the grid_size (GH, GW) x tile_size (TH, TW) >= image_size (H, W) + + Args: + imgs: batch of 2D images with shape (B, C, H, W) or (C, H, W). + grid_size: number of tiles to be cropped in each direction (GH, GW) + even_tile_size: Determine if the width and height of the tiles must be even. + + Returns: + tensor with tiles (B, GH, GW, C, TH, TW). B = 1 in case of a single image is provided. + tensor with the padded batch of 2D imageswith shape (B, C, H', W'). + + """ + batch: torch.Tensor = imgs # B x C x H x W + + # compute stride and kernel size + h, w = batch.shape[-2:] + kernel_vert: int = math.ceil(h / grid_size[0]) + kernel_horz: int = math.ceil(w / grid_size[1]) + + if even_tile_size: + kernel_vert += 1 if kernel_vert % 2 else 0 + kernel_horz += 1 if kernel_horz % 2 else 0 + + # add padding (with that kernel size we could need some extra cols and rows...) + pad_vert = kernel_vert * grid_size[0] - h + pad_horz = kernel_horz * grid_size[1] - w + + # add the padding in the last coluns and rows + if pad_vert > batch.shape[-2] or pad_horz > batch.shape[-1]: + raise ValueError("Cannot compute tiles on the image according to the given grid size") + + if pad_vert > 0 or pad_horz > 0: + batch = F.pad(batch, [0, pad_horz, 0, pad_vert], mode="reflect") # B x C x H' x W' + + # compute tiles + c: int = batch.shape[-3] + tiles: torch.Tensor = ( + batch.unfold(1, c, c) # unfold(dimension, size, step) + .unfold(2, kernel_vert, kernel_vert) + .unfold(3, kernel_horz, kernel_horz) + .squeeze(1) + ).contiguous() # GH x GW x C x TH x TW + + if tiles.shape[-5] != grid_size[0]: + raise AssertionError + + if tiles.shape[-4] != grid_size[1]: + raise AssertionError + + return tiles, batch + + +def _compute_interpolation_tiles(padded_imgs: torch.Tensor, tile_size: Tuple[int, int]) -> torch.Tensor: + r"""Compute interpolation tiles on a properly padded set of images. + + Note that images must be padded. So, the tile_size (TH, TW) * grid_size (GH, GW) = image_size (H, W) + + Args: + padded_imgs: batch of 2D images with shape (B, C, H, W) already padded to extract tiles + of size (TH, TW). + tile_size: shape of the current tiles (TH, TW). + + Returns: + tensor with the interpolation tiles (B, 2GH, 2GW, C, TH/2, TW/2). + + """ + if padded_imgs.dim() != 4: + raise AssertionError("Images Tensor must be 4D.") + + if padded_imgs.shape[-2] % tile_size[0] != 0: + raise AssertionError("Images are not correctly padded.") + + if padded_imgs.shape[-1] % tile_size[1] != 0: + raise AssertionError("Images are not correctly padded.") + + # tiles to be interpolated are built by dividing in 4 each already existing + interp_kernel_vert: int = tile_size[0] // 2 + interp_kernel_horz: int = tile_size[1] // 2 + + c: int = padded_imgs.shape[-3] + interp_tiles: torch.Tensor = ( + padded_imgs.unfold(1, c, c) + .unfold(2, interp_kernel_vert, interp_kernel_vert) + .unfold(3, interp_kernel_horz, interp_kernel_horz) + .squeeze(1) + ).contiguous() # 2GH x 2GW x C x TH/2 x TW/2 + + if interp_tiles.shape[-3] != c: + raise AssertionError + + if interp_tiles.shape[-2] != tile_size[0] / 2: + raise AssertionError + + if interp_tiles.shape[-1] != tile_size[1] / 2: + raise AssertionError + + return interp_tiles + + +def _my_histc(tiles: torch.Tensor, bins: int) -> torch.Tensor: + return _torch_histc_cast(tiles, bins=bins, min=0, max=1) + + +def _compute_luts( + tiles_x_im: torch.Tensor, num_bins: int = 256, clip: float = 40.0, diff: bool = False +) -> torch.Tensor: + r"""Compute luts for a batched set of tiles. + + Same approach as in OpenCV (https://github.com/opencv/opencv/blob/master/modules/imgproc/src/clahe.cpp) + + Args: + tiles_x_im: set of tiles per image to apply the lut. (B, GH, GW, C, TH, TW) + num_bins: number of bins. default: 256 + clip: threshold value for contrast limiting. If it is 0 then the clipping is disabled. + diff: denote if the differentiable histagram will be used. Default: False + + Returns: + Lut for each tile (B, GH, GW, C, 256). + + """ + if tiles_x_im.dim() != 6: + raise AssertionError("Tensor must be 6D.") + + b, gh, gw, c, th, tw = tiles_x_im.shape + pixels: int = th * tw + tiles: torch.Tensor = tiles_x_im.view(-1, pixels) # test with view # T x (THxTW) + if not diff: + if torch.jit.is_scripting(): + histos = torch.stack([_torch_histc_cast(tile, bins=num_bins, min=0, max=1) for tile in tiles]) + else: + histos = torch.stack(list(map(_my_histc, tiles, [num_bins] * len(tiles)))) + else: + bins: torch.Tensor = torch.linspace(0, 1, num_bins, device=tiles.device) + histos = histogram(tiles, bins, torch.tensor(0.001)).squeeze() + histos *= pixels + + if clip > 0.0: + max_val: float = max(clip * pixels // num_bins, 1) + histos.clamp_(max=max_val) + clipped: torch.Tensor = pixels - histos.sum(1) + residual: torch.Tensor = torch.remainder(clipped, num_bins) + redist: torch.Tensor = (clipped - residual).div(num_bins) + histos += redist[None].transpose(0, 1) + # trick to avoid using a loop to assign the residual + v_range: torch.Tensor = torch.arange(num_bins, device=histos.device) + mat_range: torch.Tensor = v_range.repeat(histos.shape[0], 1) + histos += mat_range < residual[None].transpose(0, 1) + + lut_scale: float = (num_bins - 1) / pixels + luts: torch.Tensor = torch.cumsum(histos, 1) * lut_scale + luts = luts.clamp(0, num_bins - 1) + if not diff: + luts = luts.floor() # to get the same values as converting to int maintaining the type + luts = luts.view((b, gh, gw, c, num_bins)) + return luts + + +def _map_luts(interp_tiles: torch.Tensor, luts: torch.Tensor) -> torch.Tensor: + r"""Assign the required luts to each tile. + + Args: + interp_tiles: set of interpolation tiles. (B, 2GH, 2GW, C, TH/2, TW/2) + luts: luts for each one of the original tiles. (B, GH, GW, C, 256) + + Returns: + mapped luts (B, 2GH, 2GW, 4, C, 256) + + """ + if interp_tiles.dim() != 6: + raise AssertionError("interp_tiles tensor must be 6D.") + + if luts.dim() != 5: + raise AssertionError("luts tensor must be 5D.") + + # gh, gw -> 2x the number of tiles used to compute the histograms + # th, tw -> /2 the sizes of the tiles used to compute the histograms + num_imgs, gh, gw, c, _, _ = interp_tiles.shape + + # precompute idxs for non corner regions (doing it in cpu seems slightly faster) + j_idxs = torch.empty(0, 4, dtype=torch.long) + if gh > 2: + j_floor = torch.arange(1, gh - 1).view(gh - 2, 1).div(2, rounding_mode="trunc") + j_idxs = torch.tensor([[0, 0, 1, 1], [-1, -1, 0, 0]] * ((gh - 2) // 2)) # reminder + j_idxs[:, 0:2] -= 1 + j_idxs += j_floor + + i_idxs = torch.empty(0, 4, dtype=torch.long) + if gw > 2: + i_floor = torch.arange(1, gw - 1).view(gw - 2, 1).div(2, rounding_mode="trunc") + i_idxs = torch.tensor([[0, 1, 0, 1], [-1, 0, -1, 0]] * ((gw - 2) // 2)) # reminder + i_idxs[:, [0, 2]] -= 1 + i_idxs += i_floor + + # selection of luts to interpolate each patch + # create a tensor with dims: interp_patches height and width x 4 x num channels x bins in the histograms + # the tensor is init to -1 to denote non init hists + luts_x_interp_tiles: torch.Tensor = torch.full( # B x GH x GW x 4 x C x 256 + (num_imgs, gh, gw, 4, c, luts.shape[-1]), -1, dtype=interp_tiles.dtype, device=interp_tiles.device + ) + # corner regions + luts_x_interp_tiles[:, 0 :: gh - 1, 0 :: gw - 1, 0] = luts[:, 0 :: max(gh // 2 - 1, 1), 0 :: max(gw // 2 - 1, 1)] + # border region (h) + luts_x_interp_tiles[:, 1:-1, 0 :: gw - 1, 0] = luts[:, j_idxs[:, 0], 0 :: max(gw // 2 - 1, 1)] + luts_x_interp_tiles[:, 1:-1, 0 :: gw - 1, 1] = luts[:, j_idxs[:, 2], 0 :: max(gw // 2 - 1, 1)] + # border region (w) + luts_x_interp_tiles[:, 0 :: gh - 1, 1:-1, 0] = luts[:, 0 :: max(gh // 2 - 1, 1), i_idxs[:, 0]] + luts_x_interp_tiles[:, 0 :: gh - 1, 1:-1, 1] = luts[:, 0 :: max(gh // 2 - 1, 1), i_idxs[:, 1]] + # internal region + luts_x_interp_tiles[:, 1:-1, 1:-1, :] = luts[ + :, j_idxs.repeat(max(gh - 2, 1), 1, 1).permute(1, 0, 2), i_idxs.repeat(max(gw - 2, 1), 1, 1) + ] + + return luts_x_interp_tiles + + +def _compute_equalized_tiles(interp_tiles: torch.Tensor, luts: torch.Tensor) -> torch.Tensor: + r"""Equalize the tiles. + + Args: + interp_tiles: set of interpolation tiles, values must be in the range [0, 1]. + (B, 2GH, 2GW, C, TH/2, TW/2) + luts: luts for each one of the original tiles. (B, GH, GW, C, 256) + + Returns: + equalized tiles (B, 2GH, 2GW, C, TH/2, TW/2) + + """ + if interp_tiles.dim() != 6: + raise AssertionError("interp_tiles tensor must be 6D.") + + if luts.dim() != 5: + raise AssertionError("luts tensor must be 5D.") + + mapped_luts: torch.Tensor = _map_luts(interp_tiles, luts) # Bx2GHx2GWx4xCx256 + + # gh, gw -> 2x the number of tiles used to compute the histograms + # th, tw -> /2 the sizes of the tiles used to compute the histograms + num_imgs, gh, gw, c, th, tw = interp_tiles.shape + + # equalize tiles + flatten_interp_tiles: torch.Tensor = (interp_tiles * 255).long().flatten(-2, -1) # B x GH x GW x 4 x C x (THxTW) + flatten_interp_tiles = flatten_interp_tiles.unsqueeze(-3).expand(num_imgs, gh, gw, 4, c, th * tw) + preinterp_tiles_equalized = ( + torch.gather(mapped_luts, 5, flatten_interp_tiles) # B x GH x GW x 4 x C x TH x TW + .to(interp_tiles) + .reshape(num_imgs, gh, gw, 4, c, th, tw) + ) + + # interp tiles + tiles_equalized: torch.Tensor = torch.zeros_like(interp_tiles) + + # compute the interpolation weights (shapes are 2 x TH x TW because they must be applied to 2 interp tiles) + ih = ( + torch.arange(2 * th - 1, -1, -1, dtype=interp_tiles.dtype, device=interp_tiles.device) + .div(2.0 * th - 1)[None] + .transpose(-2, -1) + .expand(2 * th, tw) + ) + ih = ih.unfold(0, th, th).unfold(1, tw, tw) # 2 x 1 x TH x TW + iw = ( + torch.arange(2 * tw - 1, -1, -1, dtype=interp_tiles.dtype, device=interp_tiles.device) + .div(2.0 * tw - 1) + .expand(th, 2 * tw) + ) + iw = iw.unfold(0, th, th).unfold(1, tw, tw) # 1 x 2 x TH x TW + + # compute row and column interpolation weights + tiw = iw.expand((gw - 2) // 2, 2, th, tw).reshape(gw - 2, 1, th, tw).unsqueeze(0) # 1 x GW-2 x 1 x TH x TW + tih = ih.repeat((gh - 2) // 2, 1, 1, 1).unsqueeze(1) # GH-2 x 1 x 1 x TH x TW + + # internal regions + tl, tr, bl, br = preinterp_tiles_equalized[:, 1:-1, 1:-1].unbind(3) + t = torch.addcmul(tr, tiw, torch.sub(tl, tr)) + b = torch.addcmul(br, tiw, torch.sub(bl, br)) + tiles_equalized[:, 1:-1, 1:-1] = torch.addcmul(b, tih, torch.sub(t, b)) + + # corner regions + tiles_equalized[:, 0 :: gh - 1, 0 :: gw - 1] = preinterp_tiles_equalized[:, 0 :: gh - 1, 0 :: gw - 1, 0] + + # border region (h) + t, b, _, _ = preinterp_tiles_equalized[:, 1:-1, 0].unbind(2) + tiles_equalized[:, 1:-1, 0] = torch.addcmul(b, tih.squeeze(1), torch.sub(t, b)) + t, b, _, _ = preinterp_tiles_equalized[:, 1:-1, gh - 1].unbind(2) + tiles_equalized[:, 1:-1, gh - 1] = torch.addcmul(b, tih.squeeze(1), torch.sub(t, b)) + + # border region (w) + left, right, _, _ = preinterp_tiles_equalized[:, 0, 1:-1].unbind(2) + tiles_equalized[:, 0, 1:-1] = torch.addcmul(right, tiw, torch.sub(left, right)) + left, right, _, _ = preinterp_tiles_equalized[:, gw - 1, 1:-1].unbind(2) + tiles_equalized[:, gw - 1, 1:-1] = torch.addcmul(right, tiw, torch.sub(left, right)) + + # same type as the input + return tiles_equalized.div(255.0) + + +@perform_keep_shape_image +def equalize_clahe( + input: torch.Tensor, + clip_limit: float = 40.0, + grid_size: Tuple[int, int] = (8, 8), + slow_and_differentiable: bool = False, +) -> torch.Tensor: + r"""Apply clahe equalization on the input tensor. + + .. image:: _static/img/equalize_clahe.png + + NOTE: Lut computation uses the same approach as in OpenCV, in next versions this can change. + + Args: + input: images tensor to equalize with values in the range [0, 1] and shape :math:`(*, C, H, W)`. + clip_limit: threshold value for contrast limiting. If 0 clipping is disabled. + grid_size: number of tiles to be cropped in each direction (GH, GW). + slow_and_differentiable: flag to select implementation + + Returns: + Equalized image or images with shape as the input. + + Examples: + >>> img = torch.rand(1, 10, 20) + >>> res = equalize_clahe(img) + >>> res.shape + torch.Size([1, 10, 20]) + + >>> img = torch.rand(2, 3, 10, 20) + >>> res = equalize_clahe(img) + >>> res.shape + torch.Size([2, 3, 10, 20]) + + """ + if not isinstance(clip_limit, float): + raise TypeError(f"Input clip_limit type is not float. Got {type(clip_limit)}") + + if not isinstance(grid_size, tuple): + raise TypeError(f"Input grid_size type is not Tuple. Got {type(grid_size)}") + + if len(grid_size) != 2: + raise TypeError(f"Input grid_size is not a Tuple with 2 elements. Got {len(grid_size)}") + + if isinstance(grid_size[0], float) or isinstance(grid_size[1], float): + raise TypeError("Input grid_size type is not valid, must be a Tuple[int, int].") + + if grid_size[0] <= 0 or grid_size[1] <= 0: + raise ValueError(f"Input grid_size elements must be positive. Got {grid_size}") + + imgs: torch.Tensor = input # B x C x H x W + + # hist_tiles: torch.Tensor # B x GH x GW x C x TH x TW # not supported by JIT + # img_padded: torch.Tensor # B x C x H' x W' # not supported by JIT + # the size of the tiles must be even in order to divide them into 4 tiles for the interpolation + hist_tiles, img_padded = _compute_tiles(imgs, grid_size, True) + tile_size: Tuple[int, int] = (hist_tiles.shape[-2], hist_tiles.shape[-1]) + interp_tiles: torch.Tensor = _compute_interpolation_tiles(img_padded, tile_size) # B x 2GH x 2GW x C x TH/2 x TW/2 + luts: torch.Tensor = _compute_luts( + hist_tiles, clip=clip_limit, diff=slow_and_differentiable + ) # B x GH x GW x C x 256 + equalized_tiles: torch.Tensor = _compute_equalized_tiles(interp_tiles, luts) # B x 2GH x 2GW x C x TH/2 x TW/2 + + # reconstruct the images form the tiles + # try permute + contiguous + view + eq_imgs: torch.Tensor = equalized_tiles.permute(0, 3, 1, 4, 2, 5).reshape_as(img_padded) + h, w = imgs.shape[-2:] + eq_imgs = eq_imgs[..., :h, :w] # crop imgs if they were padded + + # remove batch if the input was not in batch form + if input.dim() != eq_imgs.dim(): + eq_imgs = eq_imgs.squeeze(0) + + return eq_imgs diff --git a/kornia/enhance/histogram.py b/kornia/enhance/histogram.py new file mode 100644 index 0000000..eb8bba7 --- /dev/null +++ b/kornia/enhance/histogram.py @@ -0,0 +1,273 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Optional, Tuple + +import torch + + +def marginal_pdf( + values: torch.Tensor, bins: torch.Tensor, sigma: torch.Tensor, epsilon: float = 1e-10 +) -> Tuple[torch.Tensor, torch.Tensor]: + """Calculate the marginal probability distribution function of the input based on the number of histogram bins. + + Args: + values: shape [BxNx1]. + bins: shape [NUM_BINS]. + sigma: shape [1], gaussian smoothing factor. + epsilon: scalar, for numerical stability. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: + - torch.Tensor: shape [BxN]. + - torch.Tensor: shape [BxNxNUM_BINS]. + + """ + if not isinstance(values, torch.Tensor): + raise TypeError(f"Input values type is not a torch.Tensor. Got {type(values)}") + + if not isinstance(bins, torch.Tensor): + raise TypeError(f"Input bins type is not a torch.Tensor. Got {type(bins)}") + + if not isinstance(sigma, torch.Tensor): + raise TypeError(f"Input sigma type is not a torch.Tensor. Got {type(sigma)}") + + if not values.dim() == 3: + raise ValueError(f"Input values must be a of the shape BxNx1. Got {values.shape}") + + if not bins.dim() == 1: + raise ValueError(f"Input bins must be a of the shape NUM_BINS. Got {bins.shape}") + + if not sigma.dim() == 0: + raise ValueError(f"Input sigma must be a of the shape 1. Got {sigma.shape}") + + residuals = values - bins.unsqueeze(0).unsqueeze(0) + kernel_values = torch.exp(-0.5 * (residuals / sigma).pow(2)) + + pdf = torch.mean(kernel_values, dim=1) + normalization = torch.sum(pdf, dim=1).unsqueeze(1) + epsilon + pdf = pdf / normalization + + return pdf, kernel_values + + +def joint_pdf(kernel_values1: torch.Tensor, kernel_values2: torch.Tensor, epsilon: float = 1e-10) -> torch.Tensor: + """Calculate the joint probability distribution function of the input tensors based on the number of histogram bins. + + Args: + kernel_values1: shape [BxNxNUM_BINS]. + kernel_values2: shape [BxNxNUM_BINS]. + epsilon: scalar, for numerical stability. + + Returns: + shape [BxNUM_BINSxNUM_BINS]. + + """ + if not isinstance(kernel_values1, torch.Tensor): + raise TypeError(f"Input kernel_values1 type is not a torch.Tensor. Got {type(kernel_values1)}") + + if not isinstance(kernel_values2, torch.Tensor): + raise TypeError(f"Input kernel_values2 type is not a torch.Tensor. Got {type(kernel_values2)}") + + if not kernel_values1.dim() == 3: + raise ValueError(f"Input kernel_values1 must be a of the shape BxN. Got {kernel_values1.shape}") + + if not kernel_values2.dim() == 3: + raise ValueError(f"Input kernel_values2 must be a of the shape BxN. Got {kernel_values2.shape}") + + if kernel_values1.shape != kernel_values2.shape: + raise ValueError( + "Inputs kernel_values1 and kernel_values2 must have the same shape." + f" Got {kernel_values1.shape} and {kernel_values2.shape}" + ) + + joint_kernel_values = torch.matmul(kernel_values1.transpose(1, 2), kernel_values2) + normalization = torch.sum(joint_kernel_values, dim=(1, 2)).view(-1, 1, 1) + epsilon + pdf = joint_kernel_values / normalization + + return pdf + + +def histogram(x: torch.Tensor, bins: torch.Tensor, bandwidth: torch.Tensor, epsilon: float = 1e-10) -> torch.Tensor: + """Estimate the histogram of the input torch.Tensor. + + The calculation uses kernel density estimation which requires a bandwidth (smoothing) parameter. + + Args: + x: Input torch.Tensor to compute the histogram with shape :math:`(B, D)`. + bins: The number of bins to use the histogram :math:`(N_{bins})`. + bandwidth: Gaussian smoothing factor with shape shape [1]. + epsilon: A scalar, for numerical stability. + + Returns: + Computed histogram of shape :math:`(B, N_{bins})`. + + Examples: + >>> x = torch.rand(1, 10) + >>> bins = torch.torch.linspace(0, 255, 128) + >>> hist = histogram(x, bins, bandwidth=torch.tensor(0.9)) + >>> hist.shape + torch.Size([1, 128]) + + """ + pdf, _ = marginal_pdf(x.unsqueeze(2), bins, bandwidth, epsilon) + + return pdf + + +def histogram2d( + x1: torch.Tensor, x2: torch.Tensor, bins: torch.Tensor, bandwidth: torch.Tensor, epsilon: float = 1e-10 +) -> torch.Tensor: + """Estimate the 2d histogram of the input torch.Tensor. + + The calculation uses kernel density estimation which requires a bandwidth (smoothing) parameter. + + Args: + x1: Input torch.Tensor to compute the histogram with shape :math:`(B, D1)`. + x2: Input torch.Tensor to compute the histogram with shape :math:`(B, D2)`. + bins: The number of bins to use the histogram :math:`(N_{bins})`. + bandwidth: Gaussian smoothing factor with shape shape [1]. + epsilon: A scalar, for numerical stability. Default: 1e-10. + + Returns: + Computed histogram of shape :math:`(B, N_{bins}), N_{bins})`. + + Examples: + >>> x1 = torch.rand(2, 32) + >>> x2 = torch.rand(2, 32) + >>> bins = torch.torch.linspace(0, 255, 128) + >>> hist = histogram2d(x1, x2, bins, bandwidth=torch.tensor(0.9)) + >>> hist.shape + torch.Size([2, 128, 128]) + + """ + _, kernel_values1 = marginal_pdf(x1.unsqueeze(2), bins, bandwidth, epsilon) + _, kernel_values2 = marginal_pdf(x2.unsqueeze(2), bins, bandwidth, epsilon) + + pdf = joint_pdf(kernel_values1, kernel_values2) + + return pdf + + +def image_histogram2d( + image: torch.Tensor, + min: float = 0.0, + max: float = 255.0, + n_bins: int = 256, + bandwidth: Optional[float] = None, + centers: Optional[torch.Tensor] = None, + return_pdf: bool = False, + kernel: str = "triangular", + eps: float = 1e-10, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Estimate the histogram of the input image(s). + + The calculation uses triangular kernel density estimation. + + Args: + image: Input torch.Tensor to compute the histogram with shape + :math:`(H, W)`, :math:`(C, H, W)` or :math:`(B, C, H, W)`. + min: Lower end of the interval (inclusive). + max: Upper end of the interval (inclusive). Ignored when + :attr:`centers` is specified. + n_bins: The number of histogram bins. Ignored when + :attr:`centers` is specified. + bandwidth: Smoothing factor. If not specified or equal to -1, + :math:`(bandwidth = (max - min) / n_bins)`. + centers: Centers of the bins with shape :math:`(n_bins,)`. + If not specified or empty, it is calculated as centers of + equal width bins of [min, max] range. + return_pdf: If True, also return probability densities for + each bin. + kernel: kernel to perform kernel density estimation + ``(`triangular`, `gaussian`, `uniform`, `epanechnikov`)``. + eps: epsilon for numerical stability. + + Returns: + Computed histogram of shape :math:`(bins)`, :math:`(C, bins)`, + :math:`(B, C, bins)`. + Computed probability densities of shape :math:`(bins)`, :math:`(C, bins)`, + :math:`(B, C, bins)`, if return_pdf is ``True``. torch.Tensor of torch.zeros with shape + of the histogram otherwise. + + """ + if image is not None and not isinstance(image, torch.Tensor): + raise TypeError(f"Input image type is not a torch.Tensor. Got {type(image)}.") + + if centers is not None and not isinstance(centers, torch.Tensor): + raise TypeError(f"Bins' centers type is not a torch.Tensor. Got {type(centers)}.") + + if centers is not None and len(centers.shape) > 0 and centers.dim() != 1: + raise ValueError(f"Bins' centers must be a torch.Tensor of the shape (n_bins,). Got {centers.shape}.") + + if not isinstance(min, float): + raise TypeError(f"Type of lower end of the range is not a float. Got {type(min)}.") + + if not isinstance(max, float): + raise TypeError(f"Type of upper end of the range is not a float. Got {type(min)}.") + + if not isinstance(n_bins, int): + raise TypeError(f"Type of number of bins is not an int. Got {type(n_bins)}.") + + if bandwidth is not None and not isinstance(bandwidth, float): + raise TypeError(f"Bandwidth type is not a float. Got {type(bandwidth)}.") + + if not isinstance(return_pdf, bool): + raise TypeError(f"Return_pdf type is not a bool. Got {type(return_pdf)}.") + + if bandwidth is None: + bandwidth = (max - min) / n_bins + + if centers is None: + centers = min + bandwidth * (torch.arange(n_bins, device=image.device, dtype=image.dtype) + 0.5) + centers = centers.reshape(-1, 1, 1, 1, 1) + + u = torch.abs(image.unsqueeze(0) - centers) / bandwidth + + if kernel == "gaussian": + kernel_values = torch.exp(-0.5 * u**2) + elif kernel in ("triangular", "uniform", "epanechnikov"): + # compute the mask and cast to floating point + mask = (u <= 1).to(u.dtype) + if kernel == "triangular": + kernel_values = (1.0 - u) * mask + elif kernel == "uniform": + kernel_values = mask + else: # kernel == "epanechnikov" + kernel_values = (1.0 - u**2) * mask + else: + raise ValueError(f"Kernel must be 'triangular', 'gaussian', 'uniform' or 'epanechnikov'. Got {kernel}.") + + hist = torch.sum(kernel_values, dim=(-2, -1)).permute(1, 2, 0) + + if return_pdf: + normalization = torch.sum(hist, dim=-1, keepdim=True) + eps + pdf = hist / normalization + if image.dim() == 2: + hist = hist.squeeze() + pdf = pdf.squeeze() + elif image.dim() == 3: + hist = hist.squeeze(0) + pdf = pdf.squeeze(0) + return hist, pdf + + if image.dim() == 2: + hist = hist.squeeze() + elif image.dim() == 3: + hist = hist.squeeze(0) + + return hist, torch.zeros_like(hist) diff --git a/kornia/enhance/integral.py b/kornia/enhance/integral.py new file mode 100644 index 0000000..abfce9c --- /dev/null +++ b/kornia/enhance/integral.py @@ -0,0 +1,173 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Optional, Tuple + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SHAPE + + +def integral_tensor(input: torch.Tensor, dim: Optional[Tuple[int, ...]] = None) -> torch.Tensor: + """Calculate integral of the input torch.Tensor. + + The algorithm computes the integral image by summing over the specified dimensions. + + In case dim is specified, the contained dimensions must be unique and sorted in ascending order + and not exceed the number of dimensions of the input torch.Tensor. + + Args: + input: the input torch.Tensor with shape :math:`(*, D)`. Where D is the number of dimensions. + dim: the dimension to be summed. + + Returns: + Integral torch.Tensor for the input torch.Tensor with shape :math:`(*, D)`. + + Examples: + >>> input = torch.ones(3, 5) + >>> output = integral_tensor(input, (-2, -1)) + >>> output + tensor([[ 1., 2., 3., 4., 5.], + [ 2., 4., 6., 8., 10.], + [ 3., 6., 9., 12., 15.]]) + + """ + KORNIA_CHECK_SHAPE(input, ["*", "D"]) + + if dim is None: + dim = (-1,) + + KORNIA_CHECK(len(dim) > 0, "dim must be a non-empty tuple.") + KORNIA_CHECK(len(dim) <= len(input.shape), "dim must be a tuple of length <= input.shape.") + + output = input + for i in dim: + output = output.cumsum(i) + return output + + +def integral_image(image: torch.Tensor) -> torch.Tensor: + r"""Calculate integral of the input image torch.Tensor. + + This particular version sums over the last two dimensions. + + Args: + image: the input image torch.Tensor with shape :math:`(*, H, W)`. + + Returns: + Integral torch.Tensor for the input image torch.Tensor with shape :math:`(*, H, W)`. + + Examples: + >>> input = torch.ones(1, 5, 5) + >>> output = integral_image(input) + >>> output + tensor([[[ 1., 2., 3., 4., 5.], + [ 2., 4., 6., 8., 10.], + [ 3., 6., 9., 12., 15.], + [ 4., 8., 12., 16., 20.], + [ 5., 10., 15., 20., 25.]]]) + + """ + KORNIA_CHECK_SHAPE(image, ["*", "H", "W"]) + + return integral_tensor(image, (-2, -1)) + + +class IntegralTensor(nn.Module): + r"""Calculates integral of the input torch.Tensor. + + Args: + image: the input torch.Tensor with shape :math:`(B,C,H,W)`. + + Returns: + Integral torch.Tensor for the input torch.Tensor with shape :math:`(B,C,H,W)`. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + Examples: + >>> input = torch.ones(3, 5) + >>> dim = (-2, -1) + >>> output = IntegralTensor(dim)(input) + >>> output + tensor([[ 1., 2., 3., 4., 5.], + [ 2., 4., 6., 8., 10.], + [ 3., 6., 9., 12., 15.]]) + + """ + + def __init__(self, dim: Optional[Tuple[int, ...]] = None) -> None: + super().__init__() + self.dim = dim + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Compute cumulative sums of the input tensor along configured axes. + + Args: + input: Tensor to integrate. The operation supports arbitrary rank, + and dimensions are accumulated according to ``self.dim``. + + Returns: + A tensor with the same shape as ``input`` containing cumulative + sums. If ``self.dim`` is ``None``, integration is applied to the + last dimension by :func:`integral_tensor`. + """ + return integral_tensor(input, self.dim) + + +class IntegralImage(nn.Module): + """Calculates integral of the input image torch.Tensor. + + This particular version sums over the last two dimensions. + + Args: + image: the input image torch.Tensor with shape :math:`(B,C,H,W)`. + + Returns: + Integral torch.Tensor for the input image torch.Tensor with shape :math:`(B,C,H,W)`. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + Examples: + >>> input = torch.ones(1, 5, 5) + >>> output = IntegralImage()(input) + >>> output + tensor([[[ 1., 2., 3., 4., 5.], + [ 2., 4., 6., 8., 10.], + [ 3., 6., 9., 12., 15.], + [ 4., 8., 12., 16., 20.], + [ 5., 10., 15., 20., 25.]]]) + + """ + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Compute a 2D integral image by summing over height and width. + + Args: + input: Tensor whose last two dimensions represent spatial axes, + typically shaped :math:`(*, H, W)`. + + Returns: + A tensor of the same shape as ``input`` where each location stores + the cumulative sum over the top-left rectangle ending at that + location. + """ + return integral_image(input) diff --git a/kornia/enhance/jpeg.py b/kornia/enhance/jpeg.py new file mode 100644 index 0000000..a4a5eb1 --- /dev/null +++ b/kornia/enhance/jpeg.py @@ -0,0 +1,778 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import math +from typing import Union + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.color import rgb_to_ycbcr, ycbcr_to_rgb +from kornia.constants import pi +from kornia.core.check import ( + KORNIA_CHECK, + KORNIA_CHECK_IS_TENSOR, + KORNIA_CHECK_SHAPE, +) +from kornia.geometry.transform.affwarp import rescale +from kornia.image.utils import perform_keep_shape_image + +_DCT8_CACHE: dict[ + tuple[Union[torch.dtype, None], Union[str, torch.device, None]], tuple[torch.Tensor, torch.Tensor] +] = {} + +__all__ = ["JPEGCodecDifferentiable", "jpeg_codec_differentiable"] + + +def _differentiable_polynomial_rounding(input: torch.Tensor) -> torch.Tensor: + """Differentiable rounding. + + Args: + input: Input tensor of any shape to be rounded. + + Returns: + Pseudo rounded tensor of the same shape as input tensor. + + """ + input_round = input.round() + return input_round + (input - input_round) ** 3 + + +def _differentiable_polynomial_floor(input: torch.Tensor) -> torch.Tensor: + """Perform floor via a differentiable operation. + + Args: + input: Input tensor of any shape to be floored. + + Returns: + Pseudo rounded tensor of the same shape as input tensor. + + """ + input_floor = input.floor() + return input_floor + (input - 0.5 - input_floor) ** 3 + + +def _differentiable_clipping( + input: torch.Tensor, + min_val: Union[float, None] = None, + max_val: Union[float, None] = None, + scale: float = 0.02, +) -> torch.Tensor: + """Clip via a differentiable and soft approximation of the clipping operation. + + Args: + input: Input tensor of any shape. + min_val: Minimum value. + max_val: Maximum value. + scale: Scale value. Default 0.02. + + Returns: + Clipped output tensor of the same shape as the input tensor. + + """ + output: torch.Tensor = input.clone() + if max_val is not None: + output[output > max_val] = -scale * (torch.exp(-output[output > max_val] + max_val) - 1.0) + max_val + if min_val is not None: + output[output < min_val] = scale * (torch.exp(output[output < min_val] - min_val) - 1.0) + min_val + return output + + +def _get_default_qt_y(device: Union[str, torch.device, None], dtype: Union[torch.dtype, None]) -> torch.Tensor: + """Generate default Quantization table of Y channel.""" + return torch.tensor( + [ + [16, 11, 10, 16, 24, 40, 51, 61], + [12, 12, 14, 19, 26, 58, 60, 55], + [14, 13, 16, 24, 40, 57, 69, 56], + [14, 17, 22, 29, 51, 87, 80, 62], + [18, 22, 37, 56, 68, 109, 103, 77], + [24, 35, 55, 64, 81, 104, 113, 92], + [49, 64, 78, 87, 103, 121, 120, 101], + [72, 92, 95, 98, 112, 100, 103, 99], + ], + device=device, + dtype=dtype, + ) + + +def _get_default_qt_c(device: Union[str, torch.device, None], dtype: Union[torch.dtype, None]) -> torch.Tensor: + """Generate default Quantization table of C channels.""" + return torch.tensor( + [ + [17, 18, 24, 47, 99, 99, 99, 99], + [18, 21, 26, 66, 99, 99, 99, 99], + [24, 26, 56, 99, 99, 99, 99, 99], + [47, 66, 99, 99, 99, 99, 99, 99], + [99, 99, 99, 99, 99, 99, 99, 99], + [99, 99, 99, 99, 99, 99, 99, 99], + [99, 99, 99, 99, 99, 99, 99, 99], + [99, 99, 99, 99, 99, 99, 99, 99], + ], + device=device, + dtype=dtype, + ) + + +def _patchify_8x8(input: torch.Tensor) -> torch.Tensor: + """Extract non-overlapping 8 x 8 patches from the given input image. + + Args: + input (torch.Tensor): Input image of the shape :math:`(B, H, W)`. + + Returns: + output (torch.Tensor): Image patchify of the shape :math:`(B, N, 8, 8)`. + + """ + # Get input shape + B, H, W = input.shape + # Patchify to shape [B, N, H // 8, W // 8] + output: torch.Tensor = input.view(B, H // 8, 8, W // 8, 8).permute(0, 1, 3, 2, 4).reshape(B, -1, 8, 8) + return output + + +def _unpatchify_8x8(input: torch.Tensor, H: int, W: int) -> torch.Tensor: + """Reverse non-overlapping 8 x 8 patching. + + Args: + input (torch.Tensor): Input image of the shape :math:`(B, N, 8, 8)`. + H: height of resulting torch.Tensor. + W: width of resulting torch.Tensor. + + Returns: + output (torch.Tensor): Image patchify of the shape :math:`(B, H, W)`. + + """ + # Get input shape + B, _N = input.shape[:2] + # Unpatch to [B, H, W] + output: torch.Tensor = input.view(B, H // 8, W // 8, 8, 8).permute(0, 1, 3, 2, 4).reshape(B, H, W) + return output + + +def _dct_8x8(input: torch.Tensor) -> torch.Tensor: + """Perform an 8 x 8 discrete cosine transform. + + Args: + input (torch.Tensor): Patched input torch.tensor of the shape :math:`(B, N, 8, 8)`. + + Returns: + output (torch.Tensor): DCT output torch.tensor of the shape :math:`(B, N, 8, 8)`. + + """ + # Get dtype and device + dtype: Union[torch.dtype, None] = input.dtype + device: Union[str, torch.device, None] = input.device + dct_tensor, dct_scale = _get_dct8_basis_scale(dtype, device) + # Apply DCT + output: torch.Tensor = dct_scale[None, None] * torch.tensordot(input - 128.0, dct_tensor) + return output + + +def _idct_8x8(input: torch.Tensor) -> torch.Tensor: + """Perform an 8 x 8 inverse discrete cosine transform. + + Args: + input (torch.Tensor): Patched input torch.tensor of the shape :math:`(B, N, 8, 8)`. + + Returns: + output (torch.Tensor): IDCT output torch.tensor of the shape :math:`(B, N, 8, 8)`. + + """ + dtype = input.dtype + device = input.device + + idx = torch.arange(8, dtype=dtype, device=device) + spatial_idx = idx.unsqueeze(0) + freq_idx = idx.unsqueeze(1) + + basis = torch.cos((2.0 * spatial_idx + 1.0) * freq_idx * pi / 16.0) + alpha = torch.ones(8, dtype=dtype, device=device) + alpha[0] = 1.0 / (2**0.5) + dct_scale = torch.outer(alpha, alpha) + input = input * dct_scale + + tmp = input @ basis + output = (tmp.transpose(-1, -2) @ basis).transpose(-1, -2) + + output = output * 0.25 + 128.0 + return output + + +def _jpeg_quality_to_scale( + compression_strength: torch.Tensor, +) -> torch.Tensor: + """Convert a given JPEG quality to the scaling factor. + + Args: + compression_strength (torch.Tensor): Compression strength ranging from 0 to 100. Any shape is supported. + + Returns: + scale (torch.Tensor): Scaling factor to be applied to quantization matrix. Same shape as input. + + """ + # Get scale + scale: torch.Tensor = _differentiable_polynomial_floor( + torch.where( + compression_strength < 50, + 5000.0 / compression_strength, + 200.0 - 2.0 * compression_strength, + ) + ) + return scale + + +def _quantize( + input: torch.Tensor, + jpeg_quality: torch.Tensor, + quantization_table: torch.Tensor, +) -> torch.Tensor: + """Perform quantization. + + Args: + input (torch.Tensor): Input torch.tensor of the shape :math:`(B, N, 8, 8)`. + jpeg_quality (torch.Tensor): Compression strength to be applied, shape is :math:`(B)`. + quantization_table (torch.Tensor): Quantization table of the shape :math:`(1, 8, 8)` or :math:`(B, 8, 8)`. + + Returns: + output (torch.Tensor): Quantized output torch.tensor of the shape :math:`(B, N, 8, 8)`. + + """ + # Scale quantization table + quantization_table_scaled: torch.Tensor = ( + quantization_table[:, None] * _jpeg_quality_to_scale(jpeg_quality)[:, None, None, None] + ) + # Perform scaling + quantization_table = _differentiable_polynomial_floor( + _differentiable_clipping((quantization_table_scaled + 50.0) / 100.0, 1, 255) + ) + output: torch.Tensor = input / quantization_table + # Perform rounding + output = _differentiable_polynomial_rounding(output) + return output + + +def _dequantize( + input: torch.Tensor, + jpeg_quality: torch.Tensor, + quantization_table: torch.Tensor, +) -> torch.Tensor: + """Perform dequantization. + + Args: + input (torch.Tensor): Input torch.tensor of the shape :math:`(B, N, 8, 8)`. + jpeg_quality (torch.Tensor): Compression strength to be applied, shape is :math:`(B)`. + quantization_table (torch.Tensor): Quantization table of the shape :math:`(1, 8, 8)` or :math:`(B, 8, 8)`. + + Returns: + output (torch.Tensor): Quantized output torch.tensor of the shape :math:`(B, N, 8, 8)`. + + """ + # Scale quantization table + quantization_table_scaled: torch.Tensor = ( + quantization_table[:, None] * _jpeg_quality_to_scale(jpeg_quality)[:, None, None, None] + ) + # Perform scaling + output: torch.Tensor = input * _differentiable_polynomial_floor( + _differentiable_clipping((quantization_table_scaled + 50.0) / 100.0, 1, 255) + ) + return output + + +def _chroma_subsampling(input_ycbcr: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Implement chroma subsampling. + + Args: + input_ycbcr (torch.Tensor): YCbCr input torch.tensor of the shape :math:`(B, 3, H, W)`. + + Returns: + output_y (torch.Tensor): Y component (not-subsampled), shape is :math:`(B, H, W)`. + output_cb (torch.Tensor): Cb component (subsampled), shape is :math:`(B, H // 2, W // 2)`. + output_cr (torch.Tensor): Cr component (subsampled), shape is :math:`(B, H // 2, W // 2)`. + + """ + # Get components + output_y: torch.Tensor = input_ycbcr[:, 0] + output_cb: torch.Tensor = input_ycbcr[:, 1] + output_cr: torch.Tensor = input_ycbcr[:, 2] + # Perform average pooling of Cb and Cr channels + output_cb = rescale( + output_cb[:, None], + factor=0.5, + interpolation="bilinear", + align_corners=False, + antialias=True, + ) + output_cr = rescale( + output_cr[:, None], + factor=0.5, + interpolation="bilinear", + align_corners=False, + antialias=True, + ) + return output_y, output_cb[:, 0], output_cr[:, 0] + + +def _chroma_upsampling(input_c: torch.Tensor) -> torch.Tensor: + """Perform chroma upsampling. + + Args: + input_c (torch.Tensor): Cb or Cr component to be upsampled of the shape :math:`(B, H, W)`. + + Returns: + output_c (torch.Tensor): Upsampled C(b or r) component of the shape :math:`(B, H * 2, W * 2)`. + + """ + # Upsample component + output_c: torch.Tensor = rescale( + input_c[:, None], + factor=2.0, + interpolation="bilinear", + align_corners=False, + antialias=False, + ) + return output_c[:, 0] + + +def _jpeg_encode( + image_rgb: torch.Tensor, + jpeg_quality: torch.Tensor, + quantization_table_y: torch.Tensor, + quantization_table_c: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Perform JPEG encoding. + + Args: + image_rgb (torch.Tensor): RGB input images of the shape :math:`(B, 3, H, W)`. + jpeg_quality (torch.Tensor): Compression strength of the shape :math:`(B)`. + quantization_table_y (torch.Tensor): Quantization table for Y channel. + quantization_table_c (torch.Tensor): Quantization table for C channels. + + Returns: + y_encoded (torch.Tensor): Encoded Y component of the shape :math:`(B, N, 8, 8)`. + cb_encoded (torch.Tensor): Encoded Cb component of the shape :math:`(B, N, 8, 8)`. + cr_encoded (torch.Tensor): Encoded Cr component of the shape :math:`(B, N, 8, 8)`. + + """ + # Convert RGB image to YCbCr. + image_ycbcr: torch.Tensor = rgb_to_ycbcr(image_rgb) + # Scale pixel-range to [0, 255] + image_ycbcr = 255.0 * image_ycbcr + # Perform chroma subsampling + input_y, input_cb, input_cr = _chroma_subsampling(image_ycbcr) + # Patchify, DCT, and rounding + input_y, input_cb, input_cr = ( + _patchify_8x8(input_y), + _patchify_8x8(input_cb), + _patchify_8x8(input_cr), + ) + dct_y = _dct_8x8(input_y) + dct_cb_cr = _dct_8x8(torch.cat((input_cb, input_cr), dim=1)) + y_encoded: torch.Tensor = _quantize( + dct_y, + jpeg_quality, + quantization_table_y, + ) + cb_encoded, cr_encoded = _quantize( + dct_cb_cr, + jpeg_quality, + quantization_table_c, + ).chunk(2, dim=1) + return y_encoded, cb_encoded, cr_encoded + + +def _jpeg_decode( + input_y: torch.Tensor, + input_cb: torch.Tensor, + input_cr: torch.Tensor, + jpeg_quality: torch.Tensor, + H: int, + W: int, + quantization_table_y: torch.Tensor, + quantization_table_c: torch.Tensor, +) -> torch.Tensor: + """Perform JPEG decoding. + + Args: + input_y (torch.Tensor): Compressed Y component of the shape :math:`(B, N, 8, 8)`. + input_cb (torch.Tensor): Compressed Cb component of the shape :math:`(B, N, 8, 8)`. + input_cr (torch.Tensor): Compressed Cr component of the shape :math:`(B, N, 8, 8)`. + jpeg_quality (torch.Tensor): Compression strength of the shape :math:`(B)`. + H (int): Original image height. + W (int): Original image width. + quantization_table_y (torch.Tensor): Quantization table for Y channel. + quantization_table_c (torch.Tensor): Quantization table for C channels. + + Returns: + rgb_decoded (torch.Tensor): Decompressed RGB image of the shape :math:`(B, 3, H, W)`. + + """ + # Dequantize inputs + input_y = _dequantize( + input_y, + jpeg_quality, + quantization_table_y, + ) + input_cb_cr = _dequantize( + torch.cat((input_cb, input_cr), dim=1), + jpeg_quality, + quantization_table_c, + ) + # Perform inverse DCT + idct_y: torch.Tensor = _idct_8x8(input_y) + idct_cb, idct_cr = _idct_8x8(input_cb_cr).chunk(2, dim=1) + # Reverse patching + image_y: torch.Tensor = _unpatchify_8x8(idct_y, H, W) + image_cb: torch.Tensor = _unpatchify_8x8(idct_cb, H // 2, W // 2) + image_cr: torch.Tensor = _unpatchify_8x8(idct_cr, H // 2, W // 2) + # Perform chroma upsampling + image_cb = _chroma_upsampling(image_cb) + image_cr = _chroma_upsampling(image_cr) + # Back to [0, 1] pixel-range + image_ycbcr: torch.Tensor = torch.stack((image_y, image_cb, image_cr), dim=1) / 255.0 + # Convert back to RGB space. + rgb_decoded: torch.Tensor = ycbcr_to_rgb(image_ycbcr) + return rgb_decoded + + +def _perform_padding(image: torch.Tensor) -> tuple[torch.Tensor, int, int]: + """Pad a given image to be dividable by 16. + + Args: + image: Image of the shape :math:`(*, 3, H, W)`. + + Returns: + image_padded: Padded image of the shape :math:`(*, 3, H_{new}, W_{new})`. + h_pad: Padded pixels along the horizontal axis. + w_pad: Padded pixels along the vertical axis. + + """ + # Get spatial dimensions of the image + H, W = image.shape[-2:] + # Compute horizontal and vertical padding + h_pad: int = math.ceil(H / 16) * 16 - H + w_pad: int = math.ceil(W / 16) * 16 - W + # Perform padding (we follow JPEG and F.pad only the bottom and right side of the image) + image_padded: torch.Tensor = F.pad(image, (0, w_pad, 0, h_pad), "replicate") + return image_padded, h_pad, w_pad + + +@perform_keep_shape_image +def jpeg_codec_differentiable( + input: torch.Tensor, + jpeg_quality: torch.Tensor, + quantization_table_y: torch.Tensor | None = None, + quantization_table_c: torch.Tensor | None = None, +) -> torch.Tensor: + r"""Differentiable JPEG encoding-decoding module. + + Based on :cite:`reich2024` :cite:`shin2017`, we perform differentiable JPEG encoding-decoding as follows: + + .. image:: _static/img/jpeg_codec_differentiable.png + + .. math:: + + \text{JPEG}_{\text{diff}}(I, q, QT_{y}, QT_{c}) = \hat{I} + + Where: + - :math:`I` is the original image to be coded. + - :math:`q` is the JPEG quality controlling the compression strength. + - :math:`QT_{y}` is the luma quantization table. + - :math:`QT_{c}` is the chroma quantization table. + - :math:`\hat{I}` is the resulting JPEG encoded-decoded image. + + .. note::: + The input (and output) pixel range is :math:`[0, 1]`. In case you want to handle normalized images you are + required to first perform denormalization followed by normalizing the output images again. + + Note, that this implementation models the encoding-decoding mapping of JPEG in a differentiable setting, + however, does not allow the excess of the JPEG-coded byte file itself. + For more details please refer to :cite:`reich2024`. + + This implementation is not meant for data loading. For loading JPEG images please refer to `kornia.io`. + There we provide an optimized Rust implementation for fast JPEG loading. + + Args: + input: the RGB image to be coded. + jpeg_quality: JPEG quality in the range :math:`[0, 100]` controlling the compression strength. + quantization_table_y: quantization table for Y channel. Default: `None`, which will load the standard + quantization table. + quantization_table_c: quantization table for C channels. Default: `None`, which will load the standard + quantization table. + + Shape: + - input: :math:`(*, 3, H, W)`. + - jpeg_quality: :math:`(1)` or :math:`(B)` (if used batch dim. needs to match w/ input). + - quantization_table_y: :math:`(8, 8)` or :math:`(B, 8, 8)` (if used batch dim. needs to match w/ input). + - quantization_table_c: :math:`(8, 8)` or :math:`(B, 8, 8)` (if used batch dim. needs to match w/ input). + + Return: + JPEG coded image of the shape :math:`(B, 3, H, W)` + + Example: + To perform JPEG coding with the standard quantization tables just provide a JPEG quality + + >>> img = torch.rand(3, 3, 64, 64, requires_grad=True, dtype=torch.float) + >>> jpeg_quality = torch.tensor((99.0, 25.0, 1.0), requires_grad=True) + >>> img_jpeg = jpeg_codec_differentiable(img, jpeg_quality) + >>> img_jpeg.sum().backward() + + You also have the option to provide custom quantization tables + + >>> img = torch.rand(3, 3, 64, 64, requires_grad=True, dtype=torch.float) + >>> jpeg_quality = torch.tensor((99.0, 25.0, 1.0), requires_grad=True) + >>> quantization_table_y = torch.randint(1, 256, size=(3, 8, 8), dtype=torch.float) + >>> quantization_table_c = torch.randint(1, 256, size=(3, 8, 8), dtype=torch.float) + >>> img_jpeg = jpeg_codec_differentiable(img, jpeg_quality, quantization_table_y, quantization_table_c) + >>> img_jpeg.sum().backward() + + In case you want to control the quantization purly base on the quantization tables use a JPEG quality of 99.5. + Setting the JPEG quality to 99.5 leads to a QT scaling of 1, see Eq. 2 of :cite:`reich2024` for details. + + >>> img = torch.rand(3, 3, 64, 64, requires_grad=True, dtype=torch.float) + >>> jpeg_quality = torch.ones(3) * 99.5 + >>> quantization_table_y = torch.randint(1, 256, size=(3, 8, 8), dtype=torch.float) + >>> quantization_table_c = torch.randint(1, 256, size=(3, 8, 8), dtype=torch.float) + >>> img_jpeg = jpeg_codec_differentiable(img, jpeg_quality, quantization_table_y, quantization_table_c) + >>> img_jpeg.sum().backward() + + """ + # Check that inputs are tensors + KORNIA_CHECK_IS_TENSOR(input) + KORNIA_CHECK_IS_TENSOR(jpeg_quality) + # Get device and dtype + dtype: Union[torch.dtype, None] = input.dtype + device: Union[str, torch.device, None] = input.device + # Use default QT if QT is not given + quantization_table_y = _get_default_qt_y(device, dtype) if quantization_table_y is None else quantization_table_y + quantization_table_c = _get_default_qt_c(device, dtype) if quantization_table_c is None else quantization_table_c + KORNIA_CHECK_IS_TENSOR(quantization_table_y) + KORNIA_CHECK_IS_TENSOR(quantization_table_c) + # Check shape of inputs + KORNIA_CHECK_SHAPE(input, ["*", "3", "H", "W"]) + KORNIA_CHECK_SHAPE(jpeg_quality, ["B"]) + # Add batch dimension to quantization tables if needed + if quantization_table_y.ndim == 2: + quantization_table_y = quantization_table_y.unsqueeze(dim=0) + if quantization_table_c.ndim == 2: + quantization_table_c = quantization_table_c.unsqueeze(dim=0) + # Check resulting shape of quantization tables + KORNIA_CHECK_SHAPE(quantization_table_y, ["B", "8", "8"]) + KORNIA_CHECK_SHAPE(quantization_table_c, ["B", "8", "8"]) + # Check value range of JPEG quality + KORNIA_CHECK( + (jpeg_quality.amin().item() >= 0.0) and (jpeg_quality.amax().item() <= 100.0), + f"JPEG quality is out of range. Expected range is [0, 100], " + f"got [{jpeg_quality.amin().item()}, {jpeg_quality.amax().item()}]. Consider clipping jpeg_quality.", + ) + # Pad the image to a shape dividable by 16 + input, h_pad, w_pad = _perform_padding(input) + # Get height and shape + H, W = input.shape[-2:] + # Check matching batch dimensions + if quantization_table_y.shape[0] != 1: + KORNIA_CHECK( + quantization_table_y.shape[0] == input.shape[0], + f"Batch dimensions do not match. " + f"Got {input.shape[0]} images and {quantization_table_y.shape[0]} quantization tables (Y).", + ) + if quantization_table_c.shape[0] != 1: + KORNIA_CHECK( + quantization_table_c.shape[0] == input.shape[0], + f"Batch dimensions do not match. " + f"Got {input.shape[0]} images and {quantization_table_c.shape[0]} quantization tables (C).", + ) + if jpeg_quality.shape[0] != 1: + KORNIA_CHECK( + jpeg_quality.shape[0] == input.shape[0], + f"Batch dimensions do not match. Got {input.shape[0]} images and {jpeg_quality.shape[0]} JPEG qualities.", + ) + # keep jpeg_quality same device as input torch.Tensor + jpeg_quality = jpeg_quality.to(device, dtype) + # Quantization tables to same device and dtype as input image + quantization_table_y = quantization_table_y.to(device, dtype) + quantization_table_c = quantization_table_c.to(device, dtype) + # Perform encoding + y_encoded, cb_encoded, cr_encoded = _jpeg_encode( + image_rgb=input, + jpeg_quality=jpeg_quality, + quantization_table_c=quantization_table_c, + quantization_table_y=quantization_table_y, + ) + image_rgb_jpeg: torch.Tensor = _jpeg_decode( + input_y=y_encoded, + input_cb=cb_encoded, + input_cr=cr_encoded, + jpeg_quality=jpeg_quality, + H=H, + W=W, + quantization_table_c=quantization_table_c, + quantization_table_y=quantization_table_y, + ) + # Clip coded image + image_rgb_jpeg = _differentiable_clipping(input=image_rgb_jpeg, min_val=0.0, max_val=255.0) + # Crop the image again to the original shape + image_rgb_jpeg = image_rgb_jpeg[..., : H - h_pad, : W - w_pad] + return image_rgb_jpeg + + +def _get_dct8_basis_scale( + dtype: Union[torch.dtype, None], device: Union[str, torch.device, None] +) -> tuple[torch.Tensor, torch.Tensor]: + key = (dtype, device) + if key not in _DCT8_CACHE: + i = torch.arange(8, dtype=dtype, device=device) + freq = (2.0 * i + 1.0)[:, None] * i[None, :] * (pi / 16.0) + basis_1d = torch.cos(freq) + dct_tensor = basis_1d[:, None, :, None] * basis_1d[None, :, None, :] + alpha = torch.ones(8, dtype=dtype, device=device) + alpha[0] = 1.0 / (2**0.5) + dct_scale = torch.outer(alpha, alpha) * 0.25 + _DCT8_CACHE[key] = (dct_tensor, dct_scale) + return _DCT8_CACHE[key] + + +class JPEGCodecDifferentiable(nn.Module): + r"""Differentiable JPEG encoding-decoding module. + + Based on :cite:`reich2024` :cite:`shin2017`, we perform differentiable JPEG encoding-decoding as follows: + + .. math:: + + \text{JPEG}_{\text{diff}}(I, q, QT_{y}, QT_{c}) = \hat{I} + + Where: + - :math:`I` is the original image to be coded. + - :math:`q` is the JPEG quality controlling the compression strength. + - :math:`QT_{y}` is the luma quantization table. + - :math:`QT_{c}` is the chroma quantization table. + - :math:`\hat{I}` is the resulting JPEG encoded-decoded image. + + .. image:: _static/img/jpeg_codec_differentiable.png + + .. note:: + The input (and output) pixel range is :math:`[0, 1]`. In case you want to handle normalized images you are + required to first perform denormalization followed by normalizing the output images again. + + Note, that this implementation models the encoding-decoding mapping of JPEG in a differentiable setting, + however, does not allow the excess of the JPEG-coded byte file itself. + For more details please refer to :cite:`reich2024`. + + This implementation is not meant for data loading. For loading JPEG images please refer to `kornia.io`. + There we provide an optimized Rust implementation for fast JPEG loading. + + Args: + quantization_table_y: quantization table for Y channel. Default: `None`, which will load the standard + quantization table. + quantization_table_c: quantization table for C channels. Default: `None`, which will load the standard + quantization table. + + Shape: + - quantization_table_y: :math:`(8, 8)` or :math:`(B, 8, 8)` (if used batch dim. needs to match w/ image_rgb). + - quantization_table_c: :math:`(8, 8)` or :math:`(B, 8, 8)` (if used batch dim. needs to match w/ image_rgb). + - image_rgb: :math:`(*, 3, H, W)`. + - jpeg_quality: :math:`(1)` or :math:`(B)` (if used batch dim. needs to match w/ image_rgb). + + Example: + You can use the differentiable JPEG module with standard quantization tables by + + >>> diff_jpeg_module = JPEGCodecDifferentiable() + >>> img = torch.rand(2, 3, 32, 32, requires_grad=True, dtype=torch.float) + >>> jpeg_quality = torch.tensor((99.0, 1.0), requires_grad=True) + >>> img_jpeg = diff_jpeg_module(img, jpeg_quality) + >>> img_jpeg.sum().backward() + + You can also specify custom quantization tables to be used by + + >>> quantization_table_y = torch.randint(1, 256, size=(2, 8, 8), dtype=torch.float) + >>> quantization_table_c = torch.randint(1, 256, size=(2, 8, 8), dtype=torch.float) + >>> diff_jpeg_module = JPEGCodecDifferentiable(quantization_table_y, quantization_table_c) + >>> img = torch.rand(2, 3, 32, 32, requires_grad=True, dtype=torch.float) + >>> jpeg_quality = torch.tensor((99.0, 1.0), requires_grad=True) + >>> img_jpeg = diff_jpeg_module(img, jpeg_quality) + >>> img_jpeg.sum().backward() + + In case you want to learn the quantization tables just pass parameters `nn.Parameter` + + >>> quantization_table_y = torch.nn.Parameter(torch.randint(1, 256, size=(2, 8, 8), dtype=torch.float)) + >>> quantization_table_c = torch.nn.Parameter(torch.randint(1, 256, size=(2, 8, 8), dtype=torch.float)) + >>> diff_jpeg_module = JPEGCodecDifferentiable(quantization_table_y, quantization_table_c) + >>> img = torch.rand(2, 3, 32, 32, requires_grad=True, dtype=torch.float) + >>> jpeg_quality = torch.tensor((99.0, 1.0), requires_grad=True) + >>> img_jpeg = diff_jpeg_module(img, jpeg_quality) + >>> img_jpeg.sum().backward() + + """ + + def __init__( + self, + quantization_table_y: torch.Tensor | nn.Parameter | None = None, + quantization_table_c: torch.Tensor | nn.Parameter | None = None, + ) -> None: + super().__init__() + # Get default quantization tables if needed + quantization_table_y = _get_default_qt_y(None, None) if quantization_table_y is None else quantization_table_y + quantization_table_c = _get_default_qt_c(None, None) if quantization_table_c is None else quantization_table_c + if isinstance(quantization_table_y, nn.Parameter): + self.register_parameter("quantization_table_y", quantization_table_y) + else: + self.register_buffer("quantization_table_y", quantization_table_y) + if isinstance(quantization_table_c, nn.Parameter): + self.register_parameter("quantization_table_c", quantization_table_c) + else: + self.register_buffer("quantization_table_c", quantization_table_c) + + def forward( + self, + image_rgb: torch.Tensor, + jpeg_quality: torch.Tensor, + ) -> torch.Tensor: + """Apply differentiable JPEG compression and decompression. + + The method runs an RGB image through the internal JPEG codec pipeline + using the module's quantization tables, then reconstructs an RGB tensor. + Quantization tables are moved to the input device and dtype before + encoding/decoding. + + Args: + image_rgb: Input RGB tensor with shape :math:`(*, 3, H, W)`. + jpeg_quality: JPEG quality factor tensor. It can be scalar or + batched, and must be broadcast-compatible with the leading + dimensions of ``image_rgb`` as required by + :func:`jpeg_codec_differentiable`. + + Returns: + Reconstructed RGB tensor after differentiable JPEG processing, with + the same shape as ``image_rgb``. + """ + device = image_rgb.device + dtype = image_rgb.dtype + # Move quantization tables to the same device and dtype as input + # and store it in the local variables created in init + quantization_table_y = self.quantization_table_y.to(device, dtype) + quantization_table_c = self.quantization_table_c.to(device, dtype) + # Perform encoding-decoding + image_rgb_jpeg: torch.Tensor = jpeg_codec_differentiable( + image_rgb, + jpeg_quality=jpeg_quality, + quantization_table_c=quantization_table_c, + quantization_table_y=quantization_table_y, + ) + return image_rgb_jpeg diff --git a/kornia/enhance/normalize.py b/kornia/enhance/normalize.py new file mode 100644 index 0000000..3e701c9 --- /dev/null +++ b/kornia/enhance/normalize.py @@ -0,0 +1,352 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""nn.Module containing functionals for intensity normalisation.""" + +from typing import List, Tuple, Union + +import torch +from torch import nn + +from kornia.image.utils import perform_keep_shape_image + +__all__ = ["Denormalize", "Normalize", "denormalize", "normalize", "normalize_min_max"] + + +class Normalize(nn.Module): + r"""Normalize a torch.Tensor image with mean and standard deviation. + + .. math:: + \text{input[channel] = (input[channel] - mean[channel]) / std[channel]} + + Where `mean` is :math:`(M_1, ..., M_n)` and `std` :math:`(S_1, ..., S_n)` for `n` channels, + + Args: + mean: Mean for each channel. + std: Standard deviations for each channel. + + Shape: + - Input: Image torch.Tensor of size :math:`(*, C, ...)`. + - Output: Normalised torch.Tensor with same size as input :math:`(*, C, ...)`. + + Examples: + >>> x = torch.rand(1, 4, 3, 3) + >>> out = Normalize(0.0, 255.)(x) + >>> out.shape + torch.Size([1, 4, 3, 3]) + + >>> x = torch.rand(1, 4, 3, 3) + >>> mean = torch.zeros(4) + >>> std = 255. * torch.ones(4) + >>> out = Normalize(mean, std)(x) + >>> out.shape + torch.Size([1, 4, 3, 3]) + + """ + + def __init__( + self, + mean: Union[torch.Tensor, Tuple[float], List[float], float], + std: Union[torch.Tensor, Tuple[float], List[float], float], + ) -> None: + super().__init__() + + if isinstance(mean, (int, float)): + mean = torch.tensor([mean]) + + if isinstance(std, (int, float)): + std = torch.tensor([std]) + + if isinstance(mean, (tuple, list)): + mean = torch.tensor(mean)[None] + + if isinstance(std, (tuple, list)): + std = torch.tensor(std)[None] + + self.mean = mean + self.std = std + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Normalize an input tensor channel-wise with this module's statistics. + + This method is a thin wrapper over :func:`normalize`, reusing the + ``mean`` and ``std`` values stored in the module constructor. + + Args: + input: Tensor to normalize, typically with shape :math:`(*, C, ...)`, + where ``*`` represents optional leading dimensions (for example + batch) and ``C`` is the channel dimension. + + Returns: + A tensor with the same shape as ``input`` whose channel values are + normalized by ``(x - mean) / std``. + """ + return normalize(input, self.mean, self.std) + + def __repr__(self) -> str: + repr = f"(mean={self.mean}, std={self.std})" + return self.__class__.__name__ + repr + + +def normalize(data: torch.Tensor, mean: torch.Tensor, std: torch.Tensor) -> torch.Tensor: + r"""Normalize an image/video torch.Tensor with mean and standard deviation. + + .. math:: + \text{input[channel] = (input[channel] - mean[channel]) / std[channel]} + + Where `mean` is :math:`(M_1, ..., M_n)` and `std` :math:`(S_1, ..., S_n)` for `n` channels, + + Args: + data: Image torch.Tensor of size :math:`(B, C, *)`. + mean: Mean for each channel. + std: Standard deviations for each channel. + + Return: + Normalised torch.Tensor with same size as input :math:`(B, C, *)`. + + Examples: + >>> x = torch.rand(1, 4, 3, 3) + >>> out = normalize(x, torch.tensor([0.0]), torch.tensor([255.])) + >>> out.shape + torch.Size([1, 4, 3, 3]) + + >>> x = torch.rand(1, 4, 3, 3) + >>> mean = torch.zeros(4) + >>> std = 255. * torch.ones(4) + >>> out = normalize(x, mean, std) + >>> out.shape + torch.Size([1, 4, 3, 3]) + + """ + shape = data.shape + + if torch.onnx.is_in_onnx_export(): + if not isinstance(mean, torch.Tensor) or not isinstance(std, torch.Tensor): + raise ValueError("Only torch.Tensor is accepted when converting to ONNX.") + if mean.shape[0] != 1 or std.shape[0] != 1: + raise ValueError( + "Batch dimension must be one for broadcasting when converting to ONNX." + f"Try changing mean shape and std shape from ({mean.shape}, {std.shape}) to (1, C) or (1, C, 1, 1)." + ) + else: + if isinstance(mean, float): + mean = torch.tensor([mean] * shape[1], device=data.device, dtype=data.dtype) + + if isinstance(std, float): + std = torch.tensor([std] * shape[1], device=data.device, dtype=data.dtype) + + # Allow broadcast on channel dimension + if mean.shape and mean.shape[0] != 1: + if mean.shape[0] != data.shape[1] and mean.shape[:2] != data.shape[:2]: + raise ValueError(f"mean length and number of channels do not match. Got {mean.shape} and {data.shape}.") + + # Allow broadcast on channel dimension + if std.shape and std.shape[0] != 1: + if std.shape[0] != data.shape[1] and std.shape[:2] != data.shape[:2]: + raise ValueError(f"std length and number of channels do not match. Got {std.shape} and {data.shape}.") + + mean = torch.as_tensor(mean, device=data.device, dtype=data.dtype) + std = torch.as_tensor(std, device=data.device, dtype=data.dtype) + + mean = mean[..., None] + std = std[..., None] + + out: torch.Tensor = (data.view(shape[0], shape[1], -1) - mean) / std + + return out.view(shape) + + +class Denormalize(nn.Module): + r"""Denormalize a torch.Tensor image with mean and standard deviation. + + .. math:: + \text{input[channel] = (input[channel] * std[channel]) + mean[channel]} + + Where `mean` is :math:`(M_1, ..., M_n)` and `std` :math:`(S_1, ..., S_n)` for `n` channels, + + Args: + mean: Mean for each channel. + std: Standard deviations for each channel. + + Shape: + - Input: Image torch.Tensor of size :math:`(*, C, ...)`. + - Output: Denormalised torch.Tensor with same size as input :math:`(*, C, ...)`. + + Examples: + >>> x = torch.rand(1, 4, 3, 3) + >>> out = Denormalize(0.0, 255.)(x) + >>> out.shape + torch.Size([1, 4, 3, 3]) + + >>> x = torch.rand(1, 4, 3, 3, 3) + >>> mean = torch.zeros(1, 4) + >>> std = 255. * torch.ones(1, 4) + >>> out = Denormalize(mean, std)(x) + >>> out.shape + torch.Size([1, 4, 3, 3, 3]) + + """ + + def __init__(self, mean: Union[torch.Tensor, float], std: Union[torch.Tensor, float]) -> None: + super().__init__() + + self.mean = mean + self.std = std + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Restore scale/offset from a tensor normalized by mean and std. + + This method delegates to :func:`denormalize` using this module's stored + ``mean`` and ``std`` parameters. + + Args: + input: Tensor to denormalize, commonly shaped :math:`(*, C, ...)` + with channel dimension ``C``. + + Returns: + A tensor with the same shape as ``input`` where each channel is + transformed by ``x * std + mean``. + """ + return denormalize(input, self.mean, self.std) + + def __repr__(self) -> str: + repr = f"(mean={self.mean}, std={self.std})" + return self.__class__.__name__ + repr + + +def denormalize(data: torch.Tensor, mean: Union[torch.Tensor, float], std: Union[torch.Tensor, float]) -> torch.Tensor: + r"""Denormalize an image/video torch.Tensor with mean and standard deviation. + + .. math:: + \text{input[channel] = (input[channel] * std[channel]) + mean[channel]} + + Where `mean` is :math:`(M_1, ..., M_n)` and `std` :math:`(S_1, ..., S_n)` for `n` channels, + + Args: + data: Image torch.Tensor of size :math:`(B, C, *)`. + mean: Mean for each channel. + std: Standard deviations for each channel. + + Return: + Denormalised torch.Tensor with same size as input :math:`(B, C, *)`. + + Examples: + >>> x = torch.rand(1, 4, 3, 3) + >>> out = denormalize(x, 0.0, 255.) + >>> out.shape + torch.Size([1, 4, 3, 3]) + + >>> x = torch.rand(1, 4, 3, 3, 3) + >>> mean = torch.zeros(1, 4) + >>> std = 255. * torch.ones(1, 4) + >>> out = denormalize(x, mean, std) + >>> out.shape + torch.Size([1, 4, 3, 3, 3]) + + """ + shape = data.shape + + if torch.onnx.is_in_onnx_export(): + if not isinstance(mean, torch.Tensor) or not isinstance(std, torch.Tensor): + raise ValueError("Only torch.Tensor is accepted when converting to ONNX.") + if mean.shape[0] != 1 or std.shape[0] != 1: + raise ValueError("Batch dimension must be one for broadcasting when converting to ONNX.") + else: + if isinstance(mean, float): + mean = torch.tensor([mean] * shape[1], device=data.device, dtype=data.dtype) + + if isinstance(std, float): + std = torch.tensor([std] * shape[1], device=data.device, dtype=data.dtype) + + # Allow broadcast on channel dimension + if mean.shape and mean.shape[0] != 1: + if mean.shape[0] != data.shape[-3] and mean.shape[:2] != data.shape[:2]: + raise ValueError(f"mean length and number of channels do not match. Got {mean.shape} and {data.shape}.") + + # Allow broadcast on channel dimension + if std.shape and std.shape[0] != 1: + if std.shape[0] != data.shape[-3] and std.shape[:2] != data.shape[:2]: + raise ValueError(f"std length and number of channels do not match. Got {std.shape} and {data.shape}.") + + mean = torch.as_tensor(mean, device=data.device, dtype=data.dtype) + std = torch.as_tensor(std, device=data.device, dtype=data.dtype) + + if mean.dim() == 1: + mean = mean.view(1, -1, *([1] * (data.dim() - 2))) + # If the torch.Tensor is >1D (e.g., (B, C)), reshape to (B, C, 1, ...) + else: + while len(mean.shape) < data.dim(): + mean = mean.unsqueeze(-1) + + if std.dim() == 1: + std = std.view(1, -1, *([1] * (data.dim() - 2))) + else: + while len(std.shape) < data.dim(): + std = std.unsqueeze(-1) + + return torch.addcmul(mean, data, std) + + +@perform_keep_shape_image +def normalize_min_max( + input: torch.Tensor, min_val: float = 0.0, max_val: float = 1.0, eps: float = 1e-6 +) -> torch.Tensor: + r"""Normalise an image/video torch.Tensor by MinMax and re-scales the value between a range. + + The data is normalised using the following formulation: + + .. math:: + y_i = (b - a) * \frac{x_i - \text{min}(x)}{\text{max}(x) - \text{min}(x)} + a + + where :math:`a` is :math:`\text{min_val}` and :math:`b` is :math:`\text{max_val}`. + + Args: + input: The image torch.Tensor to be normalised with shape :math:`(*, C, H, W)`. + min_val: The minimum value for the new range. + max_val: The maximum value for the new range. + eps: Float number to avoid zero division. + + Returns: + The normalised image torch.Tensor with same shape as input :math:`(*, C, H, W)`. + + Example: + >>> x = torch.rand(1, 5, 3, 3) + >>> x_norm = normalize_min_max(x, min_val=-1., max_val=1.) + >>> x_norm.min() + tensor(-1.) + >>> x_norm.max() + tensor(1.0000) + + """ + if not isinstance(input, torch.Tensor): + raise TypeError(f"data should be a torch.Tensor. Got: {type(input)}.") + + if not isinstance(min_val, float): + raise TypeError(f"'min_val' should be a float. Got: {type(min_val)}.") + + if not isinstance(max_val, float): + raise TypeError(f"'max_val' should be a float. Got: {type(max_val)}.") + + shape = input.shape + B, C = shape[0], shape[1] + + x_reshaped = input.view(B, C, -1) + x_min = x_reshaped.min(-1, keepdim=True)[0] # Shape: (B, C, 1) + x_max = x_reshaped.max(-1, keepdim=True)[0] # Shape: (B, C, 1) + + x_out = (max_val - min_val) * (x_reshaped - x_min) / (x_max - x_min + eps) + min_val + return x_out.view(shape) diff --git a/kornia/enhance/rescale.py b/kornia/enhance/rescale.py new file mode 100644 index 0000000..a21f854 --- /dev/null +++ b/kornia/enhance/rescale.py @@ -0,0 +1,54 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Union + +import torch +from torch import nn + + +class Rescale(nn.Module): + r"""Initialize the Rescale operator. + + Args: + factor: The scaling factor. Could be a float or a 0-d torch.Tensor. + + """ + + def __init__(self, factor: Union[float, torch.Tensor]) -> None: + super().__init__() + if isinstance(factor, float): + self.factor = torch.tensor(factor) + else: + if not isinstance(factor, torch.Tensor) or factor.ndim != 0: + raise TypeError(f"Expected factor to be a float or a 0-d torch.Tensor, got {factor}.") + self.factor = factor + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Multiply the input tensor by the configured scaling factor. + + Args: + input: Tensor to be rescaled. In image workflows this is commonly + shaped as :math:`(*, C, H, W)`, where ``*`` denotes optional + leading dimensions (for example batch), ``C`` is channel count, + and ``H``/``W`` are spatial dimensions. + + Returns: + A tensor with the same shape as ``input``, where each element is + multiplied by ``self.factor``. + """ + return input * self.factor diff --git a/kornia/enhance/shift_rgb.py b/kornia/enhance/shift_rgb.py new file mode 100644 index 0000000..9e634d0 --- /dev/null +++ b/kornia/enhance/shift_rgb.py @@ -0,0 +1,36 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import torch + +from kornia.core.check import KORNIA_CHECK_IS_COLOR, KORNIA_CHECK_IS_TENSOR + + +def shift_rgb(image: torch.Tensor, r_shift: torch.Tensor, g_shift: torch.Tensor, b_shift: torch.Tensor) -> torch.Tensor: + """Shift rgb channels. + + Shift each image's channel by either r_shift for red, g_shift for green and b_shift for blue channels. + """ + KORNIA_CHECK_IS_TENSOR(image) + KORNIA_CHECK_IS_COLOR(image, f"with shape {image.shape}") + + shifts = [r_shift, g_shift, b_shift] + + shifted = (image + torch.stack(shifts, dim=1).view(-1, 3, 1, 1).to(image)).clamp_(min=0, max=1) + + return shifted diff --git a/kornia/enhance/threshold.py b/kornia/enhance/threshold.py new file mode 100644 index 0000000..b8c7258 --- /dev/null +++ b/kornia/enhance/threshold.py @@ -0,0 +1,152 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from enum import IntEnum +from typing import Union + +import torch +from torch import Tensor +from torch.nn import Module + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR + + +class ThresholdType(IntEnum): + """Threshold types compatible with OpenCV fixed thresholding types. + + Note: THRESH_OTSU is intentionally not supported in this PR. + """ + + THRESH_BINARY = 0 + THRESH_BINARY_INV = 1 + THRESH_TRUNC = 2 + THRESH_TOZERO = 3 + THRESH_TOZERO_INV = 4 + + # OpenCV uses 8 for OTSU, reserved for follow-up PR. + THRESH_OTSU = 8 + + +def threshold( + input: Tensor, + thresh: Union[float, Tensor], + maxval: Union[float, Tensor] = 255.0, + type: Union[int, ThresholdType] = ThresholdType.THRESH_BINARY, +) -> Tensor: + """Apply a fixed-level threshold to each element in the input tensor. + + Implements OpenCV-like behavior for the following threshold types: + - THRESH_BINARY + - THRESH_BINARY_INV + - THRESH_TRUNC + - THRESH_TOZERO + - THRESH_TOZERO_INV + + Args: + input: Image tensor of shape (..., H, W). Typically (B, C, H, W). + thresh: Threshold value (scalar or tensor broadcastable to input). + maxval: Maximum value used with binary thresholding types. + type: Threshold type. + + Returns: + Thresholded tensor with same shape/dtype/device as `input`. + + Raises: + NotImplementedError: if THRESH_OTSU flag is passed. + ValueError: if type is not supported. + """ + KORNIA_CHECK_IS_TENSOR(input) + + t = int(type) + + # Detect if OTSU flag is present (opencv allows OR-ing) + if t & int(ThresholdType.THRESH_OTSU): + raise NotImplementedError("THRESH_OTSU is not implemented yet. Please use a fixed threshold type.") + + KORNIA_CHECK( + t in {int(x) for x in ThresholdType if x != ThresholdType.THRESH_OTSU}, + f"Unsupported threshold type: {type}. Supported: BINARY, BINARY_INV, TRUNC, TOZERO, TOZERO_INV.", + ) + + # Make thresh/maxval tensors on same device/dtype for safe broadcasting + thresh_t = thresh + if not isinstance(thresh_t, Tensor): + thresh_t = torch.tensor(thresh_t, device=input.device, dtype=input.dtype) + else: + thresh_t = thresh_t.to(device=input.device, dtype=input.dtype) + + maxval_t = maxval + if not isinstance(maxval_t, Tensor): + maxval_t = torch.tensor(maxval_t, device=input.device, dtype=input.dtype) + else: + maxval_t = maxval_t.to(device=input.device, dtype=input.dtype) + + mask = input > thresh_t + zeros = torch.zeros_like(input) + + if t == int(ThresholdType.THRESH_BINARY): + return torch.where(mask, maxval_t, zeros) + + if t == int(ThresholdType.THRESH_BINARY_INV): + return torch.where(mask, zeros, maxval_t) + + if t == int(ThresholdType.THRESH_TRUNC): + return torch.minimum(input, thresh_t) + + if t == int(ThresholdType.THRESH_TOZERO): + return torch.where(mask, input, zeros) + + if t == int(ThresholdType.THRESH_TOZERO_INV): + return torch.where(mask, zeros, input) + + # Should never reach here due to KORNIA_CHECK above + raise ValueError(f"Unsupported threshold type: {type}") + + +class Threshold(Module): + """Module wrapper for `kornia.enhance.threshold`.""" + + def __init__( + self, + thresh: float, + maxval: float = 255.0, + type: Union[int, ThresholdType] = ThresholdType.THRESH_BINARY, + ) -> None: + super().__init__() + self.thresh = float(thresh) + self.maxval = float(maxval) + self.type = int(type) + + def forward(self, input: Tensor) -> Tensor: + """Apply thresholding with this module's configured parameters. + + This method delegates to :func:`threshold` and reuses the threshold + value, maximum replacement value, and thresholding mode stored on the + module instance. + + Args: + input: Input tensor to threshold. Typical image inputs are shaped + :math:`(*, C, H, W)`, but any tensor shape supported by + :func:`threshold` is accepted. + + Returns: + A tensor with the same shape as ``input`` where values are remapped + according to ``self.type`` using ``self.thresh`` and ``self.maxval``. + """ + return threshold(input, self.thresh, self.maxval, self.type) diff --git a/kornia/enhance/zca.py b/kornia/enhance/zca.py new file mode 100644 index 0000000..3bbf50b --- /dev/null +++ b/kornia/enhance/zca.py @@ -0,0 +1,396 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import List, Optional, Tuple + +import torch +from torch import nn + +from kornia.core.utils import _torch_svd_cast + +__all__ = ["ZCAWhitening", "linear_transform", "zca_mean", "zca_whiten"] + + +class ZCAWhitening(nn.Module): + r"""Compute the ZCA whitening matrix transform and the mean vector and applies the transform to the data. + + The data torch.Tensor is flattened, and the mean :math:`\mathbf{\mu}` + and covariance matrix :math:`\mathbf{\Sigma}` are computed from + the flattened data :math:`\mathbf{X} \in \mathbb{R}^{N \times D}`, where + :math:`N` is the sample size and :math:`D` is flattened dimensionality + (e.g. for a torch.Tensor with size 5x3x2x2 :math:`N = 5` and :math:`D = 12`). The ZCA whitening + transform is given by: + + .. math:: + + \mathbf{X}_{\text{zca}} = (\mathbf{X - \mu})(US^{-\frac{1}{2}}U^T)^T + + where :math:`U` are the eigenvectors of :math:`\Sigma` and :math:`S` contain the corresponding + eigenvalues of :math:`\Sigma`. After the transform is applied, the output is reshaped to same shape. + + Args: + dim: Determines the dimension that represents the samples axis. + eps: a small number used for numerical stability. + unbiased: Whether to use the biased estimate of the covariance matrix. + compute_inv: Compute the inverse transform matrix. + detach_transforms: Detaches gradient from the ZCA fitting. + + shape: + - x: :math:`(D_0,...,D_{\text{dim}},...,D_N)` is a batch of N-D tensors. + - x_whiten: :math:`(D_0,...,D_{\text{dim}},...,D_N)` same shape as input. + + .. note:: + See a working example `here `__. + + Examples: + >>> x = torch.tensor([[0,1],[1,0],[-1,0],[0,-1]], dtype = torch.float32) + >>> zca = ZCAWhitening().fit(x) + >>> x_whiten = zca(x) + >>> zca = ZCAWhitening() + >>> x_whiten = zca(x, include_fit = True) # Includes the fitting step + >>> x_whiten = zca(x) # Can run now without the fitting set + >>> # Enable backprop through ZCA fitting process + >>> zca = ZCAWhitening(detach_transforms = False) + >>> x_whiten = zca(x, include_fit = True) # Includes the fitting step + + Note: + This implementation uses :py:meth:`~torch.svd` which yields NaNs in the backwards step + if the singular values are not unique. See `here `_ for + more information. + + References: + [1] `Stanford PCA & ZCA whitening tutorial `_ + + """ + + def __init__( + self, + dim: int = 0, + eps: float = 1e-6, + unbiased: bool = True, + detach_transforms: bool = True, + compute_inv: bool = False, + ) -> None: + super().__init__() + + self.dim = dim + self.eps = eps + self.unbiased = unbiased + self.detach_transforms = detach_transforms + self.compute_inv = compute_inv + + self.fitted = False + + self.mean_vector: torch.Tensor + self.transform_matrix: torch.Tensor + self.transform_inv: Optional[torch.Tensor] + + def fit(self, x: torch.Tensor) -> "ZCAWhitening": + r"""Fit ZCA whitening matrices to the data. + + Args: + x: Input data. + + Returns: + Returns a fitted ZCAWhiten object instance. + + """ + T, mean, T_inv = zca_mean(x, self.dim, self.unbiased, self.eps, self.compute_inv) + + self.mean_vector = mean + self.transform_matrix = T + if T_inv is None: + self.transform_inv = torch.empty([0]) + else: + self.transform_inv = T_inv + + if self.detach_transforms: + self.mean_vector = self.mean_vector.detach() + self.transform_matrix = self.transform_matrix.detach() + self.transform_inv = self.transform_inv.detach() + + self.fitted = True + + return self + + def forward(self, x: torch.Tensor, include_fit: bool = False) -> torch.Tensor: + r"""Apply the whitening transform to the data. + + Args: + x: Input data. + include_fit: Indicates whether to fit the data as part of the forward pass. + + Returns: + The transformed data. + + """ + if include_fit: + self.fit(x) + + if not self.fitted: + raise RuntimeError("Needs to be fitted first before running. Please call fit or set include_fit to True.") + + x_whiten = linear_transform(x, self.transform_matrix, self.mean_vector, self.dim) + + return x_whiten + + def inverse_transform(self, x: torch.Tensor) -> torch.Tensor: + r"""Apply the inverse transform to the whitened data. + + Args: + x: Whitened data. + + Returns: + Original data. + + """ + if not self.fitted: + raise RuntimeError("Needs to be fitted first before running. Please call fit or set include_fit to True.") + + if not self.compute_inv: + raise RuntimeError("Did not compute inverse ZCA. Please set compute_inv to True") + + if self.transform_inv is None: + raise TypeError("The transform inverse should be a torch.Tensor. Gotcha None.") + + mean_inv: torch.Tensor = -self.mean_vector.mm(self.transform_matrix) + + y = linear_transform(x, self.transform_inv, mean_inv) + + return y + + +def zca_mean( + inp: torch.Tensor, dim: int = 0, unbiased: bool = True, eps: float = 1e-6, return_inverse: bool = False +) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + r"""Compute the ZCA whitening matrix and mean vector. + + The output can be used with :py:meth:`~kornia.color.linear_transform`. + See :class:`~kornia.color.ZCAWhitening` for details. + + Args: + inp: input data torch.Tensor. + dim: Specifies the dimension that serves as the samples dimension. + unbiased: Whether to use the unbiased estimate of the covariance matrix. + eps: a small number used for numerical stability. + return_inverse: Whether to return the inverse ZCA transform. + + Shapes: + - inp: :math:`(D_0,...,D_{\text{dim}},...,D_N)` is a batch of N-D tensors. + - transform_matrix: :math:`(\Pi_{d=0,d\neq \text{dim}}^N D_d, \Pi_{d=0,d\neq \text{dim}}^N D_d)` + - mean_vector: :math:`(1, \Pi_{d=0,d\neq \text{dim}}^N D_d)` + - inv_transform: same shape as the transform matrix + + Returns: + A tuple containing the ZCA matrix and the mean vector. If return_inverse is set to True, + then it returns the inverse ZCA matrix, otherwise it returns None. + + .. note:: + See a working example `here `__. + + Examples: + >>> x = torch.tensor([[0,1],[1,0],[-1,0],[0,-1]], dtype = torch.float32) + >>> transform_matrix, mean_vector,_ = zca_mean(x) # Returns transformation matrix and data mean + >>> x = torch.rand(3,20,2,2) + >>> transform_matrix, mean_vector, inv_transform = zca_mean(x, dim = 1, return_inverse = True) + >>> # transform_matrix.size() equals (12,12) and the mean vector.size equal (1,12) + + """ + if not isinstance(inp, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(inp)}") + + if not isinstance(eps, float): + raise TypeError(f"eps type is not a float. Got{type(eps)}") + + if not isinstance(unbiased, bool): + raise TypeError(f"unbiased type is not bool. Got{type(unbiased)}") + + if not isinstance(dim, int): + raise TypeError(f"Argument 'dim' must be of type int. Got {type(dim)}") + + if not isinstance(return_inverse, bool): + raise TypeError(f"Argument return_inverse must be of type bool {type(return_inverse)}") + + inp_size = inp.size() + + if dim >= len(inp_size) or dim < -len(inp_size): + raise IndexError( + f"Dimension out of range (expected to be in range of [{-len(inp_size)},{len(inp_size) - 1}], but got {dim}" + ) + + if dim < 0: + dim = len(inp_size) + dim + + feat_dims = torch.cat([torch.arange(0, dim), torch.arange(dim + 1, len(inp_size))]) + + new_order: List[int] = torch.cat([torch.tensor([dim]), feat_dims]).tolist() + + inp_permute = inp.permute(new_order) + + N = inp_size[dim] + feature_sizes = torch.tensor(inp_size[0:dim] + inp_size[dim + 1 : :]) + num_features: int = int(torch.prod(feature_sizes).item()) + + mean: torch.Tensor = torch.mean(inp_permute, dim=0, keepdim=True) + + mean = mean.reshape((1, num_features)) + + inp_center_flat: torch.Tensor = inp_permute.reshape((N, num_features)) - mean + + cov = inp_center_flat.t().mm(inp_center_flat) + + if unbiased: + cov = cov / float(N - 1) + else: + cov = cov / float(N) + + U, S, _ = _torch_svd_cast(cov) + + S = S.reshape(-1, 1) + S_inv_root: torch.Tensor = torch.rsqrt(S + eps) + T: torch.Tensor = (U).mm(S_inv_root * U.t()) + + T_inv: Optional[torch.Tensor] = None + if return_inverse: + T_inv = (U).mm(torch.sqrt(S + eps) * U.t()) + + return T, mean, T_inv + + +def zca_whiten(inp: torch.Tensor, dim: int = 0, unbiased: bool = True, eps: float = 1e-6) -> torch.Tensor: + r"""Apply ZCA whitening transform. + + See :class:`~kornia.color.ZCAWhitening` for details. + + Args: + inp: input data torch.Tensor. + dim: Specifies the dimension that serves as the samples dimension. + unbiased: Whether to use the unbiased estimate of the covariance matrix. + eps: a small number used for numerical stability. + + Returns: + Whiten Input data. + + .. note:: + See a working example `here `__. + + Examples: + >>> x = torch.tensor([[0,1],[1,0],[-1,0]], dtype = torch.float32) + >>> zca_whiten(x) + tensor([[ 0.0000, 1.1547], + [ 1.0000, -0.5773], + [-1.0000, -0.5773]]) + + """ + if not isinstance(inp, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(inp)}") + + if not isinstance(eps, float): + raise TypeError(f"eps type is not a float. Got{type(eps)}") + + if not isinstance(unbiased, bool): + raise TypeError(f"unbiased type is not bool. Got{type(unbiased)}") + + if not isinstance(dim, int): + raise TypeError(f"Argument 'dim' must be of type int. Got {type(dim)}") + + transform, mean, _ = zca_mean(inp, dim, unbiased, eps, False) + + inp_whiten = linear_transform(inp, transform, mean, dim) + + return inp_whiten + + +def linear_transform( + inp: torch.Tensor, transform_matrix: torch.Tensor, mean_vector: torch.Tensor, dim: int = 0 +) -> torch.Tensor: + r"""Given a transformation matrix and a mean vector, this function will flatten the input + torch.Tensor along the given dimension and subtract the mean vector from it. Then the dot + product with the transformation matrix will be computed and then the resulting torch.Tensor + is reshaped to the original input shape. + + .. math:: + + \mathbf{X}_{T} = (\mathbf{X - \mu})(T) + + Args: + inp: Input data :math:`X`. + transform_matrix: Transform matrix :math:`T`. + mean_vector: mean vector :math:`\mu`. + dim: Batch dimension. + + Shapes: + - inp: :math:`(D_0,...,D_{\text{dim}},...,D_N)` is a batch of N-D tensors. + - transform_matrix: :math:`(\Pi_{d=0,d\neq \text{dim}}^N D_d, \Pi_{d=0,d\neq \text{dim}}^N D_d)` + - mean_vector: :math:`(1, \Pi_{d=0,d\neq \text{dim}}^N D_d)` + + Returns: + Transformed data. + + Example: + >>> # Example where dim = 3 + >>> inp = torch.ones((10,3,4,5)) + >>> transform_mat = torch.ones((10*3*4,10*3*4)) + >>> mean = 2*torch.ones((1,10*3*4)) + >>> out = linear_transform(inp, transform_mat, mean, 3) + >>> print(out.shape, out.unique()) # Should a be (10,3,4,5) torch.tensor of -120s + torch.Size([10, 3, 4, 5]) tensor([-120.]) + + >>> # Example where dim = 0 + >>> inp = torch.ones((10,2)) + >>> transform_mat = torch.ones((2,2)) + >>> mean = torch.zeros((1,2)) + >>> out = linear_transform(inp, transform_mat, mean) + >>> print(out.shape, out.unique()) # Should a be (10,2) torch.tensor of 2s + torch.Size([10, 2]) tensor([2.]) + + """ # noqa: D205 + inp_size = inp.size() + + if dim >= len(inp_size) or dim < -len(inp_size): + raise IndexError( + f"Dimension out of range (expected to be in range of [{-len(inp_size)},{len(inp_size) - 1}], but got {dim}" + ) + + if dim < 0: + dim = len(inp_size) + dim + + feat_dims = torch.cat([torch.arange(0, dim), torch.arange(dim + 1, len(inp_size))]) + + perm = torch.cat([torch.tensor([dim]), feat_dims]) + perm_inv = torch.argsort(perm) + + new_order: List[int] = perm.tolist() + inv_order: List[int] = perm_inv.tolist() + + feature_sizes = torch.tensor(inp_size[0:dim] + inp_size[dim + 1 : :]) + num_features: int = int(torch.prod(feature_sizes).item()) + + inp_permute = inp.permute(new_order) + inp_flat = inp_permute.reshape((-1, num_features)) + + inp_center = inp_flat - mean_vector + inp_transformed = inp_center.mm(transform_matrix) + + inp_transformed = inp_transformed.reshape(inp_permute.size()) + + inp_transformed = inp_transformed.permute(inv_order) + + return inp_transformed diff --git a/kornia/feature/__init__.py b/kornia/feature/__init__.py new file mode 100644 index 0000000..16a595f --- /dev/null +++ b/kornia/feature/__init__.py @@ -0,0 +1,195 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Feature — Feature detection, description, and matching for Kornia. + +This subpackage provides modules for keypoint detection, descriptors, and feature matching. +""" + +from .affine_shape import LAFAffineShapeEstimator, LAFAffNetShapeEstimator, PatchAffineShapeEstimator +from .aliked import ALIKED, ALIKEDFeatures +from .dedode import DeDoDe +from .defmo import DeFMO +from .disk import DISK, DISKFeatures +from .hardnet import HardNet, HardNet8 +from .hynet import TLU, FilterResponseNorm2d, HyNet +from .integrated import ( + GFTTAffNetHardNet, + HesAffNetHardNet, + KeyNetAffNetHardNet, + KeyNetHardNet, + LAFDescriptor, + LightGlueMatcher, + LocalFeature, + LocalFeatureMatcher, + SIFTFeature, + SIFTFeatureScaleSpace, + get_laf_descriptors, +) +from .keynet import KeyNet, KeyNetDetector +from .laf import ( + KORNIA_CHECK_LAF, + denormalize_laf, + ellipse_to_laf, + extract_patches_from_pyramid, + extract_patches_simple, + get_laf_center, + get_laf_orientation, + get_laf_scale, + laf_from_center_scale_ori, + laf_from_three_points, + laf_is_inside_image, + laf_to_boundary_points, + laf_to_three_points, + make_upright, + normalize_laf, + perspective_transform_lafs, + rotate_laf, + scale_laf, + set_laf_orientation, +) +from .lightglue import LightGlue +from .lightglue_onnx import OnnxLightGlue +from .loftr import LoFTR +from .matching import ( + DescriptorMatcher, + GeometryAwareDescriptorMatcher, + match_adalam, + match_fginn, + match_mnn, + match_nn, + match_smnn, + match_snn, +) +from .mkd import MKDDescriptor +from .orientation import LAFOrienter, OriNet, PatchDominantGradientOrientation +from .responses import ( + BlobDoG, + BlobDoGSingle, + BlobHessian, + CornerGFTT, + CornerHarris, + dog_response, + dog_response_single, + gftt_response, + harris_response, + hessian_response, +) +from .scale_space_detector import MultiResolutionDetector, PassLAF, ScaleSpaceDetector +from .siftdesc import DenseSIFTDescriptor, SIFTDescriptor +from .sold2 import SOLD2, SOLD2_detector +from .sosnet import SOSNet +from .tfeat import TFeat +from .xfeat import InterpolateSparse2d, XFeat, XFeatModel + +__all__ = [ + "ALIKED", + "DISK", + "KORNIA_CHECK_LAF", + "SOLD2", + "TLU", + "ALIKEDFeatures", + "BlobDoG", + "BlobDoGSingle", + "BlobHessian", + "CornerGFTT", + "CornerHarris", + "DISKFeatures", + "DeDoDe", + "DeFMO", + "DenseSIFTDescriptor", + "DescriptorMatcher", + "DescriptorMatcher", + "FilterResponseNorm2d", + "GFTTAffNetHardNet", + "GFTTAffNetHardNet", + "GeometryAwareDescriptorMatcher", + "HardNet", + "HardNet8", + "HesAffNetHardNet", + "HyNet", + "InterpolateSparse2d", + "KeyNet", + "KeyNet", + "KeyNetAffNetHardNet", + "KeyNetDetector", + "KeyNetHardNet", + "LAFAffNetShapeEstimator", + "LAFAffineShapeEstimator", + "LAFDescriptor", + "LAFDescriptor", + "LAFOrienter", + "LightGlue", + "LightGlueMatcher", + "LoFTR", + "LocalFeature", + "LocalFeature", + "LocalFeatureMatcher", + "LocalFeatureMatcher", + "MKDDescriptor", + "MultiResolutionDetector", + "OnnxLightGlue", + "OriNet", + "PassLAF", + "PatchAffineShapeEstimator", + "PatchDominantGradientOrientation", + "SIFTDescriptor", + "SIFTFeature", + "SIFTFeature", + "SIFTFeatureScaleSpace", + "SOLD2_detector", + "SOSNet", + "ScaleSpaceDetector", + "TFeat", + "XFeat", + "XFeatModel", + "denormalize_laf", + "dog_response", + "dog_response_single", + "ellipse_to_laf", + "extract_patches_from_pyramid", + "extract_patches_simple", + "get_laf_center", + "get_laf_descriptors", + "get_laf_descriptors", + "get_laf_orientation", + "get_laf_scale", + "gftt_response", + "harris_response", + "hessian_response", + "laf_from_center_scale_ori", + "laf_from_three_points", + "laf_is_inside_image", + "laf_to_boundary_points", + "laf_to_three_points", + "make_upright", + "match_adalam", + "match_fginn", + "match_mnn", + "match_mnn", + "match_nn", + "match_nn", + "match_smnn", + "match_smnn", + "match_snn", + "match_snn", + "normalize_laf", + "perspective_transform_lafs", + "rotate_laf", + "scale_laf", + "set_laf_orientation", +] diff --git a/kornia/feature/adalam/__init__.py b/kornia/feature/adalam/__init__.py new file mode 100644 index 0000000..89dfafd --- /dev/null +++ b/kornia/feature/adalam/__init__.py @@ -0,0 +1,23 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Feature Adalam — AdaLAM feature matching for Kornia. + +This subpackage provides AdaLAM filtering and matching utilities. +""" + +from .adalam import AdalamFilter, get_adalam_default_config, match_adalam diff --git a/kornia/feature/adalam/adalam.py b/kornia/feature/adalam/adalam.py new file mode 100644 index 0000000..2828e17 --- /dev/null +++ b/kornia/feature/adalam/adalam.py @@ -0,0 +1,316 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Integrated from original AdaLAM repo +# https://github.com/cavalli1234/AdaLAM +# Copyright (c) 2020, Luca Cavalli + +from typing import Optional, Tuple, Union + +import torch + +from kornia.core.check import KORNIA_CHECK_LAF, KORNIA_CHECK_SHAPE +from kornia.feature.laf import get_laf_center, get_laf_orientation, get_laf_scale + +from .core import AdalamConfig, _no_match, adalam_core +from .utils import dist_matrix + + +def get_adalam_default_config() -> AdalamConfig: + """Return default Adalam Config.""" + return AdalamConfig( + area_ratio=100, + search_expansion=4, + ransac_iters=128, + min_inliers=6, + min_confidence=200, + orientation_difference_threshold=30, + scale_rate_threshold=1.5, + detected_scale_rate_threshold=5, + refit=True, + force_seed_mnn=True, + device=torch.device("cpu"), + ) + + +def match_adalam( + desc1: torch.Tensor, + desc2: torch.Tensor, + lafs1: torch.Tensor, + lafs2: torch.Tensor, + config: Optional[AdalamConfig] = None, + hw1: Optional[Tuple[int, int]] = None, + hw2: Optional[Tuple[int, int]] = None, + dm: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Perform descriptor matching, followed by AdaLAM filtering. + + See :cite:`AdaLAM2020` for more details. + + If the distance matrix dm is not provided, :py:func:`torch.cdist` is used. + + Args: + desc1: Batch of descriptors of a shape :math:`(B1, D)`. + desc2: Batch of descriptors of a shape :math:`(B2, D)`. + lafs1: LAFs of a shape :math:`(1, B1, 2, 3)`. + lafs2: LAFs of a shape :math:`(1, B2, 2, 3)`. + config: dict with AdaLAM config + dm: torch.Tensor containing the distances from each descriptor in desc1 + to each descriptor in desc2, shape of :math:`(B1, B2)`. + hw1: Height/width of image. + hw2: Height/width of image. + + + Return: + - Descriptor distance of matching descriptors, shape of :math:`(B3, 1)`. + - Long torch.Tensor indexes of matching descriptors in desc1 and desc2. Shape: :math:`(B3, 2)`, + where 0 <= B3 <= B1. + + """ + KORNIA_CHECK_SHAPE(desc1, ["B", "DIM"]) + KORNIA_CHECK_SHAPE(desc2, ["B", "DIM"]) + KORNIA_CHECK_LAF(lafs1) + KORNIA_CHECK_LAF(lafs2) + config_ = get_adalam_default_config() + if config is None: + config_["device"] = desc1.device + else: + config_ = get_adalam_default_config() + for key, val in config.items(): + if key not in config_.keys(): + print( + f"WARNING: custom configuration contains a key which is not recognized ({key}). " + f"Known configurations are {list(config_.keys())}." + ) + continue + # TypedDict does not support variable names. https://stackoverflow.com/a/59583427/1983544 + config_[key] = val # type: ignore + adalam_object = AdalamFilter(config_) + idxs, quality = adalam_object.match_and_filter( + get_laf_center(lafs1).reshape(-1, 2), + get_laf_center(lafs2).reshape(-1, 2), + desc1, + desc2, + hw1, + hw2, + get_laf_orientation(lafs1).reshape(-1), + get_laf_orientation(lafs2).reshape(-1), + get_laf_scale(lafs1).reshape(-1), + get_laf_scale(lafs2).reshape(-1), + return_dist=True, + ) + return quality, idxs + + +class AdalamFilter: + """Implement the AdaLAM (Adaptive Locally-Affine Matching) filter for outlier rejection. + + This class wraps the AdaLAM algorithm to filter feature matches based on + local affine consistency. + + Args: + custom_config: Optional configuration object for AdaLAM parameters. + """ + + def __init__(self, custom_config: Optional[AdalamConfig] = None) -> None: + """Wrap the method AdaLAM for outlier filtering. + + init args: + custom_config: dictionary overriding the default configuration. Missing parameters are kept as default. + See documentation of DEFAULT_CONFIG for specific explanations on the accepted parameters. + """ + if custom_config is not None: + self.config = custom_config + else: + self.config = get_adalam_default_config() + + def filter_matches( + self, + k1: torch.Tensor, + k2: torch.Tensor, + putative_matches: torch.Tensor, + scores: torch.Tensor, + mnn: Optional[torch.Tensor] = None, + im1shape: Optional[Tuple[int, int]] = None, + im2shape: Optional[Tuple[int, int]] = None, + o1: Optional[torch.Tensor] = None, + o2: Optional[torch.Tensor] = None, + s1: Optional[torch.Tensor] = None, + s2: Optional[torch.Tensor] = None, + return_dist: bool = False, + ) -> Union[Tuple[torch.Tensor, torch.Tensor], torch.Tensor]: + """Call the core functionality of AdaLAM, i.e. just outlier filtering. + + No sanity check is performed on the inputs. + + Args: + k1: keypoint locations in the source image, in pixel coordinates. + Expected a float32 torch.Tensor with shape (num_keypoints_in_source_image, 2). + k2: keypoint locations in the destination image, in pixel coordinates. + Expected a float32 torch.Tensor with shape (num_keypoints_in_destination_image, 2). + putative_matches: Initial set of putative matches to be filtered. + The current implementation assumes that these are unfiltered nearest neighbor matches, + so it requires this to be a list of indices a_i such that the source keypoint i is + associated to the destination keypoint a_i. For now to use AdaLAM on different inputs a + workaround on the input format is required. + Expected a long torch.Tensor with shape (num_keypoints_in_source_image,). + scores: Confidence scores on the putative_matches. Usually holds Lowe's ratio scores. + mnn: A mask indicating which putative matches are also mutual nearest neighbors. See documentation on + 'force_seed_mnn' in the DEFAULT_CONFIG. If None, it disables the mutual nearest neighbor filtering on + seed point selection. Expected a bool torch.Tensor with shape (num_keypoints_in_source_image,) + im1shape: Shape of the source image. If None, it is inferred from keypoints max and min, at the cost of + wasted runtime. So please provide it. Expected a tuple with (width, height) or (height, width) + of source image + im2shape: Shape of the destination image. If None, it is inferred from keypoints max and min, at the cost + of wasted runtime. So please provide it. Expected a tuple with (width, height) or (height, width) + of destination image + o1: keypoint orientations in degrees. They can be None if 'orientation_difference_threshold' in config + is set to None. See documentation on 'orientation_difference_threshold' in the DEFAULT_CONFIG. + Expected a float32 torch.Tensor with shape (num_keypoints_in_source/destination_image,) + o2: same as o1 but for destination. + s1: keypoint scales. They can be None if 'scale_rate_threshold' in config is set to None. + See documentation on 'scale_rate_threshold' in the DEFAULT_CONFIG. + Expected a float32 torch.Tensor with shape (num_keypoints_in_source/destination_image,) + s2: same as s1 but for destination. + return_dist: if True, inverse confidence value is also outputted. + + Returns: + Filtered putative matches. + A long torch.Tensor with shape (num_filtered_matches, 2) with indices of corresponding + keypoints in k1 and k2. + + """ + with torch.no_grad(): + return adalam_core( + k1, + k2, + fnn12=putative_matches, + scores1=scores, + mnn=mnn, + im1shape=im1shape, + im2shape=im2shape, + o1=o1, + o2=o2, + s1=s1, + s2=s2, + config=self.config, + return_dist=return_dist, + ) + + def match_and_filter( + self, + k1: torch.Tensor, + k2: torch.Tensor, + d1: torch.Tensor, + d2: torch.Tensor, + im1shape: Optional[Tuple[int, int]] = None, + im2shape: Optional[Tuple[int, int]] = None, + o1: Optional[torch.Tensor] = None, + o2: Optional[torch.Tensor] = None, + s1: Optional[torch.Tensor] = None, + s2: Optional[torch.Tensor] = None, + return_dist: bool = False, + ) -> Union[Tuple[torch.Tensor, torch.Tensor], torch.Tensor]: + """Match and filter with AdaLAM. + + This function: + + - performs some elementary sanity check on the inputs; + - wraps input arrays into torch tensors and loads to GPU if necessary; + - extracts nearest neighbors; + - finds mutual nearest neighbors if required; + - finally calls AdaLAM filtering. + + Args: + k1: keypoint locations in the source image, in pixel coordinates. + Expected an array with shape (num_keypoints_in_source_image, 2). + k2: keypoint locations in the destination image, in pixel coordinates. + Expected an array with shape (num_keypoints_in_destination_image, 2). + d1: descriptors in the source image. + Expected an array with shape (num_keypoints_in_source_image, descriptor_size). + d2: descriptors in the destination image. + Expected an array with shape (num_keypoints_in_destination_image, descriptor_size). + im1shape: Shape of the source image. If None, it is inferred from keypoints max and min, at the cost of + wasted runtime. So please provide it. Expected a tuple with (width, height) or (height, width) + of source image + im2shape: Shape of the destination image. If None, it is inferred from keypoints max and min, at the cost + of wasted runtime. So please provide it. Expected a tuple with (width, height) or (height, width) + of destination image + o1: keypoint orientations in degrees. They can be None if 'orientation_difference_threshold' in config + is set to None. See documentation on 'orientation_difference_threshold' in the DEFAULT_CONFIG. + Expected an array with shape (num_keypoints_in_source/destination_image,) + o2: Same as o1 for destination. + s1: keypoint scales. They can be None if 'scale_rate_threshold' in config is set to None. + See documentation on 'scale_rate_threshold' in the DEFAULT_CONFIG. + Expected an array with shape (num_keypoints_in_source/destination_image,) + s2: Same as s1 for destination. + return_dist: if True, inverse confidence value is also outputted. + + Returns: + Filtered putative matches. + A long torch.Tensor with shape (num_filtered_matches, 2) with indices of corresponding + keypoints in k1 and k2. + + """ + if s1 is None or s2 is None: + if self.config["scale_rate_threshold"] is not None: + raise AttributeError( + "Current configuration considers keypoint scales for filtering, but scales have not been provided.\n" # noqa: E501 + "Please either provide scales or set 'scale_rate_threshold' to None to disable scale filtering" + ) + if o1 is None or o2 is None: + if self.config["orientation_difference_threshold"] is not None: + raise AttributeError( + "Current configuration considers keypoint orientations for filtering, but orientations have not been provided.\n" # noqa: E501 + "Please either provide orientations or set 'orientation_difference_threshold' to None to disable orientations filtering" # noqa: E501 + ) + _k1 = torch.as_tensor(k1, device=self.config["device"], dtype=torch.float32) + _k2 = torch.as_tensor(k2, device=self.config["device"], dtype=torch.float32) + _d1 = torch.as_tensor(d1, device=self.config["device"], dtype=torch.float32) + _d2 = torch.as_tensor(d2, device=self.config["device"], dtype=torch.float32) + if o1 is not None: + _o1 = torch.as_tensor(o1, device=self.config["device"], dtype=torch.float32) + _o2 = torch.as_tensor(o2, device=self.config["device"], dtype=torch.float32) + else: + _o1, _o2 = o1, o2 + if s1 is not None: + _s1 = torch.as_tensor(s1, device=self.config["device"], dtype=torch.float32) + _s2 = torch.as_tensor(s2, device=self.config["device"], dtype=torch.float32) + else: + _s1, _s2 = s1, s2 + + if (len(_d2) <= 1) or (len(_d1) <= 1): + idxs, dists = _no_match(_d1) + if return_dist: + return idxs, dists + return idxs + + distmat = dist_matrix(_d1, _d2, is_normalized=False) + dd12, nn12 = torch.topk(distmat, k=2, dim=1, largest=False) # (n1, 2) + + putative_matches = nn12[:, 0] + scores = dd12[:, 0] / dd12[:, 1].clamp_min_(1e-3) + + if self.config["force_seed_mnn"]: + _dd21, nn21 = torch.min(distmat, dim=0) # (n2,) + mnn = nn21[putative_matches] == torch.arange(_k1.shape[0], device=self.config["device"]) + else: + mnn = None + + return self.filter_matches( + _k1, _k2, putative_matches, scores, mnn, im1shape, im2shape, _o1, _o2, _s1, _s2, return_dist + ) diff --git a/kornia/feature/adalam/core.py b/kornia/feature/adalam/core.py new file mode 100644 index 0000000..c9bab83 --- /dev/null +++ b/kornia/feature/adalam/core.py @@ -0,0 +1,427 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math +from typing import Optional, Tuple, Union + +import torch +from typing_extensions import NotRequired, TypedDict + +from .ransac import ransac +from .utils import dist_matrix, orientation_diff + + +class AdalamConfig(TypedDict): + """A config structure for the Adalam model. + + Args: + area_ratio: Ratio between seed circle area and image area. Higher values produce more seeds with smaller + neighborhoods + search_expansion: Expansion factor of the seed circle radius for the purpose of collecting neighborhoods. + Increases neighborhood radius without changing seed distribution + ransac_iters: Fixed number of inner GPU-RANSAC iterations + min_inliers: Minimum number of inliers required to accept inliers coming from a neighborhood + min_confidence: Threshold used by the confidence-based GPU-RANSAC + orientation_difference_threshold: Maximum difference in orientations for a point to be accepted in a + neighborhood. Set to None to disable the use of keypoint orientations + scale_rate_threshold: Maximum difference (ratio) in scales for a point to be accepted in a neighborhood. Set + to None to disable the use of keypoint scales + detected_scale_rate_threshold: Prior on maximum possible scale change detectable in image couples. Affinities + with higher scale changes are regarded as outliers + refit: Whether to perform refitting at the end of the RANSACs. Generally improves accuracy at the cost of + runtime + force_seed_mnn: Whether to consider only MNN for the purpose of selecting seeds. Generally improves accuracy + at the cost of runtime + device: Union[str, torch.device, None] to be used for running AdaLAM. Use GPU if available. + mnn: Default None. You can provide a MNN mask in input to skip MNN computation and still get the improvement. + + """ + + area_ratio: NotRequired[int] + search_expansion: NotRequired[int] + ransac_iters: NotRequired[int] + min_inliers: NotRequired[int] + min_confidence: NotRequired[int] + orientation_difference_threshold: NotRequired[int] + scale_rate_threshold: NotRequired[float] + detected_scale_rate_threshold: NotRequired[int] + refit: NotRequired[bool] + force_seed_mnn: NotRequired[bool] + device: NotRequired[torch.device] + mnn: NotRequired[torch.Tensor] + + +def _no_match(dm: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Output empty tensors. + + Returns: + - Descriptor distance of matching descriptors, shape of :math:`(0, 1)`. + - Long torch.Tensor indexes of matching descriptors in desc1 and desc2, shape of :math:`(0, 2)`. + + """ + dists = torch.empty(0, 1, device=dm.device, dtype=dm.dtype) + idxs = torch.empty(0, 2, device=dm.device, dtype=torch.long) + return dists, idxs + + +def select_seeds( + dist1: torch.Tensor, + R1: Union[float, torch.Tensor], + scores1: torch.Tensor, + fnn12: torch.Tensor, + mnn: Optional[torch.Tensor], +) -> Tuple[torch.Tensor, torch.Tensor]: + """Select seed correspondences among the set of available matches. + + dist1: Precomputed distance matrix between keypoints in image I_1 + R1: Base radius of neighborhoods in image I_1 + scores1: Confidence scores on the putative_matches. Usually holds Lowe's ratio scores. + fnn12: Matches between keypoints of I_1 and I_2. + The i-th entry of fnn12 is j if and only if keypoint k_i in image I_1 is matched to keypoint k_j in image I_2 + mnn: A mask indicating which putative matches are also mutual nearest neighbors. See documentation on + 'force_seed_mnn' in the DEFAULT_CONFIG. + If None, it disables the mutual nearest neighbor filtering on seed point selection. + Expected a bool torch.Tensor with shape (num_keypoints_in_source_image,) + + Returns: + Indices of seed points. + + im1seeds: Keypoint index of chosen seeds in image I_1 + im2seeds: Keypoint index of chosen seeds in image I_2 + + """ + im1neighmap = dist1 < R1**2 # (n1, n1) + # find out who scores higher than whom + im1scorescomp = scores1.unsqueeze(1) > scores1.unsqueeze(0) # (n1, n1) + # find out who scores higher than all of its neighbors: seed points + if mnn is not None: + im1bs = (~torch.any(im1neighmap & im1scorescomp & mnn.unsqueeze(0), dim=1)) & mnn & (scores1 < 0.8**2) # (n1,) + else: + im1bs = (~torch.any(im1neighmap & im1scorescomp, dim=1)) & (scores1 < 0.8**2) + + # collect all seeds in both images and the 1NN of the seeds of the other image + im1seeds = torch.where(im1bs)[0] # (n1bs) index format + im2seeds = fnn12[im1bs] # (n1bs) index format + return im1seeds, im2seeds + + +def extract_neighborhood_sets( + o1: Optional[torch.Tensor], + o2: Optional[torch.Tensor], + s1: Optional[torch.Tensor], + s2: Optional[torch.Tensor], + dist1: torch.Tensor, + im1seeds: torch.Tensor, + im2seeds: torch.Tensor, + k1: torch.Tensor, + k2: torch.Tensor, + R1: Union[float, torch.Tensor], + R2: Union[float, torch.Tensor], + fnn12: torch.Tensor, + ORIENTATION_THR: float, + SCALE_RATE_THR: float, + SEARCH_EXP: float, + MIN_INLIERS: float, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Assign keypoints to seed points. + + This checks both the distance and the agreement of the local transformation + if available. + + o1: Orientations of keypoints in image I_1 + o2: Orientations of keypoints in image I_2 + s1: Scales of keypoints in image I_1 + s2: Scales of keypoints in image I_2 + dist1: Precomputed distance matrix between keypoints in image I_1 + im1seeds: Keypoint index of chosen seeds in image I_1 + im2seeds: Keypoint index of chosen seeds in image I_2 + k1: Keypoint locations in image I_1 + k2: Keypoint locations in image I_2 + R1: Base radius of neighborhoods in image I_1 + R2: Base radius of neighborhoods in image I_2 + fnn12: Matches between keypoints of I_1 and I_2. + The i-th entry of fnn12 is j if and only if keypoint k_i in image I_1 is matched to keypoint k_j in image I_2 + ORIENTATION_THR: Maximum deviation of orientation with respect to seed S_i to keep a keypoint in i-th neighborhood + SCALE_RATE_THR: Maximum deviation of scale with respect to seed S_i to keep a keypoint in i-th neighborhood + SEARCH_EXP: Expansion rate for both radii R1 and R2 to consider inclusion of neighboring keypoints + MIN_INLIERS: Minimum number of inliers to keep a seed point. This is used as an early filter here + to remove already seeds with not enough samples to ever pass this threshold. + + Returns: + Local neighborhoods assignments: + + local_neighs_mask: Boolean matrix of size (num_seeds, num_keypoints). + Entry (i, j) is True iff keypoint j was assigned to seed i. + rdims: Number of keypoints included in the neighborhood for each seed + im1seeds: Keypoint index of chosen seeds in image I_1 + im2seeds: Keypoint index of chosen seeds in image I_2 + + """ + dst1 = dist1[im1seeds, :] + dst2 = dist_matrix(k2[fnn12[im1seeds]], k2[fnn12]) + + # initial candidates are matches which are close to the same seed in both images + local_neighs_mask = (dst1 < (SEARCH_EXP * R1) ** 2) & (dst2 < (SEARCH_EXP * R2) ** 2) + + # If requested, also their orientation delta should be compatible with that of the corresponding seed + if ORIENTATION_THR is not None and ORIENTATION_THR < 180 and (o1 is not None) and (o2 is not None): + relo = orientation_diff(o1, o2[fnn12]) + orientation_diffs = torch.abs(orientation_diff(relo.unsqueeze(0), relo[im1seeds].unsqueeze(1))) + local_neighs_mask = local_neighs_mask & (orientation_diffs < ORIENTATION_THR) + + # If requested, also their scale delta should be compatible with that of the corresponding seed + if SCALE_RATE_THR is not None and (SCALE_RATE_THR < 10) and (s1 is not None) and (s2 is not None): + rels = s2[fnn12] / s1 + scale_rates = rels[im1seeds].unsqueeze(1) / rels.unsqueeze(0) + local_neighs_mask = ( + local_neighs_mask & (scale_rates < SCALE_RATE_THR) & (scale_rates > 1 / SCALE_RATE_THR) + ) # (ns, n1) + + # count how many keypoints ended up in each neighborhood + numn1 = torch.sum(local_neighs_mask, dim=1) + # and only keep the torch.ones that have enough points + valid_seeds = numn1 >= MIN_INLIERS + + local_neighs_mask = local_neighs_mask[valid_seeds, :] + + rdims = numn1[valid_seeds] + + return local_neighs_mask, rdims, im1seeds[valid_seeds], im2seeds[valid_seeds] + + +def extract_local_patterns( + fnn12: torch.Tensor, + fnn_to_seed_local_consistency_map_corr: torch.Tensor, + k1: torch.Tensor, + k2: torch.Tensor, + im1seeds: torch.Tensor, + im2seeds: torch.Tensor, + scores: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Prepare local neighborhoods around each seed for the parallel RANSACs. + + This involves two steps: 1) Collect + all selected keypoints and refer them with respect to their seed point 2) Sort keypoints by score for the + progressive sampling to pick the best samples first. + + fnn12: Matches between keypoints of I_1 and I_2. + The i-th entry of fnn12 is j if and only if keypoint k_i in image I_1 is matched to keypoint k_j in image I_2 + fnn_to_seed_local_consistency_map_corr: Boolean matrix of size (num_seeds, num_keypoints). + Entry (i, j) is True iff keypoint j was assigned to seed i. + k1: Keypoint locations in image I_1 + k2: Keypoint locations in image I_2 + im1seeds: Keypoint index of chosen seeds in image I_1 + im2seeds: Keypoint index of chosen seeds in image I_2 + scores: Scores to rank correspondences by confidence. + Lower scores are assumed to be more confident, consistently with Lowe's ratio scores. + Note: scores should be between 0 and 1 for this function to work as expected. + + Returns: + All information required for running the parallel RANSACs. + Data is formatted so that all inputs for different RANSACs are concatenated + along the same dimension to support different input sizes. + + im1loc: Keypoint locations in image I_1 for each RANSAC sample. + im2loc: Keypoint locations in image I_2 for each RANSAC sample. + ransidx: Integer identifier of the RANSAC problem. + This allows to distinguish inputs belonging to the same problem. + tokp1: Index of the original keypoint in image I_1 for each RANSAC sample. + tokp2: Index of the original keypoint in image I_2 for each RANSAC sample. + + """ + # first get an indexing representation of the assignments: + # - ransidx holds the index of the seed for each assignment + # - tokp1 holds the index of the keypoint in image I_1 for each assignment + ransidx, tokp1 = torch.where(fnn_to_seed_local_consistency_map_corr) + # - and of course tokp2 holds the index of the corresponding keypoint in image I_2 + tokp2 = fnn12[tokp1] + + # Now take the locations in the image of each considered keypoint ... + im1abspattern = k1[tokp1] + im2abspattern = k2[tokp2] + + # ... and subtract the location of its corresponding seed to get relative coordinates + im1loc = im1abspattern - k1[im1seeds[ransidx]] + im2loc = im2abspattern - k2[im2seeds[ransidx]] + + # Finally we need to sort keypoints by scores in a way that assignments to the same seed are close together + # To achieve this we assume scores lie in (0, 1) and add the integer index of the corresponding seed + expanded_local_scores = scores[tokp1] + ransidx.type(scores.dtype) + + sorting_perm = torch.argsort(expanded_local_scores) + + return im1loc[sorting_perm], im2loc[sorting_perm], ransidx, tokp1[sorting_perm], tokp2[sorting_perm] + + +def adalam_core( + k1: torch.Tensor, + k2: torch.Tensor, + fnn12: torch.Tensor, + scores1: torch.Tensor, + config: AdalamConfig, + mnn: Optional[torch.Tensor] = None, + im1shape: Optional[Tuple[int, int]] = None, + im2shape: Optional[Tuple[int, int]] = None, + o1: Optional[torch.Tensor] = None, + o2: Optional[torch.Tensor] = None, + s1: Optional[torch.Tensor] = None, + s2: Optional[torch.Tensor] = None, + return_dist: bool = False, +) -> Union[Tuple[torch.Tensor, torch.Tensor], torch.Tensor]: + """Call the core functionality of AdaLAM, i.e. just outlier filtering. + + No sanity check is performed on the inputs. + + Args: + k1: keypoint locations in the source image, in pixel coordinates. + Expected a float32 torch.Tensor with shape (num_keypoints_in_source_image, 2). + k2: keypoint locations in the destination image, in pixel coordinates. + Expected a float32 torch.Tensor with shape (num_keypoints_in_destination_image, 2). + fnn12: Initial set of putative matches to be filtered. + The current implementation assumes that these are unfiltered nearest neighbor matches, + so it requires this to be a list of indices a_i such that the source keypoint i is associated to the + destination keypoint a_i. For now to use AdaLAM on different inputs a workaround on the input format is + required. Expected a long torch.Tensor with shape (num_keypoints_in_source_image,). + scores1: Confidence scores on the putative_matches. Usually holds Lowe's ratio scores. + config: Adalam configuration. + mnn: A mask indicating which putative matches are also mutual nearest neighbors. See documentation on + 'force_seed_mnn' in the DEFAULT_CONFIG. If None, it disables the mutual nearest neighbor filtering on seed + point selection. Expected a bool torch.Tensor with shape (num_keypoints_in_source_image,) + im1shape: Shape of the source image. If None, it is inferred from keypoints max and min, at the cost of wasted + runtime. So please provide it. Expected a tuple with (width, height) or (height, width) of source + image + im2shape: Shape of the destination image. If None, it is inferred from keypoints max and min, at the cost of + wasted runtime. So please provide it. Expected a tuple with (width, height) or (height, width) of + destination image + o1: keypoint orientations in degrees. They can be None if 'orientation_difference_threshold' in config is + set to None. See documentation on 'orientation_difference_threshold' in the DEFAULT_CONFIG. + Expected a float32 torch.Tensor with shape (num_keypoints_in_source/destination_image,) + o2: Same as o1 but for destination. + s1: keypoint scales. They can be None if 'scale_rate_threshold' in config is set to None. + See documentation on 'scale_rate_threshold' in the DEFAULT_CONFIG. + Expected a float32 torch.Tensor with shape (num_keypoints_in_source/destination_image,) + s2: Same as s1 but for destination. + return_dist: if True, inverse confidence value is also outputted. Default is False + + Returns: + idxs: A long torch.Tensor with shape (num_filtered_matches, 2) with indices of corresponding + keypoints in k1 and k2. + dists: inverse confidence ratio. + + """ + AREA_RATIO = config["area_ratio"] + SEARCH_EXP = config["search_expansion"] + RANSAC_ITERS = config["ransac_iters"] + MIN_INLIERS = config["min_inliers"] + MIN_CONF = config["min_confidence"] + ORIENTATION_THR = config["orientation_difference_threshold"] + SCALE_RATE_THR = config["scale_rate_threshold"] + REFIT = config["refit"] + + if isinstance(im1shape, tuple): + _im1shape = torch.tensor(im1shape, device=k1.device, dtype=k1.dtype) + else: + k1mins = k1.min(dim=0).values + k1maxs = k1.max(dim=0).values + _im1shape = k1maxs - k1mins + + if isinstance(im2shape, tuple): + _im2shape = torch.tensor(im2shape, device=k2.device, dtype=k2.dtype) + else: + k2mins = k2.min(dim=0).values + k2maxs = k2.max(dim=0).values + _im2shape = k2maxs - k2mins + + # Compute seed selection radii to be invariant to image rescaling + R1 = torch.sqrt(torch.prod(_im1shape[:2]) / AREA_RATIO / math.pi) + R2 = torch.sqrt(torch.prod(_im2shape[:2]) / AREA_RATIO / math.pi) + + # Precompute the inner distances of keypoints in image I_1 + dist1 = dist_matrix(k1, k1) + + # Select seeds + im1seeds, im2seeds = select_seeds(dist1, R1, scores1, fnn12, mnn) + + # Find the neighboring and coherent keyopints consistent with each seed + local_neighs_mask, rdims, im1seeds, im2seeds = extract_neighborhood_sets( + o1, + o2, + s1, + s2, + dist1, + im1seeds, + im2seeds, + k1, + k2, + R1, + R2, + fnn12, + ORIENTATION_THR, + SCALE_RATE_THR, + SEARCH_EXP, + MIN_INLIERS, + ) + + if rdims.shape[0] == 0: + # No seed point survived. Just output ratio-test matches. This should happen very rarely. + score_mask = scores1 <= 0.95 + absolute_im1idx = torch.where(score_mask)[0] + if len(absolute_im1idx) > 0: + absolute_im2idx = fnn12[absolute_im1idx] + out_scores = scores1[score_mask].reshape(-1, 1) + idxs = torch.stack([absolute_im1idx, absolute_im2idx], dim=1) + else: + idxs, out_scores = _no_match(scores1) + if return_dist: + return idxs, out_scores + else: + return idxs + + # Format neighborhoods for parallel RANSACs + im1loc, im2loc, ransidx, tokp1, tokp2 = extract_local_patterns( + fnn12, local_neighs_mask, k1, k2, im1seeds, im2seeds, scores1 + ) + im1loc = im1loc / (R1 * SEARCH_EXP) + im2loc = im2loc / (R2 * SEARCH_EXP) + + # Run the parallel confidence-based RANSACs to perform local affine verification + inlier_idx, _, inl_confidence, inlier_counts = ransac( + xsamples=im1loc, ysamples=im2loc, rdims=rdims, iters=RANSAC_ITERS, refit=REFIT, config=dict(config) + ) + + conf = inl_confidence[ransidx[inlier_idx]] + cnt = inlier_counts[ransidx[inlier_idx]].float() + dist_ratio = 1.0 / conf + passed_inliers_mask = (conf >= MIN_CONF) & (cnt * (1 - dist_ratio) >= MIN_INLIERS) + accepted_inliers = inlier_idx[passed_inliers_mask] + accepted_dist = dist_ratio[passed_inliers_mask] + + absolute_im1idx = tokp1[accepted_inliers] + absolute_im2idx = tokp2[accepted_inliers] + + final_matches = torch.stack([absolute_im1idx, absolute_im2idx], dim=1) + if final_matches.shape[0] > 1: + # https://stackoverflow.com/a/72005790 + final_matches, idxs, counts = torch.unique(final_matches, dim=0, return_inverse=True, return_counts=True) + _, ind_sorted = torch.sort(idxs) + cum_sum = counts.cumsum(0) + cum_sum = torch.cat((torch.tensor([0], dtype=cum_sum.dtype, device=cum_sum.device), cum_sum[:-1])) + first_indicies = ind_sorted[cum_sum] + accepted_dist = accepted_dist[first_indicies] + if return_dist: + return final_matches, accepted_dist.reshape(-1, 1) + return final_matches diff --git a/kornia/feature/adalam/ransac.py b/kornia/feature/adalam/ransac.py new file mode 100644 index 0000000..13896c2 --- /dev/null +++ b/kornia/feature/adalam/ransac.py @@ -0,0 +1,206 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Tuple, Union + +import torch + +from .utils import arange_sequence, batch_2x2_ellipse, batch_2x2_inv, draw_first_k_couples, piecewise_arange + + +def stable_sort_residuals(residuals: torch.Tensor, ransidx: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Sort residuals.""" + logres = torch.log(residuals + 1e-10) + minlogres = torch.min(logres) + maxlogres = torch.max(logres) + + sorting_score = ransidx.unsqueeze(0).float() + 0.99 * (logres - minlogres) / (maxlogres - minlogres) + + sorting_idxes = torch.argsort(sorting_score, dim=-1) # (niters, numsamples) + + iters_range = torch.arange(residuals.shape[0], device=residuals.device) + + return residuals[iters_range.unsqueeze(-1), sorting_idxes], sorting_idxes + + +def group_sum_and_cumsum( + scores_mat: torch.Tensor, end_group_idx: torch.Tensor, group_idx: Union[torch.Tensor, slice, None] = None +) -> Tuple[torch.Tensor, Union[torch.Tensor, None]]: + """Calculate cumulative sum over group.""" + cumulative_scores = torch.cumsum(scores_mat, dim=1) + ending_cumusums = cumulative_scores[:, end_group_idx] + shifted_ending_cumusums = torch.cat( + [ + torch.zeros(size=(ending_cumusums.shape[0], 1), dtype=ending_cumusums.dtype, device=scores_mat.device), + ending_cumusums[:, :-1], + ], + dim=1, + ) + grouped_sums = ending_cumusums - shifted_ending_cumusums + + if group_idx is not None: + grouped_cumsums = cumulative_scores - shifted_ending_cumusums[:, group_idx] + return grouped_sums, grouped_cumsums + return grouped_sums, None + + +def confidence_based_inlier_selection( + residuals: torch.Tensor, + ransidx: torch.Tensor, + rdims: torch.Tensor, + idxoffsets: torch.Tensor, + dv: torch.device, + min_confidence: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Select inliers from confidence scores.""" + numransacs = rdims.shape[0] + numiters = residuals.shape[0] + + sorted_res, sorting_idxes = stable_sort_residuals(residuals, ransidx) + sorted_res_sqr = sorted_res**2 + + too_perfect_fits = sorted_res_sqr <= 1e-8 + end_rans_indexing = torch.cumsum(rdims, dim=0) - 1 + + _, inv_indices, res_dup_counts = torch.unique_consecutive( + sorted_res_sqr.half().float(), dim=1, return_counts=True, return_inverse=True + ) + + duplicates_per_sample = res_dup_counts[inv_indices] + inlier_weights = (1.0 / duplicates_per_sample).repeat(numiters, 1) + inlier_weights[too_perfect_fits] = 0.0 + + balanced_rdims, weights_cumsums = group_sum_and_cumsum(inlier_weights, end_rans_indexing, ransidx) + if not isinstance(weights_cumsums, torch.Tensor): + raise TypeError("Expected the `weights_cumsums` to be a torch.Tensor!") + + progressive_inl_rates = weights_cumsums.float() / (balanced_rdims.repeat_interleave(rdims, dim=1)).float() + + good_inl_mask = (sorted_res_sqr * min_confidence <= progressive_inl_rates) | too_perfect_fits + + inlier_weights[~good_inl_mask] = 0.0 + inlier_counts_matrix, _ = group_sum_and_cumsum(inlier_weights, end_rans_indexing) + + inl_counts, inl_iters = torch.max(inlier_counts_matrix.long(), dim=0) + + relative_inl_idxes = arange_sequence(inl_counts) + inl_ransidx = torch.arange(numransacs, device=dv).repeat_interleave(inl_counts) + inl_sampleidx = sorting_idxes[inl_iters.repeat_interleave(inl_counts), idxoffsets[inl_ransidx] + relative_inl_idxes] + highest_accepted_sqr_residuals = sorted_res_sqr[inl_iters, idxoffsets + inl_counts - 1] + expected_extra_inl = ( + balanced_rdims[inl_iters, torch.arange(numransacs, device=dv)].float() * highest_accepted_sqr_residuals + ) + return inl_ransidx, inl_sampleidx, inl_counts, inl_iters, inl_counts.float() / expected_extra_inl + + +def sample_padded_inliers( + xsamples: torch.Tensor, + ysamples: torch.Tensor, + inlier_counts: torch.Tensor, + inl_ransidx: torch.Tensor, + inl_sampleidx: torch.Tensor, + numransacs: int, + dv: torch.device, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Sample from padded inliers.""" + maxinliers = int(torch.max(inlier_counts).item()) + dtype = xsamples.dtype + padded_inlier_x = torch.zeros(size=(numransacs, maxinliers, 2), device=dv, dtype=dtype) + padded_inlier_y = torch.zeros(size=(numransacs, maxinliers, 2), device=dv, dtype=dtype) + + padded_inlier_x[inl_ransidx, piecewise_arange(inl_ransidx)] = xsamples[inl_sampleidx] + padded_inlier_y[inl_ransidx, piecewise_arange(inl_ransidx)] = ysamples[inl_sampleidx] + + return padded_inlier_x, padded_inlier_y + + +def ransac( + xsamples: torch.Tensor, + ysamples: torch.Tensor, + rdims: torch.Tensor, + config: Dict[str, Any], + iters: int = 128, + refit: bool = True, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Run ransac.""" + DET_THR = config["detected_scale_rate_threshold"] + MIN_CONFIDENCE = config["min_confidence"] + dv: torch.device = config["device"] + + numransacs = rdims.shape[0] + ransidx = torch.arange(numransacs, device=dv).repeat_interleave(rdims) + idxoffsets = torch.cat([torch.tensor([0], device=dv), torch.cumsum(rdims[:-1], dim=0)], dim=0) + + rand_samples_rel = draw_first_k_couples(iters, rdims, dv) + rand_samples_abs = rand_samples_rel + idxoffsets + sampled_x = torch.transpose( + xsamples[rand_samples_abs], dim0=1, dim1=2 + ) # (niters, 2, numransacs, 2) -> (niters, numransacs, 2, 2) + sampled_y = torch.transpose(ysamples[rand_samples_abs], dim0=1, dim1=2) + + # minimal fit for sampled_x @ A^T = sampled_y + affinities_fit = torch.transpose(batch_2x2_inv(sampled_x, check_dets=True) @ sampled_y, -1, -2) + if not refit: + eigenvals, _eigenvecs = batch_2x2_ellipse(affinities_fit) + bad_ones = (eigenvals[..., 1] < 1 / DET_THR**2) | (eigenvals[..., 0] > DET_THR**2) + affinities_fit[bad_ones] = torch.eye(2, device=dv) + y_pred = (affinities_fit[:, ransidx] @ xsamples.unsqueeze(-1)).squeeze(-1) + + residuals = torch.norm(y_pred - ysamples, dim=-1) # (niters, numsamples) + + inl_ransidx, inl_sampleidx, inl_counts, inl_iters, inl_confidence = confidence_based_inlier_selection( + residuals, ransidx, rdims, idxoffsets, dv=dv, min_confidence=MIN_CONFIDENCE + ) + + if len(inl_sampleidx) == 0: + # If no inliers have been found, there is nothing to re-fit! + refit = False + + if not refit: + return ( + inl_sampleidx, + affinities_fit[inl_iters, torch.arange(inl_iters.shape[0], device=dv)], + inl_confidence, + inl_counts, + ) + + # Organize inliers found into a matrix for efficient GPU re-fitting. + # Cope with the irregular number of inliers per sample by padding with torch.zeros + padded_inlier_x, padded_inlier_y = sample_padded_inliers( + xsamples, ysamples, inl_counts, inl_ransidx, inl_sampleidx, numransacs, dv + ) + + # A @ pad_x.T = pad_y.T + # A = pad_y.T @ pad_x @ (pad_x.T @ pad_x)^-1 + refit_affinity = ( + padded_inlier_y.transpose(-2, -1) + @ padded_inlier_x + @ batch_2x2_inv(padded_inlier_x.transpose(-2, -1) @ padded_inlier_x, check_dets=True) + ) + + # Filter out degenerate affinities with large scale changes + eigenvals, _eigenvecs = batch_2x2_ellipse(refit_affinity) + bad_ones = (eigenvals[..., 1] < 1 / DET_THR**2) | (eigenvals[..., 0] > DET_THR**2) + refit_affinity[bad_ones] = torch.eye(2, device=dv, dtype=refit_affinity.dtype) + y_pred = (refit_affinity[ransidx] @ xsamples.unsqueeze(-1)).squeeze(-1) + + residuals = torch.norm(y_pred - ysamples, dim=-1) + + inl_ransidx, inl_sampleidx, inl_counts, inl_iters, inl_confidence = confidence_based_inlier_selection( + residuals.unsqueeze(0), ransidx, rdims, idxoffsets, dv=dv, min_confidence=MIN_CONFIDENCE + ) + return inl_sampleidx, refit_affinity, inl_confidence, inl_counts diff --git a/kornia/feature/adalam/utils.py b/kornia/feature/adalam/utils.py new file mode 100644 index 0000000..b0f0a60 --- /dev/null +++ b/kornia/feature/adalam/utils.py @@ -0,0 +1,177 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math +from typing import Tuple + +import torch + + +def arange_sequence(ranges: torch.Tensor) -> torch.Tensor: + """Return a sequence of the ranges specified by the argument. + + Example: + [2, 5, 1, 2] -> [0, 1, 0, 1, 2, 3, 4, 0, 0, 1] + + """ + maxcnt = torch.max(ranges).item() + numuni = ranges.shape[0] + complete_ranges = torch.arange(maxcnt, device=ranges.device).unsqueeze(0).expand(numuni, -1) + + return complete_ranges[complete_ranges < ranges.unsqueeze(-1)] + + +def dist_matrix(d1: torch.Tensor, d2: torch.Tensor, is_normalized: bool = False) -> torch.Tensor: + """Distance between two tensors.""" + if is_normalized: + return 2 - 2.0 * d1 @ d2.t() + x_norm = (d1**2).sum(1).view(-1, 1) + y_norm = (d2**2).sum(1).view(1, -1) + # print(x_norm, y_norm) + distmat = x_norm + y_norm - 2.0 * d1 @ d2.t() + # distmat[torch.isnan(distmat)] = np.inf + return distmat + + +def orientation_diff(o1: torch.Tensor, o2: torch.Tensor) -> torch.Tensor: + """Orientation difference between two tensors.""" + diff = o2 - o1 + diff[diff < -180] += 360 + diff[diff >= 180] -= 360 + return diff + + +def piecewise_arange(piecewise_idxer: torch.Tensor) -> torch.Tensor: + """Count repeated indices. + + Example: + [0, 0, 0, 3, 3, 3, 3, 1, 1, 2] -> [0, 1, 2, 0, 1, 2, 3, 0, 1, 0] + """ + dv = piecewise_idxer.device + # print(piecewise_idxer) + uni: torch.Tensor + uni, counts = torch.unique_consecutive(piecewise_idxer, return_counts=True) + # print(counts) + maxcnt = int(torch.max(counts).item()) + numuni = uni.shape[0] + tmp = torch.zeros(size=(numuni, maxcnt), device=dv).bool() + ranges = torch.arange(maxcnt, device=dv).unsqueeze(0).expand(numuni, -1) + tmp[ranges < counts.unsqueeze(-1)] = True + return ranges[tmp] + + +def batch_2x2_inv(m: torch.Tensor, check_dets: bool = False) -> torch.Tensor: + """Return inverse of batch of 2x2 matrices.""" + a = m[..., 0, 0] + b = m[..., 0, 1] + c = m[..., 1, 0] + d = m[..., 1, 1] + minv = torch.empty_like(m) + det = a * d - b * c + if check_dets: + det[torch.abs(det) < 1e-10] = 1e-10 + minv[..., 0, 0] = d + minv[..., 1, 1] = a + minv[..., 0, 1] = -b + minv[..., 1, 0] = -c + return minv / det.unsqueeze(-1).unsqueeze(-1) + + +def batch_2x2_Q(m: torch.Tensor) -> torch.Tensor: + """Return Q of batch of 2x2 matrices.""" + return batch_2x2_inv(batch_2x2_invQ(m), check_dets=True) + + +def batch_2x2_invQ(m: torch.Tensor) -> torch.Tensor: + """Return inverse Q of batch of 2x2 matrices.""" + return m @ m.transpose(-1, -2) + + +def batch_2x2_det(m: torch.Tensor) -> torch.Tensor: + """Return determinant of batch of 2x2 matrices.""" + a = m[..., 0, 0] + b = m[..., 0, 1] + c = m[..., 1, 0] + d = m[..., 1, 1] + return a * d - b * c + + +def batch_2x2_ellipse(m: torch.Tensor, *, eps: float = 0.0) -> Tuple[torch.Tensor, torch.Tensor]: + """Return Eigenvalues and Eigenvectors of batch of 2x2 matrices.""" + am = m[..., 0, 0] + bm = m[..., 0, 1] + cm = m[..., 1, 0] + dm = m[..., 1, 1] + + a = am * am + bm * bm + b = am * cm + bm * dm + d = cm * cm + dm * dm + + trh = 0.5 * (a + d) + diff = 0.5 * (a - d) + + # stable hypot + sqrtdisc = torch.hypot(diff, b) + + e1 = trh + sqrtdisc + e2 = trh - sqrtdisc + if eps > 0: + e1 = e1.clamp(min=eps) + e2 = e2.clamp(min=eps) + else: + e1 = e1.clamp(min=0.0) + e2 = e2.clamp(min=0.0) + eigenvals = torch.stack([e1, e2], dim=-1) + + theta = 0.5 * torch.atan2(2.0 * b, a - d) + c = torch.cos(theta) + s = torch.sin(theta) + + ev1 = torch.stack([c, s], dim=-1) # (...,2) + ev2 = torch.stack([-s, c], dim=-1) # orthogonal (...,2) + eigenvecs = torch.stack([ev1, ev2], dim=-1) # (...,2,2) columns are eigenvectors + return eigenvals, eigenvecs + + +def draw_first_k_couples(k: int, rdims: torch.Tensor, dv: torch.device) -> torch.Tensor: + """Return first k couples. + + Exhaustive search over the first n samples: + * n(n+1)/2 = n2/2 + n/2 couples + Max n for which we can exhaustively sample with k couples: + * n2/2 + n/2 = k + * n = sqrt(1/4 + 2k)-1/2 = (sqrt(8k+1)-1)/2 + """ + max_exhaustive_search = int(math.sqrt(2 * k + 0.25) - 0.5) + residual_search = int(k - max_exhaustive_search * (max_exhaustive_search + 1) / 2) + + repeats = torch.cat( + [ + torch.arange(max_exhaustive_search, dtype=torch.long, device=dv) + 1, + torch.tensor([residual_search], dtype=torch.long, device=dv), + ] + ) + idx_sequence = torch.stack([repeats.repeat_interleave(repeats), arange_sequence(repeats)], dim=-1) + return torch.remainder(idx_sequence.unsqueeze(-1), rdims) + + +def random_samples_indices(iters: int, rdims: torch.Tensor, dv: torch.device) -> torch.Tensor: + """Randomly sample indices of torch.Tensor.""" + rands = torch.rand(size=(iters, 2, rdims.shape[0]), device=dv) + scaled_rands = rands * (rdims - 1e-8).float() + rand_samples_rel = scaled_rands.long() + return rand_samples_rel diff --git a/kornia/feature/affine_shape.py b/kornia/feature/affine_shape.py new file mode 100644 index 0000000..e603749 --- /dev/null +++ b/kornia/feature/affine_shape.py @@ -0,0 +1,247 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math +from typing import Dict, Optional + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK_LAF, KORNIA_CHECK_SHAPE +from kornia.filters.kernels import get_gaussian_kernel2d +from kornia.filters.sobel import SpatialGradient + +from .laf import ( + ellipse_to_laf, + extract_patches_from_pyramid, + get_laf_orientation, + get_laf_scale, + make_upright, + scale_laf, + set_laf_orientation, +) + +urls: Dict[str, str] = {} +urls["affnet"] = "https://github.com/ducha-aiki/affnet/raw/master/pretrained/AffNet.pth" + + +class PatchAffineShapeEstimator(nn.Module): + r"""Module, which estimates the second moment matrix of the patch gradients. + + The method determines the affine shape of the local feature as in :cite:`baumberg2000`. + + Args: + patch_size: the input image patch size. + eps: for safe division. + + """ + + def __init__(self, patch_size: int = 19, eps: float = 1e-10) -> None: + super().__init__() + self.patch_size: int = patch_size + self.gradient: nn.Module = SpatialGradient("sobel", 1) + self.eps: float = eps + sigma: float = float(self.patch_size) / math.sqrt(2.0) + self.weighting: torch.Tensor = get_gaussian_kernel2d((self.patch_size, self.patch_size), (sigma, sigma), True) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(patch_size={self.patch_size}, eps={self.eps})" + + def forward(self, patch: torch.Tensor) -> torch.Tensor: + """Run forward. + + Args: + patch: :math:`(B, 1, H, W)` + + Returns: + torch.Tensor: ellipse_shape :math:`(B, 1, 3)` + + """ + KORNIA_CHECK_SHAPE(patch, ["B", "1", "H", "W"]) + self.weighting = self.weighting.to(patch.dtype).to(patch.device) + grads: torch.Tensor = self.gradient(patch) * self.weighting + # unpack the edges + gx: torch.Tensor = grads[:, :, 0] + gy: torch.Tensor = grads[:, :, 1] + # abc == 1st axis, mixture, 2nd axis. Ellipse_shape is a 2nd moment matrix. + ellipse_shape = torch.cat( + [ + gx.pow(2).mean(dim=2).mean(dim=2, keepdim=True), + (gx * gy).mean(dim=2).mean(dim=2, keepdim=True), + gy.pow(2).mean(dim=2).mean(dim=2, keepdim=True), + ], + dim=2, + ) + + # Now lets detect degenerate cases: when 2 or 3 elements are close to zero (e.g. if patch is completely black + bad_mask = ((ellipse_shape < self.eps).float().sum(dim=2, keepdim=True) >= 2).to(ellipse_shape.dtype) + # We will replace degenerate shape with circular shapes. + circular_shape = torch.tensor([1.0, 0.0, 1.0]).to(ellipse_shape.device).to(ellipse_shape.dtype).view(1, 1, 3) + ellipse_shape = ellipse_shape * (1.0 - bad_mask) + circular_shape * bad_mask + # normalization + ellipse_shape = ellipse_shape / ellipse_shape.max(dim=2, keepdim=True)[0] + return ellipse_shape + + +class LAFAffineShapeEstimator(nn.Module): + """Module, which extracts patches using input images and local affine frames (LAFs). + + Then runs :class:`~kornia.feature.PatchAffineShapeEstimator` on patches to estimate LAFs shape. + + Then original LAF shape is replaced with estimated one. The original LAF orientation is not preserved, + so it is recommended to first run LAFAffineShapeEstimator and then LAFOrienter, + + + Args: + patch_size: the input image patch size. + affine_shape_detector: Patch affine shape estimator, :class:`~kornia.feature.PatchAffineShapeEstimator`. + preserve_orientation: if True, the original orientation is preserved. + + """ # pylint: disable + + def __init__( + self, patch_size: int = 32, affine_shape_detector: Optional[nn.Module] = None, preserve_orientation: bool = True + ) -> None: + super().__init__() + self.patch_size = patch_size + self.affine_shape_detector = affine_shape_detector or PatchAffineShapeEstimator(self.patch_size) + self.preserve_orientation = preserve_orientation + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}" + f"(patch_size={self.patch_size}, " + f"affine_shape_detector={self.affine_shape_detector}, " + f"preserve_orientation={self.preserve_orientation})" + ) + + def forward(self, laf: torch.Tensor, img: torch.Tensor) -> torch.Tensor: + """Run forward. + + Args: + laf: :math:`(B, N, 2, 3)` + img: :math:`(B, 1, H, W)` + + Returns: + LAF_out: :math:`(B, N, 2, 3)` + + """ + KORNIA_CHECK_LAF(laf) + KORNIA_CHECK_SHAPE(img, ["B", "1", "H", "W"]) + B, N = laf.shape[:2] + PS: int = self.patch_size + patches: torch.Tensor = extract_patches_from_pyramid(img, make_upright(laf), PS, True).view(-1, 1, PS, PS) + ellipse_shape: torch.Tensor = self.affine_shape_detector(patches) + ellipses = torch.cat([laf.view(-1, 2, 3)[..., 2].unsqueeze(1), ellipse_shape], dim=2).view(B, N, 5) + scale_orig = get_laf_scale(laf) + if self.preserve_orientation: + ori_orig = get_laf_orientation(laf) + laf_out = ellipse_to_laf(ellipses) + ellipse_scale = get_laf_scale(laf_out) + laf_out = scale_laf(laf_out, scale_orig / ellipse_scale) + if self.preserve_orientation: + laf_out = set_laf_orientation(laf_out, ori_orig) + return laf_out + + +class LAFAffNetShapeEstimator(nn.Module): + """Module, which extracts patches using input images and local affine frames (LAFs). + + Then runs AffNet on patches to estimate LAFs shape. This is based on the original code from paper + "Repeatability Is Not Enough: Learning Discriminative Affine Regions via Discriminability"". + See :cite:`AffNet2018` for more details. + + Then original LAF shape is replaced with estimated one. The original LAF orientation is not preserved, + so it is recommended to first run LAFAffineShapeEstimator and then LAFOrienter. + + Args: + pretrained: Download and set pretrained weights to the model. + + """ + + def __init__(self, pretrained: bool = False, preserve_orientation: bool = True) -> None: + super().__init__() + self.features = nn.Sequential( + nn.Conv2d(1, 16, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(16, affine=False), + nn.ReLU(), + nn.Conv2d(16, 16, kernel_size=3, stride=1, padding=1, bias=False), + nn.BatchNorm2d(16, affine=False), + nn.ReLU(), + nn.Conv2d(16, 32, kernel_size=3, stride=2, padding=1, bias=False), + nn.BatchNorm2d(32, affine=False), + nn.ReLU(), + nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1, bias=False), + nn.BatchNorm2d(32, affine=False), + nn.ReLU(), + nn.Conv2d(32, 64, kernel_size=3, stride=2, padding=1, bias=False), + nn.BatchNorm2d(64, affine=False), + nn.ReLU(), + nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1, bias=False), + nn.BatchNorm2d(64, affine=False), + nn.ReLU(), + nn.Dropout(0.25), + nn.Conv2d(64, 3, kernel_size=8, stride=1, padding=0, bias=True), + nn.Tanh(), + nn.AdaptiveAvgPool2d(1), + ) + self.patch_size = 32 + # use torch.hub to load pretrained model + if pretrained: + pretrained_dict = torch.hub.load_state_dict_from_url(urls["affnet"], map_location=torch.device("cpu")) + self.load_state_dict(pretrained_dict["state_dict"], strict=False) + self.preserve_orientation = preserve_orientation + self.eval() + + @staticmethod + def _normalize_input(x: torch.Tensor, eps: float = 1e-6) -> torch.Tensor: + """Normalize the input by batch.""" + sp, mp = torch.std_mean(x, dim=(-3, -2, -1), keepdim=True) + # WARNING: we need to .detach() input, otherwise the gradients produced by + # the patches extractor with F.grid_sample are very noisy, making the detector + # training totally unstable. + return (x - mp.detach()) / (sp.detach() + eps) + + def forward(self, laf: torch.Tensor, img: torch.Tensor) -> torch.Tensor: + """Run forward. + + Args: + laf: :math:`(B, N, 2, 3)` + img: :math:`(B, 1, H, W)` + + Returns: + LAF_out: :math:`(B, N, 2, 3)` + + """ + KORNIA_CHECK_LAF(laf) + KORNIA_CHECK_SHAPE(img, ["B", "1", "H", "W"]) + B, N = laf.shape[:2] + PS: int = self.patch_size + patches: torch.Tensor = extract_patches_from_pyramid(img, make_upright(laf), PS, True).view(-1, 1, PS, PS) + xy = self.features(self._normalize_input(patches)).view(-1, 3) + a1 = torch.cat([1.0 + xy[:, 0].reshape(-1, 1, 1), 0 * xy[:, 0].reshape(-1, 1, 1)], dim=2) + a2 = torch.cat([xy[:, 1].reshape(-1, 1, 1), 1.0 + xy[:, 2].reshape(-1, 1, 1)], dim=2) + new_laf_no_center = torch.cat([a1, a2], dim=1).reshape(B, N, 2, 2) + new_laf = torch.cat([new_laf_no_center, laf[:, :, :, 2:3]], dim=3) + scale_orig = get_laf_scale(laf) + if self.preserve_orientation: + ori_orig = get_laf_orientation(laf) + ellipse_scale = get_laf_scale(new_laf) + laf_out = scale_laf(make_upright(new_laf), scale_orig / ellipse_scale) + if self.preserve_orientation: + laf_out = set_laf_orientation(laf_out, ori_orig) + return laf_out diff --git a/kornia/feature/aliked/__init__.py b/kornia/feature/aliked/__init__.py new file mode 100644 index 0000000..f28636b --- /dev/null +++ b/kornia/feature/aliked/__init__.py @@ -0,0 +1,21 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Feature ALIKED — ALIKED local feature detector and descriptor.""" + +from .aliked import ALIKED, ALIKEDFeatures +from .deform_conv2d import deform_conv2d diff --git a/kornia/feature/aliked/aliked.py b/kornia/feature/aliked/aliked.py new file mode 100644 index 0000000..a72b10e --- /dev/null +++ b/kornia/feature/aliked/aliked.py @@ -0,0 +1,1035 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Adapted from the ALIKED implementation in LightGlue +# (https://github.com/cvg/LightGlue/blob/main/lightglue/aliked.py) +# which itself is based on the original ALIKED code by Zhao Xiaoming et al.: +# https://github.com/Shiaoming/ALIKED +# +# BSD 3-Clause License +# +# Copyright (c) 2022, Zhao Xiaoming +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# Authors: +# Xiaoming Zhao, Xingming Wu, Weihai Chen, Peter C.Y. Chen, Qingsong Xu, and +# Zhengguo Li + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable, Optional, Tuple + +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn.modules.utils import _pair + +from kornia.color import grayscale_to_rgb +from kornia.geometry.subpix import nms2d + +from .deform_conv2d import deform_conv2d + +# --------------------------------------------------------------------------- +# Output data structure +# --------------------------------------------------------------------------- + + +@dataclass +class ALIKEDFeatures: + r"""Keypoints, descriptors and scores detected by ALIKED for a single image. + + Since ALIKED detects a varying number of keypoints per image, + ``ALIKEDFeatures`` is not batched. + + Args: + keypoints: pixel coordinates ``(N, 2)`` as ``[x, y]``. + descriptors: L2-normalised descriptors ``(N, D)``. + keypoint_scores: detection confidence scores ``(N,)``. + """ + + keypoints: torch.Tensor + descriptors: torch.Tensor + keypoint_scores: torch.Tensor + + @property + def n(self) -> int: + """Number of detected keypoints.""" + return self.keypoints.shape[0] + + def to(self, *args: Any, **kwargs: Any) -> ALIKEDFeatures: + """Move all tensors to a new device / dtype.""" + return ALIKEDFeatures( + self.keypoints.to(*args, **kwargs), + self.descriptors.to(*args, **kwargs), + self.keypoint_scores.to(*args, **kwargs), + ) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _conv1x1(in_planes: int, out_planes: int, stride: int = 1) -> nn.Conv2d: + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +def _conv3x3(in_planes: int, out_planes: int, stride: int = 1, groups: int = 1, dilation: int = 1) -> nn.Conv2d: + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=dilation, + groups=groups, + bias=False, + dilation=dilation, + ) + + +def get_patches(tensor: torch.Tensor, required_corners: torch.Tensor, ps: int) -> torch.Tensor: + """Extract ps x ps patches centred at required_corners from a CHW tensor.""" + c, h, w = tensor.shape + corner = (required_corners - ps / 2 + 1).long() + corner[:, 0] = corner[:, 0].clamp(min=0, max=w - 1 - ps) + corner[:, 1] = corner[:, 1].clamp(min=0, max=h - 1 - ps) + offset = torch.arange(0, ps) + x, y = torch.meshgrid(offset, offset, indexing="ij") + patches = torch.stack((x, y)).permute(2, 1, 0).unsqueeze(2) + patches = patches.to(corner) + corner[None, None] + pts = patches.reshape(-1, 2) + sampled = tensor.permute(1, 2, 0)[tuple(pts.T)[::-1]] + sampled = sampled.reshape(ps, ps, -1, c) + return sampled.permute(2, 3, 0, 1) + + +# --------------------------------------------------------------------------- +# LAF helpers +# --------------------------------------------------------------------------- + + +def _affine_from_cov(cov: torch.Tensor) -> torch.Tensor: + """Build a 2x2 affine matrix from a batch of 2x2 covariance matrices. + + The affine is ``U @ diag(sqrt(eigenvalues))``, where ``U`` contains the + orthonormal eigenvectors of ``cov`` as *columns*. This gives an ellipse + whose axes align with the principal directions of the covariance. + + Args: + cov: symmetric positive-semi-definite matrices ``(N, 2, 2)``. + + Returns: + Affine matrices ``(N, 2, 2)``. + """ + # eigh returns eigenvalues sorted ascending; columns of eigenvectors are evecs. + # torch.linalg.eigh is not implemented for float16/bfloat16 on CUDA, so promote + # to float32 and cast the result back. + orig_dtype = cov.dtype + if cov.dtype in (torch.float16, torch.bfloat16): + cov = cov.float() + eigenvalues, eigenvectors = torch.linalg.eigh(cov) + scales = eigenvalues.clamp(min=1e-8).sqrt() # (N, 2) + # Each column of eigenvectors multiplied by the corresponding scale. + return (eigenvectors * scales[:, None, :]).to(orig_dtype) # (N, 2, 2) + + +def _laf_from_kpts_and_affine( + keypoints_px: torch.Tensor, + affine: torch.Tensor, +) -> torch.Tensor: + """Assemble a ``(N, 2, 3)`` LAF from pixel keypoints and 2x2 affine matrices. + + Args: + keypoints_px: ``(N, 2)`` pixel coordinates ``[x, y]``. + affine: ``(N, 2, 2)`` affine (rotation+scale) part of the LAF. + + Returns: + LAF tensor ``(N, 2, 3)`` following the kornia convention + ``[[a, b, cx], [c, d, cy]]``. + """ + centers = keypoints_px.unsqueeze(-1) # (N, 2, 1) + return torch.cat([affine, centers], dim=-1) # (N, 2, 3) + + +# --------------------------------------------------------------------------- +# Differentiable Keypoint Detection (DKD) +# --------------------------------------------------------------------------- + + +class DKD(nn.Module): + """Differentiable keypoint detection from a score map. + + Args: + radius: soft detection radius; NMS kernel size is ``2 * radius + 1``. + top_k: if ``> 0`` return exactly the ``top_k`` highest-scoring + keypoints; otherwise threshold mode is used. + scores_th: score threshold when ``top_k <= 0``. + If ``> 0`` keep keypoints with ``score > scores_th``; + otherwise keep keypoints above the per-image mean score. + n_limit: maximum number of keypoints returned in threshold mode. + """ + + def __init__( + self, + radius: int = 2, + top_k: int = 0, + scores_th: float = 0.2, + n_limit: int = 20000, + ) -> None: + super().__init__() + self.radius = radius + self.top_k = top_k + self.scores_th = scores_th + self.n_limit = n_limit + self.kernel_size = 2 * self.radius + 1 + self.temperature = 0.1 + self.unfold = nn.Unfold(kernel_size=self.kernel_size, padding=self.radius) + x = torch.linspace(-self.radius, self.radius, self.kernel_size) + hw_grid = torch.stack(torch.meshgrid([x, x], indexing="ij")).view(2, -1).t()[:, [1, 0]] + self.register_buffer("hw_grid", hw_grid) + + def forward( + self, + scores_map: torch.Tensor, + sub_pixel: bool = True, + image_size: Optional[torch.Tensor] = None, + return_affine: bool = False, + ) -> tuple[list[torch.Tensor], ...]: + """Detect keypoints from a score map. + + Args: + scores_map: ``(B, 1, H, W)`` detection score map. + sub_pixel: whether to apply soft-argmax sub-pixel refinement. + image_size: optional ``(B, 2)`` tensor of valid image sizes ``(W, H)`` + for border masking. + return_affine: if ``True``, also return per-keypoint 2x2 local affine + matrices estimated from the soft-argmax weight covariance. + + Returns: + A 3-tuple ``(keypoints, keypoint_scores, score_dispersities)`` — or a + 4-tuple ``(..., local_affines)`` when ``return_affine=True``. + Each ``keypoints[i]`` is ``(N_i, 2)`` normalised to ``[-1, 1]``. + Each ``local_affines[i]`` is ``(N_i, 2, 2)`` (only when ``sub_pixel=True`` + and ``return_affine=True``; otherwise identity matrices are returned). + """ + b, _c, h, w = scores_map.shape + scores_nograd = scores_map.detach() + nms_scores = nms2d(scores_nograd, (self.kernel_size, self.kernel_size)) + + # remove border + nms_scores[:, :, : self.radius, :] = 0 + nms_scores[:, :, :, : self.radius] = 0 + if image_size is not None: + for i in range(b): + iw, ih = image_size[i].long() + nms_scores[i, :, ih.item() - self.radius :, :] = 0 + nms_scores[i, :, :, iw.item() - self.radius :] = 0 + else: + nms_scores[:, :, -self.radius :, :] = 0 + nms_scores[:, :, :, -self.radius :] = 0 + + if self.top_k > 0: + topk = torch.topk(nms_scores.view(b, -1), self.top_k) + indices_keypoints = [topk.indices[i] for i in range(b)] + else: + if self.scores_th > 0: + masks = nms_scores > self.scores_th + if masks.sum() == 0: + th = scores_nograd.reshape(b, -1).mean(dim=1) + masks = nms_scores > th.reshape(b, 1, 1, 1) + else: + th = scores_nograd.reshape(b, -1).mean(dim=1) + masks = nms_scores > th.reshape(b, 1, 1, 1) + masks = masks.reshape(b, -1) + + indices_keypoints = [] + scores_view = scores_nograd.reshape(b, -1) + for mask, scores in zip(masks, scores_view): + indices = mask.nonzero()[:, 0] + if len(indices) > self.n_limit: + kpts_sc = scores[indices] + sort_idx = kpts_sc.sort(descending=True)[1] + indices = indices[sort_idx[: self.n_limit]] + indices_keypoints.append(indices) + + wh = torch.tensor([w - 1, h - 1], device=scores_nograd.device, dtype=scores_nograd.dtype) + + keypoints = [] + scoredispersitys = [] + kptscores = [] + local_affines: list[torch.Tensor] = [] + if sub_pixel: + patches = self.unfold(scores_map) # B x (kernel**2) x (H*W) + for b_idx in range(b): + patch = patches[b_idx].t() # (H*W) x (kernel**2) + indices_kpt = indices_keypoints[b_idx] + patch_scores = patch[indices_kpt] # M x (kernel**2) + keypoints_xy_nms = torch.stack( + [indices_kpt % w, torch.div(indices_kpt, w, rounding_mode="trunc")], + dim=1, + ) + + max_v = patch_scores.max(dim=1).values.detach()[:, None] + x_exp = ((patch_scores - max_v) / self.temperature).exp() + x_sum = x_exp.sum(dim=1, keepdim=True) + + xy_residual = x_exp @ self.hw_grid / x_sum # type: ignore[operator] + + hw_grid_dist2 = ( + torch.norm( + (self.hw_grid[None, :, :] - xy_residual[:, None, :]) / self.radius, # type: ignore[index] + dim=-1, + ) + ** 2 + ) + scoredispersity = (x_exp * hw_grid_dist2).sum(dim=1) / x_exp.sum(dim=1) + + keypoints_xy = keypoints_xy_nms + xy_residual + keypoints_xy = keypoints_xy / wh * 2 - 1 # -> [-1, 1] + + kptscore = F.grid_sample( + scores_map[b_idx].unsqueeze(0), + keypoints_xy.view(1, 1, -1, 2), + mode="bilinear", + align_corners=True, + )[0, 0, 0, :] + + keypoints.append(keypoints_xy) + scoredispersitys.append(scoredispersity) + kptscores.append(kptscore) + + if return_affine: + # Weighted covariance of hw_grid positions under soft-argmax weights. + # w_i = x_exp_i / sum(x_exp), mu = xy_residual (already computed). + # cov = sum_i w_i (g_i - mu)(g_i - mu)^T -> (N, 2, 2) + W = x_exp / x_sum # (N, K²) normalised weights + delta = self.hw_grid[None] - xy_residual[:, None] # type: ignore[index] # (N, K², 2) + cov = torch.einsum("ni,nij,nik->njk", W, delta, delta) # (N, 2, 2) + local_affines.append(_affine_from_cov(cov)) + else: + for b_idx in range(b): + indices_kpt = indices_keypoints[b_idx] + keypoints_xy_nms = torch.stack( + [indices_kpt % w, torch.div(indices_kpt, w, rounding_mode="trunc")], + dim=1, + ) + keypoints_xy = keypoints_xy_nms / wh * 2 - 1 + kptscore = F.grid_sample( + scores_map[b_idx].unsqueeze(0), + keypoints_xy.view(1, 1, -1, 2), + mode="bilinear", + align_corners=True, + )[0, 0, 0, :] + keypoints.append(keypoints_xy) + scoredispersitys.append(torch.zeros_like(kptscore)) + kptscores.append(kptscore) + if return_affine: + # No soft-argmax weights available; fall back to identity. + N_i = keypoints_xy.shape[0] + local_affines.append( + torch.eye(2, device=scores_map.device, dtype=scores_map.dtype).unsqueeze(0).expand(N_i, -1, -1) + ) + + if return_affine: + return keypoints, kptscores, scoredispersitys, local_affines + return keypoints, kptscores, scoredispersitys + + +# --------------------------------------------------------------------------- +# Image padding helper +# --------------------------------------------------------------------------- + + +class InputPadder: + """Pad an image so that both spatial dimensions are divisible by ``divis_by``.""" + + def __init__(self, h: int, w: int, divis_by: int = 8) -> None: + self.ht = h + self.wd = w + pad_ht = (((h // divis_by) + 1) * divis_by - h) % divis_by + pad_wd = (((w // divis_by) + 1) * divis_by - w) % divis_by + self._pad = [pad_wd // 2, pad_wd - pad_wd // 2, pad_ht // 2, pad_ht - pad_ht // 2] + + def pad(self, x: torch.Tensor) -> torch.Tensor: + """Pad *x* (BCHW).""" + return F.pad(x, self._pad, mode="replicate") + + def unpad(self, x: torch.Tensor) -> torch.Tensor: + """Remove the padding added by :meth:`pad`.""" + ht, wd = x.shape[-2], x.shape[-1] + c = [self._pad[2], ht - self._pad[3], self._pad[0], wd - self._pad[1]] + return x[..., c[0] : c[1], c[2] : c[3]] + + +# --------------------------------------------------------------------------- +# Building blocks +# --------------------------------------------------------------------------- + + +class DeformableConv2d(nn.Module): + """Deformable conv2d (DCNv1 or DCNv2) without torchvision dependency. + + Uses the pure-PyTorch :func:`~kornia.feature.aliked.deform_conv2d.deform_conv2d` + implementation that matches ``torchvision.ops.deform_conv2d``. + + Args: + in_channels: number of input channels. + out_channels: number of output channels. + kernel_size: spatial kernel size. + stride: convolution stride. + padding: zero-padding size. + bias: whether to add a learnable bias. + mask: if ``True`` use DCNv2 modulation mask (3 * k*k offsets instead of 2). + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int = 3, + stride: int = 1, + padding: int = 1, + dilation: int = 1, + bias: bool = False, + mask: bool = False, + ) -> None: + super().__init__() + self.stride = stride + self.padding = padding + self.dilation = dilation + self.mask = mask + self.channel_num = 3 * kernel_size * kernel_size if mask else 2 * kernel_size * kernel_size + self.offset_conv = nn.Conv2d( + in_channels, + self.channel_num, + kernel_size=kernel_size, + stride=stride, + padding=self.padding, + dilation=dilation, + bias=True, + ) + self.regular_conv = nn.Conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + padding=self.padding, + dilation=dilation, + bias=bias, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run the module forward pass. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + h, w = x.shape[2:] + max_offset = max(h, w) / 4.0 + out = self.offset_conv(x) + if self.mask: + o1, o2, mask_w = torch.chunk(out, 3, dim=1) + offset = torch.cat((o1, o2), dim=1) + mask_w = torch.sigmoid(mask_w) + else: + offset = out + mask_w = None + offset = offset.clamp(-max_offset, max_offset) + return deform_conv2d( + input=x, + offset=offset, + weight=self.regular_conv.weight, + bias=self.regular_conv.bias, + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + mask=mask_w, + ) + + +def get_conv( + inplanes: int, + planes: int, + kernel_size: int = 3, + stride: int = 1, + padding: int = 1, + bias: bool = False, + conv_type: str = "conv", + mask: bool = False, +) -> nn.Module: + """Return a standard or deformable conv2d layer.""" + if conv_type == "conv": + return nn.Conv2d(inplanes, planes, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias) + if conv_type == "dcn": + return DeformableConv2d( + inplanes, + planes, + kernel_size=kernel_size, + stride=stride, + padding=_pair(padding)[0], + bias=bias, + mask=mask, + ) + raise TypeError(f"Unknown conv_type: {conv_type!r}. Expected 'conv' or 'dcn'.") + + +class ConvBlock(nn.Module): + """Two-layer conv block with BN and an activation gate.""" + + def __init__( + self, + in_channels: int, + out_channels: int, + gate: Optional[Callable[..., nn.Module]] = None, + norm_layer: Optional[Callable[..., nn.Module]] = None, + conv_type: str = "conv", + mask: bool = False, + ) -> None: + super().__init__() + self.gate = nn.ReLU(inplace=True) if gate is None else gate + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self.conv1 = get_conv(in_channels, out_channels, kernel_size=3, conv_type=conv_type, mask=mask) + self.bn1 = norm_layer(out_channels) + self.conv2 = get_conv(out_channels, out_channels, kernel_size=3, conv_type=conv_type, mask=mask) + self.bn2 = norm_layer(out_channels) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run the module forward pass. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + x = self.gate(self.bn1(self.conv1(x))) + return self.gate(self.bn2(self.conv2(x))) + + +class ResBlock(nn.Module): + """Residual block (BasicBlock variant) supporting deformable convolutions.""" + + expansion: int = 1 + + def __init__( + self, + inplanes: int, + planes: int, + stride: int = 1, + downsample: Optional[nn.Module] = None, + groups: int = 1, + base_width: int = 64, + dilation: int = 1, + gate: Optional[Callable[..., nn.Module]] = None, + norm_layer: Optional[Callable[..., nn.Module]] = None, + conv_type: str = "conv", + mask: bool = False, + ) -> None: + super().__init__() + if gate is None: + self.gate = nn.ReLU(inplace=True) + else: + self.gate = gate + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError("ResBlock only supports groups=1 and base_width=64") + if dilation > 1: + raise NotImplementedError("Dilation > 1 not supported in ResBlock") + self.conv1 = get_conv(inplanes, planes, kernel_size=3, conv_type=conv_type, mask=mask) + self.bn1 = norm_layer(planes) + self.conv2 = get_conv(planes, planes, kernel_size=3, conv_type=conv_type, mask=mask) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run the module forward pass. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + identity = x + out = self.gate(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + if self.downsample is not None: + identity = self.downsample(x) + out = self.gate(out + identity) + return out + + +# --------------------------------------------------------------------------- +# Sparse Descriptor with Deformable Heads (SDDH) +# --------------------------------------------------------------------------- + + +class SDDH(nn.Module): + """Sparse descriptor head using deformable sampling positions. + + Args: + dims: input feature dimension. + kernel_size: patch size used to estimate sampling offsets. + n_pos: number of deformable sampling positions per keypoint. + gate: activation function for the offset network. + conv2d: if ``True`` use a 1x1 conv to aggregate features; otherwise + use a learnable weighted sum (``agg_weights``). + mask: if ``True`` use DCNv2-style attention weighting on samples. + """ + + def __init__( + self, + dims: int, + kernel_size: int = 3, + n_pos: int = 8, + gate: Optional[nn.Module] = None, + conv2d: bool = False, + mask: bool = False, + ) -> None: + super().__init__() + if gate is None: + gate = nn.ReLU() + self.kernel_size = kernel_size + self.n_pos = n_pos + self.conv2d = conv2d + self.mask = mask + self.get_patches_func = get_patches + + self.channel_num = 3 * n_pos if mask else 2 * n_pos + self.offset_conv = nn.Sequential( + nn.Conv2d(dims, self.channel_num, kernel_size=kernel_size, stride=1, padding=0, bias=True), + gate, + nn.Conv2d(self.channel_num, self.channel_num, kernel_size=1, stride=1, padding=0, bias=True), + ) + self.sf_conv = nn.Conv2d(dims, dims, kernel_size=1, stride=1, padding=0, bias=False) + + if not conv2d: + agg_weights = nn.Parameter(torch.rand(n_pos, dims, dims)) + self.register_parameter("agg_weights", agg_weights) + else: + self.convM = nn.Conv2d(dims * n_pos, dims, kernel_size=1, stride=1, padding=0, bias=False) + + def forward( + self, + x: torch.Tensor, + keypoints: list[torch.Tensor], + ) -> tuple[list[torch.Tensor], list[torch.Tensor]]: + """Compute descriptors at keypoint locations. + + Args: + x: dense feature map ``(B, C, H, W)``. + keypoints: list of ``(N_i, 2)`` normalised keypoint coordinates ``[-1, 1]``. + + Returns: + A pair ``(descriptors, offsets)`` where each element is a list of length B. + """ + b, c, h, w = x.shape + wh = torch.tensor([[w - 1, h - 1]], device=x.device, dtype=x.dtype) + max_offset = max(h, w) / 4.0 + + offsets_list = [] + descriptors = [] + + for ib in range(b): + xi, kptsi = x[ib], keypoints[ib] + kptsi_wh = (kptsi / 2 + 0.5) * wh + N_kpts = len(kptsi) + + if self.kernel_size > 1: + patch = self.get_patches_func(xi, kptsi_wh.long(), self.kernel_size) + else: + kptsi_wh_long = kptsi_wh.long() + patch = xi[:, kptsi_wh_long[:, 1], kptsi_wh_long[:, 0]].permute(1, 0).reshape(N_kpts, c, 1, 1) + + offset = self.offset_conv(patch).clamp(-max_offset, max_offset) + + if self.mask: + offset = offset[:, :, 0, 0].view(N_kpts, 3, self.n_pos).permute(0, 2, 1) + mask_w = torch.sigmoid(offset[:, :, -1]) + offset = offset[:, :, :-1] + else: + offset = offset[:, :, 0, 0].view(N_kpts, 2, self.n_pos).permute(0, 2, 1) + mask_w = None + offsets_list.append(offset) + + pos = kptsi_wh.unsqueeze(1) + offset # [N_kpts, n_pos, 2] + pos = 2.0 * pos / wh[None] - 1 + pos = pos.reshape(1, N_kpts * self.n_pos, 1, 2) + + features = F.grid_sample(xi.unsqueeze(0), pos, mode="bilinear", align_corners=True) + features = features.reshape(c, N_kpts, self.n_pos, 1).permute(1, 0, 2, 3) + + if mask_w is not None: + features = torch.einsum("ncpo,np->ncpo", features, mask_w) + + features = torch.selu_(self.sf_conv(features)).squeeze(-1) # [N_kpts, C, n_pos] + + if not self.conv2d: + descs = torch.einsum("ncp,pcd->nd", features, self.agg_weights) # codespell:ignore + else: + features = features.reshape(N_kpts, -1)[:, :, None, None] + descs = self.convM(features).squeeze(-1).squeeze(-1) + + descs = F.normalize(descs, p=2.0, dim=1) + descriptors.append(descs) + + return descriptors, offsets_list + + +# --------------------------------------------------------------------------- +# ALIKED main module +# --------------------------------------------------------------------------- + +_ALIKED_CFGS: dict[str, tuple[int, int, int, int, int, int, int]] = { + # c1, c2, c3, c4, dim, K, M + "aliked-t16": (8, 16, 32, 64, 64, 3, 16), + "aliked-n16": (16, 32, 64, 128, 128, 3, 16), + "aliked-n16rot": (16, 32, 64, 128, 128, 3, 16), + "aliked-n32": (16, 32, 64, 128, 128, 3, 32), +} + +_CHECKPOINT_URL = "https://github.com/Shiaoming/ALIKED/raw/main/models/{}.pth" + + +class ALIKED(nn.Module): + r"""ALIKED local feature detector and descriptor. + + ALIKED (Adaptive Local Image KEypoint Detection) combines a multi-scale + ResNet backbone with deformable descriptor sampling (SDDH) and a + differentiable keypoint detector (DKD). + + See :cite:`zhao2023aliked` for details. + + .. image:: _static/img/ALIKED.png + + Args: + model_name: backbone configuration, one of + ``'aliked-t16'``, ``'aliked-n16'``, ``'aliked-n16rot'``, ``'aliked-n32'``. + max_num_keypoints: maximum number of keypoints to detect. + ``-1`` means no limit (threshold-based mode). + detection_threshold: minimum detection score in threshold mode. + nms_radius: NMS radius (kernel size ``= 2 * nms_radius + 1``). + + Example: + >>> aliked = ALIKED.from_pretrained('aliked-n16') # doctest: +SKIP + >>> images = torch.rand(1, 3, 256, 256) # doctest: +SKIP + >>> features = aliked(images) # doctest: +SKIP + """ + + n_limit_max: int = 20000 + + def __init__( + self, + model_name: str = "aliked-n16", + max_num_keypoints: int = -1, + detection_threshold: float = 0.2, + nms_radius: int = 2, + ) -> None: + super().__init__() + + if model_name not in _ALIKED_CFGS: + raise ValueError(f"Unknown model_name {model_name!r}. Choose from {list(_ALIKED_CFGS)}.") + + self.model_name = model_name + self.max_num_keypoints = max_num_keypoints + self.detection_threshold = detection_threshold + self.nms_radius = nms_radius + + c1, c2, c3, c4, dim, K, M = _ALIKED_CFGS[model_name] + conv_types = ["conv", "conv", "dcn", "dcn"] + conv2d = False + mask = False + + self.pool2 = nn.AvgPool2d(kernel_size=2, stride=2) + self.pool4 = nn.AvgPool2d(kernel_size=4, stride=4) + self.norm = nn.BatchNorm2d + self.gate = nn.SELU(inplace=True) + + self.block1 = ConvBlock(3, c1, self.gate, self.norm, conv_type=conv_types[0]) + self.block2 = self._make_resblock(c1, c2, conv_types[1], mask) + self.block3 = self._make_resblock(c2, c3, conv_types[2], mask) + self.block4 = self._make_resblock(c3, c4, conv_types[3], mask) + + self.conv1 = _conv1x1(c1, dim // 4) + self.conv2 = _conv1x1(c2, dim // 4) + self.conv3 = _conv1x1(c3, dim // 4) + self.conv4 = _conv1x1(dim, dim // 4) # note: c4 == dim for all configs + + self.upsample2 = nn.Upsample(scale_factor=2, mode="bilinear", align_corners=True) + self.upsample4 = nn.Upsample(scale_factor=4, mode="bilinear", align_corners=True) + self.upsample8 = nn.Upsample(scale_factor=8, mode="bilinear", align_corners=True) + self.upsample32 = nn.Upsample(scale_factor=32, mode="bilinear", align_corners=True) + + self.score_head = nn.Sequential( + _conv1x1(dim, 8), + self.gate, + _conv3x3(8, 4), + self.gate, + _conv3x3(4, 4), + self.gate, + _conv3x3(4, 1), + ) + self.desc_head = SDDH(dim, K, M, gate=self.gate, conv2d=conv2d, mask=mask) + self.dkd = DKD( + radius=nms_radius, + top_k=-1 if detection_threshold > 0 else max_num_keypoints, + scores_th=detection_threshold, + n_limit=max_num_keypoints if max_num_keypoints > 0 else self.n_limit_max, + ) + + def _make_resblock(self, c_in: int, c_out: int, conv_type: str, mask: bool) -> ResBlock: + return ResBlock( + c_in, + c_out, + stride=1, + downsample=nn.Conv2d(c_in, c_out, 1), + gate=self.gate, + norm_layer=self.norm, + conv_type=conv_type, + mask=mask, + ) + + def extract_dense_map(self, image: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Run the backbone and return ``(feature_map, score_map)``. + + Args: + image: ``(B, 3, H, W)`` float image. Inputs are internally padded to + a multiple of 32 and the padding is removed before returning. + + Returns: + ``feature_map``: ``(B, dim, H, W)`` L2-normalised dense features. + ``score_map``: ``(B, 1, H, W)`` detection score map in ``(0, 1)``. + """ + div_by = 2**5 + padder = InputPadder(image.shape[-2], image.shape[-1], div_by) + image = padder.pad(image) + + x1 = self.block1(image) + x2 = self.block2(self.pool2(x1)) + x3 = self.block3(self.pool4(x2)) + x4 = self.block4(self.pool4(x3)) + + x1 = self.gate(self.conv1(x1)) + x2 = self.gate(self.conv2(x2)) + x3 = self.gate(self.conv3(x3)) + x4 = self.gate(self.conv4(x4)) + + x2_up = self.upsample2(x2) + x3_up = self.upsample8(x3) + x4_up = self.upsample32(x4) + x1234 = torch.cat([x1, x2_up, x3_up, x4_up], dim=1) + + score_map = torch.sigmoid(self.score_head(x1234)) + feature_map = F.normalize(x1234, p=2, dim=1) + + feature_map = padder.unpad(feature_map) + score_map = padder.unpad(score_map) + + return feature_map, score_map + + def forward( + self, + images: torch.Tensor, + image_size: Optional[torch.Tensor] = None, + ) -> list[ALIKEDFeatures]: + """Detect and describe local features in a batch of images. + + Args: + images: ``(B, 3, H, W)`` float images (can be grayscale ``(B, 1, H, W)`` + — will be broadcast to 3 channels automatically). + image_size: optional ``(B, 2)`` tensor of valid ``(W, H)`` for + border masking when images are padded to a common size. + + Returns: + A list of :class:`ALIKEDFeatures` of length B, one per image. + Keypoints are in pixel coordinates ``[x, y]``. + """ + if images.shape[1] == 1: + images = grayscale_to_rgb(images) + + feature_map, score_map = self.extract_dense_map(images) + keypoints, kptscores, _scoredispersitys = self.dkd(score_map, image_size=image_size) + descriptors, _offsets = self.desc_head(feature_map, keypoints) + + B = images.shape[0] + _, _, h, w = images.shape + wh = torch.tensor([w - 1, h - 1], device=images.device, dtype=images.dtype) + + results = [] + for i in range(B): + # Convert normalised [-1,1] coords back to pixel coordinates + kps_px = wh * (keypoints[i] + 1) / 2.0 + results.append(ALIKEDFeatures(kps_px, descriptors[i], kptscores[i])) + return results + + def forward_laf( + self, + img: torch.Tensor, + mask: Optional[torch.Tensor] = None, + compute_affine: bool = True, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Detect and describe local features, returning results in kornia LAF format. + + Local Affine Frames are estimated from the soft-argmax weight covariance + computed inside :class:`DKD`: the 2x2 affine matrix captures the dominant + orientation and scale of each detected keypoint without any additional + network parameters. + + All per-image tensors are zero-padded along the keypoint dimension so + that the outputs are proper batched tensors. + + Args: + img: image to extract features with shape :math:`(B,C,H,W)`. + mask: optional spatial mask ``(B, 1, H, W)`` with values in + ``[0, 1]``; the score map is multiplied by this mask before + keypoint detection so that features are suppressed in masked + regions. + compute_affine: if ``True`` (default), estimate the 2x2 affine shape + of each LAF using ``torch.linalg.eigh`` on the soft-argmax + covariance. Set to ``False`` to skip the eigendecomposition and + return identity affines, which is faster and avoids the linalg + call entirely (useful when only keypoint positions are needed). + + Returns: + - Detected local affine frames with shape :math:`(B,N,2,3)`. + - Response function values for corresponding LAFs with shape :math:`(B,N,1)`. + - Local descriptors of shape :math:`(B,N,D)`. + + """ + if img.shape[1] == 1: + img = grayscale_to_rgb(img) + + feature_map, score_map = self.extract_dense_map(img) + + if mask is not None: + # Resize mask to score map resolution and apply. + mask_rs = F.interpolate( + mask.to(score_map.dtype), size=score_map.shape[-2:], mode="bilinear", align_corners=True + ) + score_map = score_map * mask_rs + + dkd_out = self.dkd(score_map, return_affine=compute_affine) + if compute_affine: + keypoints, kptscores, _scoredispersitys, local_affines = dkd_out # type: ignore[misc] + else: + keypoints, kptscores, _scoredispersitys = dkd_out # type: ignore[misc] + local_affines = None + descriptors, _offsets = self.desc_head(feature_map, keypoints) + + B, _, H, W = img.shape + wh = torch.tensor([W - 1, H - 1], device=img.device, dtype=img.dtype) + + lafs_list: list[torch.Tensor] = [] + for i in range(B): + kps_px = wh * (keypoints[i] + 1) / 2.0 # (N, 2) + if local_affines is not None: + affine_i = local_affines[i] + else: + # Identity affine: both axes are unit-scale, no rotation. + n = kps_px.shape[0] + affine_i = torch.eye(2, device=img.device, dtype=img.dtype).unsqueeze(0).expand(n, -1, -1) + laf_i = _laf_from_kpts_and_affine(kps_px, affine_i) # (N, 2, 3) + lafs_list.append(laf_i) + + # Pad to the maximum number of keypoints in the batch. + n_max = max(laf.shape[0] for laf in lafs_list) if lafs_list else 0 + + def _pad(t: torch.Tensor, n: int, fill: float = 0.0) -> torch.Tensor: + pad_n = n - t.shape[0] + if pad_n == 0: + return t + pad_shape = (pad_n,) + t.shape[1:] + return torch.cat([t, t.new_full(pad_shape, fill)], dim=0) + + lafs = torch.stack([_pad(laf, n_max) for laf in lafs_list]) # (B, N, 2, 3) + responses = torch.stack([_pad(s, n_max).unsqueeze(-1) for s in kptscores]) # (B, N, 1) + descs = torch.stack([_pad(d, n_max) for d in descriptors]) # (B, N, D) + + return lafs, responses, descs + + @classmethod + def from_pretrained( + cls, + model_name: str = "aliked-n16", + max_num_keypoints: int = -1, + detection_threshold: float = 0.2, + nms_radius: int = 2, + device: Optional[torch.device] = None, + ) -> ALIKED: + """Load a pretrained ALIKED model from the official checkpoint repository. + + Args: + model_name: one of ``'aliked-t16'``, ``'aliked-n16'``, + ``'aliked-n16rot'``, ``'aliked-n32'``. + max_num_keypoints: passed to :class:`ALIKED` constructor. + detection_threshold: passed to :class:`ALIKED` constructor. + nms_radius: passed to :class:`ALIKED` constructor. + device: target device; defaults to CPU. + + Returns: + Pretrained :class:`ALIKED` in eval mode. + """ + if device is None: + device = torch.device("cpu") + model = cls( + model_name=model_name, + max_num_keypoints=max_num_keypoints, + detection_threshold=detection_threshold, + nms_radius=nms_radius, + ).to(device) + url = _CHECKPOINT_URL.format(model_name) + state_dict = torch.hub.load_state_dict_from_url(url, map_location=device) + model.load_state_dict(state_dict, strict=False) + model.eval() + return model diff --git a/kornia/feature/aliked/deform_conv2d.py b/kornia/feature/aliked/deform_conv2d.py new file mode 100644 index 0000000..b898a19 --- /dev/null +++ b/kornia/feature/aliked/deform_conv2d.py @@ -0,0 +1,212 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Pure-PyTorch implementation of deform_conv2d matching torchvision.ops.deform_conv2d. + +Implements both DCNv1 (mask=None) and DCNv2 (mask provided) as described in: + - Deformable Convolutional Networks (https://arxiv.org/abs/1703.06211) + - Deformable ConvNets v2 (https://arxiv.org/abs/1811.11168) +""" + +from __future__ import annotations + +from typing import Optional, Tuple, Union + +import torch +from torch import Tensor + + +def deform_conv2d( + input: Tensor, + offset: Tensor, + weight: Tensor, + bias: Optional[Tensor] = None, + stride: Union[int, Tuple[int, int]] = (1, 1), + padding: Union[int, Tuple[int, int]] = (0, 0), + dilation: Union[int, Tuple[int, int]] = (1, 1), + mask: Optional[Tensor] = None, +) -> Tensor: + """Deformable 2D convolution (DCNv1/v2) without torchvision dependency. + + Matches the signature and behaviour of ``torchvision.ops.deform_conv2d``. + + Args: + input: input feature map ``(B, C_in, H_in, W_in)``. + offset: offset tensor ``(B, 2 * n_offset_groups * kH * kW, H_out, W_out)``. + Layout per kernel point: ``(dy, dx)`` interleaved. + weight: convolution kernel ``(C_out, C_in / groups, kH, kW)``. + bias: optional bias ``(C_out,)``. + stride: convolution stride ``(sH, sW)``. + padding: zero-padding ``(pH, pW)``. + dilation: kernel dilation ``(dH, dW)``. + mask: optional DCNv2 modulation mask + ``(B, n_offset_groups * kH * kW, H_out, W_out)`` in ``[0, 1]``. + + Returns: + Output feature map ``(B, C_out, H_out, W_out)``. + """ + B, C_in, H_in, W_in = input.shape + C_out, C_in_per_group, kH, kW = weight.shape + + if isinstance(padding, int): + padding = (padding, padding) + if isinstance(dilation, int): + dilation = (dilation, dilation) + if isinstance(stride, int): + stride = (stride, stride) + + sH, sW = stride + pH, pW = padding + dH, dW = dilation + + groups = C_in // C_in_per_group + K = kH * kW + n_off_grps = offset.shape[1] // (2 * K) + c_per_off_grp = C_in // n_off_grps + + H_out = (H_in + 2 * pH - dH * (kH - 1) - 1) // sH + 1 + W_out = (W_in + 2 * pW - dW * (kW - 1) - 1) // sW + 1 + + # parse offsets: interleaved (dy, dx) per kernel point per group + offset = offset.reshape(B, n_off_grps, 2 * K, H_out, W_out) + off_y = offset[:, :, 0::2, :, :] # [B, n_off_grps, K, H_out, W_out] + off_x = offset[:, :, 1::2, :, :] + + if mask is not None: + mask = mask.reshape(B, n_off_grps, K, H_out, W_out) + + # base sampling grid + grid_oh = torch.arange(H_out, device=input.device, dtype=input.dtype) + grid_ow = torch.arange(W_out, device=input.device, dtype=input.dtype) + + base_h = (grid_oh * sH - pH).reshape(1, 1, 1, H_out, 1) + base_w = (grid_ow * sW - pW).reshape(1, 1, 1, 1, W_out) + + kh_offs = torch.arange(kH, device=input.device, dtype=input.dtype) * dH + kw_offs = torch.arange(kW, device=input.device, dtype=input.dtype) * dW + kern_h = kh_offs.repeat_interleave(kW).reshape(1, 1, K, 1, 1) + kern_w = kw_offs.repeat(kH).reshape(1, 1, K, 1, 1) + + # Absolute fractional sampling positions [B, n_off_grps, K, H_out, W_out] + sample_h = base_h + kern_h + off_y + sample_w = base_w + kern_w + off_x + + # bilinear interpolation with zero-padding + sampled = _bilinear_sample(input, sample_h, sample_w, n_off_grps, c_per_off_grp) + # sampled: [B, n_off_grps, c_per_off_grp, K, H_out, W_out] + + # apply mask (DCNv2 modulation) + if mask is not None: + sampled = sampled * mask.unsqueeze(2) # broadcast over channels + + # reshape for grouped matmul + sampled = sampled.reshape(B, C_in, K, H_out, W_out) + + C_out_per_group = C_out // groups + sampled = sampled.reshape(B, groups, C_in_per_group, K, H_out * W_out) + weight_flat = weight.reshape(groups, C_out_per_group, C_in_per_group, K) + + out = torch.einsum("bgckn,gock->bgon", sampled, weight_flat) + out = out.reshape(B, C_out, H_out, W_out) + + if bias is not None: + out = out + bias.reshape(1, -1, 1, 1) + + return out + + +def _bilinear_sample( + input: Tensor, + sample_h: Tensor, + sample_w: Tensor, + n_off_grps: int, + c_per_off_grp: int, +) -> Tensor: + """Sample input at fractional (sample_h, sample_w) positions with zero-padding. + + Uses the same corner-validity bilinear interpolation as torchvision's + deformable-conv C++ kernel. + + Args: + input: ``(B, C_in, H_in, W_in)`` + sample_h: ``(B, n_off_grps, K, H_out, W_out)`` + sample_w: ``(B, n_off_grps, K, H_out, W_out)`` + n_off_grps: number of offset groups. + c_per_off_grp: input channels per offset group. + + Returns: + Sampled tensor ``(B, n_off_grps, c_per_off_grp, K, H_out, W_out)``. + """ + B, _C_in, H_in, W_in = input.shape + + h0 = sample_h.floor() + w0 = sample_w.floor() + + lh = sample_h - h0 + lw = sample_w - w0 + + h0 = h0.long() + w0 = w0.long() + h1 = h0 + 1 + w1 = w0 + 1 + + def _valid(h: Tensor, w: Tensor) -> Tensor: + return ((h >= 0) & (h < H_in) & (w >= 0) & (w < W_in)).to(input.dtype) + + m00 = _valid(h0, w0) + m01 = _valid(h0, w1) + m10 = _valid(h1, w0) + m11 = _valid(h1, w1) + + h0c = h0.clamp(0, max(H_in - 1, 0)) + h1c = h1.clamp(0, max(H_in - 1, 0)) + w0c = w0.clamp(0, max(W_in - 1, 0)) + w1c = w1.clamp(0, max(W_in - 1, 0)) + + # input -> [B, n_off_grps, c_per_off_grp, H_in * W_in] + input_flat = input.reshape(B, n_off_grps, c_per_off_grp, H_in * W_in) + + flat_size = sample_h.shape[2] * sample_h.shape[3] * sample_h.shape[4] # K*H_out*W_out + + def _gather(hh: Tensor, ww: Tensor) -> Tensor: + idx = (hh * W_in + ww).reshape(B, n_off_grps, flat_size) + idx = idx.unsqueeze(2).expand(-1, -1, c_per_off_grp, -1) + return torch.gather(input_flat, 3, idx) + + v00 = _gather(h0c, w0c) + v01 = _gather(h0c, w1c) + v10 = _gather(h1c, w0c) + v11 = _gather(h1c, w1c) + + K = sample_h.shape[2] + H_out = sample_h.shape[3] + W_out = sample_h.shape[4] + shape = (B, n_off_grps, c_per_off_grp, K, H_out, W_out) + + v00 = v00.reshape(shape) + v01 = v01.reshape(shape) + v10 = v10.reshape(shape) + v11 = v11.reshape(shape) + + lh = lh.unsqueeze(2) + lw = lw.unsqueeze(2) + m00 = m00.unsqueeze(2) + m01 = m01.unsqueeze(2) + m10 = m10.unsqueeze(2) + m11 = m11.unsqueeze(2) + + return v00 * m00 * (1 - lh) * (1 - lw) + v01 * m01 * (1 - lh) * lw + v10 * m10 * lh * (1 - lw) + v11 * m11 * lh * lw diff --git a/kornia/feature/dedode/__init__.py b/kornia/feature/dedode/__init__.py new file mode 100644 index 0000000..e1bdf0e --- /dev/null +++ b/kornia/feature/dedode/__init__.py @@ -0,0 +1,25 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Feature DeDoDe — DeDoDe feature detector for Kornia. + +This subpackage provides the DeDoDe keypoint detector and related utilities. +""" + +from .dedode import DeDoDe + +__all__ = ["DeDoDe"] diff --git a/kornia/feature/dedode/decoder.py b/kornia/feature/dedode/decoder.py new file mode 100644 index 0000000..ef8e1fe --- /dev/null +++ b/kornia/feature/dedode/decoder.py @@ -0,0 +1,177 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Optional, Tuple + +import torch +from torch import nn + + +class Decoder(nn.Module): + """Implement the decoder module for DeDoDe feature extraction. + + Args: + layers: The architectural layers for the decoder. + super_resolution: Whether to apply super-resolution upsampling. + num_prototypes: The number of prototypes for the output head. + """ + + def __init__(self, layers: Any, *args, super_resolution: bool = False, num_prototypes: int = 1, **kwargs) -> None: # type: ignore[no-untyped-def] + super().__init__(*args, **kwargs) + self.layers = layers + self.scales = self.layers.keys() + self.super_resolution = super_resolution + self.num_prototypes = num_prototypes + + def forward( + self, features: torch.Tensor, context: Optional[torch.Tensor] = None, scale: Optional[int] = None + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + features: Input value used by this method. + context: Input value used by this method. + scale: Input value used by this method. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + if context is not None: + features = torch.cat((features, context), dim=1) + stuff = self.layers[scale](features) + logits, context = stuff[:, : self.num_prototypes], stuff[:, self.num_prototypes :] + return logits, context + + +class ConvRefiner(nn.Module): + """Implement a convolutional refiner for DeDoDe feature extraction. + + Args: + in_dim: The input dimension for the refiner. + hidden_dim: The hidden dimension for the refiner. + out_dim: The output dimension for the refiner. + dw: Whether to use depthwise convolution. + kernel_size: The kernel size for the convolution. + hidden_blocks: The number of hidden blocks in the refiner. + amp: Whether to use automatic mixed precision. + residual: Whether to use residual connections. + amp_dtype: The data type for automatic mixed precision. + """ + + def __init__( # type: ignore[no-untyped-def] + self, + in_dim=6, + hidden_dim=16, + out_dim=2, + dw=True, + kernel_size=5, + hidden_blocks=5, + amp=True, + residual=False, + amp_dtype=torch.float16, + ): + super().__init__() + self.block1 = self.create_block( + in_dim, + hidden_dim, + dw=False, + kernel_size=1, + ) + self.hidden_blocks = nn.Sequential( + *[ + self.create_block( + hidden_dim, + hidden_dim, + dw=dw, + kernel_size=kernel_size, + ) + for hb in range(hidden_blocks) + ] + ) + self.out_conv = nn.Conv2d(hidden_dim, out_dim, 1, 1, 0) + self.amp = amp + self.amp_dtype = amp_dtype + self.residual = residual + + def create_block( # type: ignore[no-untyped-def] + self, + in_dim, + out_dim, + dw=True, + kernel_size=5, + bias=True, + norm_type=nn.BatchNorm2d, + ): + """Create one decoder block for the requested scale. + + Args: + in_dim: Input value used by this method. + out_dim: Input value used by this method. + dw: Input value used by this method. + kernel_size: Input value used by this method. + bias: Input value used by this method. + norm_type: Input value used by this method. + + Returns: + Decoder block configured for the requested input and output channel counts. + """ + num_groups = 1 if not dw else in_dim + if dw: + if out_dim % in_dim != 0: + raise Exception("outdim must be divisible by indim for depthwise") + conv1 = nn.Conv2d( + in_dim, + out_dim, + kernel_size=kernel_size, + stride=1, + padding=kernel_size // 2, + groups=num_groups, + bias=bias, + ) + norm = norm_type(out_dim) if norm_type is nn.BatchNorm2d else norm_type(num_channels=out_dim) + relu = nn.ReLU(inplace=True) + conv2 = nn.Conv2d(out_dim, out_dim, 1, 1, 0) + return nn.Sequential(conv1, norm, relu, conv2) + + def forward(self, feats: torch.Tensor) -> torch.Tensor: + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + feats: Input value used by this method. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + _b, _c, _hs, _ws = feats.shape + # AMP is intentionally scoped to "cuda" only: float16 autocast on CPU is not + # supported by PyTorch and a no-op on MPS. Use amp_dtype=torch.float32 on + # non-CUDA devices to disable AMP entirely. + with torch.autocast("cuda", enabled=self.amp, dtype=self.amp_dtype): + x0 = self.block1(feats) + x = self.hidden_blocks(x0) + if self.residual: + x = (x + x0) / 1.4 + x = self.out_conv(x) + return x diff --git a/kornia/feature/dedode/dedode.py b/kornia/feature/dedode/dedode.py new file mode 100644 index 0000000..b05e8e3 --- /dev/null +++ b/kornia/feature/dedode/dedode.py @@ -0,0 +1,264 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Literal, Optional, Tuple + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK_SHAPE +from kornia.enhance.normalize import Normalize + +from .dedode_models import DeDoDeDescriptor, DeDoDeDetector, get_descriptor, get_detector +from .utils import dedode_denormalize_pixel_coordinates, sample_keypoints + +urls: Dict[str, Dict[str, str]] = { + "detector": { + "L-upright": "https://github.com/Parskatt/DeDoDe/releases/download/dedode_pretrained_models/dedode_detector_L.pth", + "L-C4": "https://github.com/georg-bn/rotation-steerers/releases/download/release-2/dedode_detector_C4.pth", + "L-SO2": "https://github.com/georg-bn/rotation-steerers/releases/download/release-2/dedode_detector_SO2.pth", + "L-C4-v2": "https://github.com/Parskatt/DeDoDe/releases/download/v2/dedode_detector_L_v2.pth", + }, + "descriptor": { + "B-upright": "https://github.com/Parskatt/DeDoDe/releases/download/dedode_pretrained_models/dedode_descriptor_B.pth", + "B-C4": "https://github.com/georg-bn/rotation-steerers/releases/download/release-2/B_C4_Perm_descriptor_setting_C.pth", + "B-SO2": "https://github.com/georg-bn/rotation-steerers/releases/download/release-2/B_SO2_Spread_descriptor_setting_C.pth", + "G-upright": "https://github.com/Parskatt/DeDoDe/releases/download/dedode_pretrained_models/dedode_descriptor_G.pth", + "G-C4": "https://github.com/georg-bn/rotation-steerers/releases/download/release-2/G_C4_Perm_descriptor_setting_C.pth", + "G-SO2": "https://github.com/georg-bn/rotation-steerers/releases/download/release-2/G_SO2_Spread_descriptor_setting_C.pth", + }, +} + + +class DeDoDe(nn.Module): + r"""nn.Module which detects and/or describes local features in an image using the DeDode method. + + See :cite:`edstedt2024dedode` for details. + + .. note:: DeDode takes ImageNet normalized images as input (not in range [0, 1]). + + Args: + detector_model: The detector model kind. Available options are: `L`. + descriptor_model: The descriptor model kind. Available options are: `G` or `B` + amp_dtype: The automatic mixed precision desired. + + Example: + >>> dedode = DeDoDe.from_pretrained(detector_weights="L-C4-v2", descriptor_weights="B-upright") + >>> images = torch.randn(1, 3, 256, 256) + >>> keypoints, scores = dedode.detect(images) + >>> descriptions = dedode.describe(images, keypoints = keypoints) + >>> keypoints, scores, features = dedode(images) # alternatively do both + + """ + + # TODO: implement steerers and mnn matchers + def __init__( + self, + detector_model: Literal["L"] = "L", + descriptor_model: Literal["G", "B"] = "G", + amp_dtype: torch.dtype = torch.float16, + ) -> None: + super().__init__() + self.detector: DeDoDeDetector = get_detector(detector_model, amp_dtype) + self.descriptor: DeDoDeDescriptor = get_descriptor(descriptor_model, amp_dtype) + self.normalizer = Normalize(torch.tensor([0.485, 0.456, 0.406]), std=torch.tensor([0.229, 0.224, 0.225])) + + def forward( + self, + images: torch.Tensor, + n: Optional[int] = 10_000, + apply_imagenet_normalization: bool = True, + pad_if_not_divisible: bool = True, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Detect and describe keypoints in the input images. + + Args: + images: A torch.Tensor of shape :math:`(B, 3, H, W)` containing the ImageNet-Normalized input images. + n: The number of keypoints to detect. + apply_imagenet_normalization: Whether to apply ImageNet normalization to the input images. + pad_if_not_divisible: F.pad image shape if not evenly divisible. + + Returns: + keypoints: A torch.Tensor of shape :math:`(B, N, 2)` containing the detected keypoints in the image range, + unlike `.detect()` function. + + scores: A torch.Tensor of shape :math:`(B, N)` containing the scores of the detected keypoints. + + descriptions: A torch.Tensor of shape :math:`(B, N, DIM)` containing the descriptions + of the detected keypoints. DIM is 256 for B and 512 for G. + + """ + if apply_imagenet_normalization: + images = self.normalizer(images) + _B, _C, H, W = images.shape + if pad_if_not_divisible: + pd_h = 14 - H % 14 if H % 14 > 0 else 0 + pd_w = 14 - W % 14 if W % 14 > 0 else 0 + images = F.pad(images, (0, pd_w, 0, pd_h), value=0.0) + keypoints, scores = self.detect(images, n=n, apply_imagenet_normalization=False, crop_h=H, crop_w=W) + descriptions = self.describe(images, keypoints, apply_imagenet_normalization=False, crop_h=H, crop_w=W) + return dedode_denormalize_pixel_coordinates(keypoints, H, W), scores, descriptions + + @torch.inference_mode() + def detect( + self, + images: torch.Tensor, + n: Optional[int] = 10_000, + apply_imagenet_normalization: bool = True, + pad_if_not_divisible: bool = True, + crop_h: Optional[int] = None, + crop_w: Optional[int] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Detect keypoints in the input images. + + .. note:: + This method unconditionally sets the model to eval mode via ``self.train(False)`` + so that BatchNorm and Dropout behave deterministically. This is intentional: the + detector is only used at inference time and its statistics must be frozen. + + Args: + images: A torch.Tensor of shape :math:`(B, 3, H, W)` containing the input images. + n: The number of keypoints to detect. + apply_imagenet_normalization: Whether to apply ImageNet normalization to the input images. + pad_if_not_divisible: F.pad image shape if not evenly divisible. + crop_h: The height of the crop to be used for detection. If None, the full image is used. + crop_w: The width of the crop to be used for detection. If None, the full image is used. + + Returns: + keypoints: A torch.Tensor of shape :math:`(B, N, 2)` containing the detected keypoints, + normalized to the range :math:`[-1, 1]`. + scores: A torch.Tensor of shape :math:`(B, N)` containing the scores of the detected keypoints. + + """ + KORNIA_CHECK_SHAPE(images, ["B", "3", "H", "W"]) + self.train(False) + B, _C, H, W = images.shape + if pad_if_not_divisible: + h, w = images.shape[2:] + pd_h = 14 - h % 14 if h % 14 > 0 else 0 + pd_w = 14 - w % 14 if w % 14 > 0 else 0 + images = F.pad(images, (0, pd_w, 0, pd_h), value=0.0) + if apply_imagenet_normalization: + images = self.normalizer(images) + logits = self.detector.forward(images) + # Remove the padding, if any + logits = logits[..., :H, :W] + if crop_h is not None and crop_w is not None: + logits = logits[..., :crop_h, :crop_w] + H, W = crop_h, crop_w + scoremap = logits.reshape(B, H * W).softmax(dim=-1).reshape(B, H, W) + keypoints, confidence = sample_keypoints(scoremap, num_samples=n) + return keypoints, confidence + + @torch.inference_mode() + def describe( + self, + images: torch.Tensor, + keypoints: Optional[torch.Tensor] = None, + apply_imagenet_normalization: bool = True, + pad_if_not_divisible: bool = True, + crop_h: Optional[int] = None, + crop_w: Optional[int] = None, + ) -> torch.Tensor: + """Describe keypoints in the input images. If keypoints are not provided, returns the dense descriptors. + + .. note:: + This method unconditionally sets the model to eval mode via ``self.train(False)`` + so that BatchNorm and Dropout behave deterministically. This is intentional: the + descriptor is only used at inference time and its statistics must be frozen. + + Args: + images: A torch.Tensor of shape :math:`(B, 3, H, W)` containing the input images. + keypoints: An optional torch.Tensor of shape :math:`(B, N, 2)` containing the detected keypoints. + apply_imagenet_normalization: Whether to apply ImageNet normalization to the input images. + pad_if_not_divisible: Zero-pad the image so H and W are divisible by 14. Required when + using the ``G`` descriptor backed by DINOv2 (patch size 14). Ignored for ``B``. + crop_h: The height of the crop to be used for description. If None, the full image is used. + crop_w: The width of the crop to be used for description. If None, the full image is used. + + Returns: + descriptions: A torch.Tensor of shape :math:`(B, N, DIM)` containing the descriptions + of the detected keypoints. + If the dense descriptors are requested, the shape is :math:`(B, DIM, H, W)`. + + """ + KORNIA_CHECK_SHAPE(images, ["B", "3", "H", "W"]) + _B, _C, H, W = images.shape + if keypoints is not None: + KORNIA_CHECK_SHAPE(keypoints, ["B", "N", "2"]) + if apply_imagenet_normalization: + images = self.normalizer(images) + if pad_if_not_divisible: + pd_h = 14 - H % 14 if H % 14 > 0 else 0 + pd_w = 14 - W % 14 if W % 14 > 0 else 0 + images = F.pad(images, (0, pd_w, 0, pd_h), value=0.0) + self.train(False) + descriptions = self.descriptor.forward(images) + if crop_h is not None and crop_w is not None: + descriptions = descriptions[..., :crop_h, :crop_w] + H, W = crop_h, crop_w + + if keypoints is not None: + described_keypoints = F.grid_sample( + descriptions.float(), keypoints[:, None], mode="bilinear", align_corners=False + )[:, :, 0].mT + return described_keypoints + return descriptions + + @classmethod + def from_pretrained( + cls, + detector_weights: str = "L-C4-v2", + descriptor_weights: str = "G-upright", + amp_dtype: torch.dtype = torch.float16, + ) -> nn.Module: + r"""Load a pretrained model. + + Args: + detector_weights: The weights to load for the detector. + One of 'L-upright' (original paper, https://arxiv.org/abs/2308.08479), + 'L-C4', 'L-SO2' (from steerers, better for rotations, https://arxiv.org/abs/2312.02152), + 'L-C4-v2' (from dedode v2, better at rotations, less clustering, https://arxiv.org/abs/2404.08928). + Default is 'L-C4-v2'. + descriptor_weights: The weights to load for the descriptor. + One of 'B-upright','G-upright' (original paper, https://arxiv.org/abs/2308.08479), + 'B-C4', 'B-SO2', 'G-C4', 'G-SO2' (from steerers, better for rotations, https://arxiv.org/abs/2312.02152). + Default is 'G-upright'. + amp_dtype: the dtype to use for the model. One of torch.float16 or torch.float32. + Default is torch.float16, suitable for CUDA. Use torch.float32 for CPU or MPS + + Returns: + The pretrained model. + + """ + # The model architecture kind is encoded as the first character of the weight name, + # e.g. "L-C4-v2" -> "L", "G-upright" -> "G". All current checkpoints follow this + # convention intentionally so that new variants only need a new URL entry. + model: DeDoDe = cls( + detector_model=detector_weights[0], # type: ignore[arg-type] + descriptor_model=descriptor_weights[0], # type: ignore[arg-type] + amp_dtype=amp_dtype, + ) + model.detector.load_state_dict( + torch.hub.load_state_dict_from_url(urls["detector"][detector_weights], map_location=torch.device("cpu")) + ) + model.descriptor.load_state_dict( + torch.hub.load_state_dict_from_url(urls["descriptor"][descriptor_weights], map_location=torch.device("cpu")) + ) + model.eval() + return model diff --git a/kornia/feature/dedode/dedode_models.py b/kornia/feature/dedode/dedode_models.py new file mode 100644 index 0000000..c7cd40a --- /dev/null +++ b/kornia/feature/dedode/dedode_models.py @@ -0,0 +1,207 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch +from torch import nn + +from .decoder import ConvRefiner, Decoder +from .descriptor import DeDoDeDescriptor +from .detector import DeDoDeDetector +from .encoder import VGG19, VGG_DINOv2 + + +def dedode_detector_L(amp_dtype: torch.dtype = torch.float16) -> DeDoDeDetector: + """Get DeDoDe descriptor of type L.""" + NUM_PROTOTYPES = 1 + residual = True + hidden_blocks = 8 + amp = True + conv_refiner = nn.ModuleDict( + { + "8": ConvRefiner( + 512, + 512, + 256 + NUM_PROTOTYPES, + hidden_blocks=hidden_blocks, + residual=residual, + amp=amp, + amp_dtype=amp_dtype, + ), + "4": ConvRefiner( + 256 + 256, + 256, + 128 + NUM_PROTOTYPES, + hidden_blocks=hidden_blocks, + residual=residual, + amp=amp, + amp_dtype=amp_dtype, + ), + "2": ConvRefiner( + 128 + 128, + 128, + 64 + NUM_PROTOTYPES, + hidden_blocks=hidden_blocks, + residual=residual, + amp=amp, + amp_dtype=amp_dtype, + ), + "1": ConvRefiner( + 64 + 64, + 64, + 1 + NUM_PROTOTYPES, + hidden_blocks=hidden_blocks, + residual=residual, + amp=amp, + amp_dtype=amp_dtype, + ), + } + ) + encoder = VGG19(amp=amp, amp_dtype=amp_dtype) + decoder = Decoder(conv_refiner) + model = DeDoDeDetector(encoder=encoder, decoder=decoder) + return model + + +def dedode_descriptor_B(amp_dtype: torch.dtype = torch.float16) -> DeDoDeDescriptor: + """Get DeDoDe descriptor of type B.""" + NUM_PROTOTYPES = 256 # == descriptor size + residual = True + hidden_blocks = 5 + amp = True + conv_refiner = nn.ModuleDict( + { + "8": ConvRefiner( + 512, + 512, + 256 + NUM_PROTOTYPES, + hidden_blocks=hidden_blocks, + residual=residual, + amp=amp, + amp_dtype=amp_dtype, + ), + "4": ConvRefiner( + 256 + 256, + 256, + 128 + NUM_PROTOTYPES, + hidden_blocks=hidden_blocks, + residual=residual, + amp=amp, + amp_dtype=amp_dtype, + ), + "2": ConvRefiner( + 128 + 128, + 64, + 32 + NUM_PROTOTYPES, + hidden_blocks=hidden_blocks, + residual=residual, + amp=amp, + amp_dtype=amp_dtype, + ), + "1": ConvRefiner( + 64 + 32, + 32, + 1 + NUM_PROTOTYPES, + hidden_blocks=hidden_blocks, + residual=residual, + amp=amp, + amp_dtype=amp_dtype, + ), + } + ) + encoder = VGG19(amp=amp, amp_dtype=amp_dtype) + decoder = Decoder(conv_refiner, num_prototypes=NUM_PROTOTYPES) + model = DeDoDeDescriptor(encoder=encoder, decoder=decoder) + return model + + +def dedode_descriptor_G(amp_dtype: torch.dtype = torch.float16) -> DeDoDeDescriptor: + """Get DeDoDe descriptor of type G.""" + NUM_PROTOTYPES = 256 # == descriptor size + residual = True + hidden_blocks = 5 + amp = True + conv_refiner = nn.ModuleDict( + { + "14": ConvRefiner( + 1024, + 768, + 512 + NUM_PROTOTYPES, + hidden_blocks=hidden_blocks, + residual=residual, + amp=amp, + amp_dtype=amp_dtype, + ), + "8": ConvRefiner( + 512 + 512, + 512, + 256 + NUM_PROTOTYPES, + hidden_blocks=hidden_blocks, + residual=residual, + amp=amp, + amp_dtype=amp_dtype, + ), + "4": ConvRefiner( + 256 + 256, + 256, + 128 + NUM_PROTOTYPES, + hidden_blocks=hidden_blocks, + residual=residual, + amp=amp, + amp_dtype=amp_dtype, + ), + "2": ConvRefiner( + 128 + 128, + 64, + 32 + NUM_PROTOTYPES, + hidden_blocks=hidden_blocks, + residual=residual, + amp=amp, + amp_dtype=amp_dtype, + ), + "1": ConvRefiner( + 64 + 32, + 32, + 1 + NUM_PROTOTYPES, + hidden_blocks=hidden_blocks, + residual=residual, + amp=amp, + amp_dtype=amp_dtype, + ), + } + ) + vgg_kwargs = {"amp": amp, "amp_dtype": amp_dtype} + dinov2_kwargs = {"amp": amp, "amp_dtype": amp_dtype, "dinov2_weights": None} + encoder = VGG_DINOv2(vgg_kwargs=vgg_kwargs, dinov2_kwargs=dinov2_kwargs) + decoder = Decoder(conv_refiner, num_prototypes=NUM_PROTOTYPES) + model = DeDoDeDescriptor(encoder=encoder, decoder=decoder) + return model + + +def get_detector(kind: str = "L", amp_dtype: torch.dtype = torch.float16) -> DeDoDeDetector: + """Get DeDoDe detector.""" + if kind == "L": + return dedode_detector_L(amp_dtype) + raise ValueError(f"Unknown detector kind: {kind}") + + +def get_descriptor(kind: str = "B", amp_dtype: torch.dtype = torch.float16) -> DeDoDeDescriptor: + """Get DeDoDe descriptor.""" + if kind == "B": + return dedode_descriptor_B(amp_dtype) + if kind == "G": + return dedode_descriptor_G(amp_dtype) + raise ValueError(f"Unknown descriptor kind: {kind}") diff --git a/kornia/feature/dedode/descriptor.py b/kornia/feature/dedode/descriptor.py new file mode 100644 index 0000000..9c4f37d --- /dev/null +++ b/kornia/feature/dedode/descriptor.py @@ -0,0 +1,66 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch +import torch.nn.functional as F +from torch import nn + + +class DeDoDeDescriptor(nn.Module): + """Implement the DeDoDe descriptor for learning local feature representations. + + Args: + encoder: The backbone encoder module. + decoder: The decoder module for descriptor generation. + """ + + def __init__(self, encoder: nn.Module, decoder: nn.Module, *args, **kwargs) -> None: # type: ignore[no-untyped-def] + super().__init__(*args, **kwargs) + self.encoder = encoder + self.decoder = decoder + + def forward( + self, + images: torch.Tensor, + ) -> torch.Tensor: + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + images: Input value used by this method. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + features, sizes = self.encoder(images) + context = None + descriptions = None + scales = self.decoder.scales + for idx, (feature_map, scale) in enumerate(zip(reversed(features), scales)): + if idx == 0: + descriptions, context = self.decoder(feature_map, scale=scale, context=context) + else: + delta_descriptions, context = self.decoder(feature_map, scale=scale, context=context) + descriptions = descriptions + delta_descriptions + if idx < len(scales) - 1: + size = sizes[-(idx + 2)] + descriptions = F.interpolate(descriptions, size=size, mode="bilinear", align_corners=False) + context = F.interpolate(context, size=size, mode="bilinear", align_corners=False) + return descriptions diff --git a/kornia/feature/dedode/detector.py b/kornia/feature/dedode/detector.py new file mode 100644 index 0000000..1681494 --- /dev/null +++ b/kornia/feature/dedode/detector.py @@ -0,0 +1,67 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch +import torch.nn.functional as F +from torch import nn + + +class DeDoDeDetector(nn.Module): + """Implement the DeDoDe detector for keypoint learning and localization. + + Args: + encoder: The backbone encoder module. + decoder: The decoder module for probability map generation. + """ + + def __init__(self, encoder: nn.Module, decoder: nn.Module, *args, **kwargs) -> None: # type: ignore[no-untyped-def] + super().__init__(*args, **kwargs) + self.encoder = encoder + self.decoder = decoder + + def forward( + self, + images: torch.Tensor, + ) -> torch.Tensor: + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + images: Input value used by this method. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + dtype = images.dtype + features, sizes = self.encoder(images) + context = None + logits = None + scales = ["8", "4", "2", "1"] + for idx, (feature_map, scale) in enumerate(zip(reversed(features), scales)): + delta_logits, context = self.decoder(feature_map, context=context, scale=scale) + if logits is None: + logits = delta_logits + else: + logits = logits + delta_logits.float() # ensure float (need bf16 doesn't have f.interpolate) + if idx < len(scales) - 1: + size = sizes[-(idx + 2)] + logits = F.interpolate(logits, size=size, mode="bicubic", align_corners=False) + context = F.interpolate(context.float(), size=size, mode="bilinear", align_corners=False) + return logits.to(dtype) # type: ignore diff --git a/kornia/feature/dedode/encoder.py b/kornia/feature/dedode/encoder.py new file mode 100644 index 0000000..a39768d --- /dev/null +++ b/kornia/feature/dedode/encoder.py @@ -0,0 +1,146 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Optional, Tuple + +import torch +from torch import nn + +from .vgg import vgg19_bn + + +class VGG19(nn.Module): + """Implement the VGG19 encoder for feature encoding. + + Args: + amp: Whether to use automatic mixed precision. + amp_dtype: The data type for automatic mixed precision. + """ + + def __init__(self, amp: bool = False, amp_dtype: torch.dtype = torch.float16) -> None: + super().__init__() + self.layers = nn.ModuleList(vgg19_bn().features[:40]) # type: ignore + # Maxpool layers: 6, 13, 26, 39 + self.amp = amp + self.amp_dtype = amp_dtype + + def forward(self, x: torch.Tensor, **kwargs): # type: ignore[no-untyped-def] + # AMP is intentionally scoped to "cuda" only: float16 autocast on CPU is not + # supported by PyTorch and a no-op on MPS. Use amp_dtype=torch.float32 on + # non-CUDA devices to disable AMP entirely. + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + **kwargs: Additional keyword arguments accepted for compatibility with the shared encoder interface. They + are not used by this VGG19 batch-normalized backbone wrapper. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + with torch.autocast("cuda", enabled=self.amp, dtype=self.amp_dtype): + feats = [] + sizes = [] + for layer in self.layers: + if isinstance(layer, nn.MaxPool2d): + feats.append(x) + sizes.append(x.shape[-2:]) + x = layer(x) + return feats, sizes + + +class FrozenDINOv2(nn.Module): + """Implement a wrapper for a frozen DINOv2 vision transformer backbone.""" + + def __init__(self, amp: bool = True, amp_dtype: torch.dtype = torch.float16, dinov2_weights: Optional[Any] = None): + super().__init__() + if dinov2_weights is None: + dinov2_weights = torch.hub.load_state_dict_from_url( + "https://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_pretrain.pth", map_location="cpu" + ) + from .transformer import vit_large + + vit_kwargs = dict( + img_size=518, + patch_size=14, + init_values=1.0, + ffn_layer="mlp", + block_chunks=0, + ) + dinov2_vitl14 = vit_large(**vit_kwargs).eval() + dinov2_vitl14.load_state_dict(dinov2_weights) + self.amp = amp + self.amp_dtype = amp_dtype + if self.amp: + dinov2_vitl14 = dinov2_vitl14.to(self.amp_dtype) + self.dinov2_vitl14 = [dinov2_vitl14] # ugly hack to not show parameters to DDP + + def forward(self, x: torch.Tensor): # type: ignore[no-untyped-def] + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + B, _C, H, W = x.shape + if self.dinov2_vitl14[0].device != x.device: + self.dinov2_vitl14[0] = self.dinov2_vitl14[0].to(x.device).to(self.amp_dtype) + with torch.inference_mode(): + dinov2_features_16 = self.dinov2_vitl14[0].forward_features(x.to(self.amp_dtype)) + features_16 = dinov2_features_16["x_norm_patchtokens"].permute(0, 2, 1).reshape(B, 1024, H // 14, W // 14) + return [features_16.clone()], [(H // 14, W // 14)] # clone from inference mode to use in autograd + + +class VGG_DINOv2(nn.Module): + """Implement a hybrid encoder combining VGG and DINOv2 features.""" + + def __init__(self, vgg_kwargs=None, dinov2_kwargs=None): # type: ignore[no-untyped-def] + if (vgg_kwargs is None) or (dinov2_kwargs is None): + raise ValueError("Input kwargs please") + super().__init__() + self.vgg = VGG19(**vgg_kwargs) + self.frozen_dinov2 = FrozenDINOv2(**dinov2_kwargs) + + def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, Tuple[int, int]]: + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + feats_vgg, sizes_vgg = self.vgg(x) + feat_dinov2, size_dinov2 = self.frozen_dinov2(x) + return feats_vgg + feat_dinov2, sizes_vgg + size_dinov2 diff --git a/kornia/feature/dedode/transformer/__init__.py b/kornia/feature/dedode/transformer/__init__.py new file mode 100644 index 0000000..1532a73 --- /dev/null +++ b/kornia/feature/dedode/transformer/__init__.py @@ -0,0 +1,27 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Feature DeDoDe Transformer — Transformer backbones for DeDoDe feature detector. + +This subpackage provides transformer models for DeDoDe feature extraction. +""" + +from .dinov2 import vit_large + +__all__ = [ + "vit_large", +] diff --git a/kornia/feature/dedode/transformer/dinov2.py b/kornia/feature/dedode/transformer/dinov2.py new file mode 100644 index 0000000..5170fc2 --- /dev/null +++ b/kornia/feature/dedode/transformer/dinov2.py @@ -0,0 +1,469 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/main/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py + +import math +from functools import partial +from typing import Callable, Sequence, Tuple, Union + +import torch +import torch.utils.checkpoint +from torch import nn +from torch.nn.init import trunc_normal_ + +from kornia.core.check import KORNIA_CHECK + +from .layers import MemEffAttention, Mlp, PatchEmbed, SwiGLUFFNFused +from .layers import NestedTensorBlock as Block + + +def named_apply(fn: Callable, module: nn.Module, name="", depth_first=True, include_root=False) -> nn.Module: + """Apply named function to module.""" + if not depth_first and include_root: + fn(module=module, name=name) + for child_name, child_module in module.named_children(): + child_name = ".".join((name, child_name)) if name else child_name + named_apply(fn=fn, module=child_module, name=child_name, depth_first=depth_first, include_root=True) + if depth_first and include_root: + fn(module=module, name=name) + return module + + +class BlockChunk(nn.ModuleList): + """Implement a container for sequential execution of transformer block chunks.""" + + def forward(self, x): + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + for b in self: + x = b(x) + return x + + +class DinoVisionTransformer(nn.Module): + """Implement the DINOv2 Vision Transformer architecture.""" + + def __init__( + self, + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4.0, + qkv_bias=True, + ffn_bias=True, + proj_bias=True, + drop_path_rate=0.0, + drop_path_uniform=False, + init_values=None, # for layerscale: None or 0 => no layerscale + embed_layer=PatchEmbed, + act_layer=nn.GELU, + block_fn=Block, + ffn_layer="mlp", + block_chunks=1, + ): + """Construct dino vision transformer. + + Args: + img_size (int, tuple): input image size + patch_size (int, tuple): patch size + in_chans (int): number of input channels + embed_dim (int): embedding dimension + depth (int): depth of transformer + num_heads (int): number of attention heads + mlp_ratio (int): ratio of mlp hidden dim to embedding dim + qkv_bias (bool): enable bias for qkv if True + proj_bias (bool): enable bias for proj in attn if True + ffn_bias (bool): enable bias for ffn if True + drop_path_rate (float): stochastic depth rate + drop_path_uniform (bool): apply uniform drop rate across blocks + init_values (float): layer-scale init values + embed_layer (nn.Module): patch embedding layer + act_layer (nn.Module): MLP activation layer + block_fn (nn.Module): transformer block class + ffn_layer (str): "mlp", "swiglu", "swiglufused" or "identity" + block_chunks: (int) split block sequence into block_chunks units for FSDP wrap + + """ + super().__init__() + norm_layer = partial(nn.LayerNorm, eps=1e-6) + + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + self.num_tokens = 1 + self.n_blocks = depth + self.num_heads = num_heads + self.patch_size = patch_size + + self.patch_embed = embed_layer(img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) + num_patches = self.patch_embed.num_patches + + self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) + self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim)) + + if drop_path_uniform is True: + dpr = [drop_path_rate] * depth + else: + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule + + if ffn_layer == "mlp": + ffn_layer = Mlp + elif ffn_layer == "swiglufused" or ffn_layer == "swiglu": + ffn_layer = SwiGLUFFNFused + elif ffn_layer == "identity": + + def f(*args, **kwargs): + return nn.Identity() + + ffn_layer = f + else: + raise NotImplementedError + + blocks_list = [ + block_fn( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + proj_bias=proj_bias, + ffn_bias=ffn_bias, + drop_path=dpr[i], + norm_layer=norm_layer, + act_layer=act_layer, + ffn_layer=ffn_layer, + init_values=init_values, + ) + for i in range(depth) + ] + if block_chunks > 0: + self.chunked_blocks = True + chunked_blocks = [] + chunksize = depth // block_chunks + for i in range(0, depth, chunksize): + # this is to keep the block index consistent if we chunk the block list + chunked_blocks.append([nn.Identity()] * i + blocks_list[i : i + chunksize]) + self.blocks = nn.ModuleList([BlockChunk(p) for p in chunked_blocks]) + else: + self.chunked_blocks = False + self.blocks = nn.ModuleList(blocks_list) + + self.norm = norm_layer(embed_dim) + self.head = nn.Identity() + + self.mask_token = nn.Parameter(torch.zeros(1, embed_dim)) + + self.init_weights() + for param in self.parameters(): + param.requires_grad = False + + @property + def device(self): + """Return the device of the stored tensors or module parameters. + + Returns: + Device used by the model parameters. + """ + return self.cls_token.device + + def init_weights(self): + """Initialize transformer weights. + + Returns: + None. Model weights are initialized in place. + """ + trunc_normal_(self.pos_embed, std=0.02) + nn.init.normal_(self.cls_token, std=1e-6) + named_apply(init_weights_vit_timm, self) + + def interpolate_pos_encoding(self, x, w, h): + """Interpolate positional encodings to the current patch grid. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + w: Input value used by this method. + h: Input value used by this method. + + Returns: + Positional embedding tensor resized to match the current patch grid. + """ + previous_dtype = x.dtype + npatch = x.shape[1] - 1 + N = self.pos_embed.shape[1] - 1 + if npatch == N and w == h: + return self.pos_embed + pos_embed = self.pos_embed.float() + class_pos_embed = pos_embed[:, 0] + patch_pos_embed = pos_embed[:, 1:] + dim = x.shape[-1] + w0 = w // self.patch_size + h0 = h // self.patch_size + # we add a small number to avoid floating point error in the interpolation + # see discussion at https://github.com/facebookresearch/dino/issues/8 + w0, h0 = w0 + 0.1, h0 + 0.1 + + patch_pos_embed = nn.functional.interpolate( + patch_pos_embed.reshape(1, int(math.sqrt(N)), int(math.sqrt(N)), dim).permute(0, 3, 1, 2), + scale_factor=(w0 / math.sqrt(N), h0 / math.sqrt(N)), + mode="bicubic", + ) + KORNIA_CHECK(int(w0) == patch_pos_embed.shape[-2]) + KORNIA_CHECK(int(h0) == patch_pos_embed.shape[-1]) + patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim) + return torch.cat((class_pos_embed.unsqueeze(0), patch_pos_embed), dim=1).to(previous_dtype) + + def prepare_tokens_with_masks(self, x, masks=None): + """Prepare image patch tokens before transformer encoding. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + masks: Input value used by this method. + + Returns: + Token sequence prepared with class token, positional embeddings, and optional masks. + """ + _B, _nc, w, h = x.shape + x = self.patch_embed(x) + if masks is not None: + x = torch.where(masks.unsqueeze(-1), self.mask_token.to(x.dtype).unsqueeze(0), x) + + x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1) + x = x + self.interpolate_pos_encoding(x, w, h) + + return x + + def forward_features_list(self, x_list, masks_list): + """Compute transformer features for a list of image tensors. + + Args: + x_list: Input value used by this method. + masks_list: Input value used by this method. + + Returns: + List of intermediate feature dictionaries, one per input image tensor. + """ + x = [self.prepare_tokens_with_masks(x, masks) for x, masks in zip(x_list, masks_list)] + for blk in self.blocks: + x = blk(x) + + all_x = x + output = [] + for x, masks in zip(all_x, masks_list): + x_norm = self.norm(x) + output.append( + { + "x_norm_clstoken": x_norm[:, 0], + "x_norm_patchtokens": x_norm[:, 1:], + "x_prenorm": x, + "masks": masks, + } + ) + return output + + def forward_features(self, x, masks=None): + """Compute transformer features for one image tensor. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + masks: Input value used by this method. + + Returns: + Feature dictionary produced by the transformer backbone. + """ + if isinstance(x, list): + return self.forward_features_list(x, masks) + + x = self.prepare_tokens_with_masks(x, masks) + + for blk in self.blocks: + x = blk(x) + + x_norm = self.norm(x) + return { + "x_norm_clstoken": x_norm[:, 0], + "x_norm_patchtokens": x_norm[:, 1:], + "x_prenorm": x, + "masks": masks, + } + + def _get_intermediate_layers_not_chunked(self, x, n=1): + x = self.prepare_tokens_with_masks(x) + # If n is an int, take the n last blocks. If it's a list, take them + output, total_block_len = [], len(self.blocks) + blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n + for i, blk in enumerate(self.blocks): + x = blk(x) + if i in blocks_to_take: + output.append(x) + KORNIA_CHECK(len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found") + return output + + def _get_intermediate_layers_chunked(self, x, n=1): + x = self.prepare_tokens_with_masks(x) + output, i, total_block_len = [], 0, len(self.blocks[-1]) + # If n is an int, take the n last blocks. If it's a list, take them + blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n + for block_chunk in self.blocks: + for blk in block_chunk[i:]: # Passing the nn.Identity() + x = blk(x) + if i in blocks_to_take: + output.append(x) + i += 1 + KORNIA_CHECK(len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found") + return output + + def get_intermediate_layers( + self, + x: torch.Tensor, + n: Union[int, Sequence] = 1, # Layers or n last layers to take + reshape: bool = False, + return_class_token: bool = False, + norm=True, + ) -> Tuple[Union[torch.Tensor, Tuple[torch.Tensor]]]: + """Return selected intermediate transformer layer outputs. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + n: Current hourglass recursion depth. + reshape: Input value used by this method. + return_class_token: Input value used by this method. + norm: Input value used by this method. + + Returns: + Tuple or list of intermediate transformer layer outputs. + """ + if self.chunked_blocks: + outputs = self._get_intermediate_layers_chunked(x, n) + else: + outputs = self._get_intermediate_layers_not_chunked(x, n) + if norm: + outputs = [self.norm(out) for out in outputs] + class_tokens = [out[:, 0] for out in outputs] + outputs = [out[:, 1:] for out in outputs] + if reshape: + B, _, w, h = x.shape + outputs = [ + out.reshape(B, w // self.patch_size, h // self.patch_size, -1).permute(0, 3, 1, 2).contiguous() + for out in outputs + ] + if return_class_token: + return tuple(zip(outputs, class_tokens)) + return tuple(outputs) + + def forward(self, *args, is_training=False, **kwargs): + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + ret = self.forward_features(*args, **kwargs) + if is_training: + return ret + else: + return self.head(ret["x_norm_clstoken"]) + + +def init_weights_vit_timm(module: nn.Module, name: str = ""): + """ViT weight initialization, original timm impl (for reproducibility).""" + if isinstance(module, nn.Linear): + trunc_normal_(module.weight, std=0.02) + if module.bias is not None: + nn.init.zeros_(module.bias) + + +def vit_small(patch_size=16, **kwargs) -> DinoVisionTransformer: + """Return ViT Small.""" + model = DinoVisionTransformer( + patch_size=patch_size, + embed_dim=384, + depth=12, + num_heads=6, + mlp_ratio=4, + block_fn=partial(Block, attn_class=MemEffAttention), + **kwargs, + ) + return model + + +def vit_base(patch_size=16, **kwargs) -> DinoVisionTransformer: + """Return ViT Base.""" + model = DinoVisionTransformer( + patch_size=patch_size, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4, + block_fn=partial(Block, attn_class=MemEffAttention), + **kwargs, + ) + return model + + +def vit_large(patch_size=16, **kwargs) -> DinoVisionTransformer: + """Return ViT Large.""" + model = DinoVisionTransformer( + patch_size=patch_size, + embed_dim=1024, + depth=24, + num_heads=16, + mlp_ratio=4, + block_fn=partial(Block, attn_class=MemEffAttention), + **kwargs, + ) + return model + + +def vit_giant2(patch_size=16, **kwargs): + """Close to ViT-giant, with embed-dim 1536 and 24 heads => embed-dim per head 64.""" + model = DinoVisionTransformer( + patch_size=patch_size, + embed_dim=1536, + depth=40, + num_heads=24, + mlp_ratio=4, + block_fn=partial(Block, attn_class=MemEffAttention), + **kwargs, + ) + return model diff --git a/kornia/feature/dedode/transformer/layers/__init__.py b/kornia/feature/dedode/transformer/layers/__init__.py new file mode 100644 index 0000000..5b0c24a --- /dev/null +++ b/kornia/feature/dedode/transformer/layers/__init__.py @@ -0,0 +1,35 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Layers submodule for Kornia DeDoDe transformer. + +This package provides transformer layer components such as attention, blocks, MLPs, patch embedding, +and SwiGLU FFN for DeDoDe feature extraction. +""" + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +from .attention import MemEffAttention +from .block import NestedTensorBlock +from .dino_head import DINOHead +from .mlp import Mlp +from .patch_embed import PatchEmbed +from .swiglu_ffn import SwiGLUFFN, SwiGLUFFNFused diff --git a/kornia/feature/dedode/transformer/layers/attention.py b/kornia/feature/dedode/transformer/layers/attention.py new file mode 100644 index 0000000..a0e4a25 --- /dev/null +++ b/kornia/feature/dedode/transformer/layers/attention.py @@ -0,0 +1,136 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py + +import logging + +from torch import Tensor, nn + +logger = logging.getLogger("dinov2") + + +try: + from xformers.ops import fmha, memory_efficient_attention, unbind + + XFORMERS_AVAILABLE = True +except ImportError: + XFORMERS_AVAILABLE = False + + +class Attention(nn.Module): + """Implement the standard Multi-Head Self-Attention mechanism. + + Args: + dim: Total dimension of the input and output features. + num_heads: Number of attention heads. + qkv_bias: If True, add a bias term to the query, key, and value projections. + proj_bias: If True, add a bias term to the output projection. + attn_drop: Dropout probability applied to the attention weights. + proj_drop: Dropout probability applied to the output projection. + """ + + def __init__( + self, + dim: int, + num_heads: int = 8, + qkv_bias: bool = False, + proj_bias: bool = True, + attn_drop: float = 0.0, + proj_drop: float = 0.0, + ) -> None: + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim**-0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim, bias=proj_bias) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x: Tensor) -> Tensor: + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + + q, k, v = qkv[0] * self.scale, qkv[1], qkv[2] + attn = q @ k.transpose(-2, -1) + + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class MemEffAttention(Attention): + """Implement a memory-efficient version of Multi-Head Self-Attention using xFormers.""" + + def forward(self, x: Tensor, attn_bias=None) -> Tensor: # type: ignore[no-untyped-def] + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + attn_bias: Input value used by this method. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + if not XFORMERS_AVAILABLE: + if attn_bias is not None: + raise ValueError("xFormers is required for nested tensors usage") + return super().forward(x) + + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads) + + q, k, v = unbind(qkv, 2) + + x = memory_efficient_attention(q, k, v, attn_bias=attn_bias) + x = x.reshape([B, N, C]) + + x = self.proj(x) + x = self.proj_drop(x) + return x diff --git a/kornia/feature/dedode/transformer/layers/block.py b/kornia/feature/dedode/transformer/layers/block.py new file mode 100644 index 0000000..d4863cb --- /dev/null +++ b/kornia/feature/dedode/transformer/layers/block.py @@ -0,0 +1,319 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py + +import logging +from typing import Any, Callable, Dict, List, Tuple + +import torch +from torch import Tensor, nn + +from kornia.core.check import KORNIA_CHECK + +from .attention import Attention, MemEffAttention +from .drop_path import DropPath +from .layer_scale import LayerScale +from .mlp import Mlp + +logger = logging.getLogger("dinov2") + + +try: + from xformers.ops import fmha, index_select_cat, scaled_index_add + + XFORMERS_AVAILABLE = True +except ImportError: + logger.warning("xFormers not available") + XFORMERS_AVAILABLE = False + + +class Block(nn.Module): + """Implement a transformer block with attention and feed-forward sublayers. + + Args: + dim: Embedding dimension of the input and output features. + num_heads: Number of attention heads. + mlp_ratio: Expansion ratio used to compute the hidden dimension of the feed-forward network + as ``int(dim * mlp_ratio)``. + qkv_bias: If True, add a learnable bias to the query, key and value projections. + proj_bias: If True, add a learnable bias to the output projection of the attention layer. + ffn_bias: If True, add a learnable bias to the linear layers in the feed-forward network. + drop: Dropout probability applied after attention projection and inside the feed-forward network. + attn_drop: Dropout probability applied to the attention weights. + init_values: Initial value for the :class:`LayerScale` modules. If falsy, LayerScale is disabled + and an identity mapping is used instead. + drop_path: Stochastic depth probability for dropping the residual branch. + act_layer: Callable that constructs the activation layer used in the feed-forward network. + norm_layer: Callable that constructs the normalization layers applied before attention and + feed-forward sublayers. + attn_class: Callable that constructs the attention module. + ffn_layer: Callable that constructs the feed-forward network module. + """ + + def __init__( + self, + dim: int, + num_heads: int, + mlp_ratio: float = 4.0, + qkv_bias: bool = False, + proj_bias: bool = True, + ffn_bias: bool = True, + drop: float = 0.0, + attn_drop: float = 0.0, + init_values=None, + drop_path: float = 0.0, + act_layer: Callable[..., nn.Module] = nn.GELU, + norm_layer: Callable[..., nn.Module] = nn.LayerNorm, + attn_class: Callable[..., nn.Module] = Attention, + ffn_layer: Callable[..., nn.Module] = Mlp, + ) -> None: + super().__init__() + # print(f"biases: qkv: {qkv_bias}, proj: {proj_bias}, ffn: {ffn_bias}") + self.norm1 = norm_layer(dim) + self.attn = attn_class( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + proj_bias=proj_bias, + attn_drop=attn_drop, + proj_drop=drop, + ) + self.ls1 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity() + self.drop_path1 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = ffn_layer( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop, + bias=ffn_bias, + ) + self.ls2 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity() + self.drop_path2 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + self.sample_drop_ratio = drop_path + + def forward(self, x: Tensor) -> Tensor: + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + + def attn_residual_func(x: Tensor) -> Tensor: + return self.ls1(self.attn(self.norm1(x))) + + def ffn_residual_func(x: Tensor) -> Tensor: + return self.ls2(self.mlp(self.norm2(x))) + + if self.training and self.sample_drop_ratio > 0.1: + # the overhead is compensated only for a drop path rate larger than 0.1 + x = drop_add_residual_stochastic_depth( + x, + residual_func=attn_residual_func, + sample_drop_ratio=self.sample_drop_ratio, + ) + x = drop_add_residual_stochastic_depth( + x, + residual_func=ffn_residual_func, + sample_drop_ratio=self.sample_drop_ratio, + ) + elif self.training and self.sample_drop_ratio > 0.0: + x = x + self.drop_path1(attn_residual_func(x)) + x = x + self.drop_path1(ffn_residual_func(x)) # FIXME: drop_path2 + else: + x = x + attn_residual_func(x) + x = x + ffn_residual_func(x) + return x + + +def drop_add_residual_stochastic_depth( + x: Tensor, + residual_func: Callable[[Tensor], Tensor], + sample_drop_ratio: float = 0.0, +) -> Tensor: + """Add residual connection.""" + # 1) extract subset using permutation + b, _n, _d = x.shape + sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1) + brange = (torch.randperm(b, device=x.device))[:sample_subset_size] + x_subset = x[brange] + + # 2) apply residual_func to get residual + residual = residual_func(x_subset) + + x_flat = x.flatten(1) + residual = residual.flatten(1) + + residual_scale_factor = b / sample_subset_size + + # 3) add the residual + x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor) + return x_plus_residual.view_as(x) + + +def get_branges_scales(x, sample_drop_ratio=0.0): + """Add bernoulli sampled range and scale.""" + b, _n, _d = x.shape + sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1) + brange = (torch.randperm(b, device=x.device))[:sample_subset_size] + residual_scale_factor = b / sample_subset_size + return brange, residual_scale_factor + + +def add_residual(x, brange, residual, residual_scale_factor, scaling_vector=None): + """Add residual connections.""" + if scaling_vector is None: + x_flat = x.flatten(1) + residual = residual.flatten(1) + x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor) + else: + x_plus_residual = scaled_index_add( + x, brange, residual.to(dtype=x.dtype), scaling=scaling_vector, alpha=residual_scale_factor + ) + return x_plus_residual + + +attn_bias_cache: Dict[Tuple, Any] = {} + + +def get_attn_bias_and_cat(x_list, branges=None): + """Perform the index select, cat the tensors, and provide the attn_bias from cache.""" + batch_sizes = [b.shape[0] for b in branges] if branges is not None else [x.shape[0] for x in x_list] + all_shapes = tuple((b, x.shape[1]) for b, x in zip(batch_sizes, x_list)) + if all_shapes not in attn_bias_cache.keys(): + seqlens = [] + for b, x in zip(batch_sizes, x_list): + for _ in range(b): + seqlens.append(x.shape[1]) + attn_bias = fmha.BlockDiagonalMask.from_seqlens(seqlens) + attn_bias._batch_sizes = batch_sizes + attn_bias_cache[all_shapes] = attn_bias + + if branges is not None: + cat_tensors = index_select_cat([x.flatten(1) for x in x_list], branges).view(1, -1, x_list[0].shape[-1]) + else: + tensors_bs1 = tuple(x.reshape([1, -1, *x.shape[2:]]) for x in x_list) + cat_tensors = torch.cat(tensors_bs1, dim=1) + + return attn_bias_cache[all_shapes], cat_tensors + + +def drop_add_residual_stochastic_depth_list( + x_list: List[Tensor], + residual_func: Callable[[Tensor, Any], Tensor], + sample_drop_ratio: float = 0.0, + scaling_vector=None, +) -> Tensor: + """Add residual connections to a list of tensors.""" + # 1) generate random set of indices for dropping samples in the batch + branges_scales = [get_branges_scales(x, sample_drop_ratio=sample_drop_ratio) for x in x_list] + branges = [s[0] for s in branges_scales] + residual_scale_factors = [s[1] for s in branges_scales] + + # 2) get attention bias and index+concat the tensors + attn_bias, x_cat = get_attn_bias_and_cat(x_list, branges) + + # 3) apply residual_func to get residual, and split the result + residual_list = attn_bias.split(residual_func(x_cat, attn_bias=attn_bias)) # type: ignore + + outputs = [] + for x, brange, residual, residual_scale_factor in zip(x_list, branges, residual_list, residual_scale_factors): + outputs.append(add_residual(x, brange, residual, residual_scale_factor, scaling_vector).view_as(x)) + return outputs + + +class NestedTensorBlock(Block): + """Implement a Transformer block capable of processing :class:`torch.NestedTensor` inputs.""" + + def forward_nested(self, x_list: List[Tensor]) -> List[Tensor]: + """x_list contains a list of tensors to nest together and run.""" + KORNIA_CHECK(isinstance(self.attn, MemEffAttention)) + + if self.training and self.sample_drop_ratio > 0.0: + + def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor: + return self.attn(self.norm1(x), attn_bias=attn_bias) + + def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor: + return self.mlp(self.norm2(x)) + + x_list = drop_add_residual_stochastic_depth_list( + x_list, + residual_func=attn_residual_func, + sample_drop_ratio=self.sample_drop_ratio, + scaling_vector=self.ls1.gamma if isinstance(self.ls1, LayerScale) else None, + ) + x_list = drop_add_residual_stochastic_depth_list( + x_list, + residual_func=ffn_residual_func, + sample_drop_ratio=self.sample_drop_ratio, + scaling_vector=self.ls2.gamma if isinstance(self.ls1, LayerScale) else None, + ) + return x_list + else: + + def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor: + return self.ls1(self.attn(self.norm1(x), attn_bias=attn_bias)) + + def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor: + return self.ls2(self.mlp(self.norm2(x))) + + attn_bias, x = get_attn_bias_and_cat(x_list) + x = x + attn_residual_func(x, attn_bias=attn_bias) + x = x + ffn_residual_func(x) + return attn_bias.split(x) + + def forward(self, x_or_x_list): + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + x_or_x_list: Input value used by this method. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + if isinstance(x_or_x_list, Tensor): + return super().forward(x_or_x_list) + elif isinstance(x_or_x_list, list): + KORNIA_CHECK(XFORMERS_AVAILABLE, "Please install xFormers for nested tensors usage") + return self.forward_nested(x_or_x_list) + else: + raise AssertionError diff --git a/kornia/feature/dedode/transformer/layers/dino_head.py b/kornia/feature/dedode/transformer/layers/dino_head.py new file mode 100644 index 0000000..6324922 --- /dev/null +++ b/kornia/feature/dedode/transformer/layers/dino_head.py @@ -0,0 +1,101 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import torch +from torch import nn +from torch.nn.init import trunc_normal_ +from torch.nn.utils import weight_norm + + +class DINOHead(nn.Module): + """Implement the projection head for DINO-based models. + + Args: + in_dim: Input feature dimensionality. + out_dim: Output feature dimensionality of the projection head. + use_bn: Whether to use batch normalization layers in the MLP. + nlayers: Number of layers in the MLP projection head. A value smaller than 1 is clamped to 1. + hidden_dim: Hidden dimensionality used in intermediate MLP layers when ``nlayers > 1``. + bottleneck_dim: Dimensionality of the bottleneck output of the MLP before the final normalized layer. + mlp_bias: Whether to include bias terms in the MLP linear layers. + """ + + def __init__( + self, + in_dim, + out_dim, + use_bn=False, + nlayers=3, + hidden_dim=2048, + bottleneck_dim=256, + mlp_bias=True, + ): + super().__init__() + nlayers = max(nlayers, 1) + self.mlp = _build_mlp(nlayers, in_dim, bottleneck_dim, hidden_dim=hidden_dim, use_bn=use_bn, bias=mlp_bias) + self.apply(self._init_weights) + self.last_layer = weight_norm(nn.Linear(bottleneck_dim, out_dim, bias=False)) + self.last_layer.weight_g.data.fill_(1) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + + def forward(self, x): + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + x = self.mlp(x) + eps = 1e-6 if x.dtype == torch.float16 else 1e-12 + x = nn.functional.normalize(x, dim=-1, p=2, eps=eps) + x = self.last_layer(x) + return x + + +def _build_mlp(nlayers, in_dim, bottleneck_dim, hidden_dim=None, use_bn=False, bias=True): + if nlayers == 1: + return nn.Linear(in_dim, bottleneck_dim, bias=bias) + else: + layers = [nn.Linear(in_dim, hidden_dim, bias=bias)] + if use_bn: + layers.append(nn.BatchNorm1d(hidden_dim)) + layers.append(nn.GELU()) + for _ in range(nlayers - 2): + layers.append(nn.Linear(hidden_dim, hidden_dim, bias=bias)) + if use_bn: + layers.append(nn.BatchNorm1d(hidden_dim)) + layers.append(nn.GELU()) + layers.append(nn.Linear(hidden_dim, bottleneck_dim, bias=bias)) + return nn.Sequential(*layers) diff --git a/kornia/feature/dedode/transformer/layers/drop_path.py b/kornia/feature/dedode/transformer/layers/drop_path.py new file mode 100644 index 0000000..630127a --- /dev/null +++ b/kornia/feature/dedode/transformer/layers/drop_path.py @@ -0,0 +1,69 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/drop.py +from typing import Optional + +import torch +from torch import nn + + +def drop_path(x: torch.Tensor, drop_prob: Optional[float] = 0.0, training: bool = False) -> torch.Tensor: + """Apply stochastic depth sampling.""" + if drop_prob is None: + drop_path = 0.0 + if drop_prob == 0.0 or not training: + return x + keep_prob = 1.0 - drop_prob # type: ignore + shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = x.new_empty(shape).bernoulli_(keep_prob) + if keep_prob > 0.0: + random_tensor.div_(keep_prob) + output = x * random_tensor + return output + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" + + def __init__(self, drop_prob: Optional[float] = None) -> None: + super().__init__() + self.drop_prob = drop_prob + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + return drop_path(x, self.drop_prob, self.training) diff --git a/kornia/feature/dedode/transformer/layers/layer_scale.py b/kornia/feature/dedode/transformer/layers/layer_scale.py new file mode 100644 index 0000000..fdfbe01 --- /dev/null +++ b/kornia/feature/dedode/transformer/layers/layer_scale.py @@ -0,0 +1,65 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# Modified from: https://github.com/huggingface/pytorch-image-models/blob/main/timm/models/vision_transformer.py#L103-L110 + +from typing import Union + +import torch +from torch import Tensor, nn + + +class LayerScale(nn.Module): + """Implement the LayerScale mechanism for stabilizing transformer training. + + Args: + dim: Dimensionality of the input features used to define the size of the scaling parameter. + init_values: Initial value or tensor used to initialize the learnable scaling parameter ``gamma``. + inplace: If True, apply the scaling operation in-place on the input tensor. + """ + + def __init__( + self, + dim: int, + init_values: Union[float, Tensor] = 1e-5, + inplace: bool = False, + ) -> None: + super().__init__() + self.inplace = inplace + self.gamma = nn.Parameter(init_values * torch.ones(dim)) + + def forward(self, x: Tensor) -> Tensor: + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + return x.mul_(self.gamma) if self.inplace else x * self.gamma diff --git a/kornia/feature/dedode/transformer/layers/mlp.py b/kornia/feature/dedode/transformer/layers/mlp.py new file mode 100644 index 0000000..51ddcd6 --- /dev/null +++ b/kornia/feature/dedode/transformer/layers/mlp.py @@ -0,0 +1,81 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/mlp.py + + +from typing import Callable, Optional + +from torch import Tensor, nn + + +class Mlp(nn.Module): + """Implement the Multi-Layer Perceptron (MLP) used in transformer blocks. + + Args: + in_features: The number of input features. + hidden_features: The number of hidden layer features. Default: None. + out_features: The number of output features. Default: None. + act_layer: The activation function to use. Default: :class:`nn.GELU`. + drop: The dropout probability. Default: 0.0. + """ + + def __init__( + self, + in_features: int, + hidden_features: Optional[int] = None, + out_features: Optional[int] = None, + act_layer: Callable[..., nn.Module] = nn.GELU, + drop: float = 0.0, + bias: bool = True, + ) -> None: + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features, bias=bias) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features, bias=bias) + self.drop = nn.Dropout(drop) + + def forward(self, x: Tensor) -> Tensor: + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x diff --git a/kornia/feature/dedode/transformer/layers/patch_embed.py b/kornia/feature/dedode/transformer/layers/patch_embed.py new file mode 100644 index 0000000..60b844e --- /dev/null +++ b/kornia/feature/dedode/transformer/layers/patch_embed.py @@ -0,0 +1,124 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py + +from typing import Callable, Optional, Tuple, Union + +from torch import Tensor, nn + +from kornia.core.check import KORNIA_CHECK + + +def make_2tuple(x): + """Make a tuple of length 2.""" + if isinstance(x, tuple): + KORNIA_CHECK(len(x) == 2) + return x + KORNIA_CHECK(isinstance(x, int)) + return (x, x) + + +class PatchEmbed(nn.Module): + """2D image to patch embedding: (B,C,H,W) -> (B,N,D). + + Args: + img_size: Image size. + patch_size: Patch token size. + in_chans: Number of input image channels. + embed_dim: Number of linear projection output channels. + norm_layer: Normalization layer. + + """ + + def __init__( + self, + img_size: Union[int, Tuple[int, int]] = 224, + patch_size: Union[int, Tuple[int, int]] = 16, + in_chans: int = 3, + embed_dim: int = 768, + norm_layer: Optional[Callable] = None, + flatten_embedding: bool = True, + ) -> None: + super().__init__() + + image_HW = make_2tuple(img_size) + patch_HW = make_2tuple(patch_size) + patch_grid_size = ( + image_HW[0] // patch_HW[0], + image_HW[1] // patch_HW[1], + ) + + self.img_size = image_HW + self.patch_size = patch_HW + self.patches_resolution = patch_grid_size + self.num_patches = patch_grid_size[0] * patch_grid_size[1] + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.flatten_embedding = flatten_embedding + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_HW, stride=patch_HW) + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + def forward(self, x: Tensor) -> Tensor: + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + _, _, H, W = x.shape + patch_H, patch_W = self.patch_size + KORNIA_CHECK(H % patch_H == 0, f"Input image height {H} is not a multiple of patch height {patch_H}") + KORNIA_CHECK(W % patch_W == 0, f"Input image width {W} is not a multiple of patch width: {patch_W}") + + x = self.proj(x) # B C H W + H, W = x.size(2), x.size(3) + x = x.flatten(2).transpose(1, 2) # B HW C + x = self.norm(x) + if not self.flatten_embedding: + x = x.reshape(-1, H, W, self.embed_dim) # B H W C + return x + + def flops(self) -> float: + """Estimate this layer computational cost. + + Returns: + Estimated number of floating-point operations for this layer. + """ + Ho, Wo = self.patches_resolution + flops = Ho * Wo * self.embed_dim * self.in_chans * (self.patch_size[0] * self.patch_size[1]) + if self.norm is not None: + flops += Ho * Wo * self.embed_dim + return flops diff --git a/kornia/feature/dedode/transformer/layers/swiglu_ffn.py b/kornia/feature/dedode/transformer/layers/swiglu_ffn.py new file mode 100644 index 0000000..0df40fc --- /dev/null +++ b/kornia/feature/dedode/transformer/layers/swiglu_ffn.py @@ -0,0 +1,111 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Callable, Optional + +import torch.nn.functional as F +from torch import Tensor, nn + + +class SwiGLUFFN(nn.Module): + """Implement the SwiGLU Feed-Forward Network. + + This block uses the Gated Linear Unit with Swish activation, common in + modern transformer architectures like DINOv2. + + Args: + in_features: The number of input features. + hidden_features: The number of hidden features in the gating branch. + out_features: The number of output features. Default: None. + act_layer: The activation function. Default: :class:`nn.SiLU`. + drop: The dropout probability. Default: 0.0. + """ + + def __init__( + self, + in_features: int, + hidden_features: Optional[int] = None, + out_features: Optional[int] = None, + act_layer: Optional[Callable[..., nn.Module]] = None, + drop: float = 0.0, + bias: bool = True, + ) -> None: + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.w12 = nn.Linear(in_features, 2 * hidden_features, bias=bias) + self.w3 = nn.Linear(hidden_features, out_features, bias=bias) + + def forward(self, x: Tensor) -> Tensor: + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + x12 = self.w12(x) + x1, x2 = x12.chunk(2, dim=-1) + hidden = F.silu(x1) * x2 + return self.w3(hidden) + + +try: + from xformers.ops import SwiGLU + + XFORMERS_AVAILABLE = True +except ImportError: + SwiGLU = SwiGLUFFN + XFORMERS_AVAILABLE = False + + +class SwiGLUFFNFused(SwiGLU): + """Implement the fused SwiGLU activation and Feed-Forward Network. + + This implementation is optimized for training speed and memory usage. + """ + + def __init__( + self, + in_features: int, + hidden_features: Optional[int] = None, + out_features: Optional[int] = None, + act_layer: Optional[Callable[..., nn.Module]] = None, + drop: float = 0.0, + bias: bool = True, + ) -> None: + out_features = out_features or in_features + hidden_features = hidden_features or in_features + hidden_features = (int(hidden_features * 2 / 3) + 7) // 8 * 8 + super().__init__( + in_features=in_features, + hidden_features=hidden_features, + out_features=out_features, + bias=bias, + ) diff --git a/kornia/feature/dedode/utils.py b/kornia/feature/dedode/utils.py new file mode 100644 index 0000000..3708469 --- /dev/null +++ b/kornia/feature/dedode/utils.py @@ -0,0 +1,68 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Optional, Tuple, Union + +import torch +import torch.nn.functional as F + + +@torch.no_grad() +def sample_keypoints( + scoremap: torch.Tensor, + num_samples: Optional[int] = 10_000, + return_scoremap: bool = True, + increase_coverage: bool = True, +) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + """Sample keypoints from provided candidates.""" + device = scoremap.device + dtype = scoremap.dtype + B, H, W = scoremap.shape + if increase_coverage: + weights = (-(torch.linspace(-2, 2, steps=51, device=device, dtype=dtype) ** 2)).exp()[None, None] + # 10000 is just some number for maybe numerical stability, who knows. :), result is invariant anyway + local_density_x = F.conv2d((scoremap[:, None] + 1e-6) * 10000, weights[..., None, :], padding=(0, 51 // 2)) + local_density = F.conv2d(local_density_x, weights[..., None], padding=(51 // 2, 0))[:, 0] + scoremap = scoremap * (local_density + 1e-8) ** (-1 / 2) + grid = get_grid(B, H, W, device=device).reshape(B, H * W, 2) + num_samples = min(num_samples, H * W) if num_samples is not None else H * W # type: ignore + inds = torch.topk(scoremap.reshape(B, H * W), k=num_samples).indices + kps = torch.gather(grid, dim=1, index=inds[..., None].expand(B, num_samples, 2)) + if return_scoremap: + return kps, torch.gather(scoremap.reshape(B, H * W), dim=1, index=inds) + return kps + + +def get_grid(B: int, H: int, W: int, device: torch.device) -> torch.Tensor: + """Get grid of provided layout.""" + xs = (torch.arange(W, device=device) + 0.5) / W * 2 - 1 + ys = (torch.arange(H, device=device) + 0.5) / H * 2 - 1 + yy, xx = torch.meshgrid(ys, xs, indexing="ij") + base = torch.stack((xx, yy), dim=-1).reshape(1, H * W, 2) + return base.expand(B, -1, -1) + + +def dedode_denormalize_pixel_coordinates(flow: torch.Tensor, h: int, w: int) -> torch.Tensor: + """Denormalize pixel coordinates.""" + flow = torch.stack( + ( + w * (flow[..., 0] + 1) / 2, + h * (flow[..., 1] + 1) / 2, + ), + dim=-1, + ) + return flow diff --git a/kornia/feature/dedode/vgg.py b/kornia/feature/dedode/vgg.py new file mode 100644 index 0000000..5872c6f --- /dev/null +++ b/kornia/feature/dedode/vgg.py @@ -0,0 +1,279 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, List, Optional, Union, cast + +import torch +from torch import nn + + +class VGG(nn.Module): + """Implement a flexible VGG-style backbone for feature extraction. + + Args: + features: The convolutional feature extractor module. + num_classes: The number of classes for the classification head. Default: 1000. + init_weights: Whether to initialize weights using a predefined scheme. Default: True. + dropout: The dropout probability for the classifier. Default: 0.5. + """ + + def __init__( + self, features: nn.Module, num_classes: int = 1000, init_weights: bool = True, dropout: float = 0.5 + ) -> None: + super().__init__() + self.features = features + self.avgpool = nn.AdaptiveAvgPool2d((7, 7)) + self.classifier = nn.Sequential( + nn.Linear(512 * 7 * 7, 4096), + nn.ReLU(True), + nn.Dropout(p=dropout), + nn.Linear(4096, 4096), + nn.ReLU(True), + nn.Dropout(p=dropout), + nn.Linear(4096, num_classes), + ) + if init_weights: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu") + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + nn.init.normal_(m.weight, 0, 0.01) + nn.init.constant_(m.bias, 0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run this DeDoDe module forward. + + Inputs are image, feature, or token tensors used by the DeDoDe detector/descriptor pipeline. `B` denotes batch + size, `C` channels, `H` height, `W` width, `N` token count, and `D` feature dimension where those axes appear. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + x = self.features(x) + x = self.avgpool(x) + x = torch.flatten(x, 1) + x = self.classifier(x) + return x + + +def make_layers(cfg: List[Union[str, int]], batch_norm: bool = False) -> nn.Sequential: + """Make model layers.""" + layers: List[nn.Module] = [] + in_channels = 3 + for v in cfg: + if v == "M": + layers += [nn.MaxPool2d(kernel_size=2, stride=2)] + else: + v = cast(int, v) + conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1) + if batch_norm: + layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)] + else: + layers += [conv2d, nn.ReLU(inplace=True)] + in_channels = v + return nn.Sequential(*layers) + + +cfgs: Dict[str, List[Union[str, int]]] = { + "A": [64, "M", 128, "M", 256, 256, "M", 512, 512, "M", 512, 512, "M"], + "B": [64, 64, "M", 128, 128, "M", 256, 256, "M", 512, 512, "M", 512, 512, "M"], + "D": [64, 64, "M", 128, 128, "M", 256, 256, 256, "M", 512, 512, 512, "M", 512, 512, 512, "M"], + "E": [64, 64, "M", 128, 128, "M", 256, 256, 256, 256, "M", 512, 512, 512, 512, "M", 512, 512, 512, 512, "M"], +} + + +def _vgg(cfg: str, batch_norm: bool, weights: Optional[Any] = None, **kwargs: Any) -> VGG: + model = VGG(make_layers(cfgs[cfg], batch_norm=batch_norm), **kwargs) + return model + + +def vgg11(*, weights: Optional[Any] = None, **kwargs: Any) -> VGG: + """VGG-11 from `Very Deep Convolutional Networks for Large-Scale Image Recognition `__. + + Args: + weights (:class:`~torchvision.models.VGG11_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.VGG11_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + **kwargs: parameters passed to the ``torchvision.models.vgg.VGG`` + base class. Please refer to the `source code + `_ + for more details about this class. + + .. autoclass:: torchvision.models.VGG11_Weights + :members: + + """ + return _vgg("A", False, weights, **kwargs) + + +def vgg11_bn(*, weights: Optional[Any] = None, **kwargs: Any) -> VGG: + """VGG-11-BN from `Very Deep Convolutional Networks for Large-Scale Image Recognition `__. + + Args: + weights (:class:`~torchvision.models.VGG11_BN_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.VGG11_BN_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + **kwargs: parameters passed to the ``torchvision.models.vgg.VGG`` + base class. Please refer to the `source code + `_ + for more details about this class. + + .. autoclass:: torchvision.models.VGG11_BN_Weights + :members: + + """ + return _vgg("A", True, weights, **kwargs) + + +def vgg13(*, weights: Optional[Any] = None, **kwargs: Any) -> VGG: + """VGG-13 from `Very Deep Convolutional Networks for Large-Scale Image Recognition `__. + + Args: + weights (:class:`~torchvision.models.VGG13_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.VGG13_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + **kwargs: parameters passed to the ``torchvision.models.vgg.VGG`` + base class. Please refer to the `source code + `_ + for more details about this class. + + .. autoclass:: torchvision.models.VGG13_Weights + :members: + + """ + return _vgg("B", False, weights, **kwargs) + + +def vgg13_bn(*, weights: Optional[Any] = None, **kwargs: Any) -> VGG: + """VGG-13-BN from `Very Deep Convolutional Networks for Large-Scale Image Recognition `__. + + Args: + weights (:class:`~torchvision.models.VGG13_BN_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.VGG13_BN_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + **kwargs: parameters passed to the ``torchvision.models.vgg.VGG`` + base class. Please refer to the `source code + `_ + for more details about this class. + + .. autoclass:: torchvision.models.VGG13_BN_Weights + :members: + + """ + return _vgg("B", True, weights, **kwargs) + + +def vgg16(*, weights: Optional[Any] = None, **kwargs: Any) -> VGG: + """VGG-16 from `Very Deep Convolutional Networks for Large-Scale Image Recognition `__. + + Args: + weights (:class:`~torchvision.models.VGG16_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.VGG16_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + **kwargs: parameters passed to the ``torchvision.models.vgg.VGG`` + base class. Please refer to the `source code + `_ + for more details about this class. + + .. autoclass:: torchvision.models.VGG16_Weights + :members: + + """ + return _vgg("D", False, weights, **kwargs) + + +def vgg16_bn(*, weights: Optional[Any] = None, **kwargs: Any) -> VGG: + """VGG-16-BN from `Very Deep Convolutional Networks for Large-Scale Image Recognition `__. + + Args: + weights (:class:`~torchvision.models.VGG16_BN_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.VGG16_BN_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + **kwargs: parameters passed to the ``torchvision.models.vgg.VGG`` + base class. Please refer to the `source code + `_ + for more details about this class. + + .. autoclass:: torchvision.models.VGG16_BN_Weights + :members: + + """ + return _vgg("D", True, weights, **kwargs) + + +def vgg19(*, weights: Optional[Any] = None, **kwargs: Any) -> VGG: + """VGG-19 from `Very Deep Convolutional Networks for Large-Scale Image Recognition `__. + + Args: + weights (:class:`~torchvision.models.VGG19_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.VGG19_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + **kwargs: parameters passed to the ``torchvision.models.vgg.VGG`` + base class. Please refer to the `source code + `_ + for more details about this class. + + .. autoclass:: torchvision.models.VGG19_Weights + :members: + + """ + return _vgg("E", False, weights, **kwargs) + + +def vgg19_bn(*, weights: Optional[Any] = None, **kwargs: Any) -> VGG: + """VGG-19_BN from `Very Deep Convolutional Networks for Large-Scale Image Recognition `__. + + Args: + weights (:class:`~torchvision.models.VGG19_BN_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.VGG19_BN_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + **kwargs: parameters passed to the ``torchvision.models.vgg.VGG`` + base class. Please refer to the `source code + `_ + for more details about this class. + + .. autoclass:: torchvision.models.VGG19_BN_Weights + :members: + + """ + return _vgg("E", True, weights, **kwargs) diff --git a/kornia/feature/defmo.py b/kornia/feature/defmo.py new file mode 100644 index 0000000..88edf55 --- /dev/null +++ b/kornia/feature/defmo.py @@ -0,0 +1,418 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Callable, Dict, List, Optional, Type + +import torch +from torch import nn + +urls: Dict[str, str] = {} +urls["defmo_encoder"] = "http://ptak.felk.cvut.cz/personal/rozumden/defmo_saved_models/encoder_best.pt" +urls["defmo_rendering"] = "http://ptak.felk.cvut.cz/personal/rozumden/defmo_saved_models/rendering_best.pt" + + +# conv1x1, conv3x3, Bottleneck, ResNet are taken from: +# https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py + + +def conv1x1(in_planes: int, out_planes: int, stride: int = 1) -> nn.Conv2d: + """1x1 convolution.""" + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +def conv3x3(in_planes: int, out_planes: int, stride: int = 1, groups: int = 1, dilation: int = 1) -> nn.Conv2d: + """3x3 convolution with padding.""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=dilation, + groups=groups, + bias=False, + dilation=dilation, + ) + + +class Bottleneck(nn.Module): + """Implement the Bottleneck building block for ResNet. + + This block follows the ResNet V1.5 design where the stride is applied to the 3x3 convolution + instead of the first 1x1 convolution to better preserve spatial information. + + Args: + inplanes: The number of input channels. + planes: The number of intermediate channels. + stride: The stride size for the convolution. Default: 1. + downsample: An optional module to downsample the input identity. Default: None. + groups: The number of blocked connections from input channels to output channels. Default: 1. + base_width: The width of each group. Default: 64. + dilation: The spacing between kernel elements. Default: 1. + norm_layer: The normalization layer to use. Default: None. + """ + + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + + expansion: int = 4 + + def __init__( + self, + inplanes: int, + planes: int, + stride: int = 1, + downsample: Optional[nn.Module] = None, + groups: int = 1, + base_width: int = 64, + dilation: int = 1, + norm_layer: Optional[Callable[..., nn.Module]] = None, + ) -> None: + super().__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.0)) * groups + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run the bottleneck residual block. + + Args: + x: Feature tensor with shape :math:`(B, C, H, W)`, where ``B`` is batch + size, ``C`` is channel count, and ``H``/``W`` are spatial dimensions. + + Returns: + Output feature tensor after the residual branch and identity connection. + """ + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + """Implement the ResNet architecture for feature extraction. + + This implementation provides a flexible backbone used as the encoder in the DeFMO framework. + + Args: + block: The block type to use, typically :class:`Bottleneck`. + layers: A list containing the number of blocks in each of the four stages. + num_classes: The number of output classes. Default: 1000. + zero_init_residual: Whether to initialize the last batch norm in each residual branch to zero. Default: False. + groups: The number of groups for the convolution. Default: 1. + width_per_group: The width of each group. Default: 64. + replace_stride_with_dilation: A list of booleans indicating + if stride should be replaced by dilation in each stage. Default: None. + norm_layer: The normalization layer to use. Default: None. + """ + + def __init__( + self, + block: Type[Bottleneck], + layers: List[int], + num_classes: int = 1000, + zero_init_residual: bool = False, + groups: int = 1, + width_per_group: int = 64, + replace_stride_with_dilation: Optional[List[bool]] = None, + norm_layer: Optional[Callable[..., nn.Module]] = None, + ) -> None: + super().__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError( + f"replace_stride_with_dilation should be None or a 3-element tuple, got {replace_stride_with_dilation}" + ) + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2, dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2, dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2, dilate=replace_stride_with_dilation[2]) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.fc = nn.Linear(512 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu") + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with torch.zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck) and isinstance(m.bn3.weight, torch.Tensor): + nn.init.constant_(m.bn3.weight, 0) + + def _make_layer( + self, block: Type[Bottleneck], planes: int, blocks: int, stride: int = 1, dilate: bool = False + ) -> nn.Sequential: + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), norm_layer(planes * block.expansion) + ) + + layers = [] + layers.append( + block( + self.inplanes, planes, stride, downsample, self.groups, self.base_width, previous_dilation, norm_layer + ) + ) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block( + self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + dilation=self.dilation, + norm_layer=norm_layer, + ) + ) + + return nn.Sequential(*layers) + + def _forward_impl(self, x: torch.Tensor) -> torch.Tensor: + # See note [TorchScript super()] + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + x = torch.flatten(x, 1) + x = self.fc(x) + + return x + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run the ResNet backbone and classification head. + + Args: + x: Input image or feature tensor with shape :math:`(B, C, H, W)`. + + Returns: + Tensor with shape :math:`(B, N)`, where ``N`` is ``num_classes``. + """ + return self._forward_impl(x) + + +class EncoderDeFMO(nn.Module): + """Implement the Encoder module for the Deblurring Fast Moving Objects (DeFMO) model. + + The encoder extracts latent features from the concatenation of the blurred input image and + the estimated background. It uses a modified ResNet-50 backbone to accept 6-channel inputs. + + + Shape: + - Input: (B, 6, H, W) where 6 represents the concatenated blurred image and background. + - Output: A list of feature maps from different stages of the ResNet backbone. + """ + + def __init__(self) -> None: + super().__init__() + model = ResNet(Bottleneck, [3, 4, 6, 3]) # ResNet50 + modelc1 = nn.Sequential(*list(model.children())[:3]) + modelc2 = nn.Sequential(*list(model.children())[4:8]) + modelc1[0] = nn.Conv2d(6, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False) + self.net = nn.Sequential(modelc1, modelc2) + + def forward(self, input_data: torch.Tensor) -> torch.Tensor: + """Encode blurred-image and background inputs into latent features. + + Args: + input_data: Tensor with shape :math:`(B, 6, H, W)`. The 6 channels usually + concatenate RGB blurred input and RGB background estimate. + + Returns: + Latent feature tensor produced by the modified ResNet encoder. + """ + return self.net(input_data) + + +class RenderingDeFMO(nn.Module): + """Implement the Rendering module for the Deblurring Fast Moving Objects (DeFMO) model. + + This module acts as a decoder that transforms the latent features from the encoder into + a temporal sequence of sharp sub-frames, recovering the object's appearance and motion. + + Shape: + - Input: Latent feature representation from the :class:`EncoderDeFMO`. + - Output: (B, T, 4, H, W) where T is the number of sub-frames and 4 represents RGBA channels. + """ + + def __init__(self) -> None: + super().__init__() + self.tsr_steps: int = 24 + model = nn.Sequential( + nn.Conv2d(2049, 1024, kernel_size=3, stride=1, padding=1, bias=False), + nn.BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True), + nn.ReLU(inplace=True), + Bottleneck(1024, 256), + nn.PixelShuffle(2), + Bottleneck(256, 64), + nn.PixelShuffle(2), + Bottleneck(64, 16), + nn.PixelShuffle(2), + nn.Conv2d(16, 16, kernel_size=3, stride=1, padding=1, bias=False), + nn.PixelShuffle(2), + nn.Conv2d(4, 4, kernel_size=3, stride=1, padding=1, bias=True), + nn.ReLU(inplace=True), + nn.Conv2d(4, 4, kernel_size=3, stride=1, padding=1, bias=True), + ) + self.net = model + self.times = torch.linspace(0, 1, self.tsr_steps) + + def forward(self, latent: torch.Tensor) -> torch.Tensor: + """Render a temporal RGBA sequence from latent DeFMO features. + + Args: + latent: Latent feature tensor from :class:`EncoderDeFMO` with shape + :math:`(B, C, H, W)`. + + Returns: + Tensor with shape :math:`(B, T, 4, H_{out}, W_{out})`, where ``T`` is the + number of rendered time steps and 4 represents RGBA channels. + """ + times = self.times.to(latent.device).unsqueeze(0).repeat(latent.shape[0], 1) + renders = [] + for ki in range(times.shape[1]): + t_tensor = ( + times[list(range(times.shape[0])), ki] + .unsqueeze(-1) + .unsqueeze(-1) + .unsqueeze(-1) + .repeat(1, 1, latent.shape[2], latent.shape[3]) + ) + latenti = torch.cat((t_tensor, latent), 1) + result = self.net(latenti) + renders.append(result) + renders_stacked = torch.stack(renders, 1).contiguous() + renders_stacked[:, :, :4] = torch.sigmoid(renders_stacked[:, :, :4]) + return renders_stacked + + +class DeFMO(nn.Module): + """nn.Module that disentangle a fast-moving object from the background and performs deblurring. + + This is based on the original code from paper "DeFMO: Deblurring and Shape Recovery + of Fast Moving Objects". See :cite:`DeFMO2021` for more details. + + Args: + pretrained: Download and set pretrained weights to the model. Default: false. + + Returns: + Temporal super-resolution without background. + Shape: + - Input: (B, 6, H, W) + - Output: (B, S, 4, H, W) + + Examples: + >>> import kornia + >>> input = torch.rand(2, 6, 240, 320) + >>> defmo = kornia.feature.DeFMO() + >>> tsr_nobgr = defmo(input) # 2x24x4x240x320 + + """ + + def __init__(self, pretrained: bool = False) -> None: + super().__init__() + self.encoder = EncoderDeFMO() + self.rendering = RenderingDeFMO() + + # use torch.hub to load pretrained model + if pretrained: + pretrained_dict = torch.hub.load_state_dict_from_url( + urls["defmo_encoder"], map_location=torch.device("cpu") + ) + self.encoder.load_state_dict(pretrained_dict, strict=True) + pretrained_dict_ren = torch.hub.load_state_dict_from_url( + urls["defmo_rendering"], map_location=torch.device("cpu") + ) + self.rendering.load_state_dict(pretrained_dict_ren, strict=True) + self.eval() + + def forward(self, input_data: torch.Tensor) -> torch.Tensor: + """Deblur a fast-moving object into a sequence of RGBA sub-frames. + + Args: + input_data: Tensor with shape :math:`(B, 6, H, W)` containing the blurred + RGB image concatenated with an RGB background estimate. + + Returns: + Tensor with shape :math:`(B, T, 4, H, W)`, where ``T`` is the number of + temporal sub-frames and 4 stores red, green, blue, and alpha channels. + """ + latent = self.encoder(input_data) + x_out = self.rendering(latent) + return x_out diff --git a/kornia/feature/disk/__init__.py b/kornia/feature/disk/__init__.py new file mode 100644 index 0000000..1aa46b5 --- /dev/null +++ b/kornia/feature/disk/__init__.py @@ -0,0 +1,24 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Feature DISK — DISK feature detector and descriptor for Kornia. + +This subpackage provides the DISK keypoint detector and feature structures. +""" + +from .disk import DISK +from .structs import DISKFeatures diff --git a/kornia/feature/disk/_unets/__init__.py b/kornia/feature/disk/_unets/__init__.py new file mode 100644 index 0000000..e32db66 --- /dev/null +++ b/kornia/feature/disk/_unets/__init__.py @@ -0,0 +1,18 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .unet import Unet diff --git a/kornia/feature/disk/_unets/blocks.py b/kornia/feature/disk/_unets/blocks.py new file mode 100644 index 0000000..7afee19 --- /dev/null +++ b/kornia/feature/disk/_unets/blocks.py @@ -0,0 +1,147 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +import torch.nn.functional as F +from torch import nn + + +class TrivialUpsample(nn.Module): + """Bilinear upsampling layer used inside the thin DISK U-Net. + + The layer doubles spatial height and width while keeping batch size and channel count unchanged. + """ + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run this DISK component forward. + + Feature maps use `(B, C, H, W)`, where `B` is batch size, `C` channels, and `H`/`W` are spatial dimensions. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + return F.interpolate(x, scale_factor=2, mode="bilinear", align_corners=False) + + +class TrivialDownsample(nn.Module): + """Average-pooling downsampling layer used inside the thin DISK U-Net. + + The layer halves spatial height and width with a `2x2` average-pooling window. + """ + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run this DISK component forward. + + Feature maps use `(B, C, H, W)`, where `B` is batch size, `C` channels, and `H`/`W` are spatial dimensions. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + return F.avg_pool2d(x, 2) + + +class Conv(nn.Sequential): + """Convolution block used by the thin DISK U-Net. + + The block optionally applies instance normalization and a PReLU gate before a same-padded convolution. + """ + + def __init__(self, in_: int, out_: int, size: int, skip_norm_and_gate: bool = False) -> None: + norm: nn.Module + nonl: nn.Module + + if skip_norm_and_gate: + norm = nn.Sequential() + nonl = nn.Sequential() + else: + norm = nn.InstanceNorm2d(in_) + nonl = nn.PReLU(in_) + + dropout = nn.Sequential() + conv = nn.Conv2d(in_, out_, size, padding="same", bias=True) + + super().__init__(norm, nonl, dropout, conv) + + +class ThinUnetDownBlock(nn.Sequential): + """Downsampling block for the thin DISK U-Net encoder path. + + The block optionally downsamples the feature map and then applies the configured convolution block. + """ + + def __init__(self, in_: int, out_: int, size: int = 5, is_first: bool = False) -> None: + self.in_ = in_ + self.out_ = out_ + + downsample: nn.Module + if is_first: + downsample = nn.Sequential() + conv = Conv(in_, out_, size, skip_norm_and_gate=True) + else: + downsample = TrivialDownsample() + conv = Conv(in_, out_, size) + + super().__init__(downsample, conv) + + +class ThinUnetUpBlock(nn.Module): + """Upsampling block for the thin DISK U-Net decoder path. + + The block upsamples the bottom feature map, concatenates it with the horizontal skip feature map, and applies a + convolution block. + """ + + def __init__(self, bottom_: int, horizontal_: int, out_: int, size: int = 5) -> None: + super().__init__() + + self.bottom_ = bottom_ + self.horizontal_ = horizontal_ + self.cat_ = bottom_ + horizontal_ + self.out_ = out_ + + self.upsample = TrivialUpsample() + self.conv = Conv(self.cat_, self.out_, size) + + def forward(self, bot: torch.Tensor, hor: torch.Tensor) -> torch.Tensor: + """Run this DISK component forward. + + Feature maps use `(B, C, H, W)`, where `B` is batch size, `C` channels, and `H`/`W` are spatial dimensions. + + Args: + bot: Input value used by this method. + hor: Input value used by this method. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + bot_big = self.upsample(bot) + combined = torch.cat([bot_big, hor], dim=1) + + return self.conv(combined) diff --git a/kornia/feature/disk/_unets/unet.py b/kornia/feature/disk/_unets/unet.py new file mode 100644 index 0000000..034a3ff --- /dev/null +++ b/kornia/feature/disk/_unets/unet.py @@ -0,0 +1,102 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional + +import torch +from torch import nn + +from .blocks import ThinUnetDownBlock, ThinUnetUpBlock + + +class Unet(nn.Module): + """U-Net backbone used by DISK to predict dense feature maps. + + The network follows an encoder-decoder layout with skip connections. Input and output tensors use `(B, C, H, W)`, + where `B` is batch size, `C` is channel count, and `H`/`W` are spatial dimensions. + """ + + def __init__( + self, in_features: int = 1, up: Optional[list[int]] = None, down: Optional[list[int]] = None, size: int = 5 + ) -> None: + super().__init__() + if up is None: + up = [] + self.up = up + if down is None: + down = [] + self.down = down + if not len(down) == len(up) + 1: + raise ValueError("`down` must be 1 item longer than `up`") + + self.in_features = in_features + + down_dims = [in_features, *down] + self.path_down = nn.ModuleList() + for i, (d_in, d_out) in enumerate(zip(down_dims[:-1], down_dims[1:])): + down_block = ThinUnetDownBlock(d_in, d_out, size=size, is_first=i == 0) + self.path_down.append(down_block) + + bot_dims = [down[-1], *up] + hor_dims = down_dims[-2::-1] + self.path_up = nn.ModuleList() + for _, (d_bot, d_hor, d_out) in enumerate(zip(bot_dims, hor_dims, up)): + up_block = ThinUnetUpBlock(d_bot, d_hor, d_out, size=size) + self.path_up.append(up_block) + + self.n_params = 0 + for param in self.parameters(): + self.n_params += param.numel() + + def forward(self, inp: torch.Tensor) -> torch.Tensor: + """Run this DISK component forward. + + Feature maps use `(B, C, H, W)`, where `B` is batch size, `C` channels, and `H`/`W` are spatial dimensions. + + Args: + inp: Input tensor passed to the module. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + if inp.size(1) != self.in_features: + fmt = "Expected {} feature channels in input, got {}" + msg = fmt.format(self.in_features, inp.size(1)) + raise ValueError(msg) + + input_size_divisor = 2 ** len(self.up) + if (inp.size(2) % input_size_divisor != 0) or (inp.size(3) % input_size_divisor != 0): + raise ValueError( + f"Input image shape must be divisible by {input_size_divisor} (got {inp.size()}). " + "This is not inherent to DISK, but to the U-Net architecture used in pretrained models. " + "Please F.pad if necessary." + ) + + features = [inp] + for layer in self.path_down: + features.append(layer(features[-1])) + + f_bot = features[-1] + features_horizontal = features[-2::-1] + + for layer, f_hor in zip(self.path_up, features_horizontal): + f_bot = layer(f_bot, f_hor) + + return f_bot diff --git a/kornia/feature/disk/detector.py b/kornia/feature/disk/detector.py new file mode 100644 index 0000000..206bb14 --- /dev/null +++ b/kornia/feature/disk/detector.py @@ -0,0 +1,59 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch + +from kornia.geometry.subpix import nms2d + +from .structs import Keypoints + + +def heatmap_to_keypoints( + heatmap: torch.Tensor, n: int | None = None, window_size: int = 5, score_threshold: float = 0.0 +) -> list[Keypoints]: + """Inference-time nms-based detection protocol.""" + heatmap = heatmap.squeeze(1) + nms_mask = nms2d(heatmap.unsqueeze(1), (window_size, window_size), mask_only=True).squeeze(1) + nmsed = nms_mask & (heatmap > score_threshold) + + keypoints = [] + for b in range(heatmap.shape[0]): + yx = nmsed[b].nonzero(as_tuple=False) + detection_logp = heatmap[b][nmsed[b]] + xy = yx.flip((1,)) + + if n is not None: + n_ = min(n + 1, detection_logp.numel()) + # torch.kthvalue picks in ascending order and we want to pick in + # descending order, so we pick n-th smallest among -logp to get + # -threshold + minus_threshold, _indices = torch.kthvalue(-detection_logp, n_) + mask = detection_logp > -minus_threshold + + xy = xy[mask] + detection_logp = detection_logp[mask] + + # it may be that due to numerical saturation on the threshold we have + # more than n keypoints, so we need to clip them + xy = xy[:n] + detection_logp = detection_logp[:n] + + keypoints.append(Keypoints(xy, detection_logp)) + + return keypoints diff --git a/kornia/feature/disk/disk.py b/kornia/feature/disk/disk.py new file mode 100644 index 0000000..06da385 --- /dev/null +++ b/kornia/feature/disk/disk.py @@ -0,0 +1,160 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +import torch.nn.functional as F +from torch import nn + +from ._unets import Unet +from .detector import heatmap_to_keypoints +from .structs import DISKFeatures + + +class DISK(nn.Module): + r"""nn.Module which detects and described local features in an image using the DISK method. + + See :cite:`tyszkiewicz2020disk` for details. + + .. image:: _static/img/disk_outdoor_depth.jpg + + Args: + desc_dim: The dimension of the descriptor. + unet: The U-Net to use. If None, a default U-Net is used. Kornia doesn't provide the training code for DISK + so this is only useful when using a custom checkpoint trained using the code released with the paper. + The unet should take as input a torch.Tensor of shape :math:`(B, C, H, W)` and output + a torch.Tensor of shape + :math:`(B, \mathrm{desc\_dim} + 1, H, W)`. + + Example: + >>> disk = DISK.from_pretrained('depth') + >>> images = torch.rand(1, 3, 256, 256) + >>> features = disk(images) + + """ + + def __init__(self, desc_dim: int = 128, unet: None | nn.Module = None) -> None: + super().__init__() + + self.desc_dim = desc_dim + + if unet is None: + unet = Unet(in_features=3, size=5, down=[16, 32, 64, 64, 64], up=[64, 64, 64, desc_dim + 1]) + self.unet = unet + + def heatmap_and_dense_descriptors(self, images: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Return the heatmap and the dense descriptors. + + .. image:: _static/img/DISK.png + + Args: + images: The image to detect features in. Shape :math:`(B, 3, H, W)`. + + Returns: + A tuple of dense detection scores and descriptors. + Shapes are :math:`(B, 1, H, W)` and :math:`(B, D, H, W)`, where + :math:`D` is the descriptor dimension. + + """ + unet_output = self.unet(images) + + if unet_output.shape[1] != self.desc_dim + 1: + raise ValueError( + f"U-Net output has {unet_output.shape[1]} channels, but expected self.desc_dim={self.desc_dim} + 1." + ) + + descriptors = unet_output[:, : self.desc_dim] + heatmaps = unet_output[:, self.desc_dim :] + + return heatmaps, descriptors + + def forward( + self, + images: torch.Tensor, + n: int | None = None, + window_size: int = 5, + score_threshold: float = 0.0, + pad_if_not_divisible: bool = False, + ) -> list[DISKFeatures]: + """Detect features in an image, returning keypoint locations, descriptors and detection scores. + + Args: + images: The image to detect features in. Shape :math:`(B, 3, H, W)`. + n: The maximum number of keypoints to detect. If None, all keypoints are returned. + window_size: The size of the non-maxima suppression window used to filter detections. + score_threshold: The minimum score a detection must have to be returned. + See :py:class:`DISKFeatures` for details. + pad_if_not_divisible: if True, the non-16 divisible input is zero-padded to the closest 16-multiply + + Returns: + A list of length :math:`B` containing the detected features. + + """ + B = images.shape[0] + if pad_if_not_divisible: + h, w = images.shape[2:] + pd_h = 16 - h % 16 if h % 16 > 0 else 0 + pd_w = 16 - w % 16 if w % 16 > 0 else 0 + images = F.pad(images, (0, pd_w, 0, pd_h), value=0.0) + + heatmaps, descriptors = self.heatmap_and_dense_descriptors(images) + if pad_if_not_divisible: + heatmaps = heatmaps[..., :h, :w] + descriptors = descriptors[..., :h, :w] + + keypoints = heatmap_to_keypoints(heatmaps, n=n, window_size=window_size, score_threshold=score_threshold) + + features = [] + for i in range(B): + features.append(keypoints[i].merge_with_descriptors(descriptors[i])) + + return features + + @classmethod + def from_pretrained(cls, checkpoint: str = "depth", device: torch.device | None = None) -> DISK: + r"""Load a pretrained model. + + Depth model was trained using depth map supervision and is slightly more precise but biased to detect keypoints + only where SfM depth is available. Epipolar model was trained using epipolar geometry supervision and + is less precise but detects keypoints everywhere where they are matchable. The difference is especially + pronounced on thin structures and on edges of objects. + + Args: + checkpoint: The checkpoint to load. One of 'depth' or 'epipolar'. + device: The device to load the model to. + + Returns: + The pretrained model. + + """ + urls = { + "depth": "https://raw.githubusercontent.com/cvlab-epfl/disk/master/depth-save.pth", + "epipolar": "https://raw.githubusercontent.com/cvlab-epfl/disk/master/epipolar-save.pth", + } + + if checkpoint not in urls: + raise ValueError(f"Unknown pretrained model: {checkpoint}") + + if device is None: + device = torch.device("cpu") + pretrained_dict = torch.hub.load_state_dict_from_url(urls[checkpoint], map_location=device) + + model: DISK = cls().to(device) + model.load_state_dict(pretrained_dict["extractor"]) + model.eval() + return model diff --git a/kornia/feature/disk/structs.py b/kornia/feature/disk/structs.py new file mode 100644 index 0000000..70cff48 --- /dev/null +++ b/kornia/feature/disk/structs.py @@ -0,0 +1,111 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Union + +import torch +import torch.nn.functional as F + + +@dataclass +class DISKFeatures: + r"""A data structure holding DISK keypoints, descriptors and detection scores for an image. + + Since DISK detects a varying number of keypoints per image, `DISKFeatures` is not batched. + + Args: + keypoints: torch.Tensor of shape :math:`(N, 2)`, where :math:`N` is the number of keypoints. + descriptors: torch.Tensor of shape :math:`(N, D)`, where :math:`D` is the descriptor dimension. + detection_scores: torch.Tensor of shape :math:`(N,)` where the detection score can be interpreted as + the log-probability of keeping a keypoint after it has been proposed (see the paper + section *Method → Feature distribution* for details). + + """ + + keypoints: torch.Tensor + descriptors: torch.Tensor + detection_scores: torch.Tensor + + @property + def n(self) -> int: + """Return the number of stored keypoints. + + Returns: + Number of detected DISK keypoints stored in this feature container. + """ + return self.keypoints.shape[0] + + @property + def device(self) -> Union[str, torch.device, None]: + """Return the device of the stored tensors or module parameters. + + Returns: + Device used by the model parameters. + """ + return self.keypoints.device + + @property + def x(self) -> torch.Tensor: + """Accesses the x coordinates of keypoints (along image width).""" + return self.keypoints[:, 0] + + @property + def y(self) -> torch.Tensor: + """Accesses the y coordinates of keypoints (along image height).""" + return self.keypoints[:, 1] + + def to(self, *args: Any, **kwargs: Any) -> DISKFeatures: + """Call :func:`torch.Tensor.to` on each torch.tensor to move the keypoints, descriptors and detection scores to + the specified device and/or data type. + + Args: + *args: Arguments passed to :func:`torch.Tensor.to`. + **kwargs: Keyword arguments passed to :func:`torch.Tensor.to`. + + Returns: + A new DISKFeatures object with tensors of appropriate type and location. + + """ # noqa:D205 + return DISKFeatures( + self.keypoints.to(*args, **kwargs), + self.descriptors.to(*args, **kwargs), + self.detection_scores.to(*args, **kwargs), + ) + + +@dataclass +class Keypoints: + """A temporary struct used to store keypoint detections and their log-probabilities. + + After construction, merge_with_descriptors is used to select corresponding descriptors from unet output. + """ + + xys: torch.Tensor + detection_logp: torch.Tensor + + def merge_with_descriptors(self, descriptors: torch.Tensor) -> DISKFeatures: + """Select descriptors from a dense `descriptors` torch.Tensor, at locations given by `self.xys`.""" + dtype = descriptors.dtype + x, y = self.xys.T + + desc = descriptors[:, y, x].T + desc = F.normalize(desc, dim=-1) + + return DISKFeatures(self.xys.to(dtype), desc, self.detection_logp.to(dtype)) diff --git a/kornia/feature/hardnet.py b/kornia/feature/hardnet.py new file mode 100644 index 0000000..ca36c6e --- /dev/null +++ b/kornia/feature/hardnet.py @@ -0,0 +1,225 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK_SHAPE +from kornia.core.utils import is_mps_tensor_safe + +urls: Dict[str, str] = {} +urls["hardnet++"] = "https://github.com/DagnyT/hardnet/raw/master/pretrained/pretrained_all_datasets/HardNet++.pth" +urls["liberty_aug"] = ( + "https://github.com/DagnyT/hardnet/raw/master/pretrained/train_liberty_with_aug/checkpoint_liberty_with_aug.pth" +) +urls["hardnet8v2"] = "http://cmp.felk.cvut.cz/~mishkdmy/hardnet8v2.pt" + + +class HardNet(nn.Module): + r"""Module, which computes HardNet descriptors of given grayscale patches of 32x32. + + This is based on the original code from paper "Working hard to know your neighbor's + margins: Local descriptor learning loss". See :cite:`HardNet2017` for more details. + + Args: + pretrained: Download and set pretrained weights to the model. + + Returns: + torch.Tensor: HardNet descriptor of the patches. + + Shape: + - Input: :math:`(B, 1, 32, 32)` + - Output: :math:`(B, 128)` + + Examples: + >>> input = torch.rand(16, 1, 32, 32) + >>> hardnet = HardNet() + >>> descs = hardnet(input) # 16x128 + + """ + + patch_size = 32 + + def __init__(self, pretrained: bool = False) -> None: + super().__init__() + self.features = nn.Sequential( + nn.Conv2d(1, 32, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(32, affine=False), + nn.ReLU(), + nn.Conv2d(32, 32, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(32, affine=False), + nn.ReLU(), + nn.Conv2d(32, 64, kernel_size=3, stride=2, padding=1, bias=False), + nn.BatchNorm2d(64, affine=False), + nn.ReLU(), + nn.Conv2d(64, 64, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(64, affine=False), + nn.ReLU(), + nn.Conv2d(64, 128, kernel_size=3, stride=2, padding=1, bias=False), + nn.BatchNorm2d(128, affine=False), + nn.ReLU(), + nn.Conv2d(128, 128, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(128, affine=False), + nn.ReLU(), + nn.Dropout(0.3), + nn.Conv2d(128, 128, kernel_size=8, bias=False), + nn.BatchNorm2d(128, affine=False), + ) + + # use torch.hub to load pretrained model + if pretrained: + pretrained_dict = torch.hub.load_state_dict_from_url(urls["liberty_aug"], map_location=torch.device("cpu")) + self.load_state_dict(pretrained_dict["state_dict"], strict=True) + self.eval() + + @staticmethod + def _normalize_input(x: torch.Tensor, eps: float = 1e-6) -> torch.Tensor: + """Normalize the input by batch.""" + if not is_mps_tensor_safe(x): + sp, mp = torch.std_mean(x, dim=(-3, -2, -1), keepdim=True) + else: + mp = torch.mean(x, dim=(-3, -2, -1), keepdim=True) + sp = torch.std(x, dim=(-3, -2, -1), keepdim=True) + # WARNING: we need to .detach() input, otherwise the gradients produced by + # the patches extractor with F.grid_sample are very noisy, making the detector + # training totally unstable. + return (x - mp.detach()) / (sp.detach() + eps) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Compute HardNet descriptors for normalized 32x32 patches. + + Args: + input: Grayscale patch tensor of shape :math:`(B, 1, 32, 32)`. + + Returns: + L2-normalized descriptor tensor with shape :math:`(B, 128)`. + """ + KORNIA_CHECK_SHAPE(input, ["B", "1", "32", "32"]) + x_norm: torch.Tensor = self._normalize_input(input) + x_features: torch.Tensor = self.features(x_norm) + x_out = x_features.view(x_features.size(0), -1) + return F.normalize(x_out, dim=1) + + +class HardNet8(nn.Module): + r"""Module, which computes HardNet8 descriptors of given grayscale patches of 32x32. + + This is based on the original code from paper "Improving the HardNet Descriptor". + See :cite:`HardNet2020` for more details. + + Args: + pretrained: Download and set pretrained weights to the model. + + Returns: + torch.Tensor: HardNet8 descriptor of the patches. + + Shape: + - Input: :math:`(B, 1, 32, 32)` + - Output: :math:`(B, 128)` + + Examples: + >>> input = torch.rand(16, 1, 32, 32) + >>> hardnet = HardNet8() + >>> descs = hardnet(input) # 16x128 + + """ + + patch_size = 32 + + def __init__(self, pretrained: bool = False) -> None: + super().__init__() + self.features = nn.Sequential( + nn.Conv2d(1, 32, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(32, affine=False), + nn.ReLU(), + nn.Conv2d(32, 32, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(32, affine=False), + nn.ReLU(), + nn.Conv2d(32, 64, kernel_size=3, stride=2, padding=1, bias=False), + nn.BatchNorm2d(64, affine=False), + nn.ReLU(), + nn.Conv2d(64, 64, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(64, affine=False), + nn.ReLU(), + nn.Conv2d(64, 128, kernel_size=3, stride=2, padding=1, bias=False), + nn.BatchNorm2d(128, affine=False), + nn.ReLU(), + nn.Conv2d(128, 128, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(128, affine=False), + nn.ReLU(), + nn.Conv2d(128, 256, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(256, affine=False), + nn.ReLU(), + nn.Dropout(0.3), + nn.Conv2d(256, 512, kernel_size=8, bias=False), + nn.BatchNorm2d(512, affine=False), + ) + self.features.apply(self.weights_init) + self.register_buffer("components", torch.ones(512, 128, dtype=torch.float)) + self.register_buffer("mean", torch.zeros(512, dtype=torch.float)) + + # use torch.hub to load pretrained model + if pretrained: + pretrained_dict = torch.hub.load_state_dict_from_url(urls["hardnet8v2"], map_location=torch.device("cpu")) + self.load_state_dict(pretrained_dict, strict=True) + self.eval() + + @staticmethod + def weights_init(m: object) -> None: + """Initialize convolutional layers for HardNet8. + + Args: + m: Module instance passed by ``nn.Module.apply``. + """ + if isinstance(m, nn.Conv2d): + nn.init.orthogonal_(m.weight.data, gain=0.6) + if m.bias is not None: + nn.init.constant_(m.bias.data, 0.01) + + @staticmethod + def _normalize_input(x: torch.Tensor, eps: float = 1e-7) -> torch.Tensor: + """Normalize the input by batch.""" + if not is_mps_tensor_safe(x): + sp, mp = torch.std_mean(x, dim=(-3, -2, -1), keepdim=True) + else: + mp = torch.mean(x, dim=(-3, -2, -1), keepdim=True) + sp = torch.std(x, dim=(-3, -2, -1), keepdim=True) + # WARNING: we need to .detach() input, otherwise the gradients produced by + # the patches extractor with F.grid_sample are very noisy, making the detector + # training totally unstable. + return (x - mp.detach()) / (sp.detach() + eps) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Compute HardNet8 descriptors with PCA projection. + + Args: + input: Grayscale patch tensor of shape :math:`(B, 1, 32, 32)`. + + Returns: + L2-normalized descriptor tensor after learned PCA projection. + """ + KORNIA_CHECK_SHAPE(input, ["B", "1", "32", "32"]) + x_norm: torch.Tensor = self._normalize_input(input) + x_features: torch.Tensor = self.features(x_norm) + mean: torch.Tensor = torch.jit.annotate(torch.Tensor, self.mean) + components: torch.Tensor = torch.jit.annotate(torch.Tensor, self.components) + x_prePCA = F.normalize(x_features.view(x_features.size(0), -1)) + pca = torch.mm(x_prePCA - mean, components) + return F.normalize(pca, dim=1) diff --git a/kornia/feature/hynet.py b/kornia/feature/hynet.py new file mode 100644 index 0000000..b5ee2ab --- /dev/null +++ b/kornia/feature/hynet.py @@ -0,0 +1,302 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict + +import torch +from torch import nn + +urls: Dict[str, str] = {} +urls["liberty"] = "https://github.com/ducha-aiki/Key.Net-Pytorch/raw/main/model/HyNet/weights/HyNet_LIB.pth" # pylint: disable +urls["notredame"] = "https://github.com/ducha-aiki/Key.Net-Pytorch/raw/main/model/HyNet/weights/HyNet_ND.pth" # pylint: disable +urls["yosemite"] = "https://github.com/ducha-aiki/Key.Net-Pytorch/raw/main/model/HyNet/weights/HyNet_YOS.pth" # pylint: disable + + +class FilterResponseNorm2d(nn.Module): + r"""Feature Response Normalization layer from 'Filter Response Normalization Layer: Eliminating Batch Dependence + in the Training of Deep Neural Networks', see :cite:`FRN2019` for more details. + + .. math:: + y = \gamma \times \frac{x}{\sqrt{\mathrm{E}[x^2]} + |\epsilon|} + \beta + + + Args: + num_features: number of channels + eps: normalization constant + is_bias: use bias + is_scale: use scale + drop_rate: dropout rate, + is_eps_leanable: if eps is learnable + + Returns: + torch.Tensor: Normalized features + + Shape: + - Input: :math:`(B, \text{num_features}, H, W)` + - Output: :math:`(B, \text{num_features}, H, W)` + + """ # noqa: D205 + + def __init__( + self, + num_features: int, + eps: float = 1e-6, + is_bias: bool = True, + is_scale: bool = True, + is_eps_leanable: bool = False, + ) -> None: + super().__init__() + + self.num_features = num_features + self.init_eps = eps + self.is_eps_leanable = is_eps_leanable + self.is_bias = is_bias + self.is_scale = is_scale + + self.weight = nn.Parameter(torch.ones(1, num_features, 1, 1), requires_grad=True) + self.bias = nn.Parameter(torch.zeros(1, num_features, 1, 1), requires_grad=True) + if is_eps_leanable: + self.eps = nn.Parameter(torch.tensor(1), requires_grad=True) + else: + self.register_buffer("eps", torch.tensor([eps])) + self.reset_parameters() + + def reset_parameters(self) -> None: + """Reset learnable parameters to their initial values. + + Returns: + None. The module parameters are reset in place. + """ + nn.init.ones_(self.weight) + nn.init.zeros_(self.bias) + if self.is_eps_leanable: + nn.init.constant_(self.eps, self.init_eps) + + def extra_repr(self) -> str: + """Return a compact text summary for module printing. + + Returns: + Readable string summarizing the module configuration. + """ + return "num_features={num_features}, eps={init_eps}".format(**self.__dict__) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # Compute the mean norm of activations per channel. + """Run the HyNet normalization or descriptor layer. + + Patch tensors use `(B, C, H, W)`. For HyNet descriptors the expected input is usually grayscale `(B, 1, 32, 32)` + and the final descriptor has shape `(B, D)`. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + nu2 = x.pow(2).mean(dim=[2, 3], keepdim=True) + + # Perform FRN. + x = x * torch.rsqrt(nu2 + self.eps.abs()) + + # Scale and Bias + if self.is_scale: + x = self.weight * x + if self.is_bias: + x = x + self.bias + return x + + +class TLU(nn.Module): + r"""TLU layer from 'Filter Response Normalization Layer: Eliminating Batch Dependence in the Training of Deep + Neural Networks, see :cite:`FRN2019` for more details. :math:`{\tau}` is learnable per channel. + + .. math:: + y = \max(x, {\tau}) + + Args: + num_features: number of channels + + Returns: + torch.Tensor + + Shape: + - Input: :math:`(B, \text{num_features}, H, W)` + - Output: :math:`(B, \text{num_features}, H, W)` + + """ # noqa:D205 + + def __init__(self, num_features: int) -> None: + """max(y, tau) = max(y - tau, 0) + tau = ReLU(y - tau) + tau.""" + super().__init__() + self.num_features = num_features + self.tau = nn.Parameter(-torch.ones(1, num_features, 1, 1), requires_grad=True) + self.reset_parameters() + + def reset_parameters(self) -> None: + # nn.init.zeros_(self.tau) + """Reset learnable parameters to their initial values. + + Returns: + None. The module parameters are reset in place. + """ + nn.init.constant_(self.tau, -1) + + def extra_repr(self) -> str: + """Return a compact text summary for module printing. + + Returns: + Readable string summarizing the module configuration. + """ + return "num_features={num_features}".format(**self.__dict__) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run the HyNet normalization or descriptor layer. + + Patch tensors use `(B, C, H, W)`. For HyNet descriptors the expected input is usually grayscale `(B, 1, 32, 32)` + and the final descriptor has shape `(B, D)`. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + return torch.max(x, self.tau) + + +class HyNet(nn.Module): + r"""nn.Module, which computes HyNet descriptors of given grayscale patches of 32x32. + + This is based on the original code from paper + "HyNet: Learning Local Descriptor with Hybrid Similarity Measure and Triplet Loss". + See :cite:`hynet2020` for more details. + + Args: + pretrained: Download and set pretrained weights to the model. + is_bias: use bias in TLU layers + is_bias_FRN: use bias in FRN layers + dim_desc: descriptor dimensionality, + drop_rate: dropout rate, + eps_l2_norm: to avoid div by zero + + Returns: + HyNet descriptor of the patches. + + Shape: + - Input: :math:`(B, 1, 32, 32)` + - Output: :math:`(B, 128)` + + Examples: + >>> input = torch.rand(16, 1, 32, 32) + >>> hynet = HyNet() + >>> descs = hynet(input) # 16x128 + + """ + + patch_size = 32 + + def __init__( + self, + pretrained: bool = False, + is_bias: bool = True, + is_bias_FRN: bool = True, + dim_desc: int = 128, + drop_rate: float = 0.3, + eps_l2_norm: float = 1e-10, + ) -> None: + super().__init__() + self.eps_l2_norm = eps_l2_norm + self.dim_desc = dim_desc + self.drop_rate = drop_rate + self.layer1 = nn.Sequential( + FilterResponseNorm2d(1, is_bias=is_bias_FRN), + TLU(1), + nn.Conv2d(1, 32, kernel_size=3, padding=1, bias=is_bias), + FilterResponseNorm2d(32, is_bias=is_bias_FRN), + TLU(32), + ) + + self.layer2 = nn.Sequential( + nn.Conv2d(32, 32, kernel_size=3, padding=1, bias=is_bias), + FilterResponseNorm2d(32, is_bias=is_bias_FRN), + TLU(32), + ) + + self.layer3 = nn.Sequential( + nn.Conv2d(32, 64, kernel_size=3, stride=2, padding=1, bias=is_bias), + FilterResponseNorm2d(64, is_bias=is_bias_FRN), + TLU(64), + ) + + self.layer4 = nn.Sequential( + nn.Conv2d(64, 64, kernel_size=3, padding=1, bias=is_bias), + FilterResponseNorm2d(64, is_bias=is_bias_FRN), + TLU(64), + ) + self.layer5 = nn.Sequential( + nn.Conv2d(64, 128, kernel_size=3, stride=2, padding=1, bias=is_bias), + FilterResponseNorm2d(128, is_bias=is_bias_FRN), + TLU(128), + ) + + self.layer6 = nn.Sequential( + nn.Conv2d(128, 128, kernel_size=3, padding=1, bias=is_bias), + FilterResponseNorm2d(128, is_bias=is_bias_FRN), + TLU(128), + ) + + self.layer7 = nn.Sequential( + nn.Dropout(self.drop_rate), + nn.Conv2d(128, self.dim_desc, kernel_size=8, bias=False), + nn.BatchNorm2d(self.dim_desc, affine=False), + ) + + self.desc_norm = nn.LocalResponseNorm(2 * self.dim_desc, 2.0 * self.dim_desc, 0.5, 0.0) + # use torch.hub to load pretrained model + if pretrained: + pretrained_dict = torch.hub.load_state_dict_from_url(urls["liberty"], map_location=torch.device("cpu")) + self.load_state_dict(pretrained_dict, strict=True) + self.eval() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run the HyNet normalization or descriptor layer. + + Patch tensors use `(B, C, H, W)`. For HyNet descriptors the expected input is usually grayscale `(B, 1, 32, 32)` + and the final descriptor has shape `(B, D)`. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + x = self.layer5(x) + x = self.layer6(x) + x = self.layer7(x) + x = self.desc_norm(x + self.eps_l2_norm) + x = x.view(x.size(0), -1) + return x diff --git a/kornia/feature/integrated.py b/kornia/feature/integrated.py new file mode 100644 index 0000000..d90882f --- /dev/null +++ b/kornia/feature/integrated.py @@ -0,0 +1,591 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import warnings +from typing import ClassVar, Dict, List, Optional, Tuple, Union + +import torch +from torch import nn + +from kornia.color import rgb_to_grayscale +from kornia.constants import pi +from kornia.core.check import KORNIA_CHECK_LAF +from kornia.geometry.subpix import ConvQuadInterp3d +from kornia.geometry.transform import ScalePyramid + +from .affine_shape import LAFAffNetShapeEstimator +from .hardnet import HardNet +from .keynet import KeyNetDetector +from .laf import extract_patches_from_pyramid, get_laf_center, get_laf_orientation, get_laf_scale, scale_laf +from .lightglue import LightGlue +from .matching import GeometryAwareDescriptorMatcher, _no_match +from .orientation import LAFOrienter, OriNet, PassLAF +from .responses import BlobDoG, BlobDoGSingle, BlobHessian, CornerGFTT +from .scale_space_detector import ( + Detector_config, + MultiResolutionDetector, + ScaleSpaceDetector, + get_default_detector_config, +) +from .siftdesc import SIFTDescriptor + + +def get_laf_descriptors( + img: torch.Tensor, + lafs: torch.Tensor, + patch_descriptor: nn.Module, + patch_size: int = 32, + grayscale_descriptor: bool = True, +) -> torch.Tensor: + r"""Get local descriptors, corresponding to LAFs (keypoints). + + Args: + img: image features with shape :math:`(B,C,H,W)`. + lafs: local affine frames :math:`(B,N,2,3)`. + patch_descriptor: patch descriptor module, e.g. :class:`~kornia.feature.SIFTDescriptor` + or :class:`~kornia.feature.HardNet`. + patch_size: patch size in pixels, which descriptor expects. + grayscale_descriptor: True if ``patch_descriptor`` expects single-channel image. + + Returns: + Local descriptors of shape :math:`(B,N,D)` where :math:`D` is descriptor size. + + """ + KORNIA_CHECK_LAF(lafs) + patch_descriptor = patch_descriptor.to(img) + patch_descriptor.eval() + + timg: torch.Tensor = img + if lafs.shape[1] == 0: + warnings.warn(f"LAF contains no keypoints {lafs.shape}, returning empty torch.Tensor", stacklevel=1) + return torch.empty(lafs.shape[0], lafs.shape[1], 128, dtype=lafs.dtype, device=lafs.device) + if grayscale_descriptor and img.size(1) == 3: + timg = rgb_to_grayscale(img) + + patches: torch.Tensor = extract_patches_from_pyramid(timg, lafs, patch_size) + # Descriptor accepts standard torch.Tensor [B, CH, H, W], while patches are [B, N, CH, H, W] shape + # So we need to reshape a bit :) + B, N, CH, H, W = patches.size() + return patch_descriptor(patches.view(B * N, CH, H, W)).view(B, N, -1) + + +class LAFDescriptor(nn.Module): + r"""nn.Module to get local descriptors, corresponding to LAFs (keypoints). + + Internally uses :func:`~kornia.feature.get_laf_descriptors`. + + Args: + patch_descriptor_module: patch descriptor module, e.g. :class:`~kornia.feature.SIFTDescriptor` + or :class:`~kornia.feature.HardNet`. Default: :class:`~kornia.feature.HardNet`. + patch_size: patch size in pixels, which descriptor expects. + grayscale_descriptor: ``True`` if patch_descriptor expects single-channel image. + + """ + + def __init__( + self, + patch_descriptor_module: Optional[nn.Module] = None, + patch_size: int = 32, + grayscale_descriptor: bool = True, + ) -> None: + super().__init__() + if patch_descriptor_module is None: + patch_descriptor_module = HardNet(True) + self.descriptor = patch_descriptor_module + self.patch_size = patch_size + self.grayscale_descriptor = grayscale_descriptor + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}" + f"(descriptor={self.descriptor.__repr__()}, " + f"patch_size={self.patch_size}, " + f"grayscale_descriptor='{self.grayscale_descriptor})" + ) + + def forward(self, img: torch.Tensor, lafs: torch.Tensor) -> torch.Tensor: + r"""Three stage local feature detection. + + First the location and scale of interest points are determined by + detect function. Then affine shape and orientation. + + Args: + img: image features with shape :math:`(B,C,H,W)`. + lafs: local affine frames :math:`(B,N,2,3)`. + + Returns: + Local descriptors of shape :math:`(B,N,D)` where :math:`D` is descriptor size. + + """ + return get_laf_descriptors(img, lafs, self.descriptor, self.patch_size, self.grayscale_descriptor) + + +class LocalFeature(nn.Module): + """nn.Module, which combines local feature detector and descriptor. + + Args: + detector: the detection module. + descriptor: the descriptor module. + scaling_coef: multiplier for change default detector scale (e.g. it is too small for KeyNet by default) + + """ + + def __init__(self, detector: nn.Module, descriptor: LAFDescriptor, scaling_coef: float = 1.0) -> None: + super().__init__() + self.detector = detector + self.descriptor = descriptor + if scaling_coef <= 0: + raise ValueError(f"Scaling coef should be >= 0, got {scaling_coef}") + self.scaling_coef = scaling_coef + + def forward( + self, img: torch.Tensor, mask: Optional[torch.Tensor] = None + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Run forward. + + Args: + img: image to extract features with shape :math:`(B,C,H,W)`. + mask: a mask with weights where to apply the response function. + The shape must be the same as the input image. + + Returns: + - Detected local affine frames with shape :math:`(B,N,2,3)`. + - Response function values for corresponding lafs with shape :math:`(B,N,1)`. + - Local descriptors of shape :math:`(B,N,D)` where :math:`D` is descriptor size. + + """ + lafs, responses = self.detector(img, mask) + lafs = scale_laf(lafs, self.scaling_coef) + descs = self.descriptor(img, lafs) + return (lafs, responses, descs) + + +class SIFTFeature(LocalFeature): + """Convenience module, which implements DoG detector + (Root)SIFT descriptor. + + Using `kornia.feature.MultiResolutionDetector` without blur pyramid Still not as good as OpenCV/VLFeat because of + https://github.com/kornia/kornia/pull/884, + but we are working on it + """ + + def __init__( + self, + num_features: int = 8000, + upright: bool = False, + rootsift: bool = True, + device: Union[None, str, torch.device] = None, + config: Optional[Detector_config] = None, + compile_model: bool = False, + score_threshold: float = 0.0, + ) -> None: + patch_size: int = 41 + if device is None: + device = torch.device("cpu") + if config is None: + config = get_default_detector_config() + detector = MultiResolutionDetector( + BlobDoGSingle(1.0, 1.6), + num_features, + config, + ori_module=PassLAF() if upright else LAFOrienter(19), + aff_module=PassLAF(), + compile_model=compile_model, + score_threshold=score_threshold, + ).to(device) + descriptor = LAFDescriptor( + SIFTDescriptor(patch_size=patch_size, rootsift=rootsift), patch_size=patch_size, grayscale_descriptor=True + ).to(device) + super().__init__(detector, descriptor) + + +class SIFTFeatureScaleSpace(LocalFeature): + """Convenience module, which implements DoG detector + (Root)SIFT descriptor. + + Using `kornia.feature.ScaleSpaceDetector` with blur pyramid. + + Still not as good as OpenCV/VLFeat because of https://github.com/kornia/kornia/pull/884, but we are working on it + """ + + def __init__( + self, + num_features: int = 8000, + upright: bool = False, + rootsift: bool = True, + device: Union[None, str, torch.device] = None, + compile_modules: Union[bool, List[str]] = False, + ) -> None: + if device is None: + device = torch.device("cpu") + patch_size: int = 41 + detector = ScaleSpaceDetector( + num_features, + resp_module=BlobDoG(), + subpix_module=ConvQuadInterp3d(strict_maxima_bonus=0.0), + scale_pyr_module=ScalePyramid(3, 1.6, 32, double_image=True), + ori_module=PassLAF() if upright else LAFOrienter(19), + scale_space_response=True, + minima_are_also_good=True, + mr_size=6.0, + compile_modules=compile_modules, + ).to(device) + descriptor = LAFDescriptor( + SIFTDescriptor(patch_size=patch_size, rootsift=rootsift), patch_size=patch_size, grayscale_descriptor=True + ).to(device) + super().__init__(detector, descriptor) + + +class GFTTAffNetHardNet(LocalFeature): + """Convenience module, which implements GFTT detector + AffNet-HardNet descriptor.""" + + def __init__( + self, + num_features: int = 8000, + upright: bool = False, + device: Union[None, str, torch.device] = None, + compile_modules: Union[bool, List[str]] = False, + ) -> None: + if device is None: + device = torch.device("cpu") + detector = ScaleSpaceDetector( + num_features, + resp_module=CornerGFTT(), + scale_pyr_module=ScalePyramid(3, 1.6, 32, double_image=True), + ori_module=PassLAF() if upright else LAFOrienter(19), + aff_module=LAFAffNetShapeEstimator(True, preserve_orientation=False).eval(), + scale_space_response=False, + minima_are_also_good=False, + mr_size=6.0, + compile_modules=compile_modules, + ).to(device) + descriptor = LAFDescriptor(None, patch_size=32, grayscale_descriptor=True).to(device) + super().__init__(detector, descriptor) + + +class HesAffNetHardNet(LocalFeature): + """Convenience module, which implements Hessian detector + AffNet-HardNet descriptor.""" + + def __init__( + self, + num_features: int = 2048, + upright: bool = False, + device: Union[None, str, torch.device] = None, + compile_modules: Union[bool, List[str]] = False, + ) -> None: + if device is None: + device = torch.device("cpu") + detector = ScaleSpaceDetector( + num_features, + resp_module=BlobHessian(), + scale_pyr_module=ScalePyramid(3, 1.6, 32, double_image=True), + ori_module=PassLAF() if upright else LAFOrienter(19), + aff_module=LAFAffNetShapeEstimator(True, preserve_orientation=False).eval(), + scale_space_response=False, + minima_are_also_good=False, + mr_size=6.0, + compile_modules=compile_modules, + ).to(device) + descriptor = LAFDescriptor(None, patch_size=32, grayscale_descriptor=True).to(device) + super().__init__(detector, descriptor) + + +class KeyNetHardNet(LocalFeature): + """Convenience module, which implements KeyNet detector + HardNet descriptor.""" + + def __init__( + self, + num_features: int = 8000, + upright: bool = False, + device: Union[None, str, torch.device] = None, + scale_laf: float = 1.0, + compile_model: bool = False, + score_threshold: float = 0.0, + ) -> None: + if device is None: + device = torch.device("cpu") + ori_module = PassLAF() if upright else LAFOrienter(angle_detector=OriNet(True)) + detector = KeyNetDetector( + True, + num_features=num_features, + ori_module=ori_module, + compile_model=compile_model, + score_threshold=score_threshold, + ).to(device) + descriptor = LAFDescriptor(None, patch_size=32, grayscale_descriptor=True).to(device) + super().__init__(detector, descriptor, scale_laf) + + +class KeyNetAffNetHardNet(LocalFeature): + """Convenience module, which implements KeyNet detector + AffNet + HardNet descriptor. + + .. image:: _static/img/keynet_affnet.jpg + """ + + def __init__( + self, + num_features: int = 8000, + upright: bool = False, + device: Union[None, str, torch.device] = None, + scale_laf: float = 1.0, + compile_model: bool = False, + score_threshold: float = 0.0, + ) -> None: + if device is None: + device = torch.device("cpu") + ori_module = PassLAF() if upright else LAFOrienter(angle_detector=OriNet(True)) + detector = KeyNetDetector( + True, + num_features=num_features, + ori_module=ori_module, + aff_module=LAFAffNetShapeEstimator(True, preserve_orientation=False).eval(), + compile_model=compile_model, + score_threshold=score_threshold, + ).to(device) + descriptor = LAFDescriptor(None, patch_size=32, grayscale_descriptor=True).to(device) + super().__init__(detector, descriptor, scale_laf) + + +class LocalFeatureMatcher(nn.Module): + r"""nn.Module, which finds correspondences between two images based on local features. + + Args: + local_feature: Local feature detector. See :class:`~kornia.feature.GFTTAffNetHardNet`. + matcher: Descriptor matcher, see :class:`~kornia.feature.DescriptorMatcher`. + + Returns: + Dict[str, torch.Tensor]: Dictionary with image correspondences and confidence scores. + + Example: + >>> img1 = torch.rand(1, 1, 320, 200) + >>> img2 = torch.rand(1, 1, 128, 128) + >>> input = {"image0": img1, "image1": img2} + >>> gftt_hardnet_matcher = LocalFeatureMatcher( + ... GFTTAffNetHardNet(10), kornia.feature.DescriptorMatcher('snn', 0.8) + ... ) + >>> out = gftt_hardnet_matcher(input) + + """ + + def __init__(self, local_feature: nn.Module, matcher: nn.Module) -> None: + super().__init__() + self.local_feature = local_feature + self.matcher = matcher + self.eval() + + def extract_features(self, image: torch.Tensor, mask: Optional[torch.Tensor] = None) -> Dict[str, torch.Tensor]: + """Extract features from simple image.""" + lafs0, resps0, descs0 = self.local_feature(image, mask) + return {"lafs": lafs0, "responses": resps0, "descriptors": descs0} + + def no_match_output(self, device: Union[str, torch.device, None], dtype: torch.dtype) -> Dict[str, torch.Tensor]: + """Create an empty output structure when no correspondences are found. + + Args: + device: Device on which empty tensors should be allocated. + dtype: Floating-point dtype used for keypoints/LAF/confidence. + + Returns: + Dictionary with correctly typed empty tensors for all expected + output fields. + """ + return { + "keypoints0": torch.empty(0, 2, device=device, dtype=dtype), + "keypoints1": torch.empty(0, 2, device=device, dtype=dtype), + "lafs0": torch.empty(0, 0, 2, 3, device=device, dtype=dtype), + "lafs1": torch.empty(0, 0, 2, 3, device=device, dtype=dtype), + "confidence": torch.empty(0, device=device, dtype=dtype), + "batch_indexes": torch.empty(0, device=device, dtype=torch.long), + } + + def forward(self, data: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """Run forward. + + Args: + data: dictionary containing the input data in the following format: + + Keyword Args: + image0: left image with shape :math:`(N, 1, H1, W1)`. + image1: right image with shape :math:`(N, 1, H2, W2)`. + mask0 (optional): left image mask. '0' indicates a padded position :math:`(N, H1, W1)`. + mask1 (optional): right image mask. '0' indicates a padded position :math:`(N, H2, W2)`. + + Returns: + - ``keypoints0``, matching keypoints from image0 :math:`(NC, 2)`. + - ``keypoints1``, matching keypoints from image1 :math:`(NC, 2)`. + - ``confidence``, confidence score [0, 1] :math:`(NC)`. + - ``lafs0``, matching LAFs from image0 :math:`(1, NC, 2, 3)`. + - ``lafs1``, matching LAFs from image1 :math:`(1, NC, 2, 3)`. + - ``batch_indexes``, batch indexes for the keypoints and lafs :math:`(NC)`. + + """ + num_image_pairs: int = data["image0"].shape[0] + + if ("lafs0" not in data.keys()) or ("descriptors0" not in data.keys()): + # One can supply pre-extracted local features + feats_dict0: Dict[str, torch.Tensor] = self.extract_features(data["image0"]) + lafs0, descs0 = feats_dict0["lafs"], feats_dict0["descriptors"] + else: + lafs0, descs0 = data["lafs0"], data["descriptors0"] + + if ("lafs1" not in data.keys()) or ("descriptors1" not in data.keys()): + feats_dict1: Dict[str, torch.Tensor] = self.extract_features(data["image1"]) + lafs1, descs1 = feats_dict1["lafs"], feats_dict1["descriptors"] + else: + lafs1, descs1 = data["lafs1"], data["descriptors1"] + + keypoints0: torch.Tensor = get_laf_center(lafs0) + keypoints1: torch.Tensor = get_laf_center(lafs1) + + out_keypoints0: List[torch.Tensor] = [] + out_keypoints1: List[torch.Tensor] = [] + out_confidence: List[torch.Tensor] = [] + out_batch_indexes: List[torch.Tensor] = [] + out_lafs0: List[torch.Tensor] = [] + out_lafs1: List[torch.Tensor] = [] + + for batch_idx in range(num_image_pairs): + dists, idxs = self.matcher(descs0[batch_idx], descs1[batch_idx]) + if len(idxs) == 0: + continue + + current_keypoints_0 = keypoints0[batch_idx, idxs[:, 0]] + current_keypoints_1 = keypoints1[batch_idx, idxs[:, 1]] + current_lafs_0 = lafs0[batch_idx, idxs[:, 0]] + current_lafs_1 = lafs1[batch_idx, idxs[:, 1]] + + out_confidence.append(1.0 - dists) + batch_idxs = batch_idx * torch.ones(len(dists), device=keypoints0.device, dtype=torch.long) + out_keypoints0.append(current_keypoints_0) + out_keypoints1.append(current_keypoints_1) + out_lafs0.append(current_lafs_0) + out_lafs1.append(current_lafs_1) + out_batch_indexes.append(batch_idxs) + + if len(out_batch_indexes) == 0: + return self.no_match_output(data["image0"].device, data["image0"].dtype) + + return { + "keypoints0": torch.cat(out_keypoints0, dim=0).view(-1, 2), + "keypoints1": torch.cat(out_keypoints1, dim=0).view(-1, 2), + "lafs0": torch.cat(out_lafs0, dim=0).view(1, -1, 2, 3), + "lafs1": torch.cat(out_lafs1, dim=0).view(1, -1, 2, 3), + "confidence": torch.cat(out_confidence, dim=0).view(-1), + "batch_indexes": torch.cat(out_batch_indexes, dim=0).view(-1), + } + + +class LightGlueMatcher(GeometryAwareDescriptorMatcher): + """LightGlue-based matcher in kornia API. + + This is based on the original code from paper "LightGlue: Local Feature Matching at Light Speed". + See :cite:`LightGlue2023` for more details. + + Args: + feature_name: type of feature for matching, can be `disk` or `superpoint`. + params: LightGlue params. + + """ + + known_modes: ClassVar[List[str]] = [ + "aliked", + "dedodeb", + "dedodeg", + "disk", + "dog_affnet_hardnet", + "doghardnet", + "keynet_affnet_hardnet", + "sift", + "superpoint", + ] + + def __init__(self, feature_name: str = "disk", params: Optional[Dict] = None) -> None: # type: ignore + feature_name_: str = feature_name.lower() + super().__init__(feature_name_) + self.feature_name = feature_name_ + if params is None: + params = {} + self.params = params + self.matcher = LightGlue(self.feature_name, **params) + + def forward( + self, + desc1: torch.Tensor, + desc2: torch.Tensor, + lafs1: torch.Tensor, + lafs2: torch.Tensor, + hw1: Optional[Tuple[int, int]] = None, + hw2: Optional[Tuple[int, int]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Run forward. + + Args: + desc1: Batch of descriptors of a shape :math:`(B1, D)`. + desc2: Batch of descriptors of a shape :math:`(B2, D)`. + lafs1: LAFs of a shape :math:`(1, B1, 2, 3)`. + lafs2: LAFs of a shape :math:`(1, B2, 2, 3)`. + hw1: Height/width of image. + hw2: Height/width of image. + + Return: + - Descriptor distance of matching descriptors, shape of :math:`(B3, 1)`. + - Long torch.Tensor indexes of matching descriptors in desc1 and desc2, + shape of :math:`(B3, 2)` where :math:`0 <= B3 <= B1`. + + """ + if (desc1.shape[0] < 2) or (desc2.shape[0] < 2): + return _no_match(desc1) + keypoints1 = get_laf_center(lafs1) + keypoints2 = get_laf_center(lafs2) + if len(desc1.shape) == 2: + desc1 = desc1.unsqueeze(0) + if len(desc2.shape) == 2: + desc2 = desc2.unsqueeze(0) + dev = lafs1.device + if hw1 is None: + hw1_ = keypoints1.max(dim=1)[0].squeeze().flip(0) + else: + hw1_ = torch.tensor(hw1, device=dev) + if hw2 is None: + hw2_ = keypoints2.max(dim=1)[0].squeeze().flip(0) + else: + hw2_ = torch.tensor(hw2, device=dev) + ori0 = torch.deg2rad(get_laf_orientation(lafs1).reshape(1, -1)) + ori0[ori0 < 0] += 2.0 * pi + ori1 = torch.deg2rad(get_laf_orientation(lafs2).reshape(1, -1)) + ori1[ori1 < 0] += 2.0 * pi + input_dict = { + "image0": { + "keypoints": keypoints1, + "scales": get_laf_scale(lafs1).reshape(1, -1), + "oris": ori0, + "lafs": lafs1, + "descriptors": desc1, + "image_size": hw1_.flip(0).reshape(-1, 2).to(dev), + }, + "image1": { + "keypoints": keypoints2, + "lafs": lafs2, + "scales": get_laf_scale(lafs2).reshape(1, -1), + "oris": ori1, + "descriptors": desc2, + "image_size": hw2_.flip(0).reshape(-1, 2).to(dev), + }, + } + pred = self.matcher(input_dict) + matches0, mscores0 = pred["matches0"], pred["matching_scores0"] + valid = matches0 > -1 + matches = torch.stack([torch.where(valid)[1], matches0[valid]], -1) + return mscores0[valid].reshape(-1, 1), matches diff --git a/kornia/feature/keynet.py b/kornia/feature/keynet.py new file mode 100644 index 0000000..5a4f5c7 --- /dev/null +++ b/kornia/feature/keynet.py @@ -0,0 +1,227 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import List, Optional + +import torch +import torch.nn.functional as F +from torch import nn +from typing_extensions import TypedDict + +from kornia.filters import SpatialGradient +from kornia.geometry.transform import pyrdown + +from .scale_space_detector import Detector_config, MultiResolutionDetector, get_default_detector_config + + +class KeyNet_conf(TypedDict): + """Define the configuration schema for the KeyNet feature detector. + + Attributes: + num_filters: The number of filters in the convolutional layers. + num_levels: The number of levels in the image pyramid. + """ + + num_filters: int + num_levels: int + kernel_size: int + Detector_conf: Detector_config + + +keynet_default_config: KeyNet_conf = { + # Key.Net Model + "num_filters": 8, + "num_levels": 3, + "kernel_size": 5, + # Extraction Parameters + "Detector_conf": get_default_detector_config(), +} + +KeyNet_URL = "https://github.com/axelBarroso/Key.Net-Pytorch/raw/main/model/weights/keynet_pytorch.pth" + + +class _FeatureExtractor(nn.Module): + """Helper class for KeyNet. + + It loads both, the handcrafted and learnable blocks + """ + + def __init__(self) -> None: + super().__init__() + + self.hc_block = _HandcraftedBlock() + self.lb_block = _LearnableBlock() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x_hc = self.hc_block(x) + x_lb = self.lb_block(x_hc) + return x_lb + + +class _HandcraftedBlock(nn.Module): + """Helper class for KeyNet, it defines the handcrafted filters within the Key.Net handcrafted block.""" + + def __init__(self) -> None: + super().__init__() + self.spatial_gradient = SpatialGradient("sobel", 1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + sobel = self.spatial_gradient(x) + dx, dy = sobel[:, :, 0, :, :], sobel[:, :, 1, :, :] + + sobel_dx = self.spatial_gradient(dx) + dxx, dxy = sobel_dx[:, :, 0, :, :], sobel_dx[:, :, 1, :, :] + + # Only dyy is used from sobel_dy; dyx is discarded. + dyy = self.spatial_gradient(dy)[:, :, 1, :, :] + + hc_feats = torch.cat([dx, dy, dx.square(), dy.square(), dx * dy, dxy, dxy.square(), dxx, dyy, dxx * dyy], 1) + + return hc_feats + + +class _LearnableBlock(nn.Sequential): + """Helper class for KeyNet. + + It defines the learnable blocks within the Key.Net + """ + + def __init__(self, in_channels: int = 10) -> None: + super().__init__() + + self.conv0 = _KeyNetConvBlock(in_channels) + self.conv1 = _KeyNetConvBlock() + self.conv2 = _KeyNetConvBlock() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.conv2(self.conv1(self.conv0(x))) + return x + + +def _KeyNetConvBlock( + in_channels: int = 8, + out_channels: int = 8, + kernel_size: int = 5, + stride: int = 1, + padding: int = 2, + dilation: int = 1, +) -> nn.Sequential: + """Create KeyNet Conv Block. + + Default learnable convolutional block for KeyNet. + """ + return nn.Sequential( + nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding, dilation), + nn.BatchNorm2d(out_channels), + nn.ReLU(inplace=True), + ) + + +class KeyNet(nn.Module): + """Key.Net model definition -- local feature detector (response function). + + This is based on the original code + from paper "Key.Net: Keypoint Detection by Handcrafted and Learned CNN Filters". See :cite:`KeyNet2019` for + more details. + + .. image:: _static/img/KeyNet.png + + Args: + pretrained: Download and set pretrained weights to the model. + keynet_conf: Dict with initialization parameters. Do not pass it, unless you know what you are doing`. + + Returns: + KeyNet response score. + + Shape: + - Input: :math:`(B, 1, H, W)` + - Output: :math:`(B, 1, H, W)` + + """ + + def __init__(self, pretrained: bool = False, keynet_conf: Optional[KeyNet_conf] = None) -> None: + super().__init__() + if keynet_conf is None: + keynet_conf = keynet_default_config + + num_filters = keynet_conf["num_filters"] + self.num_levels = keynet_conf["num_levels"] + kernel_size = keynet_conf["kernel_size"] + padding = kernel_size // 2 + + self.feature_extractor = _FeatureExtractor() + self.last_conv = nn.Sequential( + nn.Conv2d( + in_channels=num_filters * self.num_levels, out_channels=1, kernel_size=kernel_size, padding=padding + ), + nn.ReLU(inplace=True), + ) + # use torch.hub to load pretrained model + if pretrained: + pretrained_dict = torch.hub.load_state_dict_from_url(KeyNet_URL, map_location=torch.device("cpu")) + self.load_state_dict(pretrained_dict["state_dict"], strict=True) + self.eval() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """X - input image.""" + h, w = x.shape[2], x.shape[3] + feats: List[torch.Tensor] = [self.feature_extractor(x)] + for _ in range(1, self.num_levels): + x = pyrdown(x, factor=1.2) + feats_i = self.feature_extractor(x) + feats_i = F.interpolate(feats_i, size=(h, w), mode="bilinear", align_corners=False) + feats.append(feats_i) + scores = self.last_conv(torch.cat(feats, 1)) + return scores + + +class KeyNetDetector(MultiResolutionDetector): + """Multi-scale feature detector based on KeyNet. + + This is based on the original code from paper + "Key.Net: Keypoint Detection by Handcrafted and Learned CNN Filters". + See :cite:`KeyNet2019` for more details. + + .. image:: _static/img/keynet.jpg + + Args: + pretrained: Download and set pretrained weights to the model. + num_features: Number of features to detect. + keynet_conf: Dict with initialization parameters. Do not pass it, unless you know what you are doing`. + ori_module: for local feature orientation estimation. Default: :class:`~kornia.feature.PassLAF`, + which does nothing. See :class:`~kornia.feature.LAFOrienter` for details. + aff_module: for local feature affine shape estimation. Default: :class:`~kornia.feature.PassLAF`, + which does nothing. See :class:`~kornia.feature.LAFAffineShapeEstimator` for details. + + """ + + def __init__( + self, + pretrained: bool = False, + num_features: int = 2048, + keynet_conf: Optional[KeyNet_conf] = None, + ori_module: Optional[nn.Module] = None, + aff_module: Optional[nn.Module] = None, + compile_model: bool = False, + score_threshold: float = 0.0, + ) -> None: + if keynet_conf is None: + keynet_conf = keynet_default_config + model = KeyNet(pretrained, keynet_conf) + super().__init__( + model, num_features, keynet_conf["Detector_conf"], ori_module, aff_module, compile_model, score_threshold + ) diff --git a/kornia/feature/laf.py b/kornia/feature/laf.py new file mode 100644 index 0000000..8118af8 --- /dev/null +++ b/kornia/feature/laf.py @@ -0,0 +1,634 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F + +from kornia.core.check import KORNIA_CHECK_LAF, KORNIA_CHECK_SHAPE +from kornia.geometry.conversions import angle_to_rotation_matrix, convert_points_from_homogeneous +from kornia.geometry.linalg import transform_points +from kornia.geometry.transform import pyrdown + + +def get_laf_scale(LAF: torch.Tensor) -> torch.Tensor: + """Return a scale of the LAFs. + + Args: + LAF: :math:`(B, N, 2, 3)` + + Returns: + scale :math:`(B, N, 1, 1)` + + Example: + >>> input = torch.ones(1, 5, 2, 3) # BxNx2x3 + >>> output = get_laf_scale(input) # BxNx1x1 + + """ + KORNIA_CHECK_LAF(LAF) + eps = 1e-10 + out = LAF[..., 0:1, 0:1] * LAF[..., 1:2, 1:2] - LAF[..., 1:2, 0:1] * LAF[..., 0:1, 1:2] + eps + return out.abs().sqrt() + + +def get_laf_center(LAF: torch.Tensor) -> torch.Tensor: + """Return a center (keypoint) of the LAFs. + + The convention is that center of 5-pixel image (coordinates from 0 to 4) is 2, and not 2.5. + + Args: + LAF: :math:`(B, N, 2, 3)` + + Returns: + xy :math:`(B, N, 2)` + + Example: + >>> input = torch.ones(1, 5, 2, 3) # BxNx2x3 + >>> output = get_laf_center(input) # BxNx2 + + """ + KORNIA_CHECK_LAF(LAF) + out = LAF[..., 2] + return out + + +def get_laf_orientation(LAF: torch.Tensor) -> torch.Tensor: + """Return orientation of the LAFs, in degrees. + + Args: + LAF: :math:`(B, N, 2, 3)` + + Returns: + angle in degrees :math:`(B, N, 1)` + + Example: + >>> input = torch.ones(1, 5, 2, 3) # BxNx2x3 + >>> output = get_laf_orientation(input) # BxNx1 + + """ + KORNIA_CHECK_LAF(LAF) + angle_rad = torch.atan2(LAF[..., 0, 1], LAF[..., 0, 0]) + return torch.rad2deg(angle_rad).unsqueeze(-1) + + +def rotate_laf(LAF: torch.Tensor, angles_degrees: torch.Tensor) -> torch.Tensor: + """Apply additional rotation to the LAFs. + + Compared to `set_laf_orientation`, the resulting rotation is original LAF orientation plus angles_degrees. + + Args: + LAF: :math:`(B, N, 2, 3)` + angles_degrees: :math:`(B, N, 1)` in degrees. + + Returns: + LAF oriented with angles :math:`(B, N, 2, 3)` + + """ + KORNIA_CHECK_LAF(LAF) + B, N = LAF.shape[:2] + rotmat = angle_to_rotation_matrix(angles_degrees).view(B * N, 2, 2) + out_laf = LAF.clone() + out_laf[:, :, :2, :2] = torch.bmm(LAF[:, :, :2, :2].reshape(B * N, 2, 2), rotmat).reshape(B, N, 2, 2) + return out_laf + + +def set_laf_orientation(LAF: torch.Tensor, angles_degrees: torch.Tensor) -> torch.Tensor: + """Change the orientation of the LAFs. + + Args: + LAF: :math:`(B, N, 2, 3)` + angles_degrees: :math:`(B, N, 1)` in degrees. + + Returns: + LAF oriented with angles :math:`(B, N, 2, 3)` + + """ + KORNIA_CHECK_LAF(LAF) + _B, _N = LAF.shape[:2] + ori = get_laf_orientation(LAF).reshape_as(angles_degrees) + return rotate_laf(LAF, angles_degrees - ori) + + +def laf_from_center_scale_ori( + xy: torch.Tensor, scale: Optional[torch.Tensor] = None, ori: Optional[torch.Tensor] = None +) -> torch.Tensor: + """Create a LAF from keypoint center, scale and orientation. + + Useful to create kornia LAFs from OpenCV keypoints. + + Args: + xy: :math:`(B, N, 2)`. + scale: :math:`(B, N, 1, 1)`. If not provided, scale = 1.0 is assumed + ori: angle in degrees :math:`(B, N, 1)`. If not provided orientation = 0 is assumed + + Returns: + LAF :math:`(B, N, 2, 3)` + + """ + KORNIA_CHECK_SHAPE(xy, ["B", "N", "2"]) + device = xy.device + dtype = xy.dtype + B, N = xy.shape[:2] + if scale is None: + scale = torch.ones(B, N, 1, 1, device=device, dtype=dtype) + if ori is None: + ori = torch.zeros(B, N, 1, device=device, dtype=dtype) + KORNIA_CHECK_SHAPE(scale, ["B", "N", "1", "1"]) + KORNIA_CHECK_SHAPE(ori, ["B", "N", "1"]) + unscaled_laf = torch.cat([angle_to_rotation_matrix(ori.squeeze(-1)), xy.unsqueeze(-1)], dim=-1) + laf = scale_laf(unscaled_laf, scale) + return laf + + +def scale_laf(laf: torch.Tensor, scale_coef: Union[float, torch.Tensor]) -> torch.Tensor: + """Multiplies region part of LAF ([:, :, :2, :2]) by a scale_coefficient. + + So the center, shape and orientation of the local feature stays the same, but the region area changes. + + Args: + laf: :math:`(B, N, 2, 3)` + scale_coef: broadcastable torch.Tensor or float. + + Returns: + LAF :math:`(B, N, 2, 3)` + + Example: + >>> input = torch.ones(1, 5, 2, 3) # BxNx2x3 + >>> scale = 0.5 + >>> output = scale_laf(input, scale) # BxNx2x3 + + """ + if not isinstance(scale_coef, (float, torch.Tensor)): + raise TypeError(f"scale_coef should be float or torch.Tensor. Got {type(scale_coef)}") + KORNIA_CHECK_LAF(laf) + centerless_laf = laf[:, :, :2, :2] + return torch.cat([scale_coef * centerless_laf, laf[:, :, :, 2:]], dim=3) + + +def make_upright(laf: torch.Tensor, eps: float = 1e-9) -> torch.Tensor: + """Rectify the affine matrix, so that it becomes upright. + + Args: + laf: :math:`(B, N, 2, 3)` + eps: for safe division. + + Returns: + laf: :math:`(B, N, 2, 3)` + + Example: + >>> input = torch.ones(1, 5, 2, 3) # BxNx2x3 + >>> output = make_upright(input) # BxNx2x3 + + """ + KORNIA_CHECK_LAF(laf) + det = get_laf_scale(laf) + scale = det + # The function is equivalent to doing 2x2 SVD and resetting rotation + # matrix to an identity: U, S, V = svd(LAF); LAF_upright = U * S. + b2a2 = torch.sqrt(laf[..., 0:1, 1:2] ** 2 + laf[..., 0:1, 0:1] ** 2) + eps + laf1_ell = torch.cat([(b2a2 / det).contiguous(), torch.zeros_like(det)], dim=3) + laf2_ell = torch.cat( + [ + ((laf[..., 1:2, 1:2] * laf[..., 0:1, 1:2] + laf[..., 1:2, 0:1] * laf[..., 0:1, 0:1]) / (b2a2 * det)), + (det / b2a2).contiguous(), + ], + dim=3, + ) + laf_unit_scale = torch.cat([torch.cat([laf1_ell, laf2_ell], dim=2), laf[..., :, 2:3]], dim=3) + return scale_laf(laf_unit_scale, scale) + + +def ellipse_to_laf(ells: torch.Tensor) -> torch.Tensor: + """Convert ellipse regions to LAF format. + + Ellipse (a, b, c) and upright covariance matrix [a11 a12; 0 a22] are connected + by inverse matrix square root: A = invsqrt([a b; b c]). + + See also https://github.com/vlfeat/vlfeat/blob/master/toolbox/sift/vl_frame2oell.m + + Args: + ells: torch.Tensor :math:`(B, N, 5)` of ellipses in Oxford format [x y a b c]. + + Returns: + LAF :math:`(B, N, 2, 3)` + + Example: + >>> input = torch.ones(1, 10, 5) # BxNx5 + >>> output = ellipse_to_laf(input) # BxNx2x3 + + """ + KORNIA_CHECK_SHAPE(ells, ["B", "N", "5"]) + B, N, _ = ells.shape + # Previous implementation was incorrectly using Cholesky decomp as matrix sqrt + # ell_shape = torch.cat([torch.cat([ells[..., 2:3], ells[..., 3:4]], dim=2).unsqueeze(2), + # torch.cat([ells[..., 3:4], ells[..., 4:5]], dim=2).unsqueeze(2)], dim=2).view(-1, 2, 2) + # out = torch.matrix_power(torch.cholesky(ell_shape, False), -1).view(B, N, 2, 2) + + # We will calculate 2x2 matrix square root via special case formula + # https://en.wikipedia.org/wiki/Square_root_of_a_matrix + # "The Cholesky factorization provides another particular example of square root + # which should not be confused with the unique non-negative square root." + # https://en.wikipedia.org/wiki/Square_root_of_a_2_by_2_matrix + # M = (A 0; C D) + # R = (sqrt(A) 0; C / (sqrt(A)+sqrt(D)) sqrt(D)) + a11 = ells[..., 2:3].abs().sqrt() + a12 = torch.zeros_like(a11) + a22 = ells[..., 4:5].abs().sqrt() + a21 = ells[..., 3:4] / (a11 + a22).clamp(1e-9) + A = torch.stack([a11, a12, a21, a22], dim=-1).view(B, N, 2, 2).inverse() + out = torch.cat([A, ells[..., :2].view(B, N, 2, 1)], dim=3) + return out + + +def laf_to_boundary_points(LAF: torch.Tensor, n_pts: int = 50) -> torch.Tensor: + """Convert LAFs to boundary points of the regions + center. + + Used for local features visualization, see visualize_laf function. + + Args: + LAF: :math:`(B, N, 2, 3)` + n_pts: number of points to output. + + Returns: + torch.Tensor of boundary points LAF: :math:`(B, N, n_pts, 2)` + + """ + KORNIA_CHECK_LAF(LAF) + B, N, _, _ = LAF.size() + pts = torch.cat( + [ + torch.sin(torch.linspace(0, 2 * math.pi, n_pts - 1)).unsqueeze(-1), + torch.cos(torch.linspace(0, 2 * math.pi, n_pts - 1)).unsqueeze(-1), + torch.ones(n_pts - 1, 1), + ], + dim=1, + ) + # Add origin to draw also the orientation + pts = torch.cat([torch.tensor([0.0, 0.0, 1.0]).view(1, 3), pts], dim=0).unsqueeze(0).expand(B * N, n_pts, 3) + pts = pts.to(LAF.device).to(LAF.dtype) + aux = torch.tensor([0.0, 0.0, 1.0]).view(1, 1, 3).expand(B * N, 1, 3) + HLAF = torch.cat([LAF.view(-1, 2, 3), aux.to(LAF.device).to(LAF.dtype)], dim=1) + pts_h = torch.bmm(HLAF, pts.permute(0, 2, 1)).permute(0, 2, 1) + return convert_points_from_homogeneous(pts_h.view(B, N, n_pts, 3)) + + +def get_laf_pts_to_draw(LAF: torch.Tensor, img_idx: int = 0) -> Tuple[List[int], List[int]]: + """Return list for drawing LAFs (local features). + + Args: + LAF: :math:`(B, N, 2, 3)` + img_idx: which points to output. + + Returns: + List of boundary points x, y` + + Examples: + x, y = get_laf_pts_to_draw(LAF, img_idx) + plt.figure() + plt.imshow(kornia.image.tensor_to_image(img[img_idx])) + plt.plot(x, y, 'r') + plt.show() + + """ + # TODO: Refactor doctest + KORNIA_CHECK_LAF(LAF) + pts = laf_to_boundary_points(LAF[img_idx : img_idx + 1])[0] + pts_np = pts.detach().permute(1, 0, 2).cpu() + return (pts_np[..., 0].tolist(), pts_np[..., 1].tolist()) + + +def denormalize_laf(LAF: torch.Tensor, images: torch.Tensor) -> torch.Tensor: + """De-F.normalize LAFs from scale to image scale. + + The convention is that center of 5-pixel image (coordinates from 0 to 4) is 2, and not 2.5. + + B,N,H,W = images.size() + MIN_SIZE = min(H - 1, W -1) + [a11 a21 x] + [a21 a22 y] + becomes + [a11*MIN_SIZE a21*MIN_SIZE x*(W-1)] + [a21*MIN_SIZE a22*MIN_SIZE y*(W-1)] + + Args: + LAF: :math:`(B, N, 2, 3)` + images: :math:`(B, CH, H, W)` + + Returns: + the denormalized LAF: :math:`(B, N, 2, 3)`, scale in pixels + + """ + KORNIA_CHECK_LAF(LAF) + _, _, h, w = images.size() + wf = float(w - 1) + hf = float(h - 1) + min_size = min(hf, wf) + coef = torch.ones(1, 1, 2, 3, dtype=LAF.dtype, device=LAF.device) * min_size + coef[0, 0, 0, 2] = wf + coef[0, 0, 1, 2] = hf + return coef.expand_as(LAF) * LAF + + +def normalize_laf(LAF: torch.Tensor, images: torch.Tensor) -> torch.Tensor: + """Normalize LAFs to [0,1] scale from pixel scale. + + See below: + B,N,H,W = images.size() + MIN_SIZE = min(H - 1, W -1) + [a11 a21 x] + [a21 a22 y] + becomes: + [a11/MIN_SIZE a21/MIN_SIZE x/(W-1)] + [a21/MIN_SIZE a22/MIN_SIZE y/(H-1)] + + Args: + LAF: :math:`(B, N, 2, 3)` + images: :math:`(B, CH, H, W)` + + Returns: + the denormalized LAF: :math:`(B, N, 2, 3)`, scale in image percentage (0, 1) + + """ + KORNIA_CHECK_LAF(LAF) + _, _, h, w = images.size() + wf = float(w - 1) + hf = float(h - 1) + min_size = min(hf, wf) + coef = torch.ones(1, 1, 2, 3, dtype=LAF.dtype, device=LAF.device) / min_size + coef[0, 0, 0, 2] = 1.0 / wf + coef[0, 0, 1, 2] = 1.0 / hf + return coef.expand_as(LAF) * LAF + + +def generate_patch_grid_from_normalized_LAF(img: torch.Tensor, LAF: torch.Tensor, PS: int = 32) -> torch.Tensor: + """Generate affine grid. + + Args: + img: image torch.Tensor of shape :math:`(B, CH, H, W)`. + LAF: laf with shape :math:`(B, N, 2, 3)`. + PS: patch size to be extracted. + + Returns: + grid :math:`(B*N, PS, PS, 2)` + + """ + KORNIA_CHECK_LAF(LAF) + B, N, _, _ = LAF.size() + _, ch, h, w = img.size() + + # norm, then renorm is needed for allowing detection on one resolution + # and extraction at arbitrary other + LAF_renorm = denormalize_laf(LAF, img) + + grid = F.affine_grid(LAF_renorm.view(B * N, 2, 3), [B * N, ch, PS, PS], align_corners=False) + grid[..., :, 0] = 2.0 * grid[..., :, 0].clone() / float(w - 1) - 1.0 + grid[..., :, 1] = 2.0 * grid[..., :, 1].clone() / float(h - 1) - 1.0 + return grid + + +def extract_patches_simple( + img: torch.Tensor, laf: torch.Tensor, PS: int = 32, normalize_lafs_before_extraction: bool = True +) -> torch.Tensor: + """Extract patches defined by LAFs from image torch.Tensor. + + No smoothing applied, huge aliasing (better use extract_patches_from_pyramid). + + Args: + img: images, LAFs are detected in :math:`(B, CH, H, W)`. + laf: :math:`(B, N, 2, 3)`. + PS: patch size. + normalize_lafs_before_extraction: if True, lafs are normalized to image size. + + Returns: + patches with shape :math:`(B, N, CH, PS,PS)`. + + """ + KORNIA_CHECK_LAF(laf) + if normalize_lafs_before_extraction: + nlaf = normalize_laf(laf, img) + else: + nlaf = laf + _, ch, h, w = img.size() + B, N, _, _ = laf.size() + out = [] + # for loop temporarily, to be refactored + for i in range(B): + grid = generate_patch_grid_from_normalized_LAF(img[i : i + 1], nlaf[i : i + 1], PS).to(img.device) + if img.device.type == "mps": + out.append( + F.grid_sample( + img[i : i + 1].expand(grid.size(0), ch, h, w), + grid.clamp(-1, 1), + padding_mode="zeros", + align_corners=False, + ) + ) + else: + out.append( + F.grid_sample( + img[i : i + 1].expand(grid.size(0), ch, h, w), grid, padding_mode="border", align_corners=False + ) + ) + return torch.cat(out, dim=0).view(B, N, ch, PS, PS) + + +def extract_patches_from_pyramid( + img: torch.Tensor, laf: torch.Tensor, PS: int = 32, normalize_lafs_before_extraction: bool = True +) -> torch.Tensor: + """Extract patches defined by LAFs from image torch.Tensor. + + Patches are extracted from appropriate pyramid level. + + Args: + img: images, LAFs are detected in :math:`(B, CH, H, W)`. + laf: :math:`(B, N, 2, 3)`. + PS: patch size. + normalize_lafs_before_extraction: if True, lafs are normalized to image size. + + Returns: + patches with shape :math:`(B, N, CH, PS,PS)`. + + """ + KORNIA_CHECK_LAF(laf) + if normalize_lafs_before_extraction: + nlaf = normalize_laf(laf, img) + else: + nlaf = laf + B, N, _, _ = laf.size() + _, ch, h, w = img.size() + scale = 2.0 * get_laf_scale(denormalize_laf(nlaf, img)) / float(PS) + # max_level is a compile-time constant for static image shapes. + max_level = min(h, w) // PS + pyr_idx = scale.log2().clamp(min=0.0, max=max(0, max_level - 1)).long() # (B, N, 1, 1) + out = torch.zeros(B, N, ch, PS, PS, dtype=nlaf.dtype, device=nlaf.device) + cur_img = img + # Always run at least level 0; max_level is a compile-time constant for static shapes. + num_levels = max(1, max_level) + for cur_pyr_level in range(num_levels): + _, ch_l, h_l, w_l = cur_img.size() + for i in range(B): + # torch.where avoids nonzero/data-dependent shapes → fully compilable. + level_mask = (pyr_idx[i] == cur_pyr_level).view(N, 1, 1, 1) + grid = generate_patch_grid_from_normalized_LAF(cur_img[i : i + 1], nlaf[i : i + 1], PS) + if cur_img.device.type == "mps": + patches = F.grid_sample( + cur_img[i : i + 1].expand(N, ch_l, h_l, w_l), + grid.clamp(-1, 1), + padding_mode="zeros", + align_corners=False, + ) + else: + patches = F.grid_sample( + cur_img[i : i + 1].expand(N, ch_l, h_l, w_l), grid, padding_mode="border", align_corners=False + ) + out[i] = torch.where(level_mask, patches.to(nlaf.dtype), out[i]) + if cur_pyr_level < num_levels - 1: + cur_img = pyrdown(cur_img) + # Stop early if the pyramided image is too small for further levels. + if min(cur_img.size(2), cur_img.size(3)) < PS: + break + return out + + +def laf_is_inside_image(laf: torch.Tensor, images: torch.Tensor, border: int = 0) -> torch.Tensor: + """Check if the LAF is touching or partly outside the image boundary. + + Returns the mask of LAFs, which are fully inside the image, i.e. valid. + + Args: + laf: :math:`(B, N, 2, 3)`. + images: images, lafs are detected in :math:`(B, CH, H, W)`. + border: additional border. + + Returns: + mask with shape :math:`(B, N)`. + + """ + KORNIA_CHECK_LAF(laf) + _, _, h, w = images.size() + pts = laf_to_boundary_points(laf, 12) + good_lafs_mask = ( + (pts[..., 0] >= border) * (pts[..., 0] <= w - border) * (pts[..., 1] >= border) * (pts[..., 1] <= h - border) + ) + good_lafs_mask = good_lafs_mask.min(dim=2)[0] + return good_lafs_mask + + +def laf_to_three_points(laf: torch.Tensor) -> torch.Tensor: + """Convert local affine frame(LAF) to alternative representation: coordinates of LAF center, LAF-x unit vector, + LAF-y unit vector. + + Args: + laf: :math:`(B, N, 2, 3)`. + + Returns: + threepts :math:`(B, N, 2, 3)`. + + """ # noqa:D205 + KORNIA_CHECK_LAF(laf) + three_pts = torch.stack([laf[..., 2] + laf[..., 0], laf[..., 2] + laf[..., 1], laf[..., 2]], dim=-1) + return three_pts + + +def laf_from_three_points(threepts: torch.Tensor) -> torch.Tensor: + """Convert three points to local affine frame. + + Order is (0,0), (0, 1), (1, 0). + + Args: + threepts: :math:`(B, N, 2, 3)`. + + Returns: + laf :math:`(B, N, 2, 3)`. + + """ + laf = torch.stack( + [threepts[..., 0] - threepts[..., 2], threepts[..., 1] - threepts[..., 2], threepts[..., 2]], dim=-1 + ) + return laf + + +def perspective_transform_lafs(trans_01: torch.Tensor, lafs_1: torch.Tensor) -> torch.Tensor: + r"""Apply perspective transformations to a set of local affine frames (LAFs). + + Args: + trans_01: torch.Tensor for perspective transformations of shape :math:`(B, 3, 3)`. + lafs_1: torch.Tensor of lafs of shape :math:`(B, N, 2, 3)`. + + Returns: + torch.Tensor of N-dimensional points of shape :math:`(B, N, 2, 3)`. + + Examples: + >>> rng = torch.manual_seed(0) + >>> lafs_1 = torch.rand(2, 4, 2, 3) # BxNx2x3 + >>> lafs_1 + tensor([[[[0.4963, 0.7682, 0.0885], + [0.1320, 0.3074, 0.6341]], + + [[0.4901, 0.8964, 0.4556], + [0.6323, 0.3489, 0.4017]], + + [[0.0223, 0.1689, 0.2939], + [0.5185, 0.6977, 0.8000]], + + [[0.1610, 0.2823, 0.6816], + [0.9152, 0.3971, 0.8742]]], + + + [[[0.4194, 0.5529, 0.9527], + [0.0362, 0.1852, 0.3734]], + + [[0.3051, 0.9320, 0.1759], + [0.2698, 0.1507, 0.0317]], + + [[0.2081, 0.9298, 0.7231], + [0.7423, 0.5263, 0.2437]], + + [[0.5846, 0.0332, 0.1387], + [0.2422, 0.8155, 0.7932]]]]) + >>> trans_01 = torch.eye(3).repeat(2, 1, 1) # Bx3x3 + >>> trans_01.shape + torch.Size([2, 3, 3]) + >>> lafs_0 = perspective_transform_lafs(trans_01, lafs_1) # BxNx2x3 + + """ + KORNIA_CHECK_LAF(lafs_1) + if not torch.is_tensor(trans_01): + raise TypeError("Input type is not a torch.Tensor") + + if not trans_01.device == lafs_1.device: + raise TypeError("torch.Tensor must be in the same device") + + if not trans_01.shape[0] == lafs_1.shape[0]: + raise ValueError("Input batch size must be the same for both tensors") + + if (not (trans_01.shape[-1] == 3)) or (not (trans_01.shape[-2] == 3)): + raise ValueError("Transformation should be homography") + + bs, n, _, _ = lafs_1.size() + # First, we convert LAF to points + threepts_1 = laf_to_three_points(lafs_1) + points_1 = threepts_1.permute(0, 1, 3, 2).reshape(bs, n * 3, 2) + + # First, transform the points + points_0 = transform_points(trans_01, points_1) + + # Back to LAF format + threepts_0 = points_0.view(bs, n, 3, 2).permute(0, 1, 3, 2) + return laf_from_three_points(threepts_0) diff --git a/kornia/feature/lightglue.py b/kornia/feature/lightglue.py new file mode 100644 index 0000000..7568ad1 --- /dev/null +++ b/kornia/feature/lightglue.py @@ -0,0 +1,936 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math +import warnings +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Callable, ClassVar, Dict, List, Optional, Sequence, Tuple + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK +from kornia.feature.laf import laf_to_three_points, scale_laf + +try: + from flash_attn.modules.mha import FlashCrossAttention +except ModuleNotFoundError: + FlashCrossAttention = None + +if FlashCrossAttention or hasattr(F, "scaled_dot_product_attention"): + FLASH_AVAILABLE = True +else: + FLASH_AVAILABLE = False + + +def math_clamp(x, min_, max_): # type: ignore + """Clamp a value to lie within [min, max].""" + return min(max(x, min_), max_) + + +AMP_CUSTOM_FWD_F32 = ( + torch.amp.custom_fwd(cast_inputs=torch.float32, device_type="cuda") + if hasattr(torch, "amp") and hasattr(torch.amp, "custom_fwd") + else torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +) + + +@AMP_CUSTOM_FWD_F32 +def normalize_keypoints(kpts: torch.Tensor, size: torch.Tensor) -> torch.Tensor: + """Normalize torch.Tensor of keypoints.""" + if isinstance(size, torch.Size): + size = torch.tensor(size)[None] + shift = size.float().to(kpts) / 2 + scale = size.max(1).values.float().to(kpts) / 2 + kpts = (kpts - shift[:, None]) / scale[:, None, None] + return kpts + + +def pad_to_length(x: torch.Tensor, length: int) -> Tuple[torch.Tensor, torch.Tensor]: + """Pad torch.Tensor to desired length.""" + if length <= x.shape[-2]: + return x, torch.ones_like(x[..., :1], dtype=torch.bool) + pad_tensor = torch.ones(*x.shape[:-2], length - x.shape[-2], x.shape[-1], device=x.device, dtype=x.dtype) + y = torch.cat([x, pad_tensor], dim=-2) + mask = torch.zeros(*y.shape[:-1], 1, dtype=torch.bool, device=x.device) + mask[..., : x.shape[-2], :] = True + return y, mask + + +def rotate_half(x: torch.Tensor) -> torch.Tensor: + """Apply half rotation.""" + x = x.unflatten(-1, (-1, 2)) + x1, x2 = x.unbind(dim=-1) + return torch.stack((-x2, x1), dim=-1).flatten(start_dim=-2) + + +def apply_cached_rotary_emb(freqs: torch.Tensor, t: torch.Tensor) -> torch.Tensor: + """Apply rotary embedding.""" + return (t * freqs[0]) + (rotate_half(t) * freqs[1]) + + +class LearnableFourierPositionalEncoding(nn.Module): + """Implement learnable Fourier features for positional encoding. + + This module encodes spatial coordinates into high-dimensional embeddings + using a learnable projection and periodic functions. + + Args: + M: The number of frequencies to use. + dim: The dimension of the output embedding. + F_dim: The dimension of the Fourier features. Default: None. + gamma: The scaling factor for the initialization. Default: 1.0. + """ + + def __init__(self, M: int, dim: int, F_dim: Optional[int] = None, gamma: float = 1.0) -> None: + super().__init__() + F_dim = F_dim if F_dim is not None else dim + self.gamma = gamma + self.Wr = nn.Linear(M, F_dim // 2, bias=False) + nn.init.normal_(self.Wr.weight.data, mean=0, std=self.gamma**-2) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Encode position vector.""" + projected = self.Wr(x) + cosines, sines = torch.cos(projected), torch.sin(projected) + emb = torch.stack([cosines, sines], 0).unsqueeze(-3) + return emb.repeat_interleave(2, dim=-1) + + +class TokenConfidence(nn.Module): + """Predict the confidence score for each token in the sequence. + + Args: + dim: The dimension of the input feature tokens. + """ + + def __init__(self, dim: int) -> None: + super().__init__() + self.token = nn.Sequential(nn.Linear(dim, 1), nn.Sigmoid()) + + def forward(self, desc0: torch.Tensor, desc1: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Get confidence tokens.""" + dtype = self.token[0].weight.dtype + orig_dtype = desc0.dtype + return ( + self.token(desc0.detach().to(dtype)).squeeze(-1).to(orig_dtype), + self.token(desc1.detach().to(dtype)).squeeze(-1).to(orig_dtype), + ) + + +class Attention(nn.Module): + """Implement the optimized multi-head attention mechanism for LightGlue. + + Args: + allow_flash: Whether to enable FlashAttention for increased performance. + """ + + def __init__(self, allow_flash: bool) -> None: + super().__init__() + if allow_flash and not FLASH_AVAILABLE: + warnings.warn( + "FlashAttention is not available. For optimal speed, consider installing torch >= 2.0 or flash-attn.", + stacklevel=2, + ) + self.enable_flash = allow_flash and FLASH_AVAILABLE + self.has_sdp = hasattr(F, "scaled_dot_product_attention") + if allow_flash and FlashCrossAttention: + self.flash_ = FlashCrossAttention() + if self.has_sdp: + torch.backends.cuda.enable_flash_sdp(allow_flash) + + def forward( + self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, mask: Optional[torch.Tensor] = None + ) -> torch.Tensor: + """Apply multi-head attention to query, key, and value tensors. + + Args: + q: Query tensor with shape :math:`(B, H, N_q, D_h)`, where ``B`` is batch size, + ``H`` is the number of attention heads, ``N_q`` is the number of query tokens, + and ``D_h`` is the per-head descriptor dimension. + k: Key tensor with shape :math:`(B, H, N_k, D_h)`, where ``N_k`` is the number + of key tokens. + v: Value tensor with shape :math:`(B, H, N_k, D_h)`. + mask: Optional boolean attention mask. ``True`` entries mark token pairs that + are allowed to attend to each other. + + Returns: + Attention output with shape :math:`(B, H, N_q, D_h)`. + """ + if self.enable_flash and q.device.type == "cuda": + # use torch 2.0 scaled_dot_product_attention with flash + if self.has_sdp: + args = [x.half().contiguous() for x in [q, k, v]] + v = F.scaled_dot_product_attention(*args, attn_mask=mask).to(q.dtype) # type: ignore + return v if mask is None else v.nan_to_num() + else: + KORNIA_CHECK(mask is None) + q, k, v = (x.transpose(-2, -3).contiguous() for x in [q, k, v]) + m = self.flash_(q.half(), torch.stack([k, v], 2).half()) + return m.transpose(-2, -3).to(q.dtype).clone() + elif self.has_sdp: + args = [x.contiguous() for x in [q, k, v]] + v = F.scaled_dot_product_attention(*args, attn_mask=mask) # type: ignore + return v if mask is None else v.nan_to_num() + else: + s = q.shape[-1] ** -0.5 + sim = torch.einsum("...id,...jd->...ij", q, k) * s + if mask is not None: + sim.masked_fill(~mask, -float("inf")) + attn = F.softmax(sim, -1) + return torch.einsum("...ij,...jd->...id", attn, v) + + +class SelfBlock(nn.Module): + """Implement a self-attention block to process features within a single image. + + Args: + embed_dim: The dimension of the feature embeddings. + num_heads: The number of attention heads. + flash: Whether to use FlashAttention. Default: False. + bias: Whether to include bias in the linear layers. Default: True. + """ + + def __init__(self, embed_dim: int, num_heads: int, flash: bool = False, bias: bool = True) -> None: + super().__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + KORNIA_CHECK(self.embed_dim % num_heads == 0, "Embed dimension should be dividable by num_heads") + self.head_dim = self.embed_dim // num_heads + self.Wqkv = nn.Linear(embed_dim, 3 * embed_dim, bias=bias) + self.inner_attn = Attention(flash) + self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.ffn = nn.Sequential( + nn.Linear(2 * embed_dim, 2 * embed_dim), + nn.LayerNorm(2 * embed_dim, elementwise_affine=True), + nn.GELU(), + nn.Linear(2 * embed_dim, embed_dim), + ) + + def forward( + self, + x: torch.Tensor, + encoding: torch.Tensor, + mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Update descriptors from one image using self-attention. + + Args: + x: Descriptor tensor with shape :math:`(B, N, D)`, where ``B`` is batch size, + ``N`` is the number of local features, and ``D`` is descriptor dimension. + encoding: Positional encoding tensor aligned with ``x`` and used for rotary + position embedding inside attention. + mask: Optional boolean mask limiting which descriptors may attend to each other. + + Returns: + Updated descriptor tensor with the same shape as ``x``. + """ + qkv = self.Wqkv(x) + qkv = qkv.unflatten(-1, (self.num_heads, -1, 3)).transpose(1, 2) + q, k, v = qkv[..., 0], qkv[..., 1], qkv[..., 2] + q = apply_cached_rotary_emb(encoding, q) + k = apply_cached_rotary_emb(encoding, k) + context = self.inner_attn(q, k, v, mask=mask) + message = self.out_proj(context.transpose(1, 2).flatten(start_dim=-2)) + return x + self.ffn(torch.cat([x, message], -1)) + + +class CrossBlock(nn.Module): + """Implement a cross-attention block to exchange information between two images. + + Args: + embed_dim: The dimension of the feature embeddings. + num_heads: The number of attention heads. + flash: Whether to use FlashAttention. Default: False. + bias: Whether to include bias in the linear layers. Default: True. + """ + + def __init__(self, embed_dim: int, num_heads: int, flash: bool = False, bias: bool = True) -> None: + super().__init__() + self.heads = num_heads + dim_head = embed_dim // num_heads + self.scale = dim_head**-0.5 + inner_dim = dim_head * num_heads + self.to_qk = nn.Linear(embed_dim, inner_dim, bias=bias) + self.to_v = nn.Linear(embed_dim, inner_dim, bias=bias) + self.to_out = nn.Linear(inner_dim, embed_dim, bias=bias) + self.ffn = nn.Sequential( + nn.Linear(2 * embed_dim, 2 * embed_dim), + nn.LayerNorm(2 * embed_dim, elementwise_affine=True), + nn.GELU(), + nn.Linear(2 * embed_dim, embed_dim), + ) + if flash and FLASH_AVAILABLE: + self.flash = Attention(True) + else: + self.flash = None # type: ignore + + def map_(self, func: Callable, x0: torch.Tensor, x1: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: # type: ignore + """Apply the same function to descriptor tensors from both images. + + Args: + func: Callable applied independently to ``x0`` and ``x1``. + x0: Descriptor tensor for the first image. + x1: Descriptor tensor for the second image. + + Returns: + A tuple containing ``func(x0)`` and ``func(x1)``. + """ + return func(x0), func(x1) + + def forward( + self, x0: torch.Tensor, x1: torch.Tensor, mask: Optional[torch.Tensor] = None + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Exchange information between two images using cross-attention. + + Args: + x0: Descriptor tensor from the first image with shape :math:`(B, M, D)`, + where ``M`` is the number of local features in image 0. + x1: Descriptor tensor from the second image with shape :math:`(B, N, D)`, + where ``N`` is the number of local features in image 1. + mask: Optional boolean mask with shape :math:`(B, H, M, N)` or a broadcastable + equivalent, marking valid descriptor pairs. + + Returns: + A tuple ``(x0_out, x1_out)`` with updated descriptors for both images. + """ + qk0, qk1 = self.map_(self.to_qk, x0, x1) + v0, v1 = self.map_(self.to_v, x0, x1) + qk0, qk1, v0, v1 = (t.unflatten(-1, (self.heads, -1)).transpose(1, 2) for t in (qk0, qk1, v0, v1)) + if self.flash is not None and qk0.device.type == "cuda": + m0 = self.flash(qk0, qk1, v1, mask) + m1 = self.flash(qk1, qk0, v0, mask.transpose(-1, -2) if mask is not None else None) + else: + qk0, qk1 = qk0 * self.scale**0.5, qk1 * self.scale**0.5 + sim = torch.einsum("bhid, bhjd -> bhij", qk0, qk1) + if mask is not None: + sim = sim.masked_fill(~mask, -float("inf")) + attn01 = F.softmax(sim, dim=-1) + attn10 = F.softmax(sim.transpose(-2, -1).contiguous(), dim=-1) + m0 = torch.einsum("bhij, bhjd -> bhid", attn01, v1) + m1 = torch.einsum("bhji, bhjd -> bhid", attn10.transpose(-2, -1), v0) + if mask is not None: + m0, m1 = m0.nan_to_num(), m1.nan_to_num() + m0, m1 = self.map_(lambda t: t.transpose(1, 2).flatten(start_dim=-2), m0, m1) + m0, m1 = self.map_(self.to_out, m0, m1) + x0 = x0 + self.ffn(torch.cat([x0, m0], -1)) + x1 = x1 + self.ffn(torch.cat([x1, m1], -1)) + return x0, x1 + + +class TransformerLayer(nn.Module): + """Implement a complete Transformer layer for feature matching. + + This layer typically alternates between self-attention and cross-attention + to update local feature descriptors. + """ + + def __init__(self, *args, **kwargs): # type: ignore + super().__init__() + self.self_attn = SelfBlock(*args, **kwargs) + self.cross_attn = CrossBlock(*args, **kwargs) + + def forward( + self, + desc0: torch.Tensor, + desc1: torch.Tensor, + encoding0: torch.Tensor, + encoding1: torch.Tensor, + mask0: Optional[torch.Tensor] = None, + mask1: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Run one LightGlue transformer layer over two descriptor sets. + + Args: + desc0: Descriptor tensor from image 0 with shape :math:`(B, M, D)`. + desc1: Descriptor tensor from image 1 with shape :math:`(B, N, D)`. + encoding0: Positional encoding for ``desc0``. + encoding1: Positional encoding for ``desc1``. + mask0: Optional validity mask for descriptors in image 0. + mask1: Optional validity mask for descriptors in image 1. + + Returns: + A tuple containing updated descriptor tensors for image 0 and image 1. + """ + if mask0 is not None and mask1 is not None: + return self.masked_forward(desc0, desc1, encoding0, encoding1, mask0, mask1) + else: + desc0 = self.self_attn(desc0, encoding0) + desc1 = self.self_attn(desc1, encoding1) + return self.cross_attn(desc0, desc1) + + # This part is compiled and allows padding inputs + def masked_forward( + self, + desc0: torch.Tensor, + desc1: torch.Tensor, + encoding0: torch.Tensor, + encoding1: torch.Tensor, + mask0: torch.Tensor, + mask1: torch.Tensor, + ) -> torch.Tensor: + """Run the transformer layer with explicit descriptor validity masks. + + Args: + desc0: Descriptor tensor from image 0 with shape :math:`(B, M, D)`. + desc1: Descriptor tensor from image 1 with shape :math:`(B, N, D)`. + encoding0: Positional encoding for image 0 descriptors. + encoding1: Positional encoding for image 1 descriptors. + mask0: Boolean mask for valid image 0 descriptors. + mask1: Boolean mask for valid image 1 descriptors. + + Returns: + A tuple with masked self-attention and cross-attention updates for both images. + """ + mask = mask0 & mask1.transpose(-1, -2) + mask0 = mask0 & mask0.transpose(-1, -2) + mask1 = mask1 & mask1.transpose(-1, -2) + desc0 = self.self_attn(desc0, encoding0, mask0) + desc1 = self.self_attn(desc1, encoding1, mask1) + return self.cross_attn(desc0, desc1, mask) + + +def sigmoid_log_double_softmax(sim: torch.Tensor, z0: torch.Tensor, z1: torch.Tensor) -> torch.Tensor: + """Create the log assignment matrix from logits and similarity.""" + b, m, n = sim.shape + certainties = F.logsigmoid(z0) + F.logsigmoid(z1).transpose(1, 2) + scores0 = F.log_softmax(sim, 2) + scores1 = F.log_softmax(sim.transpose(-1, -2).contiguous(), 2).transpose(-1, -2) + scores = sim.new_full((b, m + 1, n + 1), 0) + scores[:, :m, :n] = scores0 + scores1 + certainties + scores[:, :-1, -1] = F.logsigmoid(-z0.squeeze(-1)) + scores[:, -1, :-1] = F.logsigmoid(-z1.squeeze(-1)) + return scores + + +class MatchAssignment(nn.Module): + """Assign matches between two sets of local features using a dual-softmax approach. + + Args: + dim: The dimension of the feature descriptors. + """ + + def __init__(self, dim: int) -> None: + super().__init__() + self.dim = dim + self.matchability = nn.Linear(dim, 1, bias=True) + self.final_proj = nn.Linear(dim, dim, bias=True) + + def forward(self, desc0: torch.Tensor, desc1: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Build assignment matrix from descriptors.""" + mdesc0, mdesc1 = self.final_proj(desc0), self.final_proj(desc1) + _, _, d = mdesc0.shape + mdesc0, mdesc1 = mdesc0 / d**0.25, mdesc1 / d**0.25 + sim = torch.einsum("bmd,bnd->bmn", mdesc0, mdesc1) + z0 = self.matchability(desc0) + z1 = self.matchability(desc1) + scores = sigmoid_log_double_softmax(sim, z0, z1) + return scores, sim + + def get_matchability(self, desc: torch.Tensor) -> torch.Tensor: + """Estimate how likely each descriptor is to obtain a valid match. + + Args: + desc: Descriptor tensor with shape :math:`(B, N, D)`, where ``N`` is the + number of local features. + + Returns: + Matchability probabilities with shape :math:`(B, N)`. + """ + return torch.sigmoid(self.matchability(desc)).squeeze(-1) + + +def filter_matches(scores: torch.Tensor, th: float) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Obtain matches from a log assignment matrix [Bx M+1 x N+1].""" + max0, max1 = scores[:, :-1, :-1].max(2), scores[:, :-1, :-1].max(1) + m0, m1 = max0.indices, max1.indices + indices0 = torch.arange(m0.shape[1], device=m0.device)[None] + indices1 = torch.arange(m1.shape[1], device=m1.device)[None] + mutual0 = indices0 == m1.gather(1, m0) + mutual1 = indices1 == m0.gather(1, m1) + max0_exp = max0.values.exp() + zero = max0_exp.new_tensor(0) + mscores0 = torch.where(mutual0, max0_exp, zero) + mscores1 = torch.where(mutual1, mscores0.gather(1, m1), zero) + valid0 = mutual0 & (mscores0 > th) + valid1 = mutual1 & valid0.gather(1, m1) + m0 = torch.where(valid0, m0, -1) + m1 = torch.where(valid1, m1, -1) + return m0, m1, mscores0, mscores1 + + +class LightGlue(nn.Module): + """Implement the LightGlue matcher for sparse local features. + + LightGlue is a deep network that matches local features across image pairs + using a series of transformer layers and an adaptive pruning mechanism. + + Args: + features: The type of local features to match (e.g., 'superpoint', 'disk'). + conf: A configuration dictionary to override default parameters. + """ + + default_conf: ClassVar[Dict[str, Any]] = { + "name": "lightglue", # just for interfacing + "input_dim": 256, # input descriptor dimension (autoselected from weights) + "descriptor_dim": 256, + "add_scale_ori": False, + "add_laf": False, # for KeyNetAffNetHardNet + "scale_coef": 1.0, # to compensate for the SIFT scale bigger than KeyNet + "n_layers": 9, + "num_heads": 4, + "flash": True, # enable FlashAttention if available. + "mp": False, # enable mixed precision + "depth_confidence": 0.95, # early stopping, disable with -1 + "width_confidence": 0.99, # point pruning, disable with -1 + "filter_threshold": 0.1, # match threshold + "weights": None, + } + + # Point pruning involves an overhead (gather). + # Therefore, we only activate it if there are enough keypoints. + pruning_keypoint_thresholds: ClassVar[Dict[str, Any]] = { + "cpu": -1, + "mps": -1, + "cuda": 1024, + "flash": 1536, + } + required_data_keys: ClassVar[List[str]] = ["image0", "image1"] + + version: ClassVar[str] = "v0.1_arxiv" + url: ClassVar[str] = "https://github.com/cvg/LightGlue/releases/download/{}/{}_lightglue.pth" + + features: ClassVar[Dict[str, Any]] = { + "superpoint": { + "weights": "superpoint_lightglue", + "input_dim": 256, + }, + "dedodeb": { + "weights": "dedodeb_lightglue", + "input_dim": 256, + }, + "dedodeg": { + "weights": "dedodeg_lightglue", + "input_dim": 256, + }, + "disk": { + "weights": "disk_lightglue", + "input_dim": 128, + }, + "aliked": { + "weights": "aliked_lightglue", + "input_dim": 128, + }, + "raco-aliked": { + "weights": "raco_aliked_lightglue", + "input_dim": 128, + }, + "xfeat": { + "weights": "xfeat_lighterglue", + "input_dim": 64, + "descriptor_dim": 96, + "n_layers": 6, + "num_heads": 1, + "depth_confidence": -1, + "width_confidence": 0.95, + "filter_threshold": 0.1, + }, + "sift": { + "weights": "sift_lightglue", + "input_dim": 128, + "add_scale_ori": True, + }, + "dog_affnet_hardnet": { + "weights": "doghardnet", + "input_dim": 128, + "width_confidence": -1, + "depth_confidence": -1, + "add_scale_ori": True, + "scale_coef": 1.0 / 6.0, + }, + "doghardnet": { + "weights": "doghardnet", + "input_dim": 128, + "width_confidence": -1, + "depth_confidence": -1, + "add_scale_ori": True, + "scale_coef": 1.0 / 6.0, + }, + "keynet_affnet_hardnet": { + "weights": "keynet_affnet_hardnet_lightglue", + "input_dim": 128, + "width_confidence": -1, + "depth_confidence": -1, + "add_laf": True, + }, + } + + def __init__(self, features: str = "superpoint", **conf_) -> None: # type: ignore + super().__init__() + self.conf = conf = SimpleNamespace(**{**self.default_conf, **conf_}) + if features is not None: + KORNIA_CHECK(features in list(self.features.keys()), "Features keys are wrong") + for k, v in self.features[features].items(): + setattr(conf, k, v) + KORNIA_CHECK(not (self.conf.add_scale_ori and self.conf.add_laf)) # we use either scale ori, or LAF + + if conf.input_dim != conf.descriptor_dim: + self.input_proj = nn.Linear(conf.input_dim, conf.descriptor_dim, bias=True) + else: + self.input_proj = nn.Identity() # type: ignore + + head_dim = conf.descriptor_dim // conf.num_heads + self.posenc = LearnableFourierPositionalEncoding( + 2 + 2 * conf.add_scale_ori + 4 * conf.add_laf, + head_dim, + head_dim, + ) + + h, n, d = conf.num_heads, conf.n_layers, conf.descriptor_dim + self.transformers = nn.ModuleList([TransformerLayer(d, h, conf.flash) for _ in range(n)]) + self.log_assignment = nn.ModuleList([MatchAssignment(d) for _ in range(n)]) + self.token_confidence = nn.ModuleList([TokenConfidence(d) for _ in range(n - 1)]) + self.register_buffer( + "confidence_thresholds", + torch.Tensor([self.confidence_threshold(i) for i in range(self.conf.n_layers)]), + ) + state_dict = None + if features is not None: + fname = f"{conf.weights}_{self.version}.pth".replace(".", "-") + if features == "dog_affnet_hardnet": + features = "doghardnet" # new dog model is better for affnet as well + if features in ["keynet_affnet_hardnet"]: + fname = "keynet_affnet_hardnet_lightlue.pth" + url = "http://cmp.felk.cvut.cz/~mishkdmy/models/keynet_affnet_hardnet_lightlue.pth" + elif features in ["dedodeb"]: + fname = "dedodeb_lightglue.pth" + url = "http://cmp.felk.cvut.cz/~mishkdmy/models/dedodeb_lightglue.pth" + elif features in ["dedodeg"]: + fname = "dedodeg_lightglue.pth" + url = "http://cmp.felk.cvut.cz/~mishkdmy/models/dedodeg_lightglue.pth" + elif features == "xfeat": + fname = "xfeat-lighterglue.pt" + url = "https://github.com/verlab/accelerated_features/raw/main/weights/xfeat-lighterglue.pt" + else: + url = self.url.format(self.version, features.replace("-", "_")) + state_dict = torch.hub.load_state_dict_from_url(url, file_name=fname) + elif conf.weights is not None: + path = Path(__file__).parent + path = path / f"weights/{self.conf.weights}.pth" + state_dict = torch.load(str(path), map_location="cpu") + if state_dict: + # xfeat-lighterglue weights are nested under a 'matcher.' prefix + prefix = "matcher." + state_dict = {k[len(prefix) :] if k.startswith(prefix) else k: v for k, v in state_dict.items()} + # rename old state dict entries + for i in range(self.conf.n_layers): + pattern = f"self_attn.{i}", f"transformers.{i}.self_attn" + state_dict = {k.replace(*pattern): v for k, v in state_dict.items()} + pattern = f"cross_attn.{i}", f"transformers.{i}.cross_attn" + state_dict = {k.replace(*pattern): v for k, v in state_dict.items()} + self.load_state_dict(state_dict, strict=False) + print("Loaded LightGlue model") + # static lengths LightGlue is compiled for (only used with torch.compile) + self.static_lengths = None + + def compile( + self, mode: str = "reduce-overhead", static_lengths: Sequence[int] = (256, 512, 768, 1024, 1280, 1536) + ) -> None: + """Compile masked transformer layers with ``torch.compile``. + + Args: + mode: Compilation mode forwarded to ``torch.compile``. + static_lengths: Descriptor sequence lengths prepared for compiled execution. + These lengths represent possible numbers of local features. + """ + if self.conf.width_confidence != -1: + warnings.warn( + "Point pruning is partially disabled for compiled forward.", + stacklevel=2, + ) + + if ( + hasattr(torch, "_inductor") + and hasattr(torch._inductor, "cudagraph_mark_step_begin") + and torch.cuda.is_available() + ): + torch._inductor.cudagraph_mark_step_begin() + for i in range(self.conf.n_layers): + self.transformers[i].masked_forward = torch.compile( # type: ignore[assignment] + self.transformers[i].masked_forward, mode=mode, fullgraph=True + ) + + self.static_lengths = static_lengths # type: ignore + + def forward(self, data: dict) -> dict: # type: ignore + """Match keypoints and descriptors between two images. + + Input (dict): + image0: dict + keypoints: [B x M x 2] + descriptors: [B x M x D] + image: [B x C x H x W] or image_size: [B x 2] + image1: dict + keypoints: [B x N x 2] + descriptors: [B x N x D] + image: [B x C x H x W] or image_size: [B x 2] + Output (dict): + log_assignment: [B x M+1 x N+1] + matches0: [B x M] + matching_scores0: [B x M] + matches1: [B x N] + matching_scores1: [B x N] + matches: List[[Si x 2]], scores: List[[Si]] + """ + with torch.autocast(enabled=self.conf.mp, device_type="cuda"): + return self._forward(data) + + def _forward(self, data: dict) -> dict: # type: ignore + for key in self.required_data_keys: + KORNIA_CHECK(key in data, f"Missing key {key} in data") + data0, data1 = data["image0"], data["image1"] + kpts0, kpts1 = data0["keypoints"], data1["keypoints"] + b, m, _ = kpts0.shape + b, n, _ = kpts1.shape + device = kpts0.device + size0, size1 = data0.get("image_size"), data1.get("image_size") + size0 = size0 if size0 is not None else data0["image"].shape[-2:][::-1] + size1 = size1 if size1 is not None else data1["image"].shape[-2:][::-1] + + kpts0 = normalize_keypoints(kpts0, size0).clone() + kpts1 = normalize_keypoints(kpts1, size1).clone() + KORNIA_CHECK(torch.all(kpts0 >= -1).item() and torch.all(kpts0 <= 1).item(), "") # type: ignore + KORNIA_CHECK(torch.all(kpts1 >= -1).item() and torch.all(kpts1 <= 1).item(), "") # type: ignore + if self.conf.add_scale_ori: + kpts0 = torch.cat([kpts0] + [data0[k].unsqueeze(-1) for k in ("scales", "oris")], -1) + if self.conf.scale_coef != 1.0: + kpts0[..., -2] = kpts0[..., -2] * self.conf.scale_coef + kpts1 = torch.cat([kpts1] + [data1[k].unsqueeze(-1) for k in ("scales", "oris")], -1) + if self.conf.scale_coef != 1.0: + kpts1[..., -2] = kpts1[..., -2] * self.conf.scale_coef + elif self.conf.add_laf: + laf0 = scale_laf(data0["lafs"], self.conf.scale_coef) + laf1 = scale_laf(data1["lafs"], self.conf.scale_coef) + laf0 = laf_to_three_points(laf0) + laf1 = laf_to_three_points(laf1) + kpts0 = torch.cat( + [ + kpts0, + normalize_keypoints(laf0[..., 0], size0).clone().to(kpts0.dtype), + normalize_keypoints(laf0[..., 1], size0).clone().to(kpts0.dtype), + ], + -1, + ) + kpts1 = torch.cat( + [ + kpts1, + normalize_keypoints(laf1[..., 0], size1).clone().to(kpts1.dtype), + normalize_keypoints(laf1[..., 1], size1).clone().to(kpts1.dtype), + ], + -1, + ) + + desc0 = data0["descriptors"].detach().contiguous() + desc1 = data1["descriptors"].detach().contiguous() + + # Guard against empty-descriptors after pruning + if desc0.shape[1] == 0 or desc1.shape[1] == 0: + empty_m0 = torch.full((b, m), -1, device=device, dtype=torch.long) + empty_m1 = torch.full((b, n), -1, device=device, dtype=torch.long) + empty_s0 = torch.zeros((b, m), device=device) + empty_s1 = torch.zeros((b, n), device=device) + + return { + "log_assignment": torch.empty((b, 1, 1), device=device), + "matches0": empty_m0, + "matches1": empty_m1, + "matching_scores0": empty_s0, + "matching_scores1": empty_s1, + "stop": 0, + "matches": [torch.empty((0, 2), device=device, dtype=torch.long) for _ in range(b)], + "scores": [torch.empty((0,), device=device) for _ in range(b)], + "prune0": torch.ones_like(empty_s0) * self.conf.n_layers, + "prune1": torch.ones_like(empty_s1) * self.conf.n_layers, + } + + KORNIA_CHECK(desc0.shape[-1] == self.conf.input_dim, "Descriptor dimension does not match input dim in config") + KORNIA_CHECK(desc1.shape[-1] == self.conf.input_dim, "Descriptor dimension does not match input dim in config") + + if torch.is_autocast_enabled(): + desc0 = desc0.half() + desc1 = desc1.half() + + mask0, mask1 = None, None + c = max(m, n) + do_compile = self.static_lengths and c <= max(self.static_lengths) + if do_compile: + kn = min([k for k in self.static_lengths if k >= c]) + desc0, mask0 = pad_to_length(desc0, kn) + desc1, mask1 = pad_to_length(desc1, kn) + kpts0, _ = pad_to_length(kpts0, kn) + kpts1, _ = pad_to_length(kpts1, kn) + desc0 = self.input_proj(desc0) + desc1 = self.input_proj(desc1) + # cache positional embeddings + encoding0 = self.posenc(kpts0) + encoding1 = self.posenc(kpts1) + + # GNN + final_proj + assignment + do_early_stop = self.conf.depth_confidence > 0 + do_point_pruning = self.conf.width_confidence > 0 and not do_compile + pruning_th = self.pruning_min_kpts(device) + if do_point_pruning: + ind0 = torch.arange(0, m, device=device)[None] + ind1 = torch.arange(0, n, device=device)[None] + # We store the index of the layer at which pruning is detected. + prune0 = torch.ones_like(ind0) + prune1 = torch.ones_like(ind1) + token0, token1 = None, None + for i in range(self.conf.n_layers): + desc0, desc1 = self.transformers[i](desc0, desc1, encoding0, encoding1, mask0=mask0, mask1=mask1) + if i == self.conf.n_layers - 1: + continue # no early stopping or adaptive width at last layer + + if do_early_stop: + token0, token1 = self.token_confidence[i](desc0, desc1) + if self.check_if_stop(token0[..., :m, :], token1[..., :n, :], i, m + n): + break + if do_point_pruning and desc0.shape[-2] > pruning_th: + scores0 = self.log_assignment[i].get_matchability(desc0) + prunemask0 = self.get_pruning_mask(token0, scores0, i) # type: ignore + keep0 = torch.where(prunemask0)[1] + ind0 = ind0.index_select(1, keep0) + desc0 = desc0.index_select(1, keep0) + encoding0 = encoding0.index_select(-2, keep0) + prune0[:, ind0] += 1 + if do_point_pruning and desc1.shape[-2] > pruning_th: + scores1 = self.log_assignment[i].get_matchability(desc1) + prunemask1 = self.get_pruning_mask(token1, scores1, i) # type: ignore + keep1 = torch.where(prunemask1)[1] + ind1 = ind1.index_select(1, keep1) + desc1 = desc1.index_select(1, keep1) + encoding1 = encoding1.index_select(-2, keep1) + prune1[:, ind1] += 1 + + desc0, desc1 = desc0[..., :m, :], desc1[..., :n, :] + + # If adaptive width pruning removed all keypoints from either side, + # return empty matches instead of crashing in filter_matches (see #3564). + if desc0.shape[-2] == 0 or desc1.shape[-2] == 0: + matches = [torch.zeros((0, 2), dtype=torch.long, device=device) for _ in range(b)] + mscores = [torch.zeros(0, device=device) for _ in range(b)] + m0_ = torch.full((b, m), -1, device=device, dtype=torch.long) + m1_ = torch.full((b, n), -1, device=device, dtype=torch.long) + mscores0_ = torch.zeros((b, m), device=device) + mscores1_ = torch.zeros((b, n), device=device) + return { + "matches": matches, + "scores": mscores, + "matching_scores0": mscores0_, + "matching_scores1": mscores1_, + "matches0": m0_, + "matches1": m1_, + "stop": i + 1, + "prune0": prune0 if do_point_pruning else torch.ones_like(torch.arange(m, device=device)[None]), + "prune1": prune1 if do_point_pruning else torch.ones_like(torch.arange(n, device=device)[None]), + } + + scores, _ = self.log_assignment[i](desc0, desc1) + m0, m1, mscores0, mscores1 = filter_matches(scores, self.conf.filter_threshold) + + matches, mscores = [], [] + for k in range(b): + valid = m0[k] > -1 + m_indices_0 = torch.where(valid)[0] + m_indices_1 = m0[k][valid] + if do_point_pruning: + m_indices_0 = ind0[k, m_indices_0] + m_indices_1 = ind1[k, m_indices_1] + matches.append(torch.stack([m_indices_0, m_indices_1], -1)) + mscores.append(mscores0[k][valid]) + + # TODO: Remove when hloc switches to the compact format. + if do_point_pruning: + m0_ = torch.full((b, m), -1, device=m0.device, dtype=m0.dtype) + m1_ = torch.full((b, n), -1, device=m1.device, dtype=m1.dtype) + m0_[:, ind0] = torch.where(m0 == -1, -1, ind1.gather(1, m0.clamp(min=0))) + m1_[:, ind1] = torch.where(m1 == -1, -1, ind0.gather(1, m1.clamp(min=0))) + mscores0_ = torch.zeros((b, m), device=mscores0.device) + mscores1_ = torch.zeros((b, n), device=mscores1.device) + mscores0_[:, ind0] = mscores0 + mscores1_[:, ind1] = mscores1 + m0, m1, mscores0, mscores1 = m0_, m1_, mscores0_, mscores1_ + else: + prune0 = torch.ones_like(mscores0) * self.conf.n_layers + prune1 = torch.ones_like(mscores1) * self.conf.n_layers + + pred = { + "log_assignment": scores, + "matches0": m0, + "matches1": m1, + "matching_scores0": mscores0, + "matching_scores1": mscores1, + "stop": i + 1, + "matches": matches, + "scores": mscores, + "prune0": prune0, + "prune1": prune1, + } + + return pred + + def confidence_threshold(self, layer_index: int) -> float: + """Scaled confidence threshold.""" + threshold = 0.8 + 0.1 * math.exp(-4.0 * layer_index / self.conf.n_layers) + return min(max(threshold, 0), 1) + + def get_pruning_mask(self, confidences: torch.Tensor, scores: torch.Tensor, layer_index: int) -> torch.Tensor: + """Mask points which should be removed.""" + keep = scores > (1 - self.conf.width_confidence) + if confidences is not None: # Low-confidence points are never pruned. + keep |= confidences <= self.confidence_thresholds[layer_index] + return keep + + def check_if_stop( + self, + confidences0: torch.Tensor, + confidences1: torch.Tensor, + layer_index: int, + num_points: int, + ) -> torch.Tensor: + """Evaluate stopping condition.""" + confidences = torch.cat([confidences0, confidences1], -1) + threshold = self.confidence_thresholds[layer_index] + ratio_confident = 1.0 - (confidences < threshold).float().sum() / num_points + return ratio_confident > self.conf.depth_confidence + + def pruning_min_kpts(self, device: torch.device) -> int: + """Return the minimum keypoint count needed before pruning is enabled. + + Args: + device: Torch device on which descriptors are processed. + + Returns: + Minimum number of keypoints required for width pruning on that device. + A negative value disables pruning for the device. + """ + if self.conf.flash and FLASH_AVAILABLE and device.type == "cuda": + return self.pruning_keypoint_thresholds["flash"] + else: + return self.pruning_keypoint_thresholds[device.type] diff --git a/kornia/feature/lightglue_onnx/__init__.py b/kornia/feature/lightglue_onnx/__init__.py new file mode 100644 index 0000000..47fb810 --- /dev/null +++ b/kornia/feature/lightglue_onnx/__init__.py @@ -0,0 +1,23 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Feature LightGlue ONNX — ONNX-based LightGlue feature matcher for Kornia. + +This subpackage provides the OnnxLightGlue matcher and related utilities. +""" + +from .lightglue import OnnxLightGlue diff --git a/kornia/feature/lightglue_onnx/lightglue.py b/kornia/feature/lightglue_onnx/lightglue.py new file mode 100644 index 0000000..52f92ca --- /dev/null +++ b/kornia/feature/lightglue_onnx/lightglue.py @@ -0,0 +1,213 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import ClassVar, Union + +import torch + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SAME_DEVICES, KORNIA_CHECK_SHAPE + +from .utils import download_onnx_from_url, normalize_keypoints + +try: + import numpy as np + import onnxruntime as ort +except ImportError: + np = None # type: ignore + ort = None + +__all__ = ["OnnxLightGlue"] + + +class OnnxLightGlue: + r"""Wrapper for loading LightGlue-ONNX models and running inference via ONNXRuntime. + + LightGlue :cite:`LightGlue2023` performs fast descriptor-based deep keypoint matching. + This module requires `onnxruntime` to be installed. + + If you have trained your own LightGlue model, see https://github.com/fabio-sim/LightGlue-ONNX + for how to export the model to ONNX and optimize it. + + Args: + weights: Pretrained weights, or a path to your own exported ONNX model. Available pretrained weights + are ``'disk'``, ``'superpoint'``, ``'disk_fp16'``, and ``'superpoint_fp16'``. `Note that FP16 requires CUDA.` + Defaults to ``'disk_fp16'`` if ``device`` is CUDA, and ``'disk'`` if CPU. + device: Union[str, torch.device, None] to run inference on. + + """ + + MODEL_URLS: ClassVar[dict[str, str]] = { + "disk": "https://github.com/fabio-sim/LightGlue-ONNX/releases/download/v1.0.0/disk_lightglue_fused.onnx", + "superpoint": "https://github.com/fabio-sim/LightGlue-ONNX/releases/download/v1.0.0/superpoint_lightglue_fused.onnx", + "disk_fp16": "https://github.com/fabio-sim/LightGlue-ONNX/releases/download/v1.0.0/disk_lightglue_fused_fp16.onnx", + "superpoint_fp16": "https://github.com/fabio-sim/LightGlue-ONNX/releases/download/v1.0.0/superpoint_lightglue_fused_fp16.onnx", + } + + required_data_keys: ClassVar[list[str]] = ["image0", "image1"] + + def __init__(self, weights: str | None = None, device: Union[str, torch.device, None] = "cpu") -> None: + KORNIA_CHECK(ort is not None, "onnxruntime is not installed.") + KORNIA_CHECK(np is not None, "numpy is not installed.") + + device = torch.device(device) # type: ignore + self.device = device + + if device.type == "cpu": + providers = ["CPUExecutionProvider"] + elif device.type == "cuda": + providers = ["CUDAExecutionProvider", "CPUExecutionProvider"] + else: + raise ValueError(f"Unsupported device {device}") + + if weights is None: + weights = "disk_fp16" if device.type == "cuda" else "disk" + + if weights in self.MODEL_URLS: + if "fp16" in weights: + KORNIA_CHECK(device.type == "cuda", "FP16 requires CUDA.") + + url = self.MODEL_URLS[weights] + if device.type == "cpu": + url = url.replace(".onnx", "_cpu.onnx") + + weights = download_onnx_from_url(url) + + self.session = ort.InferenceSession(weights, providers=providers) + + def __call__(self, data: dict[str, dict[str, torch.Tensor]]) -> dict[str, torch.Tensor]: + """Run the ONNX LightGlue matcher. + + Args: + data: Dictionary containing image features, keypoints, descriptors, and image sizes for matching. + + Returns: + Dictionary containing ONNX LightGlue matches and matching scores. + """ + return self.forward(data) + + def forward(self, data: dict[str, dict[str, torch.Tensor]]) -> dict[str, torch.Tensor]: + r"""Match keypoints and descriptors between two images. + + The output contains the matches (the indices of the matching keypoint pairs between the first and second image) + and the corresponding confidence scores. + Only a batch size of 1 is supported. + + Args: + data: Dictionary containing both images and the keypoints and descriptors thereof. + + Returns: + Dictionary containing the matches and scores. + + ``data`` (``dict``): + ``image0`` (``dict``): + ``keypoints`` (`float32`): :math:`(1, M, 2)` + + ``descriptors`` (`float32`): :math:`(1, M, D)` + + ``image``: :math:`(1, C, H, W)` or ``image_size``: :math:`(1, 2)` + + ``image1`` (``dict``): + ``keypoints`` (`float32`): :math:`(1, N, 2)` + + ``descriptors`` (`float32`): :math:`(1, N, D)` + + ``image``: :math:`(1, C, H, W)` or ``image_size``: :math:`(1, 2)` + + ``output`` (``dict``): + ``matches`` (`int64`): :math:`(S, 2)` + + ``scores`` (`float32`): :math:`(S)` + + """ + # Input validation. + for key in self.required_data_keys: + KORNIA_CHECK(key in data, f"Missing key {key} in data") + data0, data1 = data["image0"], data["image1"] + kpts0_, kpts1_ = data0["keypoints"].contiguous(), data1["keypoints"].contiguous() + desc0, desc1 = data0["descriptors"].contiguous(), data1["descriptors"].contiguous() + KORNIA_CHECK_SAME_DEVICES([kpts0_, desc0, kpts1_, desc1], "Wrong device") + KORNIA_CHECK(kpts0_.device.type == self.device.type, "Wrong device") + KORNIA_CHECK(torch.float32 == kpts0_.dtype == kpts1_.dtype == desc0.dtype == desc1.dtype, "Wrong dtype") + KORNIA_CHECK_SHAPE(kpts0_, ["1", "M", "2"]) + KORNIA_CHECK_SHAPE(kpts1_, ["1", "N", "2"]) + KORNIA_CHECK_SHAPE(desc0, ["1", "M", "D"]) + KORNIA_CHECK_SHAPE(desc1, ["1", "N", "D"]) + KORNIA_CHECK(kpts0_.shape[1] == desc0.shape[1], "Number of keypoints does not match number of descriptors") + KORNIA_CHECK(kpts1_.shape[1] == desc1.shape[1], "Number of keypoints does not match number of descriptors") + KORNIA_CHECK(desc0.shape[2] == desc1.shape[2], "Descriptors' dimensions do not match") + + # Normalize keypoints. + size0, size1 = data0.get("image_size"), data1.get("image_size") + size0 = size0 if size0 is not None else data0["image"].shape[-2:][::-1] # type: ignore + size1 = size1 if size1 is not None else data1["image"].shape[-2:][::-1] # type: ignore + + kpts0 = normalize_keypoints(kpts0_, size=size0) # type: ignore + kpts1 = normalize_keypoints(kpts1_, size=size1) # type: ignore + + KORNIA_CHECK(torch.all(kpts0 >= -1).item() and torch.all(kpts0 <= 1).item(), "") # type: ignore + KORNIA_CHECK(torch.all(kpts1 >= -1).item() and torch.all(kpts1 <= 1).item(), "") # type: ignore + + # Inference. + lightglue_inputs = {"kpts0": kpts0, "kpts1": kpts1, "desc0": desc0, "desc1": desc1} + lightglue_outputs = ["matches0", "mscores0"] + binding = self.session.io_binding() + + for name, tensor in lightglue_inputs.items(): + binding.bind_input( + name, + device_type=self.device.type, + device_id=0, + element_type=np.float32, + shape=tuple(tensor.shape), + buffer_ptr=tensor.data_ptr(), + ) + + for name in lightglue_outputs: + binding.bind_output(name, device_type=self.device.type, device_id=0) + + self.session.run_with_iobinding(binding) + + matches, mscores = binding.get_outputs() + # Prefer DLPack-based conversion when available for zero-copy transfer between them + # The fallback path uses NumPy, which incurs a device-to-host copy and is slower. + + # ORT's io_binding.get_outputs() returns Python-level OrtValue wrappers. + # The DLPack protocol (__dlpack__ + __dlpack_device__) is only available on + # the underlying pybind11 C object (_ortvalue), not the Python wrapper itself. + # We use getattr to handle both current ORT (needs ._ortvalue) and future ORT + # versions that may expose __dlpack__ directly on the wrapper. + # Both outputs must support the full protocol before taking the zero-copy path. + _m = getattr(matches, "_ortvalue", matches) + _s = getattr(mscores, "_ortvalue", mscores) + if ( + hasattr(_m, "__dlpack__") + and hasattr(_m, "__dlpack_device__") + and hasattr(_s, "__dlpack__") + and hasattr(_s, "__dlpack_device__") + ): + outputs = { + "matches": torch.from_dlpack(_m), + "scores": torch.from_dlpack(_s), + } + else: + outputs = { + "matches": torch.from_numpy(matches.numpy()).to(self.device), + "scores": torch.from_numpy(mscores.numpy()).to(self.device), + } + return outputs diff --git a/kornia/feature/lightglue_onnx/utils/__init__.py b/kornia/feature/lightglue_onnx/utils/__init__.py new file mode 100644 index 0000000..646747c --- /dev/null +++ b/kornia/feature/lightglue_onnx/utils/__init__.py @@ -0,0 +1,24 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Feature LightGlue ONNX Utils — Utilities for ONNX-based LightGlue matcher. + +This subpackage provides helper functions for LightGlue ONNX models. +""" + +from .download import download_onnx_from_url +from .keypoints import normalize_keypoints diff --git a/kornia/feature/lightglue_onnx/utils/download.py b/kornia/feature/lightglue_onnx/utils/download.py new file mode 100644 index 0000000..6ccb79f --- /dev/null +++ b/kornia/feature/lightglue_onnx/utils/download.py @@ -0,0 +1,80 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import os +import sys +from typing import Optional +from urllib.parse import urlparse + +from torch.hub import HASH_REGEX, download_url_to_file, get_dir + + +def download_onnx_from_url( + url: str, + model_dir: Optional[str] = None, + progress: bool = True, + check_hash: bool = False, + file_name: Optional[str] = None, +) -> str: + r"""Load the ONNX model at the given URL. + + If downloaded file is a zip file, it will be automatically + decompressed. + + If the object is already present in `model_dir`, it's deserialized and + returned. + The default value of ``model_dir`` is ``/checkpoints`` where + ``hub_dir`` is the directory returned by :func:`~torch.hub.get_dir`. + + Args: + url (str): URL of the object to download + model_dir (str, optional): directory in which to save the object + progress (bool, optional): whether or not to display a progress bar to stderr. + Default: True + check_hash(bool, optional): If True, the filename part of the URL should follow the naming convention + ``filename-.ext`` where ```` is the first eight or more + digits of the SHA256 hash of the contents of the file. The hash is used to + ensure unique names and to verify the contents of the file. + Default: False + file_name (str, optional): name for the downloaded file. Filename from ``url`` will be used if not set. + + Example: + >>> model = download_onnx_from_url('https://github.com/fabio-sim/LightGlue-ONNX/releases/download/v1.0.0/disk_lightglue_fused_fp16.onnx') + + """ + if model_dir is None: + hub_dir = get_dir() + model_dir = os.path.join(hub_dir, "checkpoints") + + os.makedirs(model_dir, exist_ok=True) + + parts = urlparse(url) + filename = os.path.basename(parts.path) + if file_name is not None: + filename = file_name + cached_file = os.path.join(model_dir, filename) + if not os.path.exists(cached_file): + sys.stderr.write(f'Downloading: "{url}" to {cached_file}\n') + hash_prefix = None + if check_hash: + r = HASH_REGEX.search(filename) # r is Optional[Match[str]] + hash_prefix = r.group(1) if r else None + download_url_to_file(url, cached_file, hash_prefix, progress=progress) + + return cached_file diff --git a/kornia/feature/lightglue_onnx/utils/keypoints.py b/kornia/feature/lightglue_onnx/utils/keypoints.py new file mode 100644 index 0000000..134c9c3 --- /dev/null +++ b/kornia/feature/lightglue_onnx/utils/keypoints.py @@ -0,0 +1,30 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch + + +def normalize_keypoints(kpts: torch.Tensor, size: torch.Tensor) -> torch.Tensor: + """Normalize torch.Tensor of keypoints.""" + if isinstance(size, torch.Size): + size = torch.tensor(size)[None] + shift = size.float().to(kpts) / 2 + scale = size.max(1).values.float().to(kpts) / 2 + kpts = (kpts - shift[:, None]) / scale[:, None, None] + return kpts diff --git a/kornia/feature/loftr/__init__.py b/kornia/feature/loftr/__init__.py new file mode 100644 index 0000000..fdd0e3b --- /dev/null +++ b/kornia/feature/loftr/__init__.py @@ -0,0 +1,25 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Feature LoFTR — LoFTR feature matcher for Kornia. + +This subpackage provides the LoFTR (Detector-Free Local Feature Matching) implementation. +""" + +from __future__ import annotations + +from .loftr import LoFTR diff --git a/kornia/feature/loftr/backbone/__init__.py b/kornia/feature/loftr/backbone/__init__.py new file mode 100644 index 0000000..5578c37 --- /dev/null +++ b/kornia/feature/loftr/backbone/__init__.py @@ -0,0 +1,38 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Feature LoFTR Backbone — Backbone modules for LoFTR feature matcher. + +This subpackage provides backbone architectures for LoFTR feature extraction. +""" + +from __future__ import annotations + +from typing import Any + +from .resnet_fpn import ResNetFPN_8_2, ResNetFPN_16_4 + + +def build_backbone(config: dict[str, Any]) -> ResNetFPN_8_2 | ResNetFPN_16_4: + """Build model backbone.""" + if config["backbone_type"] == "ResNetFPN": + if config["resolution"] == (8, 2): + return ResNetFPN_8_2(config["resnetfpn"]) + elif config["resolution"] == (16, 4): + return ResNetFPN_16_4(config["resnetfpn"]) + + raise ValueError(f"LOFTR.BACKBONE_TYPE {config['backbone_type']} with res {config['resolution']} not supported.") diff --git a/kornia/feature/loftr/backbone/resnet_fpn.py b/kornia/feature/loftr/backbone/resnet_fpn.py new file mode 100644 index 0000000..91a1ed2 --- /dev/null +++ b/kornia/feature/loftr/backbone/resnet_fpn.py @@ -0,0 +1,263 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, List, Type, Union + +import torch +import torch.nn.functional as F +from torch import nn + + +def conv1x1(in_planes: int, out_planes: int, stride: int = 1) -> nn.Conv2d: + """1x1 convolution without padding.""" + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, padding=0, bias=False) + + +def conv3x3(in_planes: int, out_planes: int, stride: int = 1) -> nn.Conv2d: + """3x3 convolution with padding.""" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False) + + +class BasicBlock(nn.Module): + """Implement a basic residual block for the ResNet-FPN backbone. + + Args: + in_planes: Number of input channels. + planes: Number of output channels. + stride: Stride of the first convolution. Default: 1. + """ + + def __init__(self, in_planes: int, planes: int, stride: int = 1) -> None: + super().__init__() + self.conv1 = conv3x3(in_planes, planes, stride) + self.conv2 = conv3x3(planes, planes) + self.bn1 = nn.BatchNorm2d(planes) + self.bn2 = nn.BatchNorm2d(planes) + self.relu = nn.ReLU(inplace=True) + + if stride == 1: + self.downsample = None + else: + self.downsample = nn.Sequential(conv1x1(in_planes, planes, stride=stride), nn.BatchNorm2d(planes)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run this LoFTR component forward. + + The method processes coarse or fine matching tensors used by LoFTR. Shape symbols such as `B`, `C`, `H`, `W`, + `N`, and `D` refer to batch size, channels, height, width, token count, and feature dimension. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + y = x + y = self.relu(self.bn1(self.conv1(y))) + y = self.bn2(self.conv2(y)) + + if self.downsample is not None: + x = self.downsample(x) + + return self.relu(x + y) + + +class ResNetFPN_8_2(nn.Module): + """ResNet+FPN, output resolution are 1/8 and 1/2. + + Each block has 2 layers. + """ + + def __init__(self, config: Dict[str, Any]) -> None: + super().__init__() + # Config + block = BasicBlock + initial_dim = config["initial_dim"] + block_dims = config["block_dims"] + + # Class Variable + self.in_planes = initial_dim + + # Networks + self.conv1 = nn.Conv2d(1, initial_dim, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = nn.BatchNorm2d(initial_dim) + self.relu = nn.ReLU(inplace=True) + + self.layer1 = self._make_layer(block, block_dims[0], stride=1) # 1/2 + self.layer2 = self._make_layer(block, block_dims[1], stride=2) # 1/4 + self.layer3 = self._make_layer(block, block_dims[2], stride=2) # 1/8 + + # 3. FPN upsample + self.layer3_outconv = conv1x1(block_dims[2], block_dims[2]) + self.layer2_outconv = conv1x1(block_dims[1], block_dims[2]) + self.layer2_outconv2 = nn.Sequential( + conv3x3(block_dims[2], block_dims[2]), + nn.BatchNorm2d(block_dims[2]), + nn.LeakyReLU(), + conv3x3(block_dims[2], block_dims[1]), + ) + self.layer1_outconv = conv1x1(block_dims[0], block_dims[1]) + self.layer1_outconv2 = nn.Sequential( + conv3x3(block_dims[1], block_dims[1]), + nn.BatchNorm2d(block_dims[1]), + nn.LeakyReLU(), + conv3x3(block_dims[1], block_dims[0]), + ) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu") + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + def _make_layer(self, block: Union[Type[BasicBlock], nn.Module], dim: int, stride: int = 1) -> nn.Sequential: + layer1 = block(self.in_planes, dim, stride=stride) + layer2 = block(dim, dim, stride=1) + layers = (layer1, layer2) + + self.in_planes = dim + return nn.Sequential(*layers) + + def forward(self, x: torch.Tensor) -> List[torch.Tensor]: + # ResNet Backbone + """Run this LoFTR component forward. + + The method processes coarse or fine matching tensors used by LoFTR. Shape symbols such as `B`, `C`, `H`, `W`, + `N`, and `D` refer to batch size, channels, height, width, token count, and feature dimension. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + x0 = self.relu(self.bn1(self.conv1(x))) + x1 = self.layer1(x0) # 1/2 + x2 = self.layer2(x1) # 1/4 + x3 = self.layer3(x2) # 1/8 + + # FPN + x3_out = self.layer3_outconv(x3) + + x2_out = self.layer2_outconv(x2) + x3_out_2x = F.interpolate(x3_out, size=(x2_out.shape[2:]), mode="bilinear", align_corners=True) + x2_out = self.layer2_outconv2(x2_out + x3_out_2x) + + x1_out = self.layer1_outconv(x1) + x2_out_2x = F.interpolate(x2_out, size=(x1_out.shape[2:]), mode="bilinear", align_corners=True) + x1_out = self.layer1_outconv2(x1_out + x2_out_2x) + + return [x3_out, x1_out] + + +class ResNetFPN_16_4(nn.Module): + """ResNet+FPN, output resolution are 1/16 and 1/4. + + Each block has 2 layers. + """ + + def __init__(self, config: Dict[str, Any]) -> None: + super().__init__() + # Config + block = BasicBlock + initial_dim = config["initial_dim"] + block_dims = config["block_dims"] + + # Class Variable + self.in_planes = initial_dim + + # Networks + self.conv1 = nn.Conv2d(1, initial_dim, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = nn.BatchNorm2d(initial_dim) + self.relu = nn.ReLU(inplace=True) + + self.layer1 = self._make_layer(block, block_dims[0], stride=1) # 1/2 + self.layer2 = self._make_layer(block, block_dims[1], stride=2) # 1/4 + self.layer3 = self._make_layer(block, block_dims[2], stride=2) # 1/8 + self.layer4 = self._make_layer(block, block_dims[3], stride=2) # 1/16 + + # 3. FPN upsample + self.layer4_outconv = conv1x1(block_dims[3], block_dims[3]) + self.layer3_outconv = conv1x1(block_dims[2], block_dims[3]) + self.layer3_outconv2 = nn.Sequential( + conv3x3(block_dims[3], block_dims[3]), + nn.BatchNorm2d(block_dims[3]), + nn.LeakyReLU(), + conv3x3(block_dims[3], block_dims[2]), + ) + + self.layer2_outconv = conv1x1(block_dims[1], block_dims[2]) + self.layer2_outconv2 = nn.Sequential( + conv3x3(block_dims[2], block_dims[2]), + nn.BatchNorm2d(block_dims[2]), + nn.LeakyReLU(), + conv3x3(block_dims[2], block_dims[1]), + ) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu") + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + def _make_layer(self, block: Union[Type[BasicBlock], nn.Module], dim: int, stride: int = 1) -> nn.Sequential: + layer1 = block(self.in_planes, dim, stride=stride) + layer2 = block(dim, dim, stride=1) + layers = (layer1, layer2) + + self.in_planes = dim + return nn.Sequential(*layers) + + def forward(self, x: torch.Tensor) -> List[torch.Tensor]: + # ResNet Backbone + """Run this LoFTR component forward. + + The method processes coarse or fine matching tensors used by LoFTR. Shape symbols such as `B`, `C`, `H`, `W`, + `N`, and `D` refer to batch size, channels, height, width, token count, and feature dimension. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + x0 = self.relu(self.bn1(self.conv1(x))) + x1 = self.layer1(x0) # 1/2 + x2 = self.layer2(x1) # 1/4 + x3 = self.layer3(x2) # 1/8 + x4 = self.layer4(x3) # 1/16 + + # FPN + x4_out = self.layer4_outconv(x4) + + x4_out_2x = F.interpolate(x4_out, scale_factor=2.0, mode="bilinear", align_corners=True) + x3_out = self.layer3_outconv(x3) + x3_out = self.layer3_outconv2(x3_out + x4_out_2x) + + x3_out_2x = F.interpolate(x3_out, scale_factor=2.0, mode="bilinear", align_corners=True) + x2_out = self.layer2_outconv(x2) + x2_out = self.layer2_outconv2(x2_out + x3_out_2x) + + return [x4_out, x2_out] diff --git a/kornia/feature/loftr/loftr.py b/kornia/feature/loftr/loftr.py new file mode 100644 index 0000000..0718ffd --- /dev/null +++ b/kornia/feature/loftr/loftr.py @@ -0,0 +1,222 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Any, Optional + +import torch +from torch import nn + +from kornia.geometry import resize + +from .backbone import build_backbone +from .loftr_module import FinePreprocess, LocalFeatureTransformer +from .utils.coarse_matching import CoarseMatching +from .utils.fine_matching import FineMatching +from .utils.position_encoding import PositionEncodingSine + +urls: dict[str, str] = {} +urls["outdoor"] = "http://cmp.felk.cvut.cz/~mishkdmy/models/loftr_outdoor.ckpt" +urls["indoor_new"] = "http://cmp.felk.cvut.cz/~mishkdmy/models/loftr_indoor_ds_new.ckpt" +urls["indoor"] = "http://cmp.felk.cvut.cz/~mishkdmy/models/loftr_indoor.ckpt" + +# Comments: the config below is the one corresponding to the pretrained models +# Some do not change there anything, unless you want to retrain it. + +default_cfg = { + "backbone_type": "ResNetFPN", + "resolution": (8, 2), + "fine_window_size": 5, + "fine_concat_coarse_feat": True, + "resnetfpn": {"initial_dim": 128, "block_dims": [128, 196, 256]}, + "coarse": { + "d_model": 256, + "d_ffn": 256, + "nhead": 8, + "layer_names": ["self", "cross", "self", "cross", "self", "cross", "self", "cross"], + "attention": "linear", + "temp_bug_fix": False, + }, + "match_coarse": { + "thr": 0.2, + "border_rm": 2, + "match_type": "dual_softmax", + "dsmax_temperature": 0.1, + "skh_iters": 3, + "skh_init_bin_score": 1.0, + "skh_prefilter": True, + "train_coarse_percent": 0.4, + "train_pad_num_gt_min": 200, + }, + "fine": {"d_model": 128, "d_ffn": 128, "nhead": 8, "layer_names": ["self", "cross"], "attention": "linear"}, +} + + +class LoFTR(nn.Module): + r"""nn.Module, which finds correspondences between two images. + + This is based on the original code from paper "LoFTR: Detector-Free Local + Feature Matching with Transformers". See :cite:`LoFTR2021` for more details. + + If the distance matrix dm is not provided, :py:func:`torch.cdist` is used. + + Args: + config: Dict with initialization parameters. Do not pass it, unless you know what you are doing`. + pretrained: Download and set pretrained weights to the model. Options: 'outdoor', 'indoor'. + 'outdoor' is trained on the MegaDepth dataset and 'indoor' + on the ScanNet. + + Returns: + Dictionary with image correspondences and confidence scores. + + Example: + >>> img1 = torch.rand(1, 1, 320, 200) + >>> img2 = torch.rand(1, 1, 128, 128) + >>> input = {"image0": img1, "image1": img2} + >>> loftr = LoFTR('outdoor') + >>> out = loftr(input) + + """ + + def __init__(self, pretrained: Optional[str] = "outdoor", config: dict[str, Any] = default_cfg) -> None: + super().__init__() + # Misc + self.config = config + if pretrained == "indoor_new": + self.config["coarse"]["temp_bug_fix"] = True + # Modules + self.backbone = build_backbone(config) + self.pos_encoding = PositionEncodingSine( + config["coarse"]["d_model"], temp_bug_fix=config["coarse"]["temp_bug_fix"] + ) + self.loftr_coarse = LocalFeatureTransformer(config["coarse"]) + self.coarse_matching = CoarseMatching(config["match_coarse"]) + self.fine_preprocess = FinePreprocess(config) + self.loftr_fine = LocalFeatureTransformer(config["fine"]) + self.fine_matching = FineMatching() + self.pretrained = pretrained + if pretrained is not None: + if pretrained not in urls.keys(): + raise ValueError(f"pretrained should be None or one of {urls.keys()}") + + pretrained_dict = torch.hub.load_state_dict_from_url(urls[pretrained], map_location=torch.device("cpu")) + self.load_state_dict(pretrained_dict["state_dict"]) + self.eval() + + def forward(self, data: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + """Run forward. + + Args: + data: dictionary containing the input data in the following format: + + Keyword Args: + image0: left image with shape :math:`(N, 1, H1, W1)`. + image1: right image with shape :math:`(N, 1, H2, W2)`. + mask0 (optional): left image mask. '0' indicates a padded position :math:`(N, H1, W1)`. + mask1 (optional): right image mask. '0' indicates a padded position :math:`(N, H2, W2)`. + + Returns: + - ``keypoints0``, matching keypoints from image0 :math:`(NC, 2)`. + - ``keypoints1``, matching keypoints from image1 :math:`(NC, 2)`. + - ``confidence``, confidence score [0, 1] :math:`(NC)`. + - ``batch_indexes``, batch indexes for the keypoints and lafs :math:`(NC)`. + + """ + # 1. Local Feature CNN + _data: dict[str, torch.Tensor | int | torch.Size] = { + "bs": data["image0"].size(0), + "hw0_i": data["image0"].shape[2:], + "hw1_i": data["image1"].shape[2:], + } + + if _data["hw0_i"] == _data["hw1_i"]: # faster & better BN convergence + feats_c, feats_f = self.backbone(torch.cat([data["image0"], data["image1"]], dim=0)) + (feat_c0, feat_c1), (feat_f0, feat_f1) = feats_c.split(_data["bs"]), feats_f.split(_data["bs"]) + else: # handle different input shapes + (feat_c0, feat_f0), (feat_c1, feat_f1) = self.backbone(data["image0"]), self.backbone(data["image1"]) + + _data.update( + { + "hw0_c": feat_c0.shape[2:], + "hw1_c": feat_c1.shape[2:], + "hw0_f": feat_f0.shape[2:], + "hw1_f": feat_f1.shape[2:], + } + ) + + # 2. coarse-level loftr module + # add featmap with positional encoding, then flatten it to sequence [N, HW, C] + + # feat_c0 = rearrange(self.pos_encoding(feat_c0), 'n c h w -> n (h w) c') + # feat_c1 = rearrange(self.pos_encoding(feat_c1), 'n c h w -> n (h w) c') + feat_c0 = self.pos_encoding(feat_c0).permute(0, 2, 3, 1) + n, _h, _w, c = feat_c0.shape + feat_c0 = feat_c0.reshape(n, -1, c) + + feat_c1 = self.pos_encoding(feat_c1).permute(0, 2, 3, 1) + n1, _h1, _w1, c1 = feat_c1.shape + feat_c1 = feat_c1.reshape(n1, -1, c1) + + mask_c0 = mask_c1 = None # mask is useful in training + if "mask0" in data: + mask_c0 = resize(data["mask0"], _data["hw0_c"], interpolation="nearest").flatten(-2) + if "mask1" in data: + mask_c1 = resize(data["mask1"], _data["hw1_c"], interpolation="nearest").flatten(-2) + feat_c0, feat_c1 = self.loftr_coarse(feat_c0, feat_c1, mask_c0, mask_c1) + + # 3. match coarse-level + self.coarse_matching(feat_c0, feat_c1, _data, mask_c0=mask_c0, mask_c1=mask_c1) + + # 4. fine-level refinement + feat_f0_unfold, feat_f1_unfold = self.fine_preprocess(feat_f0, feat_f1, feat_c0, feat_c1, _data) + if feat_f0_unfold.size(0) != 0: # at least one coarse level predicted + feat_f0_unfold, feat_f1_unfold = self.loftr_fine(feat_f0_unfold, feat_f1_unfold) + + # 5. match fine-level + self.fine_matching(feat_f0_unfold, feat_f1_unfold, _data) + + rename_keys: dict[str, str] = { + "mkpts0_f": "keypoints0", + "mkpts1_f": "keypoints1", + "mconf": "confidence", + "b_ids": "batch_indexes", + } + out: dict[str, torch.Tensor] = {} + for k, v in rename_keys.items(): + _d = _data[k] + if isinstance(_d, torch.Tensor): + out[v] = _d + else: + raise TypeError(f"Expected torch.Tensor for item `{k}`. Gotcha {type(_d)}") + return out + + def load_state_dict(self, state_dict: dict[str, Any], *args: Any, **kwargs: Any) -> Any: # type: ignore[override] + """Load a checkpoint state dictionary into the module. + + Args: + state_dict: Checkpoint state dictionary whose keys are adapted to this module layout. + *args: Additional positional arguments forwarded to the parent `load_state_dict` implementation. + **kwargs: Additional keyword arguments forwarded to the parent `load_state_dict` implementation. + + Returns: + Result returned by the parent `load_state_dict` implementation. + """ + for k in list(state_dict.keys()): + if k.startswith("matcher."): + state_dict[k.replace("matcher.", "", 1)] = state_dict.pop(k) + return super().load_state_dict(state_dict, *args, **kwargs) diff --git a/kornia/feature/loftr/loftr_module/__init__.py b/kornia/feature/loftr/loftr_module/__init__.py new file mode 100644 index 0000000..4add680 --- /dev/null +++ b/kornia/feature/loftr/loftr_module/__init__.py @@ -0,0 +1,24 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Feature LoFTR Module — LoFTR module components for Kornia. + +This subpackage provides fine preprocessing and transformer modules for LoFTR. +""" + +from .fine_preprocess import FinePreprocess +from .transformer import LocalFeatureTransformer diff --git a/kornia/feature/loftr/loftr_module/fine_preprocess.py b/kornia/feature/loftr/loftr_module/fine_preprocess.py new file mode 100644 index 0000000..c6b635e --- /dev/null +++ b/kornia/feature/loftr/loftr_module/fine_preprocess.py @@ -0,0 +1,120 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Tuple + +import torch +import torch.nn.functional as F +from torch import nn + + +class FinePreprocess(nn.Module): + """Preprocess feature maps for fine-level matching in LoFTR. + + Args: + config: A dictionary containing configuration parameters for fine-level processing. + """ + + def __init__(self, config: Dict[str, Any]) -> None: + super().__init__() + + self.config = config + self.cat_c_feat = config["fine_concat_coarse_feat"] + self.W = self.config["fine_window_size"] + + d_model_c = self.config["coarse"]["d_model"] + d_model_f = self.config["fine"]["d_model"] + self.d_model_f = d_model_f + if self.cat_c_feat: + self.down_proj = nn.Linear(d_model_c, d_model_f, bias=True) + self.merge_feat = nn.Linear(2 * d_model_f, d_model_f, bias=True) + + self._reset_parameters() + + def _reset_parameters(self) -> None: + for p in self.parameters(): + if p.dim() > 1: + nn.init.kaiming_normal_(p, mode="fan_out", nonlinearity="relu") + + def forward( + self, + feat_f0: torch.Tensor, + feat_f1: torch.Tensor, + feat_c0: torch.Tensor, + feat_c1: torch.Tensor, + data: Dict[str, Any], + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Run this LoFTR component forward. + + The method processes coarse or fine matching tensors used by LoFTR. Shape symbols such as `B`, `C`, `H`, `W`, + `N`, and `D` refer to batch size, channels, height, width, token count, and feature dimension. + + Args: + feat_f0: Input value used by this method. + feat_f1: Input value used by this method. + feat_c0: Input value used by this method. + feat_c1: Input value used by this method. + data: Dictionary containing image features, keypoints, descriptors, and image sizes for matching. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + W = self.W + stride = data["hw0_f"][0] // data["hw0_c"][0] + + data.update({"W": W}) + if data["b_ids"].shape[0] == 0: + feat0 = torch.empty(0, self.W**2, self.d_model_f, device=feat_f0.device) + feat1 = torch.empty(0, self.W**2, self.d_model_f, device=feat_f0.device) + return feat0, feat1 + + # 1. unfold(crop) all local windows + feat_f0_unfold = F.unfold(feat_f0, kernel_size=(W, W), stride=stride, padding=W // 2) + + # feat_f0_unfold = rearrange(feat_f0_unfold, 'n (c ww) l -> n l ww c', ww=W**2) + n0, cww0, l0 = feat_f0_unfold.shape + c0 = cww0 // (W * W) + feat_f0_unfold = feat_f0_unfold.reshape(n0, c0, -1, l0).permute(0, 3, 2, 1) + + feat_f1_unfold = F.unfold(feat_f1, kernel_size=(W, W), stride=stride, padding=W // 2) + # feat_f1_unfold = rearrange(feat_f1_unfold, 'n (c ww) l -> n l ww c', ww=W**2) + n1, cww1, l1 = feat_f1_unfold.shape + c1 = cww1 // (W * W) + feat_f1_unfold = feat_f1_unfold.reshape(n1, c1, -1, l1).permute(0, 3, 2, 1) + + # 2. select only the predicted matches + feat_f0_unfold = feat_f0_unfold[data["b_ids"], data["i_ids"]] # [n, ww, cf] + feat_f1_unfold = feat_f1_unfold[data["b_ids"], data["j_ids"]] + + # option: use coarse-level loftr feature as context: concat and linear + if self.cat_c_feat: + feat_c_win = self.down_proj( + torch.cat([feat_c0[data["b_ids"], data["i_ids"]], feat_c1[data["b_ids"], data["j_ids"]]], 0) + ) # [2n, c] + feat_cf_win = self.merge_feat( + torch.cat( + [ + torch.cat([feat_f0_unfold, feat_f1_unfold], 0), # [2n, ww, cf] + feat_c_win.unsqueeze(1).repeat(1, W**2, 1), # [2n, ww, cf] + ], + -1, + ) + ) + feat_f0_unfold, feat_f1_unfold = torch.chunk(feat_cf_win, 2, dim=0) + + return feat_f0_unfold, feat_f1_unfold diff --git a/kornia/feature/loftr/loftr_module/linear_attention.py b/kornia/feature/loftr/loftr_module/linear_attention.py new file mode 100644 index 0000000..e58aa64 --- /dev/null +++ b/kornia/feature/loftr/loftr_module/linear_attention.py @@ -0,0 +1,138 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Linear Transformer proposed in "Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention". + +Modified from: https://github.com/idiap/fast- +transformers/blob/master/fast_transformers/attention/linear_attention.py. +""" + +from typing import Optional + +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn import Dropout + + +def elu_feature_map(x: torch.Tensor) -> torch.Tensor: + """Apply elu activation.""" + return torch.nn.functional.elu(x) + 1 + + +class LinearAttention(nn.Module): + """Implement the Linear Attention mechanism for efficient transformer processing. + + This module uses the ELU-based kernel map to approximate softmax attention. + + Args: + eps: A small value to avoid division by zero. Default: 1e-6. + """ + + def __init__(self, eps: float = 1e-6) -> None: + super().__init__() + self.feature_map = elu_feature_map + self.eps = eps + + def forward( + self, + queries: torch.Tensor, + keys: torch.Tensor, + values: torch.Tensor, + q_mask: Optional[torch.Tensor] = None, + kv_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Multi-Head linear attention proposed in "Transformers are RNNs". + + Args: + queries: [N, L, H, D] + keys: [N, S, H, D] + values: [N, S, H, D] + q_mask: [N, L] + kv_mask: [N, S] + + Returns: + queried_values: (N, L, H, D) + + """ + Q = self.feature_map(queries) + K = self.feature_map(keys) + + # set padded position to zero + if q_mask is not None: + Q = Q * q_mask[:, :, None, None] + if kv_mask is not None: + K = K * kv_mask[:, :, None, None] + values = values * kv_mask[:, :, None, None] + + v_length = values.size(1) + values = values / v_length # prevent fp16 overflow + KV = torch.einsum("nshd,nshv->nhdv", K, values) # (S,D)' @ S,V + Z = 1 / (torch.einsum("nlhd,nhd->nlh", Q, K.sum(dim=1)) + self.eps) + queried_values = torch.einsum("nlhd,nhdv,nlh->nlhv", Q, KV, Z) * v_length + + return queried_values.contiguous() + + +class FullAttention(nn.Module): + """Implement the standard Multi-Head Softmax Attention. + + Args: + use_dropout: Whether to apply dropout to the attention weights. Default: False. + attention_dropout: The dropout probability. Default: 0.1. + """ + + def __init__(self, use_dropout: bool = False, attention_dropout: float = 0.1) -> None: + super().__init__() + self.use_dropout = use_dropout + self.dropout = Dropout(attention_dropout) + + def forward( + self, + queries: torch.Tensor, + keys: torch.Tensor, + values: torch.Tensor, + q_mask: Optional[torch.Tensor] = None, + kv_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Multi-head scaled dot-product attention, a.k.a full attention. + + Args: + queries: [N, L, H, D] + keys: [N, S, H, D] + values: [N, S, H, D] + q_mask: [N, L] + kv_mask: [N, S] + + Returns: + queried_values: (N, L, H, D) + + """ + # Compute the unnormalized attention and apply the masks + QK = torch.einsum("nlhd,nshd->nlsh", queries, keys) + if kv_mask is not None and q_mask is not None: + QK.masked_fill_(~(q_mask[:, :, None, None] * kv_mask[:, None, :, None]), float("-inf")) + + # Compute the attention and the weighted average + softmax_temp = 1.0 / queries.size(3) ** 0.5 # sqrt(D) + A = F.softmax(softmax_temp * QK, dim=2) + if self.use_dropout: + A = self.dropout(A) + + queried_values = torch.einsum("nlsh,nshd->nlhd", A, values) + + return queried_values.contiguous() diff --git a/kornia/feature/loftr/loftr_module/transformer.py b/kornia/feature/loftr/loftr_module/transformer.py new file mode 100644 index 0000000..f77be52 --- /dev/null +++ b/kornia/feature/loftr/loftr_module/transformer.py @@ -0,0 +1,143 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import copy +from typing import Any, Literal, Optional + +import torch +from torch import nn + +from .linear_attention import FullAttention, LinearAttention + + +class LoFTREncoderLayer(nn.Module): + """Implement a single encoder layer for the LoFTR transformer. + + Args: + d_model: The number of expected features in the input. + nhead: The number of heads in the multi-head attention. + attention: The type of attention to use. Supported: "linear". Default: "linear". + """ + + def __init__(self, d_model: int, nhead: int, attention: Optional[Literal["linear"]] = "linear") -> None: + super().__init__() + + self.dim = d_model // nhead + self.nhead = nhead + + # multi-head attention + self.q_proj = nn.Linear(d_model, d_model, bias=False) + self.k_proj = nn.Linear(d_model, d_model, bias=False) + self.v_proj = nn.Linear(d_model, d_model, bias=False) + self.attention = LinearAttention() if attention == "linear" else FullAttention() + self.merge = nn.Linear(d_model, d_model, bias=False) + + # feed-forward network + self.mlp = nn.Sequential( + nn.Linear(d_model * 2, d_model * 2, bias=False), nn.ReLU(True), nn.Linear(d_model * 2, d_model, bias=False) + ) + + # norm and dropout + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + + def forward( + self, + x: torch.Tensor, + source: torch.Tensor, + x_mask: Optional[torch.Tensor] = None, + source_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Run forward. + + Args: + x: [N, L, C] + source: [N, S, C] + x_mask: [N, L] (optional) + source_mask: [N, S] (optional) + + """ + bs = x.size(0) + query, key, value = x, source, source + + # multi-head attention + query = self.q_proj(query).view(bs, -1, self.nhead, self.dim) # [N, L, (H, D)] + key = self.k_proj(key).view(bs, -1, self.nhead, self.dim) # [N, S, (H, D)] + value = self.v_proj(value).view(bs, -1, self.nhead, self.dim) + message = self.attention(query, key, value, q_mask=x_mask, kv_mask=source_mask) # [N, L, (H, D)] + message = self.merge(message.view(bs, -1, self.nhead * self.dim)) # [N, L, C] + message = self.norm1(message) + + # feed-forward network + message = self.mlp(torch.cat([x, message], dim=2)) + message = self.norm2(message) + + return x + message + + +class LocalFeatureTransformer(nn.Module): + """A Local Feature Transformer (LoFTR) module.""" + + def __init__(self, config: dict[str, Any]) -> None: + super().__init__() + + self.config = config + self.d_model = config["d_model"] + self.nhead = config["nhead"] + self.layer_names = config["layer_names"] + encoder_layer = LoFTREncoderLayer(config["d_model"], config["nhead"], config["attention"]) + self.layers = nn.ModuleList([copy.deepcopy(encoder_layer) for _ in range(len(self.layer_names))]) + self._reset_parameters() + + def _reset_parameters(self) -> None: + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + + def forward( + self, + feat0: torch.Tensor, + feat1: torch.Tensor, + mask0: None | torch.Tensor = None, + mask1: None | torch.Tensor = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Run forward. + + Args: + feat0: [N, L, C] + feat1: [N, S, C] + mask0: [N, L] (optional) + mask1: [N, S] (optional) + + """ + if self.d_model != feat0.size(2): + msg = "the feature number of src and transformer must be equal" + raise ValueError(msg) + + for layer, name in zip(self.layers, self.layer_names): + if name == "self": + feat0 = layer(feat0, feat0, mask0, mask0) + feat1 = layer(feat1, feat1, mask1, mask1) + elif name == "cross": + feat0 = layer(feat0, feat1, mask0, mask1) + feat1 = layer(feat1, feat0, mask1, mask0) + else: + raise KeyError + + return feat0, feat1 diff --git a/kornia/feature/loftr/utils/__init__.py b/kornia/feature/loftr/utils/__init__.py new file mode 100644 index 0000000..14c5362 --- /dev/null +++ b/kornia/feature/loftr/utils/__init__.py @@ -0,0 +1,21 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Feature LoFTR Utils — Utilities for LoFTR feature matcher. + +This subpackage provides helper functions for LoFTR modules. +""" diff --git a/kornia/feature/loftr/utils/coarse_matching.py b/kornia/feature/loftr/utils/coarse_matching.py new file mode 100644 index 0000000..0e0e669 --- /dev/null +++ b/kornia/feature/loftr/utils/coarse_matching.py @@ -0,0 +1,303 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Union + +import torch +import torch.nn.functional as F +from torch import nn + +INF = 1e9 + + +def mask_border(m: torch.Tensor, b: int, v: Union[torch.Tensor, float, bool]) -> None: + """Mask borders with value. + + Args: + m (torch.Tensor): [N, H0, W0, H1, W1] + b (int): border size. + v (m.dtype): border value. + """ + if b <= 0: + return + + m[:, :b] = v + m[:, :, :b] = v + m[:, :, :, :b] = v + m[:, :, :, :, :b] = v + m[:, -b:] = v + m[:, :, -b:] = v + m[:, :, :, -b:] = v + m[:, :, :, :, -b:] = v + + +def mask_border_with_padding( + m: torch.Tensor, bd: int, v: Union[torch.Tensor, float, bool], p_m0: torch.Tensor, p_m1: torch.Tensor +) -> None: + """Apply masking to a padded boarder.""" + if bd <= 0: + return + + m[:, :bd] = v + m[:, :, :bd] = v + m[:, :, :, :bd] = v + m[:, :, :, :, :bd] = v + + h0s, w0s = p_m0.sum(1).max(-1)[0].int(), p_m0.sum(-1).max(-1)[0].int() + h1s, w1s = p_m1.sum(1).max(-1)[0].int(), p_m1.sum(-1).max(-1)[0].int() + for b_idx, (h0, w0, h1, w1) in enumerate(zip(h0s, w0s, h1s, w1s)): + m[b_idx, h0 - bd :] = v + m[b_idx, :, w0 - bd :] = v + m[b_idx, :, :, h1 - bd :] = v + m[b_idx, :, :, :, w1 - bd :] = v + + +def compute_max_candidates(p_m0: torch.Tensor, p_m1: torch.Tensor) -> torch.Tensor: + """Compute the max candidates of all pairs within a batch. + + Args: + p_m0: padded mask 0 + p_m1: padded mask 1 + + """ + h0s, w0s = p_m0.sum(1).max(-1)[0], p_m0.sum(-1).max(-1)[0] + h1s, w1s = p_m1.sum(1).max(-1)[0], p_m1.sum(-1).max(-1)[0] + max_cand = torch.sum(torch.min(torch.stack([h0s * w0s, h1s * w1s], -1), -1)[0]) + return max_cand + + +class CoarseMatching(nn.Module): + """Perform coarse-level matching between two sets of feature maps. + + This module calculates the confidence matrix and produces coarse matches + using an optimal transport layer or a dual-softmax operator. + + Args: + config: A dictionary containing configuration parameters for the matching. + """ + + def __init__(self, config: Dict[str, Any]) -> None: + super().__init__() + self.config = config + # general config + self.thr = config["thr"] + self.border_rm = config["border_rm"] + # -- # for training fine-level LoFTR + self.train_coarse_percent = config["train_coarse_percent"] + self.train_pad_num_gt_min = config["train_pad_num_gt_min"] + + # we provide 2 options for differentiable matching + self.match_type = config["match_type"] + if self.match_type == "dual_softmax": + self.temperature = config["dsmax_temperature"] + elif self.match_type == "sinkhorn": + try: + from .superglue import log_optimal_transport + except ImportError: + raise ImportError("download superglue.py first!") from None + self.log_optimal_transport = log_optimal_transport + self.bin_score = nn.Parameter(torch.tensor(config["skh_init_bin_score"], requires_grad=True)) + self.skh_iters = config["skh_iters"] + self.skh_prefilter = config["skh_prefilter"] + else: + raise NotImplementedError + + def forward( + self, + feat_c0: torch.Tensor, + feat_c1: torch.Tensor, + data: Dict[str, torch.Tensor], + mask_c0: Optional[torch.Tensor] = None, + mask_c1: Optional[torch.Tensor] = None, + ) -> None: + """Run forward. + + Args: + feat_c0 (torch.Tensor): [N, L, C] + feat_c1 (torch.Tensor): [N, S, C] + data (dict) + mask_c0 (torch.Tensor): [N, L] (optional) + mask_c1 (torch.Tensor): [N, S] (optional) + Update: + data (dict): { + 'b_ids' (torch.Tensor): [M'], + 'i_ids' (torch.Tensor): [M'], + 'j_ids' (torch.Tensor): [M'], + 'gt_mask' (torch.Tensor): [M'], + 'mkpts0_c' (torch.Tensor): [M, 2], + 'mkpts1_c' (torch.Tensor): [M, 2], + 'mconf' (torch.Tensor): [M]} + NOTE: M' != M during training. + + """ + _, L, S, _ = feat_c0.size(0), feat_c0.size(1), feat_c1.size(1), feat_c0.size(2) + + # F.normalize + feat_c0, feat_c1 = (feat / feat.shape[-1] ** 0.5 for feat in [feat_c0, feat_c1]) + + if self.match_type == "dual_softmax": + sim_matrix = torch.einsum("nlc,nsc->nls", feat_c0, feat_c1) / self.temperature + if mask_c0 is not None and mask_c1 is not None: + sim_matrix.masked_fill_(~(mask_c0[..., None] * mask_c1[:, None]).bool(), -INF) + conf_matrix = F.softmax(sim_matrix, 1) * F.softmax(sim_matrix, 2) + + elif self.match_type == "sinkhorn": + # sinkhorn, dustbin included + sim_matrix = torch.einsum("nlc,nsc->nls", feat_c0, feat_c1) + if mask_c0 is not None and mask_c1 is not None: + sim_matrix[:, :L, :S].masked_fill_(~(mask_c0[..., None] * mask_c1[:, None]).bool(), -INF) + + # build uniform prior & use sinkhorn + log_assign_matrix = self.log_optimal_transport(sim_matrix, self.bin_score, self.skh_iters) + assign_matrix = log_assign_matrix.exp() + conf_matrix = assign_matrix[:, :-1, :-1] + + # filter prediction with dustbin score (only in evaluation mode) + if not self.training and self.skh_prefilter: + filter0 = (assign_matrix.max(dim=2)[1] == S)[:, :-1] # [N, L] + filter1 = (assign_matrix.max(dim=1)[1] == L)[:, :-1] # [N, S] + conf_matrix[filter0[..., None].repeat(1, 1, S)] = 0 + conf_matrix[filter1[:, None].repeat(1, L, 1)] = 0 + + if self.config["sparse_spvs"]: + data.update({"conf_matrix_with_bin": assign_matrix.clone()}) + + data.update({"conf_matrix": conf_matrix}) + + # predict coarse matches from conf_matrix + data.update(**self.get_coarse_match(conf_matrix, data)) + + @torch.no_grad() + def get_coarse_match(self, conf_matrix: torch.Tensor, data: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """Get corase matching. + + Args: + conf_matrix (torch.Tensor): [N, L, S] + data (dict): with keys ['hw0_i', 'hw1_i', 'hw0_c', 'hw1_c'] + + Returns: + coarse_matches (dict): { + 'b_ids' (torch.Tensor): [M'], + 'i_ids' (torch.Tensor): [M'], + 'j_ids' (torch.Tensor): [M'], + 'gt_mask' (torch.Tensor): [M'], + 'm_bids' (torch.Tensor): [M], + 'mkpts0_c' (torch.Tensor): [M, 2], + 'mkpts1_c' (torch.Tensor): [M, 2], + 'mconf' (torch.Tensor): [M]} + + """ + axes_lengths = { + "h0c": data["hw0_c"][0], + "w0c": data["hw0_c"][1], + "h1c": data["hw1_c"][0], + "w1c": data["hw1_c"][1], + } + _device = conf_matrix.device + # 1. confidence thresholding + mask = conf_matrix > self.thr + N = conf_matrix.shape[0] + mask = mask.reshape(N, axes_lengths["h0c"], axes_lengths["w0c"], axes_lengths["h1c"], axes_lengths["w1c"]) + # mask = rearrange(mask, 'b (h0c w0c) (h1c w1c) -> b h0c w0c h1c w1c', + # **axes_lengths) + if "mask0" not in data: + mask_border(mask, self.border_rm, False) + else: + mask_border_with_padding(mask, self.border_rm, False, data["mask0"], data["mask1"]) + mask = mask.reshape(N, axes_lengths["h0c"] * axes_lengths["w0c"], axes_lengths["h1c"] * axes_lengths["w1c"]) + # rearrange(mask, 'b h0c w0c h1c w1c -> b (h0c w0c) (h1c w1c)', + # **axes_lengths) + + # 2. mutual nearest + mask = ( + mask + * (conf_matrix == conf_matrix.max(dim=2, keepdim=True)[0]) + * (conf_matrix == conf_matrix.max(dim=1, keepdim=True)[0]) + ) + + # 3. find all valid coarse matches + # this only works when at most one `True` in each row + mask_v, all_j_ids = mask.max(dim=2) + b_ids, i_ids = torch.where(mask_v) + j_ids = all_j_ids[b_ids, i_ids] + mconf = conf_matrix[b_ids, i_ids, j_ids] + + # 4. Random sampling of training samples for fine-level LoFTR + # (optional) F.pad samples with gt coarse-level matches + if self.training: + # NOTE: + # The sampling is performed across all pairs in a batch without manually balancing + # #samples for fine-level increases w.r.t. batch_size + if "mask0" not in data: + num_candidates_max = mask.size(0) * max(mask.size(1), mask.size(2)) + else: + num_candidates_max = compute_max_candidates(data["mask0"], data["mask1"]) + num_matches_train = num_candidates_max * self.train_coarse_percent + num_matches_train = int(num_matches_train) + num_matches_pred = len(b_ids) + if self.train_pad_num_gt_min >= num_matches_train: + msg = "min-num-gt-F.pad should be less than num-train-matches" + raise ValueError(msg) + + # pred_indices is to select from prediction + if num_matches_pred <= num_matches_train - self.train_pad_num_gt_min: + pred_indices = torch.arange(num_matches_pred, device=_device) + else: + pred_indices = torch.randint( + num_matches_pred, (num_matches_train - self.train_pad_num_gt_min,), device=_device + ) + + # gt_pad_indices is to select from gt padding. e.g. max(3787-4800, 200) + gt_pad_indices = torch.randint( + len(data["spv_b_ids"]), + (max(num_matches_train - num_matches_pred, self.train_pad_num_gt_min),), + device=_device, + ) + mconf_gt = torch.zeros(len(data["spv_b_ids"]), device=_device) # set conf of gt paddings to all zero + + b_ids, i_ids, j_ids, mconf = ( # type: ignore + torch.cat([x[pred_indices], y[gt_pad_indices]], dim=0) + for x, y in zip( + [b_ids, data["spv_b_ids"]], + [i_ids, data["spv_i_ids"]], + [j_ids, data["spv_j_ids"]], + [mconf, mconf_gt], + ) + ) + + # These matches select patches that feed into fine-level network + coarse_matches = {"b_ids": b_ids, "i_ids": i_ids, "j_ids": j_ids} + + # 4. Update with matches in original image resolution + scale = data["hw0_i"][0] / data["hw0_c"][0] + scale0 = scale * data["scale0"][b_ids] if "scale0" in data else scale + scale1 = scale * data["scale1"][b_ids] if "scale1" in data else scale + mkpts0_c = torch.stack([i_ids % data["hw0_c"][1], i_ids // data["hw0_c"][1]], dim=1) * scale0 + mkpts1_c = torch.stack([j_ids % data["hw1_c"][1], j_ids // data["hw1_c"][1]], dim=1) * scale1 + + # These matches is the current prediction (for visualization) + coarse_matches.update( + { + "gt_mask": mconf == 0, + "m_bids": b_ids[mconf != 0], # mconf == 0 => gt matches + "mkpts0_c": mkpts0_c[mconf != 0].to(dtype=conf_matrix.dtype), + "mkpts1_c": mkpts1_c[mconf != 0].to(dtype=conf_matrix.dtype), + "mconf": mconf[mconf != 0], + } + ) + + return coarse_matches diff --git a/kornia/feature/loftr/utils/fine_matching.py b/kornia/feature/loftr/utils/fine_matching.py new file mode 100644 index 0000000..7b13639 --- /dev/null +++ b/kornia/feature/loftr/utils/fine_matching.py @@ -0,0 +1,109 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import math +from typing import Any + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.geometry.grid import create_meshgrid +from kornia.geometry.subpix import dsnt + + +class FineMatching(nn.Module): + """FineMatching with s2d paradigm.""" + + def __init__(self) -> None: + super().__init__() + + def forward(self, feat_f0: torch.Tensor, feat_f1: torch.Tensor, data: dict[str, Any]) -> None: + """Run forward. + + Args: + feat_f0 (torch.Tensor): [M, WW, C] + feat_f1 (torch.Tensor): [M, WW, C] + data (dict) + Update: + data (dict):{ + 'expec_f' (torch.Tensor): [M, 3], + 'mkpts0_f' (torch.Tensor): [M, 2], + 'mkpts1_f' (torch.Tensor): [M, 2]} + + """ + M, WW, C = feat_f0.shape + W = int(math.sqrt(WW)) + scale = data["hw0_i"][0] / data["hw0_f"][0] + self.M, self.W, self.WW, self.C, self.scale = M, W, WW, C, scale + + # corner case: if no coarse matches found + if M == 0: + if self.training: + raise ValueError("M >0, when training, see coarse_matching.py") + # logger.warning('No matches found in coarse-level.') + data.update( + { + "expec_f": torch.empty(0, 3, device=feat_f0.device, dtype=feat_f0.dtype), + "mkpts0_f": data["mkpts0_c"], + "mkpts1_f": data["mkpts1_c"], + } + ) + return + + feat_f0_picked = feat_f0[:, WW // 2, :] + sim_matrix = torch.einsum("mc,mrc->mr", feat_f0_picked, feat_f1) + softmax_temp = 1.0 / C**0.5 + heatmap = F.softmax(softmax_temp * sim_matrix, dim=1).view(-1, W, W) + + # compute coordinates from heatmap + coords_normalized = dsnt.spatial_expectation2d(heatmap[None], True)[0] # [M, 2] + grid_normalized = create_meshgrid( + W, W, normalized_coordinates=True, device=heatmap.device, dtype=heatmap.dtype + ).reshape(1, -1, 2) # [1, WW, 2] + + # compute std over + var = torch.sum(grid_normalized**2 * heatmap.view(-1, WW, 1), dim=1) - coords_normalized**2 # [M, 2] + std = torch.sum(torch.sqrt(torch.clamp(var, min=1e-10)), -1) # [M] clamp needed for numerical stability + + # for fine-level supervision + data.update({"expec_f": torch.cat([coords_normalized, std.unsqueeze(1)], -1)}) + + # compute absolute kpt coords + self.get_fine_match(coords_normalized, data) + + @torch.no_grad() + def get_fine_match(self, coords_normed: torch.Tensor, data: dict[str, Any]) -> None: + """Extract fine-level correspondences from the refined matching heatmap. + + Args: + coords_normed: Input value used by this method. + data: Dictionary containing image features, keypoints, descriptors, and image sizes for matching. + + Returns: + Dictionary updated with refined fine-level correspondences. + """ + W, _, _, scale = self.W, self.WW, self.C, self.scale + + # mkpts0_f and mkpts1_f + mkpts0_f = data["mkpts0_c"] + scale1 = scale * data["scale1"][data["b_ids"]] if "scale0" in data else scale + mkpts1_f = data["mkpts1_c"] + (coords_normed * (W // 2) * scale1)[: len(data["mconf"])] + + data.update({"mkpts0_f": mkpts0_f, "mkpts1_f": mkpts1_f}) diff --git a/kornia/feature/loftr/utils/geometry.py b/kornia/feature/loftr/utils/geometry.py new file mode 100644 index 0000000..70f1ed0 --- /dev/null +++ b/kornia/feature/loftr/utils/geometry.py @@ -0,0 +1,83 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Tuple + +import torch + + +@torch.no_grad() +def warp_kpts( + kpts0: torch.Tensor, + depth0: torch.Tensor, + depth1: torch.Tensor, + T_0to1: torch.Tensor, + K0: torch.Tensor, + K1: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Warp kpts0 from I0 to I1 with depth, K and Rt Also check covisibility and depth consistency. + + Depth is consistent if relative error < 0.2 (hard-coded). + + Args: + kpts0: [N, L, 2] - , + depth0: [N, H, W], + depth1: [N, H, W], + T_0to1: [N, 3, 4], + K0: [N, 3, 3], + K1: [N, 3, 3], + + Returns: + calculable_mask: [N, L] + warped_keypoints0: [N, L, 2] + + """ + kpts0_long = kpts0.round().long() + + # Sample depth, get calculable_mask on depth != 0 + kpts0_depth = torch.stack( + [depth0[i, kpts0_long[i, :, 1], kpts0_long[i, :, 0]] for i in range(kpts0.shape[0])], dim=0 + ) # (N, L) + nonzero_mask = kpts0_depth != 0 + + # Unproject + kpts0_h = torch.cat([kpts0, torch.ones_like(kpts0[:, :, [0]])], dim=-1) * kpts0_depth[..., None] # (N, L, 3) + kpts0_cam = K0.inverse() @ kpts0_h.transpose(2, 1) # (N, 3, L) + + # Rigid Transform + w_kpts0_cam = T_0to1[:, :3, :3] @ kpts0_cam + T_0to1[:, :3, [3]] # (N, 3, L) + w_kpts0_depth_computed = w_kpts0_cam[:, 2, :] + + # Project + w_kpts0_h = (K1 @ w_kpts0_cam).transpose(2, 1) # (N, L, 3) + w_kpts0 = w_kpts0_h[:, :, :2] / (w_kpts0_h[:, :, [2]] + 1e-4) # (N, L, 2), +1e-4 to avoid zero depth + + # Covisible Check + h, w = depth1.shape[1:3] + covisible_mask = ( + (w_kpts0[:, :, 0] > 0) * (w_kpts0[:, :, 0] < w - 1) * (w_kpts0[:, :, 1] > 0) * (w_kpts0[:, :, 1] < h - 1) + ) + w_kpts0_long = w_kpts0.long() + w_kpts0_long[~covisible_mask, :] = 0 + + w_kpts0_depth = torch.stack( + [depth1[i, w_kpts0_long[i, :, 1], w_kpts0_long[i, :, 0]] for i in range(w_kpts0_long.shape[0])], dim=0 + ) # (N, L) + consistent_mask = ((w_kpts0_depth - w_kpts0_depth_computed) / w_kpts0_depth).abs() < 0.2 + valid_mask = nonzero_mask * covisible_mask * consistent_mask + + return valid_mask, w_kpts0 diff --git a/kornia/feature/loftr/utils/position_encoding.py b/kornia/feature/loftr/utils/position_encoding.py new file mode 100644 index 0000000..a1c6237 --- /dev/null +++ b/kornia/feature/loftr/utils/position_encoding.py @@ -0,0 +1,92 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math +from typing import Tuple + +import torch +from torch import nn + + +class PositionEncodingSine(nn.Module): + """A sinusoidal position encoding that generalized to 2-dimensional images.""" + + pe: torch.Tensor + + def __init__(self, d_model: int, max_shape: Tuple[int, int] = (256, 256), temp_bug_fix: bool = True) -> None: + """Construct sinusoidal positional encoding. + + Args: + d_model: Dimensions of model input. + max_shape (tuple): for 1/8 featmap, the max length of 256 corresponds to 2048 pixels + temp_bug_fix (bool): As noted in this [issue](https://github.com/zju3dv/LoFTR/issues/41), + the original implementation of LoFTR includes a bug in the pos-enc impl, which has little impact + on the final performance. For now, we keep both impls for backward compatibility. + We will remove the buggy impl after re-training all variants of our released models. + + """ + super().__init__() + self.d_model = d_model + self.temp_bug_fix = temp_bug_fix + + pe = self._create_position_encoding(max_shape) + self.register_buffer("pe", pe, persistent=False) # [1, C, H, W] + + def _create_position_encoding(self, max_shape: Tuple[int, int]) -> torch.Tensor: + """Create a position encoding from scratch. + + For 1/8 feature map (which is standard): If the input image size is H, W (both divisible by 8), the max_shape + should be (H//8, W//8). + """ + pe = torch.zeros((self.d_model, *max_shape)) + y_position = torch.ones(max_shape).cumsum(0).float().unsqueeze(0) + x_position = torch.ones(max_shape).cumsum(1).float().unsqueeze(0) + if self.temp_bug_fix: + div_term = torch.exp( + torch.arange(0, self.d_model // 2, 2).float() * (-math.log(10000.0) / (self.d_model // 2)) + ) + else: # a buggy implementation (for backward compatibility only) + div_term = torch.exp( + torch.arange(0, self.d_model // 2, 2).float() * (-math.log(10000.0) / self.d_model // 2) + ) + div_term = div_term[:, None, None] # [C//4, 1, 1] + pe[0::4, :, :] = torch.sin(x_position * div_term) + pe[1::4, :, :] = torch.cos(x_position * div_term) + pe[2::4, :, :] = torch.sin(y_position * div_term) + pe[3::4, :, :] = torch.cos(y_position * div_term) + return pe.unsqueeze(0) + + def update_position_encoding_size(self, max_shape: Tuple[int, int]) -> None: + """Update position encoding to new max_shape. + + For 1/8 feature map (which is standard): If the input image size is H, W (both divisible by 8), the max_shape + should be (H//8, W//8). + """ + self.pe = self._create_position_encoding(max_shape).to(self.pe.device) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run forward. + + Args: + x: [N, C, H, W] + + """ + if x.size(2) > self.pe.size(2) or x.size(3) > self.pe.size(3): + max_shape = (max(x.size(2), self.pe.size(2)), max(x.size(3), self.pe.size(3))) + self.update_position_encoding_size(max_shape) + + return x + self.pe[:, :, : x.size(2), : x.size(3)] diff --git a/kornia/feature/loftr/utils/supervision.py b/kornia/feature/loftr/utils/supervision.py new file mode 100644 index 0000000..3fed900 --- /dev/null +++ b/kornia/feature/loftr/utils/supervision.py @@ -0,0 +1,169 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Any + +import torch + +from kornia.geometry.grid import create_meshgrid + +from .geometry import warp_kpts + + +# Coarse-Level supervision +@torch.no_grad() +def mask_pts_at_padded_regions(grid_pt: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + """For megadepth dataset, zero-padding exists in images.""" + n, h, w = mask.shape + mask = mask.reshape(n, h * w).unsqueeze(-1).repeat(1, 1, 2) + # from einops import repeat + # mask = repeat(mask, 'n h w -> n (h w) c', c=2) + grid_pt[~mask.bool()] = 0 + return grid_pt + + +@torch.no_grad() +def spvs_coarse(data: dict[str, Any], config: dict[str, Any]) -> None: + """Perform coarse supervision. + + Update: + data (dict): { + "conf_matrix_gt": [N, hw0, hw1], + 'spv_b_ids': [M] + 'spv_i_ids': [M] + 'spv_j_ids': [M] + 'spv_w_pt0_i': [N, hw0, 2], in original image resolution + 'spv_pt1_i': [N, hw1, 2], in original image resolution + } + + Note: + - for scannet dataset, there're 3 kinds of resolution {i, c, f} + - for megadepth dataset, there're 4 kinds of resolution {i, i_resize, c, f} + + """ + # 1. misc + device = data["image0"].device + N, _, H0, W0 = data["image0"].shape + _, _, H1, W1 = data["image1"].shape + scale = config["LOFTR"]["RESOLUTION"][0] + scale0 = scale * data["scale0"][:, None] if "scale0" in data else scale + scale1 = scale * data["scale1"][:, None] if "scale0" in data else scale + h0, w0, h1, w1 = (x // scale for x in [H0, W0, H1, W1]) + + # 2. warp grids + # create kpts in meshgrid and resize them to image resolution + grid_pt0_c = create_meshgrid(h0, w0, False, device).reshape(1, h0 * w0, 2).expand(N, h0 * w0, 2) # [N, hw, 2] + grid_pt0_i = scale0 * grid_pt0_c + grid_pt1_c = create_meshgrid(h1, w1, False, device).reshape(1, h1 * w1, 2).expand(N, h1 * w1, 2) + grid_pt1_i = scale1 * grid_pt1_c + + # mask padded region to (0, 0), so no need to manually mask conf_matrix_gt + if "mask0" in data: + grid_pt0_i = mask_pts_at_padded_regions(grid_pt0_i, data["mask0"]) + grid_pt1_i = mask_pts_at_padded_regions(grid_pt1_i, data["mask1"]) + + # warp kpts bi-directionally and resize them to coarse-level resolution + # (no depth consistency check, since it leads to worse results experimentally) + # (unhandled edge case: points with 0-depth will be warped to the left-up corner) + _, w_pt0_i = warp_kpts(grid_pt0_i, data["depth0"], data["depth1"], data["T_0to1"], data["K0"], data["K1"]) + _, w_pt1_i = warp_kpts(grid_pt1_i, data["depth1"], data["depth0"], data["T_1to0"], data["K1"], data["K0"]) + w_pt0_c = w_pt0_i / scale1 + w_pt1_c = w_pt1_i / scale0 + + # 3. check if mutual nearest neighbor + w_pt0_c_round = w_pt0_c[:, :, :].round().long() + nearest_index1 = w_pt0_c_round[..., 0] + w_pt0_c_round[..., 1] * w1 + w_pt1_c_round = w_pt1_c[:, :, :].round().long() + nearest_index0 = w_pt1_c_round[..., 0] + w_pt1_c_round[..., 1] * w0 + + # corner case: out of boundary + def out_bound_mask(pt: torch.Tensor, w: torch.Tensor, h: torch.Tensor) -> torch.Tensor: + return (pt[..., 0] < 0) + (pt[..., 0] >= w) + (pt[..., 1] < 0) + (pt[..., 1] >= h) + + nearest_index1[out_bound_mask(w_pt0_c_round, w1, h1)] = 0 + nearest_index0[out_bound_mask(w_pt1_c_round, w0, h0)] = 0 + + loop_back = torch.stack([nearest_index0[_b][_i] for _b, _i in enumerate(nearest_index1)], dim=0) + correct_0to1 = loop_back == torch.arange(h0 * w0, device=device)[None].repeat(N, 1) + correct_0to1[:, 0] = False # ignore the top-left corner + + # 4. construct a gt conf_matrix + conf_matrix_gt = torch.zeros(N, h0 * w0, h1 * w1, device=device) + b_ids, i_ids = torch.where(correct_0to1 != 0) + j_ids = nearest_index1[b_ids, i_ids] + + conf_matrix_gt[b_ids, i_ids, j_ids] = 1 + data.update({"conf_matrix_gt": conf_matrix_gt}) + + # 5. save coarse matches(gt) for training fine level + if len(b_ids) == 0: + # this won't affect fine-level loss calculation + b_ids = torch.tensor([0], device=device) + i_ids = torch.tensor([0], device=device) + j_ids = torch.tensor([0], device=device) + + data.update({"spv_b_ids": b_ids, "spv_i_ids": i_ids, "spv_j_ids": j_ids}) + + # 6. save intermediate results (for fast fine-level computation) + data.update({"spv_w_pt0_i": w_pt0_i, "spv_pt1_i": grid_pt1_i}) + + +def compute_supervision_coarse(data: dict[str, Any], config: dict[str, Any]) -> None: + """Compute coarse supervision.""" + if len(set(data["dataset_name"])) != 1: + raise ValueError("Do not support mixed datasets training!") + data_source = data["dataset_name"][0] + if data_source.lower() in ["scannet", "megadepth"]: + spvs_coarse(data, config) + else: + raise ValueError(f"Unknown data source: {data_source}") + + +# Fine-Level supervision +@torch.no_grad() +def spvs_fine(data: dict[str, Any], config: dict[str, Any]) -> None: + """Perform fine supervision. + + Update: + data (dict):{ + "expec_f_gt": [M, 2]} + """ + # 1. misc + # w_pt0_i, pt1_i = data.pop('spv_w_pt0_i'), data.pop('spv_pt1_i') + w_pt0_i, pt1_i = data["spv_w_pt0_i"], data["spv_pt1_i"] + scale = config["LOFTR"]["RESOLUTION"][1] + radius = config["LOFTR"]["FINE_WINDOW_SIZE"] // 2 + + # 2. get coarse prediction + b_ids, i_ids, j_ids = data["b_ids"], data["i_ids"], data["j_ids"] + + # 3. compute gt + scale = scale * data["scale1"][b_ids] if "scale0" in data else scale + # `expec_f_gt` might exceed the window, i.e. abs(*) > 1, which would be filtered later + expec_f_gt = (w_pt0_i[b_ids, i_ids] - pt1_i[b_ids, j_ids]) / scale / radius # [M, 2] + data.update({"expec_f_gt": expec_f_gt}) + + +def compute_supervision_fine(data: dict[str, Any], config: dict[str, Any]) -> None: + """Compute fine supervision.""" + data_source = data["dataset_name"][0] + if data_source.lower() in ["scannet", "megadepth"]: + spvs_fine(data, config) + else: + raise NotImplementedError diff --git a/kornia/feature/matching.py b/kornia/feature/matching.py new file mode 100644 index 0000000..755e2a8 --- /dev/null +++ b/kornia/feature/matching.py @@ -0,0 +1,589 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, ClassVar, Dict, List, Optional, Tuple + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK_DM_DESC, KORNIA_CHECK_SHAPE +from kornia.core.utils import is_mps_tensor_safe +from kornia.feature.laf import get_laf_center +from kornia.feature.steerers import DiscreteSteerer + +from .adalam import get_adalam_default_config, match_adalam + + +def _cdist(d1: torch.Tensor, d2: torch.Tensor) -> torch.Tensor: + r"""Compute pairwise L2 distances between rows of d1 and d2. + + Uses ``torch.cdist`` for float32/float64 on non-MPS devices. Falls back to a + manual expand-and-norm implementation for MPS tensors and for half-precision + dtypes (float16/bfloat16), since ``torch.cdist`` does not support half precision + on CUDA and may be unavailable for these dtypes elsewhere. + """ + half = (torch.float16, torch.bfloat16) + if (not is_mps_tensor_safe(d1)) and (not is_mps_tensor_safe(d2)) and d1.dtype not in half and d2.dtype not in half: + return torch.cdist(d1, d2) + d1_sq = (d1**2).sum(dim=1, keepdim=True) + d2_sq = (d2**2).sum(dim=1, keepdim=True) + dm = d1_sq.repeat(1, d2.size(0)) + d2_sq.repeat(1, d1.size(0)).t() - 2.0 * d1 @ d2.t() + dm = dm.clamp(min=0.0).sqrt() + return dm + + +def _get_default_fginn_params() -> Dict[str, Any]: + config = {"th": 0.85, "mutual": False, "spatial_th": 10.0} + return config + + +def _get_lazy_distance_matrix( + desc1: torch.Tensor, desc2: torch.Tensor, dm_: Optional[torch.Tensor] = None +) -> torch.Tensor: + """Check validity of provided distance matrix, or calculates L2-distance matrix if dm is not provided. + + Args: + desc1: Batch of descriptors of a shape :math:`(B1, D)`. + desc2: Batch of descriptors of a shape :math:`(B2, D)`. + dm_: torch.Tensor containing the distances from each descriptor in desc1 + to each descriptor in desc2, shape of :math:`(B1, B2)`. + + """ + if dm_ is None: + dm = _cdist(desc1, desc2) + else: + KORNIA_CHECK_DM_DESC(desc1, desc2, dm_) + dm = dm_ + return dm + + +def _no_match(dm: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Output empty tensors. + + Returns: + - Descriptor distance of matching descriptors, shape of :math:`(0, 1)`. + - Long torch.Tensor indexes of matching descriptors in desc1 and desc2, shape of :math:`(0, 2)`. + + """ + dists = torch.empty(0, 1, device=dm.device, dtype=dm.dtype) + idxs = torch.empty(0, 2, device=dm.device, dtype=torch.long) + return dists, idxs + + +def match_nn( + desc1: torch.Tensor, desc2: torch.Tensor, dm: Optional[torch.Tensor] = None +) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Find nearest neighbors in desc2 for each vector in desc1. + + If the distance matrix dm is not provided, :py:func:`torch.cdist` is used. + + Args: + desc1: Batch of descriptors of a shape :math:`(B1, D)`. + desc2: Batch of descriptors of a shape :math:`(B2, D)`. + dm: torch.Tensor containing the distances from each descriptor in desc1 + to each descriptor in desc2, shape of :math:`(B1, B2)`. + + Returns: + - Descriptor distance of matching descriptors, shape of :math:`(B1, 1)`. + - Long torch.Tensor indexes of matching descriptors in desc1 and desc2, shape of :math:`(B1, 2)`. + + """ + KORNIA_CHECK_SHAPE(desc1, ["B", "DIM"]) + KORNIA_CHECK_SHAPE(desc2, ["B", "DIM"]) + if (len(desc1) == 0) or (len(desc2) == 0): + return _no_match(desc1) + distance_matrix = _get_lazy_distance_matrix(desc1, desc2, dm) + match_dists, idxs_in_2 = torch.min(distance_matrix, dim=1) + idxs_in1 = torch.arange(0, idxs_in_2.size(0), device=idxs_in_2.device) + matches_idxs = torch.cat([idxs_in1.view(-1, 1), idxs_in_2.view(-1, 1)], 1) + return match_dists.view(-1, 1), matches_idxs.view(-1, 2) + + +def match_mnn( + desc1: torch.Tensor, desc2: torch.Tensor, dm: Optional[torch.Tensor] = None +) -> Tuple[torch.Tensor, torch.Tensor]: + """Find mutual nearest neighbors in desc2 for each vector in desc1. + + If the distance matrix dm is not provided, :py:func:`torch.cdist` is used. + + Args: + desc1: Batch of descriptors of a shape :math:`(B1, D)`. + desc2: Batch of descriptors of a shape :math:`(B2, D)`. + dm: torch.Tensor containing the distances from each descriptor in desc1 + to each descriptor in desc2, shape of :math:`(B1, B2)`. + + Return: + - Descriptor distance of matching descriptors, shape of. :math:`(B3, 1)`. + - Long torch.Tensor indexes of matching descriptors in desc1 and desc2, shape of :math:`(B3, 2)`, + where 0 <= B3 <= min(B1, B2) + + """ + KORNIA_CHECK_SHAPE(desc1, ["B", "DIM"]) + KORNIA_CHECK_SHAPE(desc2, ["B", "DIM"]) + if (len(desc1) == 0) or (len(desc2) == 0): + return _no_match(desc1) + distance_matrix = _get_lazy_distance_matrix(desc1, desc2, dm) + ms = min(distance_matrix.size(0), distance_matrix.size(1)) + match_dists, idxs_in_2 = torch.min(distance_matrix, dim=1) + match_dists2, idxs_in_1 = torch.min(distance_matrix, dim=0) + minsize_idxs = torch.arange(ms, device=distance_matrix.device) + + if distance_matrix.size(0) <= distance_matrix.size(1): + mutual_nns = minsize_idxs == idxs_in_1[idxs_in_2][:ms] + matches_idxs = torch.cat([minsize_idxs.view(-1, 1), idxs_in_2.view(-1, 1)], 1)[mutual_nns] + match_dists = match_dists[mutual_nns] + else: + mutual_nns = minsize_idxs == idxs_in_2[idxs_in_1][:ms] + matches_idxs = torch.cat([idxs_in_1.view(-1, 1), minsize_idxs.view(-1, 1)], 1)[mutual_nns] + match_dists = match_dists2[mutual_nns] + return match_dists.view(-1, 1), matches_idxs.view(-1, 2) + + +def match_snn( + desc1: torch.Tensor, desc2: torch.Tensor, th: float = 0.8, dm: Optional[torch.Tensor] = None +) -> Tuple[torch.Tensor, torch.Tensor]: + """Find nearest neighbors in desc2 for each vector in desc1. + + The method satisfies first to second nearest neighbor distance <= th. + + If the distance matrix dm is not provided, :py:func:`torch.cdist` is used. + + Args: + desc1: Batch of descriptors of a shape :math:`(B1, D)`. + desc2: Batch of descriptors of a shape :math:`(B2, D)`. + th: distance ratio threshold. + dm: torch.Tensor containing the distances from each descriptor in desc1 + to each descriptor in desc2, shape of :math:`(B1, B2)`. + + Return: + - Descriptor distance of matching descriptors, shape of :math:`(B3, 1)`. + - Long torch.Tensor indexes of matching descriptors in desc1 and desc2. Shape: :math:`(B3, 2)`, + where 0 <= B3 <= B1. + + """ + KORNIA_CHECK_SHAPE(desc1, ["B", "DIM"]) + KORNIA_CHECK_SHAPE(desc2, ["B", "DIM"]) + + if desc1.shape[0] == 0 or desc2.shape[0] < 2: # We cannot perform snn check, so output empty matches + return _no_match(desc1) + distance_matrix = _get_lazy_distance_matrix(desc1, desc2, dm) + vals, idxs_in_2 = torch.topk(distance_matrix, 2, dim=1, largest=False) + ratio = vals[:, 0] / vals[:, 1] + mask = ratio <= th + match_dists = ratio[mask] + if len(match_dists) == 0: + return _no_match(distance_matrix) + idxs_in1 = torch.arange(0, idxs_in_2.size(0), device=distance_matrix.device)[mask] + idxs_in_2 = idxs_in_2[:, 0][mask] + matches_idxs = torch.cat([idxs_in1.view(-1, 1), idxs_in_2.view(-1, 1)], 1) + return match_dists.view(-1, 1), matches_idxs.view(-1, 2) + + +def match_smnn( + desc1: torch.Tensor, desc2: torch.Tensor, th: float = 0.95, dm: Optional[torch.Tensor] = None +) -> Tuple[torch.Tensor, torch.Tensor]: + """Find mutual nearest neighbors in desc2 for each vector in desc1. + + the method satisfies first to second nearest neighbor distance <= th. + + If the distance matrix dm is not provided, :py:func:`torch.cdist` is used. + + Args: + desc1: Batch of descriptors of a shape :math:`(B1, D)`. + desc2: Batch of descriptors of a shape :math:`(B2, D)`. + th: distance ratio threshold. + dm: torch.Tensor containing the distances from each descriptor in desc1 + to each descriptor in desc2, shape of :math:`(B1, B2)`. + + Return: + - Descriptor distance of matching descriptors, shape of. :math:`(B3, 1)`. + - Long torch.Tensor indexes of matching descriptors in desc1 and desc2, + shape of :math:`(B3, 2)` where 0 <= B3 <= B1. + + """ + KORNIA_CHECK_SHAPE(desc1, ["B", "DIM"]) + KORNIA_CHECK_SHAPE(desc2, ["B", "DIM"]) + + if (desc1.shape[0] < 2) or (desc2.shape[0] < 2): + return _no_match(desc1) + distance_matrix = _get_lazy_distance_matrix(desc1, desc2, dm) + + dists1, idx1 = match_snn(desc1, desc2, th, distance_matrix) + dists2, idx2 = match_snn(desc2, desc1, th, distance_matrix.t()) + + if len(dists2) > 0 and len(dists1) > 0: + idx2 = idx2.flip(1) + if not is_mps_tensor_safe(idx1): + idxs_dm = torch.cdist(idx1.float(), idx2.float(), p=1.0) + else: + idxs1_rep = idx1.to(desc1).repeat_interleave(idx2.size(0), dim=0) + idxs_dm = (idx2.to(desc2).repeat(idx1.size(0), 1) - idxs1_rep).abs().sum(dim=1) + idxs_dm = idxs_dm.reshape(idx1.size(0), idx2.size(0)) + mutual_idxs1 = idxs_dm.min(dim=1)[0] < 1e-8 + mutual_idxs2 = idxs_dm.min(dim=0)[0] < 1e-8 + good_idxs1 = idx1[mutual_idxs1.view(-1)] + good_idxs2 = idx2[mutual_idxs2.view(-1)] + dists1_good = dists1[mutual_idxs1.view(-1)] + dists2_good = dists2[mutual_idxs2.view(-1)] + _, idx_upl1 = torch.sort(good_idxs1[:, 0]) + _, idx_upl2 = torch.sort(good_idxs2[:, 0]) + good_idxs1 = good_idxs1[idx_upl1] + match_dists = torch.max(dists1_good[idx_upl1], dists2_good[idx_upl2]) + matches_idxs = good_idxs1 + match_dists, matches_idxs = match_dists.view(-1, 1), matches_idxs.view(-1, 2) + else: + match_dists, matches_idxs = _no_match(distance_matrix) + return match_dists, matches_idxs + + +def match_fginn( + desc1: torch.Tensor, + desc2: torch.Tensor, + lafs1: torch.Tensor, + lafs2: torch.Tensor, + th: float = 0.8, + spatial_th: float = 10.0, + mutual: bool = False, + dm: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Find nearest neighbors in desc2 for each vector in desc1. + + The method satisfies first to second nearest neighbor distance <= th, + and assures 2nd nearest neighbor is geometrically inconsistent with the 1st one + (see :cite:`MODS2015` for more details) + + If the distance matrix dm is not provided, :py:func:`torch.cdist` is used. + + Args: + desc1: Batch of descriptors of a shape :math:`(B1, D)`. + desc2: Batch of descriptors of a shape :math:`(B2, D)`. + lafs1: LAFs of a shape :math:`(1, B1, 2, 3)`. + lafs2: LAFs of a shape :math:`(1, B2, 2, 3)`. + th: distance ratio threshold. + spatial_th: minimal distance in pixels to 2nd nearest neighbor. + mutual: also perform mutual nearest neighbor check. + dm: torch.Tensor containing the distances from each descriptor in desc1 + to each descriptor in desc2, shape of :math:`(B1, B2)`. + + Return: + - Descriptor distance of matching descriptors, shape of :math:`(B3, 1)`. + - Long torch.Tensor indexes of matching descriptors in desc1 and desc2. Shape: :math:`(B3, 2)`, + where 0 <= B3 <= B1. + + """ + KORNIA_CHECK_SHAPE(desc1, ["B", "DIM"]) + KORNIA_CHECK_SHAPE(desc2, ["B", "DIM"]) + BIG_NUMBER = 1000000.0 + + distance_matrix = _get_lazy_distance_matrix(desc1, desc2, dm) + dtype = distance_matrix.dtype + + if desc2.shape[0] < 2: # We cannot perform snn check, so output empty matches + return _no_match(distance_matrix) + + num_candidates = max(2, min(10, desc2.shape[0])) + vals_cand, idxs_in_2 = torch.topk(distance_matrix, num_candidates, dim=1, largest=False) + vals = vals_cand[:, 0] + xy2 = get_laf_center(lafs2).view(-1, 2) + candidates_xy = xy2[idxs_in_2] + kdist = torch.norm(candidates_xy - candidates_xy[0:1], p=2, dim=2) + fginn_vals = vals_cand[:, 1:] + (kdist[:, 1:] < spatial_th).to(dtype) * BIG_NUMBER + fginn_vals_best, _fginn_idxs_best = fginn_vals.min(dim=1) + + # orig_idxs = idxs_in_2.gather(1, fginn_idxs_best.unsqueeze(1))[0] + # if you need to know fginn indexes - uncomment + + vals_2nd = fginn_vals_best + idxs_in_2 = idxs_in_2[:, 0] + + ratio = vals / vals_2nd + mask = ratio <= th + match_dists = ratio[mask] + if len(match_dists) == 0: + return _no_match(distance_matrix) + idxs_in1 = torch.arange(0, idxs_in_2.size(0), device=distance_matrix.device)[mask] + idxs_in_2 = idxs_in_2[mask] + matches_idxs = torch.cat([idxs_in1.view(-1, 1), idxs_in_2.view(-1, 1)], 1) + match_dists, matches_idxs = match_dists.view(-1, 1), matches_idxs.view(-1, 2) + + if not mutual: # returning 1-way matches + return match_dists, matches_idxs + _, idxs_in_1_mut = torch.min(distance_matrix, dim=0) + good_mask = matches_idxs[:, 0] == idxs_in_1_mut[matches_idxs[:, 1]] + return match_dists[good_mask], matches_idxs[good_mask] + + +class DescriptorMatcher(nn.Module): + """nn.Module version of descriptor-only matching functions. + + This matcher only requires descriptors (no LAFs). For geometry-aware matching that uses LAFs, + see :class:`~kornia.feature.GeometryAwareDescriptorMatcher`. + + See :func:`~kornia.feature.match_nn`, :func:`~kornia.feature.match_snn`, + :func:`~kornia.feature.match_mnn` or :func:`~kornia.feature.match_smnn` for more details. + + Args: + match_mode: type of matching, can be `nn`, `snn`, `mnn`, `smnn`. + th: threshold on distance ratio, or other quality measure. + + """ + + def __init__(self, match_mode: str = "snn", th: float = 0.8) -> None: + super().__init__() + _match_mode: str = match_mode.lower() + self.known_modes = ["nn", "mnn", "snn", "smnn"] + if _match_mode not in self.known_modes: + raise NotImplementedError(f"{match_mode} is not supported. Try one of {self.known_modes}") + self.match_mode = _match_mode + self.th = th + + def forward(self, desc1: torch.Tensor, desc2: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Run forward. + + Args: + desc1: Batch of descriptors of a shape :math:`(B1, D)`. + desc2: Batch of descriptors of a shape :math:`(B2, D)`. + + Returns: + - Descriptor distance of matching descriptors, shape of :math:`(B3, 1)`. + - Long torch.Tensor indexes of matching descriptors in desc1 and desc2, + shape of :math:`(B3, 2)` where :math:`0 <= B3 <= B1`. + + """ + if self.match_mode == "nn": + out = match_nn(desc1, desc2) + elif self.match_mode == "mnn": + out = match_mnn(desc1, desc2) + elif self.match_mode == "snn": + out = match_snn(desc1, desc2, self.th) + elif self.match_mode == "smnn": + out = match_smnn(desc1, desc2, self.th) + else: + raise NotImplementedError + return out + + +class DescriptorMatcherWithSteerer(nn.Module): + """Matching that is invariant under rotations, using Steerers. + + Args: + steerer: An instance of :func:`kornia.feature.steerers.DiscreteSteerer`. + steerer_order: order of discretisation of rotation angles, e.g. 4 leads to quarter rotations. + steer_mode: can be `global`, `local`. + `global` means that the we output matches from the global rotation with most matches. + `local` means that we output matches from a distance matrix + where the distance between each descriptor pair is the minimal over rotations. + match_mode: type of matching, can be `nn`, `snn`, `mnn`, `smnn`. + WARNING: using steer_mode `global` with match_mode `nn` will lead to bad results + since `nn` doesn't generate different amount of matches depending on goodness of fit. + th: threshold on distance ratio, or other quality measure. + + Example: + >>> import kornia as K + >>> import kornia.feature as KF + >>> device = K.core.utils.get_cuda_or_mps_device_if_available() + >>> img1 = torch.randn([1, 3, 768, 768], device=device) + >>> img2 = torch.randn([1, 3, 768, 768], device=device) + >>> dedode = KF.DeDoDe.from_pretrained(detector_weights="L-C4-v2", descriptor_weights="B-SO2").to(device) + >>> steerer_order = 8 # discretisation order of rotation angles + >>> steerer = KF.steerers.DiscreteSteerer.create_dedode_default( + ... generator_type="SO2", steerer_order=steerer_order + ... ) + >>> steerer = steerer.to(device) + >>> matcher = KF.matching.DescriptorMatcherWithSteerer( + ... steerer=steerer, steerer_order=steerer_order, steer_mode="global", match_mode="smnn", th=0.98 + ... ) + >>> with torch.inference_mode(): + ... kps1, scores1, descs1 = dedode(img1, n=20_000) + ... kps2, scores2, descs2 = dedode(img2, n=20_000) + ... kps1, kps2, descs1, descs2 = kps1[0], kps2[0], descs1[0], descs2[0] + ... dists, idxs, num_rot = matcher( + ... descs1, descs2, normalize=True, subset_size=1000, + ... ) + >>> # print(f"{idxs.shape[0]} tentative matches with steered DeDoDe") + >>> # print(f"at rotation of {num_rot * 360 / steerer_order} degrees") + + """ + + def __init__( + self, + steerer: DiscreteSteerer, + steerer_order: int, + steer_mode: str = "global", + match_mode: str = "snn", + th: float = 0.8, + ) -> None: + super().__init__() + self.steerer = steerer + self.steerer_order = steerer_order + + _steer_mode: str = steer_mode.lower() + self.known_steer_modes = ["global", "local"] + if _steer_mode not in self.known_steer_modes: + raise NotImplementedError(f"{steer_mode} is not supported. Try one of {self.known_steer_modes}") + self.steer_mode = _steer_mode + _match_mode: str = match_mode.lower() + self.known_modes = ["nn", "mnn", "snn", "smnn"] + if _match_mode not in self.known_modes: + raise NotImplementedError(f"{match_mode} is not supported. Try one of {self.known_modes}") + self.match_mode = _match_mode + self.th = th + + def matching_function( + self, d1: torch.Tensor, d2: torch.Tensor, dm: Optional[torch.Tensor] = None + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Dispatch descriptor matching to the configured matching strategy. + + Args: + d1: Descriptor tensor from the first image with shape `(N1, D)`, where `N1` is descriptor count and `D` is + descriptor dimension. + d2: Descriptor tensor from the second image with shape `(N2, D)`, where `N2` is descriptor count and `D` is + descriptor dimension. + dm: Optional descriptor-distance matrix used instead of recomputing distances from `d1` and `d2`. + + Returns: + Tuple containing match distances or scores and index pairs selected by the configured matching strategy. + """ + if self.match_mode == "nn": + return match_nn(d1, d2, dm=dm) + elif self.match_mode == "mnn": + return match_mnn(d1, d2, dm=dm) + elif self.match_mode == "snn": + return match_snn(d1, d2, self.th, dm=dm) + elif self.match_mode == "smnn": + return match_smnn(d1, d2, self.th, dm=dm) + else: + raise NotImplementedError + + def forward( + self, + desc1: torch.Tensor, + desc2: torch.Tensor, + normalize: bool = False, + subset_size: Optional[int] = None, + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[int]]: + """Run forward. + + Args: + desc1: Batch of descriptors of a shape :math:`(B1, D)`. + desc2: Batch of descriptors of a shape :math:`(B2, D)`. + normalize: bool to decide whether to F.normalize descriptors to unit norm. + subset_size: If set, the subset size to use for determining optimal + number of rotations. Smaller subset size leads to faster but less + accurate matching. Only used when `self.steer_mode` is `"global"`. + + Returns: + - Descriptor distance of matching descriptors, shape of :math:`(B3, 1)`. + - Long torch.Tensor indexes of matching descriptors in desc1 and desc2, + shape of :math:`(B3, 2)` where :math:`0 <= B3 <= B1`. + - Number of global rotations from desc1 to desc2, in terms of `self.steerer_order` + (will be `None` if `self.steer_mode` is `local`). + + """ + rot1to2 = None + + if normalize: + desc1 = F.normalize(desc1, dim=-1) + desc2 = F.normalize(desc2, dim=-1) + + if self.steer_mode == "global": + if subset_size is not None: + subsample1 = torch.randperm(desc1.shape[0])[:subset_size] + subsample2 = torch.randperm(desc2.shape[0])[:subset_size] + _, _, rot1to2 = self( + desc1[subsample1], + desc2[subsample2], + normalize=normalize, + ) + desc1 = self.steerer.steer_descriptions( + desc1, + steerer_power=rot1to2, + normalize=normalize, + ) + dist, idx = self.matching_function(desc1, desc2, None) + return dist, idx, rot1to2 + dist, idx = self.matching_function(desc1, desc2, None) + rot1to2 = 0 + for r in range(1, self.steerer_order): + desc1 = self.steerer.steer_descriptions(desc1, normalize=normalize) + dist_new, idx_new = self.matching_function(desc1, desc2, None) + if idx_new.shape[0] > idx.shape[0]: + dist, idx, rot1to2 = dist_new, idx_new, r + elif self.steer_mode == "local": + dm = _cdist(desc1, desc2) + for _ in range(1, self.steerer_order): + desc1 = self.steerer.steer_descriptions(desc1, normalize=normalize) + dm_new = _cdist(desc1, desc2) + dm = torch.minimum(dm, dm_new) + dist, idx = self.matching_function(desc1, desc2, dm) + else: + raise NotImplementedError + + return dist, idx, rot1to2 + + +class GeometryAwareDescriptorMatcher(nn.Module): + """nn.Module version of geometry-aware matching functions that use LAFs (Local Affine Frames). + + Unlike :class:`~kornia.feature.DescriptorMatcher`, this matcher requires both descriptors and LAFs. + See :func:`~kornia.feature.match_fginn` or :func:`~kornia.feature.match_adalam` for more details. + + Args: + match_mode: type of matching, can be `fginn` or `adalam`. + params: dictionary of parameters for the matching function. + + """ + + known_modes: ClassVar[List[str]] = ["fginn", "adalam"] + + def __init__(self, match_mode: str = "fginn", params: Optional[Dict[str, torch.Tensor]] = None) -> None: + super().__init__() + _match_mode: str = match_mode.lower() + if _match_mode not in self.known_modes: + raise NotImplementedError(f"{match_mode} is not supported. Try one of {self.known_modes}") + self.match_mode = _match_mode + self.params = params or {} + + def forward( + self, desc1: torch.Tensor, desc2: torch.Tensor, lafs1: torch.Tensor, lafs2: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Run forward. + + Args: + desc1: Batch of descriptors of a shape :math:`(B1, D)`. + desc2: Batch of descriptors of a shape :math:`(B2, D)`. + lafs1: LAFs of a shape :math:`(1, B1, 2, 3)`. + lafs2: LAFs of a shape :math:`(1, B2, 2, 3)`. + + Returns: + - Descriptor distance of matching descriptors, shape of :math:`(B3, 1)`. + - Long torch.Tensor indexes of matching descriptors in desc1 and desc2, + shape of :math:`(B3, 2)` where :math:`0 <= B3 <= B1`. + + """ + if self.match_mode == "fginn": + params = _get_default_fginn_params() + params.update(self.params) + out = match_fginn(desc1, desc2, lafs1, lafs2, params["th"], params["spatial_th"], params["mutual"]) + elif self.match_mode == "adalam": + _params = get_adalam_default_config() + _params.update(self.params) # type: ignore[typeddict-item] + out = match_adalam(desc1, desc2, lafs1, lafs2, config=_params) + else: + raise NotImplementedError + return out diff --git a/kornia/feature/mkd.py b/kornia/feature/mkd.py new file mode 100644 index 0000000..e86d5d8 --- /dev/null +++ b/kornia/feature/mkd.py @@ -0,0 +1,713 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, List, Tuple, Union + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.constants import pi +from kornia.filters import GaussianBlur2d, SpatialGradient +from kornia.geometry.conversions import cart2pol +from kornia.geometry.grid import create_meshgrid + +# Precomputed coefficients for Von Mises kernel, given N and K(appa). +sqrt2: float = 1.4142135623730951 +COEFFS_N1_K1: List[float] = [0.38214156, 0.48090413] +COEFFS_N2_K8: List[float] = [0.14343168, 0.268285, 0.21979234] +COEFFS_N3_K8: List[float] = [0.14343168, 0.268285, 0.21979234, 0.15838885] +COEFFS: Dict[str, List[float]] = {"xy": COEFFS_N1_K1, "rhophi": COEFFS_N2_K8, "theta": COEFFS_N3_K8} + +urls: Dict[str, str] = { + k: f"https://github.com/manyids2/mkd_pytorch/raw/master/mkd_pytorch/mkd-{k}-64.pth" + for k in ["cart", "polar", "concat"] +} + + +def get_grid_dict(patch_size: int = 32) -> Dict[str, torch.Tensor]: + r"""Get cartesian and polar parametrizations of grid.""" + kgrid = create_meshgrid(height=patch_size, width=patch_size, normalized_coordinates=True) + x = kgrid[0, :, :, 0] + y = kgrid[0, :, :, 1] + rho, phi = cart2pol(x, y) + grid_dict = {"x": x, "y": y, "rho": rho, "phi": phi} + return grid_dict + + +def get_kron_order(d1: int, d2: int) -> torch.Tensor: + r"""Get order for doing kronecker product.""" + grid_d1, grid_d2 = torch.meshgrid(torch.arange(d1), torch.arange(d2), indexing="ij") + kron_order = torch.stack([grid_d1, grid_d2], dim=2).reshape(-1, 2) + return kron_order.to(torch.int64) + + +class MKDGradients(nn.Module): + r"""nn.Module, which computes gradients of given patches, stacked as [magnitudes, orientations]. + + Given gradients $g_x$, $g_y$ with respect to $x$, $y$ respectively, + - $\mathbox{mags} = $\sqrt{g_x^2 + g_y^2 + eps}$ + - $\mathbox{oris} = $\mbox{torch.tan}^{-1}(\nicefrac{g_y}{g_x})$. + + Args: + patch_size: Input patch size in pixels. + + Returns: + gradients of given patches. + + Shape: + - Input: (B, 1, patch_size, patch_size) + - Output: (B, 2, patch_size, patch_size) + + Example: + >>> patches = torch.rand(23, 1, 32, 32) + >>> gradient = MKDGradients() + >>> g = gradient(patches) # 23x2x32x32 + + """ + + def __init__(self) -> None: + super().__init__() + self.eps = 1e-8 + + self.grad = SpatialGradient(mode="diff", order=1, normalized=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Compute gradient magnitude and orientation channels for grayscale patches. + + Args: + x: Patch tensor with shape :math:`(B, 1, H, W)`, where ``B`` is batch size, + ``1`` is the grayscale channel, and ``H``/``W`` are patch height and width. + + Returns: + Tensor with shape :math:`(B, 2, H, W)`. Channel 0 stores gradient magnitude, + and channel 1 stores gradient orientation in radians. + """ + if not isinstance(x, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(x)}") + if not len(x.shape) == 4: + raise ValueError(f"Invalid input shape, we expect Bx1xHxW. Got: {x.shape}") + # Modify 'diff' gradient. Before we had lambda function, but it is not jittable + grads_xy = -self.grad(x) + gx = grads_xy[:, :, 0, :, :] + gy = grads_xy[:, :, 1, :, :] + y = torch.cat(cart2pol(gx, gy, self.eps), dim=1) + return y + + def __repr__(self) -> str: + return self.__class__.__name__ + + +class VonMisesKernel(nn.Module): + r"""nn.Module, which computes parameters of Von Mises kernel given coefficients, and embeds given patches. + + Args: + patch_size: Input patch size in pixels. + coeffs: List of coefficients. Some examples are hardcoded in COEFFS, + + Returns: + Von Mises embedding of given parametrization. + + Shape: + - Input: (B, 1, patch_size, patch_size) + - Output: (B, d, patch_size, patch_size) + + Examples: + >>> oris = torch.rand(23, 1, 32, 32) + >>> vm = VonMisesKernel(patch_size=32, + ... coeffs=[0.14343168, + ... 0.268285, + ... 0.21979234]) + >>> emb = vm(oris) # 23x7x32x32 + + """ + + def __init__(self, patch_size: int, coeffs: Union[List[Union[float, int]], Tuple[Union[float, int], ...]]) -> None: + super().__init__() + + self.patch_size = patch_size + b_coeffs = torch.tensor(coeffs) + self.register_buffer("coeffs", b_coeffs) + + # Compute parameters. + n = len(coeffs) - 1 + self.n = n + self.d = 2 * n + 1 + + # Precompute helper variables. + emb0 = torch.ones([1, 1, patch_size, patch_size]) + frange = torch.arange(n) + 1 + frange = frange.reshape(-1, 1, 1) + weights = torch.zeros([2 * n + 1]) + weights[: n + 1] = torch.sqrt(b_coeffs) + weights[n + 1 :] = torch.sqrt(b_coeffs[1:]) + weights = weights.reshape(-1, 1, 1) + self.register_buffer("emb0", emb0) + self.register_buffer("frange", frange) + self.register_buffer("weights", weights) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Embed orientation values with the Von Mises kernel basis. + + Args: + x: Orientation tensor with shape :math:`(B, 1, H, W)`, where values are + angular orientations in radians. + + Returns: + Kernel embedding with shape :math:`(B, D, H, W)`, where ``D`` is the + number of channels produced by the configured coefficient list. + """ + if not isinstance(x, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(x)}") + + if not len(x.shape) == 4 or x.shape[1] != 1: + raise ValueError(f"Invalid input shape, we expect Bx1xHxW. Got: {x.shape}") + + if not isinstance(self.emb0, torch.Tensor): + raise TypeError(f"Emb0 type is not a torch.Tensor. Got {type(x)}") + + emb0 = self.emb0.to(x).repeat(x.size(0), 1, 1, 1) + frange = self.frange.to(x) * x + emb1 = torch.cos(frange) + emb2 = torch.sin(frange) + embedding = torch.cat([emb0, emb1, emb2], dim=1) + embedding = self.weights * embedding + return embedding + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(patch_size={self.patch_size}, n={self.n}, d={self.d}, coeffs={self.coeffs})" + + +class EmbedGradients(nn.Module): + r"""nn.Module that computes gradient embedding, weighted by sqrt of magnitudes of given patches. + + Args: + patch_size: Input patch size in pixels. + relative: absolute or relative gradients. + + Returns: + Gradient embedding. + + Shape: + - Input: (B, 2, patch_size, patch_size) + - Output: (B, 7, patch_size, patch_size) + + Examples: + >>> grads = torch.rand(23, 2, 32, 32) + >>> emb_grads = EmbedGradients(patch_size=32, + ... relative=False) + >>> emb = emb_grads(grads) # 23x7x32x32 + + """ + + def __init__(self, patch_size: int = 32, relative: bool = False) -> None: + super().__init__() + self.patch_size = patch_size + self.relative = relative + self.eps = 1e-8 + + # Theta kernel for gradients. + self.kernel = VonMisesKernel(patch_size=patch_size, coeffs=COEFFS["theta"]) + + # Relative gradients. + kgrid = create_meshgrid(height=patch_size, width=patch_size, normalized_coordinates=True) + _, phi = cart2pol(kgrid[:, :, :, 0], kgrid[:, :, :, 1]) + self.register_buffer("phi", phi) + + def emb_mags(self, mags: torch.Tensor) -> torch.Tensor: + """Embed square roots of magnitudes with eps for numerical reasons.""" + mags = torch.sqrt(mags + self.eps) + return mags + + def forward(self, grads: torch.Tensor) -> torch.Tensor: + """Embed gradient magnitude and orientation into kernel descriptor channels. + + Args: + grads: Gradient tensor with shape :math:`(B, 2, H, W)`. Channel 0 contains + magnitudes, and channel 1 contains orientations. + + Returns: + Embedded gradient tensor with shape :math:`(B, D, H, W)`, where ``D`` is + the number of channels produced by the orientation kernel. + """ + if not isinstance(grads, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(grads)}") + if not len(grads.shape) == 4: + raise ValueError(f"Invalid input shape, we expect Bx2xHxW. Got: {grads.shape}") + mags = grads[:, :1, :, :] + oris = grads[:, 1:, :, :] + if self.relative: + oris = oris - self.phi.to(oris) + y = self.kernel(oris) * self.emb_mags(mags) + return y + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(patch_size={self.patch_size}, relative={self.relative})" + + +def spatial_kernel_embedding(kernel_type: str, grids: Dict[str, torch.Tensor]) -> torch.Tensor: + r"""Compute embeddings for cartesian and polar parametrizations.""" + factors = {"phi": 1.0, "rho": pi / sqrt2, "x": pi / 2, "y": pi / 2} + if kernel_type == "cart": + coeffs_ = "xy" + params_ = ["x", "y"] + elif kernel_type == "polar": + coeffs_ = "rhophi" + params_ = ["phi", "rho"] + + # Infer patch_size. + keys = list(grids.keys()) + patch_size = grids[keys[0]].shape[-1] + + # Scale appropriately. + grids_normed = {k: v * factors[k] for k, v in grids.items()} + grids_normed = {k: v.unsqueeze(0).unsqueeze(0).float() for k, v in grids_normed.items()} + + # x,y/rho,phi kernels. + vm_a = VonMisesKernel(patch_size=patch_size, coeffs=COEFFS[coeffs_]) + vm_b = VonMisesKernel(patch_size=patch_size, coeffs=COEFFS[coeffs_]) + + emb_a = vm_a(grids_normed[params_[0]]).squeeze() + emb_b = vm_b(grids_normed[params_[1]]).squeeze() + + # Final precomputed position embedding. + kron_order = get_kron_order(vm_a.d, vm_b.d) + spatial_kernel = emb_a.index_select(0, kron_order[:, 0]) * emb_b.index_select(0, kron_order[:, 1]) + return spatial_kernel + + +class ExplicitSpacialEncoding(nn.Module): + r"""nn.Module that computes explicit cartesian or polar embedding. + + Args: + kernel_type: Parametrization of kernel ``'polar'`` or ``'cart'``. + fmap_size: Input feature map size in pixels. + in_dims: Dimensionality of input feature map. + do_gmask: Apply gaussian mask. + do_l2: Apply l2-normalization. + + Returns: + Explicit cartesian or polar embedding. + + Shape: + - Input: (B, in_dims, fmap_size, fmap_size) + - Output: (B, out_dims, fmap_size, fmap_size) + + Example: + >>> emb_ori = torch.rand(23, 7, 32, 32) + >>> ese = ExplicitSpacialEncoding(kernel_type='polar', + ... fmap_size=32, + ... in_dims=7, + ... do_gmask=True, + ... do_l2=True) + >>> desc = ese(emb_ori) # 23x175x32x32 + + """ + + def __init__( + self, + kernel_type: str = "polar", + fmap_size: int = 32, + in_dims: int = 7, + do_gmask: bool = True, + do_l2: bool = True, + ) -> None: + super().__init__() + + if kernel_type not in ["polar", "cart"]: + raise NotImplementedError(f"{kernel_type} is not valid, use polar or cart).") + + self.kernel_type = kernel_type + self.fmap_size = fmap_size + self.in_dims = in_dims + self.do_gmask = do_gmask + self.do_l2 = do_l2 + self.grid = get_grid_dict(fmap_size) + self.gmask = None + + # Precompute embedding. + emb = spatial_kernel_embedding(self.kernel_type, self.grid) + + # Gaussian mask. + if self.do_gmask: + self.gmask = self.get_gmask(sigma=1.0) + emb = emb * self.gmask + + # Store precomputed embedding. + self.register_buffer("emb", emb.unsqueeze(0)) + self.d_emb: int = emb.shape[0] + self.out_dims: int = self.in_dims * self.d_emb + self.odims: int = self.out_dims + + # Store kronecker form. + emb2, idx1 = self.init_kron() + self.register_buffer("emb2", emb2) + self.register_buffer("idx1", idx1) + + def get_gmask(self, sigma: float) -> torch.Tensor: + """Compute Gaussian mask.""" + norm_rho = self.grid["rho"] / self.grid["rho"].max() + gmask = torch.exp(-1 * norm_rho**2 / sigma**2) + return gmask + + def init_kron(self) -> Tuple[torch.Tensor, torch.Tensor]: + """Initialize helper variables to calculate kronecker.""" + kron = get_kron_order(self.in_dims, self.d_emb) + _emb = torch.jit.annotate(torch.Tensor, self.emb) + emb2 = torch.index_select(_emb, 1, kron[:, 1]) + return emb2, kron[:, 0] + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply explicit spatial encoding to embedded patch features. + + Args: + x: Feature map with shape :math:`(B, C, H, W)`, where ``C`` must match + ``in_dims`` and ``H``/``W`` are expected to match ``fmap_size``. + + Returns: + Encoded descriptor tensor with shape :math:`(B, D)`, where ``D`` is the + flattened spatial-kernel descriptor dimension. + """ + if not isinstance(x, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(x)}") + if not ((len(x.shape) == 4) | (x.shape[1] == self.in_dims)): + raise ValueError(f"Invalid input shape, we expect Bx{self.in_dims}xHxW. Got: {x.shape}") + idx1 = torch.jit.annotate(torch.Tensor, self.idx1) + emb1 = torch.index_select(x, 1, idx1) + output = emb1 * self.emb2 + output = output.sum(dim=(2, 3)) + if self.do_l2: + output = F.normalize(output, dim=1) + return output + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}(" + f"kernel_type={self.kernel_type}, " + f"fmap_size={self.fmap_size}, " + f"in_dims={self.in_dims}, " + f"out_dims={self.out_dims}, " + f"do_gmask={self.do_gmask}, " + f"do_l2={self.do_l2})" + ) + + +class Whitening(nn.Module): + r"""nn.Module, performs supervised or unsupervised whitening. + + This is based on the paper "Understanding and Improving Kernel Local Descriptors". + See :cite:`mukundan2019understanding` for more details. + + Args: + xform: Variant of whitening to use. None, 'lw', 'pca', 'pcaws', 'pcawt'. + whitening_model: Dictionary with keys 'mean', 'eigvecs', 'eigvals' holding Tensors. + in_dims: Dimensionality of input descriptors. + output_dims: (int) Dimensionality reduction. + keval: Shrinkage parameter. + t: Attenuation parameter. + + Returns: + l2-normalized, whitened descriptors. + + Shape: + - Input: (B, in_dims, fmap_size, fmap_size) + - Output: (B, out_dims, fmap_size, fmap_size) + + Examples: + >>> descs = torch.rand(23, 238) + >>> whitening_model = {'pca': {'mean': torch.zeros(238), + ... 'eigvecs': torch.eye(238), + ... 'eigvals': torch.ones(238)}} + >>> whitening = Whitening(xform='pcawt', + ... whitening_model=whitening_model, + ... in_dims=238, + ... output_dims=128, + ... keval=40, + ... t=0.7) + >>> wdescs = whitening(descs) # 23x128 + + """ + + def __init__( + self, + xform: str, + whitening_model: Union[Dict[str, Dict[str, torch.Tensor]], None], + in_dims: int, + output_dims: int = 128, + keval: int = 40, + t: float = 0.7, + ) -> None: + super().__init__() + + self.xform = xform + self.in_dims = in_dims + self.keval = keval + self.t = t + self.pval = 1.0 + + # Compute true output_dims. + output_dims = min(output_dims, in_dims) + self.output_dims = output_dims + + # Initialize identity transform. + self.mean = nn.Parameter(torch.zeros(in_dims), requires_grad=True) + self.evecs = nn.Parameter(torch.eye(in_dims)[:, :output_dims], requires_grad=True) + self.evals = nn.Parameter(torch.ones(in_dims)[:output_dims], requires_grad=True) + + if whitening_model is not None: + self.load_whitening_parameters(whitening_model) + + def load_whitening_parameters(self, whitening_model: Dict[str, Dict[str, torch.Tensor]]) -> None: + """Load and adapt whitening statistics for the selected transform. + + Args: + whitening_model: Dictionary containing whitening tensors. The expected + entries are model variants such as ``"pca"`` or ``"lw"``, each holding + ``mean``, ``eigvecs`` (eigenvectors), and ``eigvals`` (eigenvalues). + """ + algo = "lw" if self.xform == "lw" else "pca" + wh_model = whitening_model[algo] + self.mean.data = wh_model["mean"] + self.evecs.data = wh_model["eigvecs"][:, : self.output_dims] + self.evals.data = wh_model["eigvals"][: self.output_dims] + + modifications = { + "pca": self._modify_pca, + "lw": self._modify_lw, + "pcaws": self._modify_pcaws, + "pcawt": self._modify_pcawt, + } + + # Call modification. + modifications[self.xform]() + + def _modify_pca(self) -> None: + """Modify powerlaw parameter.""" + self.pval = 0.5 + + def _modify_lw(self) -> None: + """No modification required.""" + + def _modify_pcaws(self) -> None: + """Shrinkage for eigenvalues.""" + alpha = self.evals[self.keval] + evals = ((1 - alpha) * self.evals) + alpha + self.evecs.data = self.evecs @ torch.diag(torch.pow(evals, -0.5)) + + def _modify_pcawt(self) -> None: + """Attenuation for eigenvalues.""" + m = -0.5 * self.t + self.evecs.data = self.evecs @ torch.diag(torch.pow(self.evals, m)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Whiten and L2-normalize local descriptors. + + Args: + x: Descriptor tensor with shape :math:`(N, D)`, where ``N`` is the number + of descriptors and ``D`` is the input descriptor dimension. + + Returns: + Whitened descriptor tensor with shape :math:`(N, O)`, where ``O`` is + ``output_dims``. + """ + if not isinstance(x, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(x)}") + if not len(x.shape) == 2: + raise ValueError(f"Invalid input shape, we expect NxD. Got: {x.shape}") + x = x - self.mean # Center the data. + x = x @ self.evecs # Apply rotation and/or scaling. + x = torch.sign(x) * torch.pow(torch.abs(x), self.pval) # Powerlaw. + return F.normalize(x, dim=1) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(xform={self.xform}, in_dims={self.in_dims}, output_dims={self.output_dims})" + + +class MKDDescriptor(nn.Module): + r"""nn.Module that computes Multiple Kernel local descriptors. + + This is based on the paper "Understanding and Improving Kernel Local Descriptors". + See :cite:`mukundan2019understanding` for more details. + + Args: + patch_size: Input patch size in pixels. + kernel_type: Parametrization of kernel ``'concat'``, ``'cart'``, ``'polar'``. + whitening: Whitening transform to apply ``None``, ``'lw'``, ``'pca'``, ``'pcawt'``, ``'pcaws'``. + training_set: Set that model was trained on ``'liberty'``, ``'notredame'``, ``'yosemite'``. + output_dims: Dimensionality reduction. + + Returns: + Explicit cartesian or polar embedding. + + Shape: + - Input: :math:`(B, in_{dims}, fmap_{size}, fmap_{size})`. + - Output: :math:`(B, out_{dims}, fmap_{size}, fmap_{size})`, + + Examples: + >>> patches = torch.rand(23, 1, 32, 32) + >>> mkd = MKDDescriptor(patch_size=32, + ... kernel_type='concat', + ... whitening='pcawt', + ... training_set='liberty', + ... output_dims=128) + >>> desc = mkd(patches) # 23x128 + + """ + + def __init__( + self, + patch_size: int = 32, + kernel_type: str = "concat", + whitening: str = "pcawt", + training_set: str = "liberty", + output_dims: int = 128, + ) -> None: + super().__init__() + + self.patch_size: int = patch_size + self.kernel_type: str = kernel_type + self.whitening: str = whitening + self.training_set: str = training_set + + self.sigma = 1.4 * (patch_size / 64) + self.smoothing = GaussianBlur2d((5, 5), (self.sigma, self.sigma), "replicate") + self.gradients = MKDGradients() + # This stupid thing needed for jitting... + polar_s: str = "polar" + cart_s: str = "cart" + self.parametrizations = [polar_s, cart_s] if self.kernel_type == "concat" else [self.kernel_type] + + # Initialize cartesian/polar embedding with absolute/relative gradients. + self.odims: int = 0 + relative_orientations = {polar_s: True, cart_s: False} + self.feats = {} + for parametrization in self.parametrizations: + gradient_embedding = EmbedGradients(patch_size=patch_size, relative=relative_orientations[parametrization]) + spatial_encoding = ExplicitSpacialEncoding( + kernel_type=parametrization, fmap_size=patch_size, in_dims=gradient_embedding.kernel.d + ) + + self.feats[parametrization] = nn.Sequential(gradient_embedding, spatial_encoding) + self.odims += spatial_encoding.odims + # Compute true output_dims. + self.output_dims: int = min(output_dims, self.odims) + + # Load supervised(lw)/unsupervised(pca) model trained on training_set. + if self.whitening is not None: + whitening_models = torch.hub.load_state_dict_from_url( + urls[self.kernel_type], map_location=torch.device("cpu") + ) + whitening_model = whitening_models[training_set] + self.whitening_layer = Whitening( + whitening, whitening_model, in_dims=self.odims, output_dims=self.output_dims + ) + self.odims = self.output_dims + self.eval() + + def forward(self, patches: torch.Tensor) -> torch.Tensor: + """Compute Multiple Kernel Descriptor (MKD) vectors for image patches. + + Args: + patches: Grayscale patch tensor with shape :math:`(B, 1, H, W)`, where + ``B`` is the number of patches and ``H``/``W`` are patch dimensions. + + Returns: + Descriptor tensor with shape :math:`(B, D)`, where ``D`` is the configured + output descriptor dimension after optional whitening. + """ + if not isinstance(patches, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(patches)}") + if not len(patches.shape) == 4: + raise ValueError(f"Invalid input shape, we expect Bx1xHxW. Got: {patches.shape}") + # Extract gradients. + g = self.smoothing(patches) + g = self.gradients(g) + + # Extract polar/cart features. + features = [] + for parametrization in self.parametrizations: + self.feats[parametrization].to(g.device) + features.append(self.feats[parametrization](g)) + + # Concatenate. + y = torch.cat(features, dim=1) + + # l2-F.normalize. + y = F.normalize(y, dim=1) + + # Whiten descriptors. + if self.whitening is not None: + y = self.whitening_layer(y) + + return y + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}(" + f"patch_size={self.patch_size}, " + f"kernel_type={self.kernel_type}, " + f"whitening={self.whitening}, " + f"training_set={self.training_set}, " + f"output_dims={self.output_dims})" + ) + + +def load_whitening_model(kernel_type: str, training_set: str) -> Dict[str, Any]: + """Load whitening model.""" + whitening_models = torch.hub.load_state_dict_from_url(urls[kernel_type], map_location=torch.device("cpu")) + whitening_model = whitening_models[training_set] + return whitening_model + + +class SimpleKD(nn.Module): + """Example to write custom Kernel Descriptors.""" + + def __init__( + self, + patch_size: int = 32, + kernel_type: str = "polar", # 'cart' 'polar' + whitening: str = "pcawt", # 'lw', 'pca', 'pcaws', 'pcawt + training_set: str = "liberty", # 'liberty', 'notredame', 'yosemite' + output_dims: int = 128, + ) -> None: + super().__init__() + + relative: bool = kernel_type == "polar" + sigma: float = 1.4 * (patch_size / 64) + self.patch_size = patch_size + # Sequence of modules. + smoothing = GaussianBlur2d((5, 5), (sigma, sigma), "replicate") + gradients = MKDGradients() + ori = EmbedGradients(patch_size=patch_size, relative=relative) + ese = ExplicitSpacialEncoding(kernel_type=kernel_type, fmap_size=patch_size, in_dims=ori.kernel.d) + wh = Whitening( + whitening, load_whitening_model(kernel_type, training_set), in_dims=ese.odims, output_dims=output_dims + ) + + self.features = nn.Sequential(smoothing, gradients, ori, ese, wh) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run the example kernel-descriptor pipeline. + + Args: + x: Grayscale patch tensor with shape :math:`(B, 1, H, W)`. + + Returns: + Descriptor tensor produced by smoothing, gradient extraction, spatial + encoding, and whitening. + """ + return self.features(x) diff --git a/kornia/feature/orientation.py b/kornia/feature/orientation.py new file mode 100644 index 0000000..0a77b3b --- /dev/null +++ b/kornia/feature/orientation.py @@ -0,0 +1,266 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Optional + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.constants import pi +from kornia.core.check import KORNIA_CHECK_LAF, KORNIA_CHECK_SHAPE +from kornia.filters import SpatialGradient, get_gaussian_discrete_kernel1d, get_gaussian_kernel2d +from kornia.geometry import rad2deg + +from .laf import extract_patches_from_pyramid, get_laf_orientation, set_laf_orientation + +urls: Dict[str, str] = {} +urls["orinet"] = "https://github.com/ducha-aiki/affnet/raw/master/pretrained/OriNet.pth" + + +class PassLAF(nn.Module): + """Dummy module to use instead of local feature orientation or affine shape estimator.""" + + def forward(self, laf: torch.Tensor, img: torch.Tensor) -> torch.Tensor: + """Run forward. + + Args: + laf: :math:`(B, N, 2, 3)` + img: :math:`(B, 1, H, W)` + + Returns: + LAF, unchanged :math:`(B, N, 2, 3)` + + """ + return laf + + +class PatchDominantGradientOrientation(nn.Module): + """Module, which estimates the dominant gradient orientation of the given patches, in radians. + + Zero angle points towards right. + + Args: + patch_size: size of the (square) input patch. + num_angular_bins: number of histogram bins. + eps: for safe division, and arctan. + + """ + + def __init__(self, patch_size: int = 32, num_angular_bins: int = 36, eps: float = 1e-8) -> None: + super().__init__() + self.patch_size = patch_size + self.num_ang_bins = num_angular_bins + self.gradient = SpatialGradient("sobel", 1) + self.eps = eps + self.angular_smooth = nn.Conv1d(1, 1, kernel_size=5, padding=2, bias=False, padding_mode="circular") + with torch.no_grad(): + self.angular_smooth.weight[:] = get_gaussian_discrete_kernel1d(5, 1.6) + sigma: float = float(self.patch_size) / 6.0 + self.weighting = get_gaussian_kernel2d((self.patch_size, self.patch_size), (sigma, sigma), True) + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}(patch_size={self.patch_size}, num_ang_bins={self.num_ang_bins}, eps={self.eps})" + ) + + def forward(self, patch: torch.Tensor) -> torch.Tensor: + """Run forward. + + Args: + patch: :math:`(B, 1, H, W)` + + Returns: + angle in radians: :math:`(B)` + + """ + KORNIA_CHECK_SHAPE(patch, ["B", "1", "H", "W"]) + _, CH, W, H = patch.size() + if (W != self.patch_size) or (H != self.patch_size) or (CH != 1): + raise TypeError( + f"input shape should be must be [Bx1x{self.patch_size}x{self.patch_size}]. Got {patch.size()}" + ) + self.weighting = self.weighting.to(patch.dtype).to(patch.device) + self.angular_smooth = self.angular_smooth.to(patch.dtype).to(patch.device) + grads: torch.Tensor = self.gradient(patch) + # unpack the edges + gx: torch.Tensor = grads[:, :, 0] + gy: torch.Tensor = grads[:, :, 1] + + mag: torch.Tensor = torch.sqrt(gx * gx + gy * gy + self.eps) * self.weighting + ori: torch.Tensor = torch.atan2(gy, gx + self.eps) + 2.0 * pi + + o_big = float(self.num_ang_bins) * (ori + 1.0 * pi) / (2.0 * pi) + bo0_big = torch.floor(o_big) + wo1_big = o_big - bo0_big + bo0_big = bo0_big % self.num_ang_bins + bo1_big = (bo0_big + 1) % self.num_ang_bins + wo0_big = (1.0 - wo1_big) * mag + wo1_big = wo1_big * mag + ang_bins_list = [] + for i in range(0, self.num_ang_bins): + ang_bins_i = F.adaptive_avg_pool2d( + (bo0_big == i).to(patch.dtype) * wo0_big + (bo1_big == i).to(patch.dtype) * wo1_big, (1, 1) + ) + ang_bins_list.append(ang_bins_i) + ang_bins = torch.cat(ang_bins_list, 1).view(-1, 1, self.num_ang_bins) + ang_bins = self.angular_smooth(ang_bins).view(-1, self.num_ang_bins) + values, indices = ang_bins.max(1) + indices_left = (self.num_ang_bins + indices - 1) % self.num_ang_bins + indices_right = (indices + 1) % self.num_ang_bins + left = torch.gather(ang_bins, 1, indices_left.reshape(-1, 1)).reshape(-1) + center = values + right = torch.gather(ang_bins, 1, indices_right.reshape(-1, 1)).reshape(-1) + c_subpix = 0.5 * (left - right) / (left + right - 2.0 * center) + angle = -((2.0 * pi * (indices.to(patch.dtype) + c_subpix) / float(self.num_ang_bins)) - pi) + return angle + + +class OriNet(nn.Module): + """Network, which estimates the canonical orientation of the given 32x32 patches, in radians. + + Zero angle points towards right. This is based on the original code from paper + "Repeatability Is Not Enough: Learning Discriminative Affine Regions via Discriminability"". + See :cite:`AffNet2018` for more details. + + Args: + pretrained: Download and set pretrained weights to the model. + eps: to avoid division by zero in atan2. + + Returns: + Angle in radians. + + Shape: + - Input: (B, 1, 32, 32) + - Output: (B) + + Examples: + >>> input = torch.rand(16, 1, 32, 32) + >>> orinet = OriNet() + >>> angle = orinet(input) # 16 + + """ + + def __init__(self, pretrained: bool = False, eps: float = 1e-8) -> None: + super().__init__() + self.features = nn.Sequential( + nn.Conv2d(1, 16, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(16, affine=False), + nn.ReLU(), + nn.Conv2d(16, 16, kernel_size=3, stride=1, padding=1, bias=False), + nn.BatchNorm2d(16, affine=False), + nn.ReLU(), + nn.Conv2d(16, 32, kernel_size=3, stride=2, padding=1, bias=False), + nn.BatchNorm2d(32, affine=False), + nn.ReLU(), + nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1, bias=False), + nn.BatchNorm2d(32, affine=False), + nn.ReLU(), + nn.Conv2d(32, 64, kernel_size=3, stride=2, padding=1, bias=False), + nn.BatchNorm2d(64, affine=False), + nn.ReLU(), + nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1, bias=False), + nn.BatchNorm2d(64, affine=False), + nn.ReLU(), + nn.Dropout(0.25), + nn.Conv2d(64, 2, kernel_size=8, stride=1, padding=1, bias=True), + nn.Tanh(), + nn.AdaptiveAvgPool2d(1), + ) + self.eps = eps + # use torch.hub to load pretrained model + if pretrained: + pretrained_dict = torch.hub.load_state_dict_from_url(urls["orinet"], map_location=torch.device("cpu")) + self.load_state_dict(pretrained_dict["state_dict"], strict=False) + self.eval() + + @staticmethod + def _normalize_input(x: torch.Tensor, eps: float = 1e-6) -> torch.Tensor: + """Normalize the input by batch.""" + sp, mp = torch.std_mean(x, dim=(-3, -2, -1), keepdim=True) + # WARNING: we need to .detach() input, otherwise the gradients produced by + # the patches extractor with F.grid_sample are very noisy, making the detector + # training totally unstable. + return (x - mp.detach()) / (sp.detach() + eps) + + def forward(self, patch: torch.Tensor) -> torch.Tensor: + """Run forward. + + Args: + patch: :math:`(B, 1, H, W)` + + Returns: + angle in radians: :math:`(B)` + + """ + xy = self.features(self._normalize_input(patch)).view(-1, 2) + angle = torch.atan2(xy[:, 0] + 1e-8, xy[:, 1] + self.eps) + return angle + + +class LAFOrienter(nn.Module): + """Module, which extracts patches using input images and local affine frames (LAFs). + + Then runs :class:`~kornia.feature.PatchDominantGradientOrientation` or + :class:`~kornia.feature.OriNet` on patches and then rotates the LAFs by the estimated angles + + Args: + patch_size: + num_angular_bins: + angle_detector: Patch orientation estimator, e.g. :class:`~kornia.feature.PatchDominantGradientOrientation` + or OriNet. + + """ # pylint: disable + + def __init__( + self, patch_size: int = 32, num_angular_bins: int = 36, angle_detector: Optional[nn.Module] = None + ) -> None: + super().__init__() + self.patch_size = patch_size + self.num_ang_bins = num_angular_bins + self.angle_detector: nn.Module + if angle_detector is None: + self.angle_detector = PatchDominantGradientOrientation(self.patch_size, self.num_ang_bins) + else: + self.angle_detector = angle_detector + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(patch_size={self.patch_size}, angle_detector={self.angle_detector})" + + def forward(self, laf: torch.Tensor, img: torch.Tensor) -> torch.Tensor: + """Run forward. + + Args: + laf: :math:`(B, N, 2, 3)` + img: :math:`(B, 1, H, W)` + + Returns: + LAF_out: :math:`(B, N, 2, 3)` + + """ + KORNIA_CHECK_LAF(laf) + KORNIA_CHECK_SHAPE(img, ["B", "C", "H", "W"]) + if laf.size(0) != img.size(0): + raise ValueError(f"Batch size of laf and img should be the same. Got {img.size(0)}, {laf.size(0)}") + B, N = laf.shape[:2] + patches: torch.Tensor = extract_patches_from_pyramid(img, laf, self.patch_size).view( + -1, 1, self.patch_size, self.patch_size + ) + angles_radians: torch.Tensor = self.angle_detector(patches).view(B, N) + prev_angle = get_laf_orientation(laf).view_as(angles_radians) + laf_out: torch.Tensor = set_laf_orientation(laf, rad2deg(angles_radians) + prev_angle) + return laf_out diff --git a/kornia/feature/responses.py b/kornia/feature/responses.py new file mode 100644 index 0000000..b5b1fbc --- /dev/null +++ b/kornia/feature/responses.py @@ -0,0 +1,467 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Optional, Union + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK_SHAPE +from kornia.filters import gaussian_blur2d, spatial_gradient + + +def _get_kernel_size(sigma: float) -> int: + ksize = int(2.0 * 4.0 * sigma + 1.0) + + # matches OpenCV, but may cause padding problem for small images + # PyTorch does not allow to F.pad more than original size. + # Therefore there is a hack in forward function + + if ksize % 2 == 0: + ksize += 1 + return ksize + + +def harris_response( + input: torch.Tensor, + k: Union[torch.Tensor, float] = 0.04, + grads_mode: str = "sobel", + sigmas: Optional[torch.Tensor] = None, +) -> torch.Tensor: + r"""Compute the Harris cornerness function. + + .. image:: _static/img/harris_response.png + + Function does not do any normalization or nms. The response map is computed according the following formulation: + + .. math:: + R = max(0, det(M) - k \cdot trace(M)^2) + + torch.where: + + .. math:: + M = \sum_{(x,y) \in W} + \begin{bmatrix} + I^{2}_x & I_x I_y \\ + I_x I_y & I^{2}_y \\ + \end{bmatrix} + + and :math:`k` is an empirically determined constant + :math:`k ∈ [ 0.04 , 0.06 ]` + + Args: + input: input image with shape :math:`(B, C, H, W)`. + k: the Harris detector free parameter. + grads_mode: can be ``'sobel'`` for standalone use or ``'diff'`` for use on Gaussian pyramid. + sigmas: coefficients to be multiplied by multichannel response. Should be shape of :math:`(B)` + It is necessary for performing non-maxima-suppression across different scale pyramid levels. + See `vlfeat `_. + + Return: + the response map per channel with shape :math:`(B, C, H, W)`. + + Example: + >>> input = torch.tensor([[[ + ... [0., 0., 0., 0., 0., 0., 0.], + ... [0., 1., 1., 1., 1., 1., 0.], + ... [0., 1., 1., 1., 1., 1., 0.], + ... [0., 1., 1., 1., 1., 1., 0.], + ... [0., 1., 1., 1., 1., 1., 0.], + ... [0., 1., 1., 1., 1., 1., 0.], + ... [0., 0., 0., 0., 0., 0., 0.], + ... ]]]) # 1x1x7x7 + >>> # compute the response map + harris_response(input, 0.04) + tensor([[[[0.0012, 0.0039, 0.0020, 0.0000, 0.0020, 0.0039, 0.0012], + [0.0039, 0.0065, 0.0040, 0.0000, 0.0040, 0.0065, 0.0039], + [0.0020, 0.0040, 0.0029, 0.0000, 0.0029, 0.0040, 0.0020], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0020, 0.0040, 0.0029, 0.0000, 0.0029, 0.0040, 0.0020], + [0.0039, 0.0065, 0.0040, 0.0000, 0.0040, 0.0065, 0.0039], + [0.0012, 0.0039, 0.0020, 0.0000, 0.0020, 0.0039, 0.0012]]]]) + + """ + # TODO: Recompute doctest + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + + if sigmas is not None: + if not isinstance(sigmas, torch.Tensor): + raise TypeError(f"sigmas type is not a torch.Tensor. Got {type(sigmas)}") + if (not len(sigmas.shape) == 1) or (sigmas.size(0) != input.size(0)): + raise ValueError(f"Invalid sigmas shape, we expect B == input.size(0). Got: {sigmas.shape}") + + gradients: torch.Tensor = spatial_gradient(input, grads_mode) + dx: torch.Tensor = gradients[:, :, 0] + dy: torch.Tensor = gradients[:, :, 1] + + # compute the structure torch.Tensor M elements + + dx2: torch.Tensor = gaussian_blur2d(dx**2, (7, 7), (1.0, 1.0)) + dy2: torch.Tensor = gaussian_blur2d(dy**2, (7, 7), (1.0, 1.0)) + dxy: torch.Tensor = gaussian_blur2d(dx * dy, (7, 7), (1.0, 1.0)) + + det_m: torch.Tensor = dx2 * dy2 - dxy * dxy + trace_m: torch.Tensor = dx2 + dy2 + + # compute the response map + scores: torch.Tensor = det_m - k * (trace_m**2) + + if sigmas is not None: + scores = scores * sigmas.pow(4).view(-1, 1, 1, 1) + + return scores + + +def gftt_response( + input: torch.Tensor, grads_mode: str = "sobel", sigmas: Optional[torch.Tensor] = None +) -> torch.Tensor: + r"""Compute the Shi-Tomasi cornerness function. + + .. image:: _static/img/gftt_response.png + + Function does not do any normalization or nms. The response map is computed according the following formulation: + + .. math:: + R = min(eig(M)) + + torch.where: + + .. math:: + M = \sum_{(x,y) \in W} + \begin{bmatrix} + I^{2}_x & I_x I_y \\ + I_x I_y & I^{2}_y \\ + \end{bmatrix} + + Args: + input: input image with shape :math:`(B, C, H, W)`. + grads_mode: can be ``'sobel'`` for standalone use or ``'diff'`` for use on Gaussian pyramid. + sigmas: coefficients to be multiplied by multichannel response. Should be shape of :math:`(B)` + It is necessary for performing non-maxima-suppression across different scale pyramid levels. + See `vlfeat `_. + + Return: + the response map per channel with shape :math:`(B, C, H, W)`. + + Example: + >>> input = torch.tensor([[[ + ... [0., 0., 0., 0., 0., 0., 0.], + ... [0., 1., 1., 1., 1., 1., 0.], + ... [0., 1., 1., 1., 1., 1., 0.], + ... [0., 1., 1., 1., 1., 1., 0.], + ... [0., 1., 1., 1., 1., 1., 0.], + ... [0., 1., 1., 1., 1., 1., 0.], + ... [0., 0., 0., 0., 0., 0., 0.], + ... ]]]) # 1x1x7x7 + >>> # compute the response map + gftt_response(input) + tensor([[[[0.0155, 0.0334, 0.0194, 0.0000, 0.0194, 0.0334, 0.0155], + [0.0334, 0.0575, 0.0339, 0.0000, 0.0339, 0.0575, 0.0334], + [0.0194, 0.0339, 0.0497, 0.0000, 0.0497, 0.0339, 0.0194], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0194, 0.0339, 0.0497, 0.0000, 0.0497, 0.0339, 0.0194], + [0.0334, 0.0575, 0.0339, 0.0000, 0.0339, 0.0575, 0.0334], + [0.0155, 0.0334, 0.0194, 0.0000, 0.0194, 0.0334, 0.0155]]]]) + + """ + # TODO: Recompute doctest + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + + gradients: torch.Tensor = spatial_gradient(input, grads_mode) + dx: torch.Tensor = gradients[:, :, 0] + dy: torch.Tensor = gradients[:, :, 1] + + dx2: torch.Tensor = gaussian_blur2d(dx**2, (7, 7), (1.0, 1.0)) + dy2: torch.Tensor = gaussian_blur2d(dy**2, (7, 7), (1.0, 1.0)) + dxy: torch.Tensor = gaussian_blur2d(dx * dy, (7, 7), (1.0, 1.0)) + + det_m: torch.Tensor = dx2 * dy2 - dxy * dxy + trace_m: torch.Tensor = dx2 + dy2 + + e1: torch.Tensor = 0.5 * (trace_m + torch.sqrt((trace_m**2 - 4 * det_m).abs())) + e2: torch.Tensor = 0.5 * (trace_m - torch.sqrt((trace_m**2 - 4 * det_m).abs())) + + scores: torch.Tensor = torch.min(e1, e2) + + if sigmas is not None: + scores = scores * sigmas.pow(4).view(-1, 1, 1, 1) + + return scores + + +def hessian_response( + input: torch.Tensor, grads_mode: str = "sobel", sigmas: Optional[torch.Tensor] = None +) -> torch.Tensor: + r"""Compute the absolute of determinant of the Hessian matrix. + + .. image:: _static/img/hessian_response.png + + Function does not do any normalization or nms. The response map is computed according the following formulation: + + .. math:: + R = det(H) + + torch.where: + + .. math:: + M = \sum_{(x,y) \in W} + \begin{bmatrix} + I_{xx} & I_{xy} \\ + I_{xy} & I_{yy} \\ + \end{bmatrix} + + Args: + input: input image with shape :math:`(B, C, H, W)`. + grads_mode: can be ``'sobel'`` for standalone use or ``'diff'`` for use on Gaussian pyramid. + sigmas: coefficients to be multiplied by multichannel response. Should be shape of :math:`(B)` + It is necessary for performing non-maxima-suppression across different scale pyramid levels. + See `vlfeat `_. + + Return: + the response map per channel with shape :math:`(B, C, H, W)`. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + Examples: + >>> input = torch.tensor([[[ + ... [0., 0., 0., 0., 0., 0., 0.], + ... [0., 1., 1., 1., 1., 1., 0.], + ... [0., 1., 1., 1., 1., 1., 0.], + ... [0., 1., 1., 1., 1., 1., 0.], + ... [0., 1., 1., 1., 1., 1., 0.], + ... [0., 1., 1., 1., 1., 1., 0.], + ... [0., 0., 0., 0., 0., 0., 0.], + ... ]]]) # 1x1x7x7 + >>> # compute the response map + hessian_response(input) + tensor([[[[0.0155, 0.0334, 0.0194, 0.0000, 0.0194, 0.0334, 0.0155], + [0.0334, 0.0575, 0.0339, 0.0000, 0.0339, 0.0575, 0.0334], + [0.0194, 0.0339, 0.0497, 0.0000, 0.0497, 0.0339, 0.0194], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0194, 0.0339, 0.0497, 0.0000, 0.0497, 0.0339, 0.0194], + [0.0334, 0.0575, 0.0339, 0.0000, 0.0339, 0.0575, 0.0334], + [0.0155, 0.0334, 0.0194, 0.0000, 0.0194, 0.0334, 0.0155]]]]) + + """ + # TODO: Recompute doctest + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + + if sigmas is not None: + if not isinstance(sigmas, torch.Tensor): + raise TypeError(f"sigmas type is not a torch.Tensor. Got {type(sigmas)}") + + if (not len(sigmas.shape) == 1) or (sigmas.size(0) != input.size(0)): + raise ValueError(f"Invalid sigmas shape, we expect B == input.size(0). Got: {sigmas.shape}") + + gradients: torch.Tensor = spatial_gradient(input, grads_mode, 2) + dxx: torch.Tensor = gradients[:, :, 0] + dxy: torch.Tensor = gradients[:, :, 1] + dyy: torch.Tensor = gradients[:, :, 2] + + scores: torch.Tensor = dxx * dyy - dxy**2 + + if sigmas is not None: + scores = scores * sigmas.pow(4).view(-1, 1, 1, 1) + + return scores + + +def dog_response(input: torch.Tensor) -> torch.Tensor: + r"""Compute the Difference-of-Gaussian response. + + Args: + input: a given the gaussian 5d torch.Tensor :math:`(B, C, D, H, W)`. + + Return: + the response map per channel with shape :math:`(B, C, D-1, H, W)`. + + """ + KORNIA_CHECK_SHAPE(input, ["B", "C", "L", "H", "W"]) + + return input[:, :, 1:] - input[:, :, :-1] + + +def dog_response_single(input: torch.Tensor, sigma1: float = 1.0, sigma2: float = 1.6) -> torch.Tensor: + r"""Compute the Difference-of-Gaussian response. + + .. image:: _static/img/dog_response_single.png + + Args: + input: a given the gaussian 4d torch.Tensor :math:`(B, C, H, W)`. + sigma1: lower gaussian sigma + sigma2: bigger gaussian sigma + + Return: + the response map per channel with shape :math:`(B, C, H, W)`. + + """ + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + ks1 = _get_kernel_size(sigma1) + ks2 = _get_kernel_size(sigma2) + g1 = gaussian_blur2d(input, (ks1, ks1), (sigma1, sigma1)) + g2 = gaussian_blur2d(input, (ks2, ks2), (sigma2, sigma2)) + return g2 - g1 + + +class BlobDoG(nn.Module): + r"""nn.Module that calculates Difference-of-Gaussians blobs. + + See + :func: `~kornia.feature.dog_response` for details. + """ + + def __init__(self) -> None: + super().__init__() + + def __repr__(self) -> str: + return self.__class__.__name__ + + def forward(self, input: torch.Tensor, sigmas: Optional[torch.Tensor] = None) -> torch.Tensor: + """Compute Difference-of-Gaussians response maps. + + Args: + input: Input tensor of shape :math:`(B, C, H, W)`. + sigmas: Unused in this wrapper; kept for API compatibility. + + Returns: + DoG response tensor from :func:`dog_response`. + """ + return dog_response(input) + + +class BlobDoGSingle(nn.Module): + r"""nn.Module that calculates Difference-of-Gaussians blobs. + + .. image:: _static/img/dog_response_single.png + + See :func:`~kornia.feature.dog_response_single` for details. + """ + + def __init__(self, sigma1: float = 1.0, sigma2: float = 1.6) -> None: + super().__init__() + self.sigma1 = sigma1 + self.sigma2 = sigma2 + + def __repr__(self) -> str: + return f"{self.__class__.__name__}, sigma1={self.sigma1}, sigma2={self.sigma2})" + + def forward(self, input: torch.Tensor, sigmas: Optional[torch.Tensor] = None) -> torch.Tensor: + """Compute single-scale Difference-of-Gaussians response. + + Args: + input: Input tensor of shape :math:`(B, C, H, W)`. + sigmas: Unused in this wrapper; kept for API compatibility. + + Returns: + Response tensor from :func:`dog_response_single` using stored + ``sigma1`` and ``sigma2``. + """ + return dog_response_single(input, self.sigma1, self.sigma2) + + +class CornerHarris(nn.Module): + r"""nn.Module that calculates Harris corners. + + .. image:: _static/img/harris_response.png + + See :func:`~kornia.feature.harris_response` for details. + """ + + k: torch.Tensor + + def __init__(self, k: Union[float, torch.Tensor], grads_mode: str = "sobel") -> None: + super().__init__() + if isinstance(k, float): + self.register_buffer("k", torch.tensor(k)) + else: + self.register_buffer("k", k) + self.grads_mode: str = grads_mode + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(k={self.k}, grads_mode={self.grads_mode})" + + def forward(self, input: torch.Tensor, sigmas: Optional[torch.Tensor] = None) -> torch.Tensor: + """Compute Harris corner response for each input channel. + + Args: + input: Input tensor of shape :math:`(B, C, H, W)`. + sigmas: Optional Gaussian smoothing values forwarded to + :func:`harris_response`. + + Returns: + Harris response map tensor. + """ + return harris_response(input, self.k, self.grads_mode, sigmas) + + +class CornerGFTT(nn.Module): + r"""nn.Module that calculates Shi-Tomasi corners. + + .. image:: _static/img/gftt_response.png + + See :func:`~kornia.feature.gftt_response` for details. + """ + + def __init__(self, grads_mode: str = "sobel") -> None: + super().__init__() + self.grads_mode: str = grads_mode + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(grads_mode={self.grads_mode})" + + def forward(self, input: torch.Tensor, sigmas: Optional[torch.Tensor] = None) -> torch.Tensor: + """Compute Shi-Tomasi (GFTT) corner response maps. + + Args: + input: Input tensor of shape :math:`(B, C, H, W)`. + sigmas: Optional smoothing sigmas for multi-scale workflows. + + Returns: + GFTT response map tensor from :func:`gftt_response`. + """ + return gftt_response(input, self.grads_mode, sigmas) + + +class BlobHessian(nn.Module): + r"""nn.Module that calculates Hessian blobs. + + .. image:: _static/img/hessian_response.png + + See :func:`~kornia.feature.hessian_response` for details. + """ + + def __init__(self, grads_mode: str = "sobel") -> None: + super().__init__() + self.grads_mode: str = grads_mode + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(grads_mode={self.grads_mode})" + + def forward(self, input: torch.Tensor, sigmas: Optional[torch.Tensor] = None) -> torch.Tensor: + """Compute Hessian determinant-like blob responses. + + Args: + input: Input tensor of shape :math:`(B, C, H, W)`. + sigmas: Optional smoothing values passed to + :func:`hessian_response`. + + Returns: + Hessian response map tensor. + """ + return hessian_response(input, self.grads_mode, sigmas) diff --git a/kornia/feature/scale_space_detector.py b/kornia/feature/scale_space_detector.py new file mode 100644 index 0000000..7288caf --- /dev/null +++ b/kornia/feature/scale_space_detector.py @@ -0,0 +1,613 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +from torch import nn +from typing_extensions import TypedDict + +from kornia.core.check import KORNIA_CHECK_SHAPE +from kornia.geometry.subpix import ( + AdaptiveQuadInterp3d, + ConvQuadInterp3d, + IterativeQuadInterp3d, + NonMaximaSuppression2d, + nms3d_minmax, +) +from kornia.geometry.transform import ScalePyramid, pyrdown, resize + +from .laf import laf_from_center_scale_ori +from .orientation import PassLAF +from .responses import BlobHessian + +# Max |sin| among the 11 boundary points sampled by laf_to_boundary_points(n_pts=12): +# angles = linspace(0, 2π, 11) → k * 2π/11 for k=0..10 +# max|sin| at k=3: sin(6π/11) ≈ 0.9898; max|cos| at k=0: cos(0) = 1.0 +# Used to inline the boundary check in _process_octave for isotropic LAFs (rotmat=eye(2)), +# avoiding CPU→GPU allocation + bmm every octave. +_MAX_ABS_SIN_12: float = math.sin(3 * 2 * math.pi / 11) # ≈ 0.9898 + + +def _scale_index_to_scale(max_coords: torch.Tensor, sigmas: torch.Tensor, num_levels: int) -> torch.Tensor: + r"""Auxiliary function for ScaleSpaceDetector. + + Converts scale level index from the subpix module to the actual + scale, using the sigmas from the ScalePyramid output. + + Args: + max_coords: torch.Tensor [BxNx3]. + sigmas: torch.Tensor [BxD], D >= 1 + num_levels: number of levels in the scale index. + + Returns: + torch.Tensor [BxNx3]. + + """ + B = max_coords.shape[0] + base_sigma = sigmas[:, 0].view(B, 1, 1) # (B, 1, 1) — per-batch base sigma + max_coords[:, :, 0:1] = base_sigma * torch.pow(2.0, max_coords[:, :, 0:1] / float(num_levels)) + return max_coords + + +def _create_octave_mask(mask: torch.Tensor, octave_shape: List[int]) -> torch.Tensor: + r"""Downsample a mask based on the given octave shape.""" + mask_shape = octave_shape[-2:] + mask_octave = F.interpolate(mask, mask_shape, mode="bilinear", align_corners=False) + return mask_octave.unsqueeze(1) + + +class ScaleSpaceDetector(nn.Module): + r"""nn.Module for differentiable local feature detection. + + As close as possible to classical local feature detectors + like Harris, Hessian-Affine or SIFT (DoG). + + It has 5 modules inside: scale pyramid generator, response ("cornerness") function, + sub-pixel localization, affine shape estimator and patch orientation estimator. + Each of those modules could be replaced with a learned custom one, as long as + they respect output shape. + + Args: + num_features: Number of features to detect. In order to keep everything batchable, + output would always have num_features output, even for completely homogeneous images. + mr_size: multiplier for local feature scale compared to the detection scale. + 6.0 is matching OpenCV 12.0 convention for SIFT. + scale_pyr_module: generates scale pyramid. See :class:`~kornia.geometry.ScalePyramid` for details. + Default: ScalePyramid(3, 1.6, 15). + resp_module: calculates ``'cornerness'`` of the pixel. + subpix_module: performs non-maximum suppression and refines keypoint location to sub-pixel / + sub-scale accuracy. See :class:`~kornia.geometry.subpix.ConvQuadInterp3d` for details. + ori_module: for local feature orientation estimation. Default:class:`~kornia.feature.PassLAF`, + which does nothing. See :class:`~kornia.feature.LAFOrienter` for details. + aff_module: for local feature affine shape estimation. Default: :class:`~kornia.feature.PassLAF`, + which does nothing. See :class:`~kornia.feature.LAFAffineShapeEstimator` for details. + minima_are_also_good: if True, then both response function minima and maxima are detected. + Useful for symmetric response functions like DoG or Hessian. Default is False. + compile_modules: selects which sub-modules to wrap with :func:`torch.compile`. + Pass ``True`` to compile every sub-module, ``False`` (default) for none, or a list + containing any subset of ``["scale_pyr", "resp", "subpix", "ori", "aff"]``. + Compiling ``subpix`` gives ~5x GPU speedup for the default + :class:`~kornia.geometry.subpix.ConvQuadInterp3d` backend by fusing its iteration loop. + The first call incurs a one-time compilation cost; subsequent calls are fast. + + """ + + def __init__( + self, + num_features: int = 500, + mr_size: float = 6.0, + scale_pyr_module: Optional[nn.Module] = None, + resp_module: Optional[nn.Module] = None, + subpix_module: Optional[nn.Module] = None, + ori_module: Optional[nn.Module] = None, + aff_module: Optional[nn.Module] = None, + minima_are_also_good: bool = False, + scale_space_response: bool = False, + compile_modules: Union[bool, List[str]] = False, + ) -> None: + super().__init__() + self.mr_size = mr_size + self.num_features = num_features + + _all_names = {"scale_pyr", "resp", "subpix", "ori", "aff"} + if compile_modules is True: + _compile_set = _all_names + elif compile_modules is False: + _compile_set = set() + else: + _compile_set = set(compile_modules) + unknown = _compile_set - _all_names + if unknown: + raise ValueError(f"Unknown module names in compile_modules: {unknown}. Valid: {_all_names}") + + if _compile_set: + # Allow torch.compile to keep data-dependent shape ops (torch.where / nonzero) + # inside the compiled graph as unbacked symbols, avoiding graph breaks and the + # 0/1-specialization recompilations that would otherwise fire whenever an octave + # first encounters zero NMS maxima (blurry/extreme-viewpoint images). + torch._dynamo.config.capture_dynamic_output_shape_ops = True + + def _maybe_compile(mod: nn.Module, name: str) -> nn.Module: + return torch.compile(mod, dynamic=True) if name in _compile_set else mod + + if scale_pyr_module is None: + extra_levels = 3 if scale_space_response else 2 + scale_pyr_module = ScalePyramid(3, 1.6, 16, extra_levels=extra_levels) + self.scale_pyr = _maybe_compile(scale_pyr_module, "scale_pyr") + if resp_module is None: + resp_module = BlobHessian() + self.resp = _maybe_compile(resp_module, "resp") + if subpix_module is None: + subpix_module = AdaptiveQuadInterp3d(strict_maxima_bonus=0.0, allow_scale_steps=True) + # Record before torch.compile wraps the module — isinstance won't match OptimizedModule. + self._is_iterative_subpix: bool = isinstance( + subpix_module, (ConvQuadInterp3d, AdaptiveQuadInterp3d, IterativeQuadInterp3d) + ) + self.subpix = _maybe_compile(subpix_module, "subpix") + if ori_module is None: + ori_module = PassLAF() + self.ori = _maybe_compile(ori_module, "ori") + if aff_module is None: + aff_module = PassLAF() + self.aff = _maybe_compile(aff_module, "aff") + self.minima_are_also_good = minima_are_also_good + # scale_space_response should be True if the response function works on scale space + # like Difference-of-Gaussians + self.scale_space_response = scale_space_response + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}(" + f"num_features={self.num_features}, " + f"mr_size={self.mr_size}, " + f"scale_pyr={self.scale_pyr.__repr__()}, " + f"resp={self.resp.__repr__()}, " + f"subpix={self.subpix.__repr__()}, " + f"ori={self.ori.__repr__()}, " + f"aff={self.aff.__repr__()})" + ) + + def _process_octave( + self, + octave: torch.Tensor, + sigmas_oct: torch.Tensor, + num_feats: int, + mask: Optional[torch.Tensor], + rotmat: torch.Tensor, + num_levels: int, + is_iterative_subpix: bool, + px_size: float, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Process one scale-space octave: response → NMS/subpix → top-K → LAF.""" + dev = octave.device + dtype = octave.dtype + B, CH, L, H, W = octave.size() + + # Run response function + if self.scale_space_response: + oct_resp = self.resp(octave, sigmas_oct.view(-1)) # (B, C, Ldog, H, W) + else: + oct_resp = self.resp(octave.permute(0, 2, 1, 3, 4).reshape(B * L, CH, H, W), sigmas_oct.view(-1)).view( + B, L, CH, H, W + ) + # Reorder to (B, CH, L, H, W) for scale-space NMS + oct_resp = oct_resp.permute(0, 2, 1, 3, 4) + scale_sigmas = sigmas_oct[:, : oct_resp.shape[2]] + + if mask is not None: + oct_mask: torch.Tensor = _create_octave_mask(mask, oct_resp.shape) + oct_resp = oct_mask * oct_resp + + # Always precompute NMS masks in one fused pass. + # - For minima_are_also_good: both masks are needed anyway. + # - Otherwise: max_nms_mask is passed to subpix (skips its internal NMS on GPU) + # and drives the sparse top-K below. + max_nms_mask: torch.Tensor + min_nms_mask: torch.Tensor + max_nms_mask, min_nms_mask = nms3d_minmax(oct_resp) + + if self.minima_are_also_good: + if is_iterative_subpix: + coord_max, response_max = self.subpix(oct_resp, precomputed_nms_mask=max_nms_mask) + coord_min, response_min = self.subpix(-oct_resp, precomputed_nms_mask=min_nms_mask) + else: + coord_max, response_max = self.subpix(oct_resp) + coord_min, response_min = self.subpix(-oct_resp) + elif is_iterative_subpix: + coord_max, response_max = self.subpix(oct_resp, precomputed_nms_mask=max_nms_mask) + else: + coord_max, response_max = self.subpix(oct_resp) + + # Zero responses at scale border levels so they never reach top-K. + # (nms3d_minmax already sets the masks False at these positions.) + response_max[:, :, 0] = 0.0 + response_max[:, :, -1] = 0.0 + + if self.minima_are_also_good: + response_min[:, :, 0] = 0.0 + response_min[:, :, -1] = 0.0 + take_min_mask = (response_min > response_max) & min_nms_mask + response_max = torch.where(take_min_mask, response_min, response_max) + coord_max = torch.where(take_min_mask.unsqueeze(2), coord_min, coord_max) + # Candidate positions: original max-NMS plus swapped min-NMS positions. + cand_mask = max_nms_mask | take_min_mask + else: + cand_mask = max_nms_mask + + # Sparse top-K: gather the small set of NMS candidates first, then run top-K + # on that (~few-thousand) set instead of the full CHxLxHxW volume (~millions). + # nms3d_minmax guarantees cand_mask is False at scale border levels already. + mask_flat = cand_mask.view(B, -1) # (B, L*H*W) + resp_flat = response_max.view(B, -1) # (B, L*H*W) + coord_flat = coord_max.view(B, 3, -1).permute(0, 2, 1) # (B, L*H*W, 3) + + if B == 1: + nms_idx = mask_flat[0].nonzero(as_tuple=True)[0] # (M,) + resp_cands = resp_flat[0][nms_idx] # (M,) + coord_cands = coord_flat[0][nms_idx] # (M, 3) + k_eff = min(num_feats, nms_idx.shape[0]) + if k_eff > 0: + resp_flat_best, local_idx = torch.topk(resp_cands, k=k_eff) + max_coords_best = coord_cands[local_idx].unsqueeze(0) # (1, k_eff, 3) + resp_flat_best = resp_flat_best.unsqueeze(0) # (1, k_eff) + else: + resp_flat_best = resp_flat.new_zeros(1, 0) + max_coords_best = coord_flat.new_zeros(1, 0, 3) + else: + # Batched fallback: mask non-candidates to -inf so they lose top-K. + fill = torch.finfo(dtype).min / 2 + resp_masked = resp_flat.masked_fill(~mask_flat, fill) + k_eff = min(num_feats, resp_masked.size(1)) + resp_flat_best, idxs = torch.topk(resp_masked, k=k_eff, dim=1) + max_coords_best = torch.gather(coord_flat, 1, idxs.unsqueeze(-1).expand(-1, -1, 3)) + + B, N = resp_flat_best.size() + + max_coords_best = _scale_index_to_scale(max_coords_best, scale_sigmas, num_levels) + + current_lafs = torch.cat( + [ + self.mr_size * max_coords_best[:, :, 0].view(B, N, 1, 1) * rotmat, + max_coords_best[:, :, 1:3].view(B, N, 2, 1), + ], + 3, + ) + + # Inline equivalent of laf_is_inside_image(scale_laf(current_lafs, 0.5), octave[:, 0], 5) + # for isotropic LAFs (rotmat = eye(2)). Avoids: scale_laf (torch.cat), and + # laf_to_boundary_points (linspace/sin/cos allocations + CPU→GPU transfer + bmm). + # For the axis-aligned isotropic case the 12-pt boundary check reduces to: + # max x-extent = max|sin| * half_s; max y-extent = max|cos| * half_s = half_s + half_s = current_lafs[:, :, 0, 0] * 0.5 + cx = current_lafs[:, :, 0, 2] + cy = current_lafs[:, :, 1, 2] + h, w = octave.shape[3], octave.shape[4] + good_mask = ( + (cx - half_s * _MAX_ABS_SIN_12 >= 5) + & (cx + half_s * _MAX_ABS_SIN_12 <= w - 5) + & (cy - half_s >= 5) + & (cy + half_s <= h - 5) + ) + resp_flat_best = resp_flat_best * good_mask.to(dev, dtype) + current_lafs.mul_(px_size) + return resp_flat_best, current_lafs + + def detect( + self, img: torch.Tensor, num_feats: int, mask: Optional[torch.Tensor] = None + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Detect local features in an image batch. + + Args: + img: Input image tensor with shape `(B, C, H, W)`. + num_feats: Number of features requested from the detector. + mask: Optional mask tensor used to restrict valid image locations. + + Returns: + Tuple containing detection scores and local affine frames. Local affine frames are usually shaped `(B, N, 2, + 3)`, where `N` is the selected feature count. + """ + dev = img.device + dtype: torch.dtype = img.dtype + sp, sigmas, _ = self.scale_pyr(img) + + # ── Hoist loop invariants ──────────────────────────────────────────── + if isinstance(self.scale_pyr.n_levels, torch.Tensor): + num_levels = int(self.scale_pyr.n_levels.item()) + elif isinstance(self.scale_pyr.n_levels, int): + num_levels = self.scale_pyr.n_levels + else: + raise TypeError( + "Expected the scale pyramid module to have `n_levels` as a torch.Tensor or int." + f"Gotcha {type(self.scale_pyr.n_levels)}" + ) + rotmat = torch.eye(2, dtype=dtype, device=dev).view(1, 1, 2, 2) + is_iterative_subpix = self._is_iterative_subpix + px_size0 = 0.5 if self.scale_pyr.double_image else 1.0 + px_sizes = [px_size0 * (2.0**i) for i in range(len(sp))] + + # ── Process octaves sequentially ──────────────────────────────────── + # All octaves are independent once the scale pyramid is built, but CUDA + # stream parallelism does not help here: subpix allocates large scatter + # tables per call, and concurrent CUDA allocations contend for the same + # device memory allocator lock. Tested: sequential ≈ parallel on GPU. + n_oct = len(sp) + results: List[Tuple[torch.Tensor, torch.Tensor]] = [ + self._process_octave( + sp[i], sigmas[i], num_feats, mask, rotmat, num_levels, is_iterative_subpix, px_sizes[i] + ) + for i in range(n_oct) + ] + + # Sort and keep best n across all octaves. + # Sparse per-octave top-K may yield fewer total candidates than num_feats + # (e.g. small images with very few NMS maxima). topk then pads with zeros + # to preserve the shape contract [B, num_feats, ...]. + responses = torch.cat([r[0] for r in results], 1) + lafs = torch.cat([r[1] for r in results], 1) + n_candidates = responses.size(1) + if n_candidates < num_feats: + pad = num_feats - n_candidates + responses = F.pad(responses, (0, pad)) + lafs = F.pad(lafs, (0, 0, 0, 0, 0, pad)) + responses, idxs = torch.topk(responses, k=num_feats, dim=1) + lafs = torch.gather(lafs, 1, idxs.unsqueeze(-1).unsqueeze(-1).expand(-1, -1, 2, 3)) + return responses, lafs + + def forward(self, img: torch.Tensor, mask: Optional[torch.Tensor] = None) -> Tuple[torch.Tensor, torch.Tensor]: + """Three stage local feature detection. + + First the location and scale of interest points are determined by detect function. + Then affine shape and orientation. + + Args: + img: image to extract features with shape [BxCxHxW] + mask: a mask with weights where to apply the response function. The shape must be the same as + the input image. + + Returns: + lafs: shape [BxNx2x3]. Detected local affine frames. + responses: shape [BxNx1]. Response function values for corresponding lafs + + """ + responses, lafs = self.detect(img, self.num_features, mask) + lafs = self.aff(lafs, img) + lafs = self.ori(lafs, img) + return lafs, responses + + +class Detector_config(TypedDict): + """Configuration for the Scale Space Detector. + + Attributes: + nms_size: The size of the Non-Maximum Suppression window. + pyramid_levels: The number of levels in the image pyramid. + """ + + nms_size: int + pyramid_levels: int + up_levels: int + scale_factor_levels: float + s_mult: float + + +def get_default_detector_config() -> Detector_config: + """Return default config.""" + # Return a shallow copy to ensure modifications outside don't affect the module-level config. + return _DEFAULT_DETECTOR_CONFIG.copy() + + +class MultiResolutionDetector(nn.Module): + """Multi-scale feature detector, based on code from KeyNet. Can be used with any response function. + + This is based on the original code from paper + "Key.Net: Keypoint Detection by Handcrafted and Learned CNN Filters". + See :cite:`KeyNet2019` for more details. + + Args: + model: response function, such as KeyNet or BlobHessian + num_features: Number of features to detect. + conf: Dict with initialization parameters. Do not pass it, unless you know what you are doing`. + ori_module: for local feature orientation estimation. Default: :class:`~kornia.feature.PassLAF`, + which does nothing. See :class:`~kornia.feature.LAFOrienter` for details. + aff_module: for local feature affine shape estimation. Default: :class:`~kornia.feature.PassLAF`, + which does nothing. See :class:`~kornia.feature.LAFAffineShapeEstimator` for details. + + """ + + def __init__( + self, + model: nn.Module, + num_features: int = 2048, + config: Optional[Detector_config] = None, + ori_module: Optional[nn.Module] = None, + aff_module: Optional[nn.Module] = None, + compile_model: bool = False, + score_threshold: float = 0.0, + ) -> None: + super().__init__() + if config is None: + config = get_default_detector_config() + # Load extraction configuration + self.num_pyramid_levels = config["pyramid_levels"] + self.num_upscale_levels = config["up_levels"] + self.scale_factor_levels = config["scale_factor_levels"] + self.mr_size = config["s_mult"] + self.nms_size = config["nms_size"] + self.score_threshold = score_threshold + nms = NonMaximaSuppression2d((self.nms_size, self.nms_size)) + self.num_features = num_features + + if compile_model: + self.model = torch.compile(model, dynamic=True) + self.nms = torch.compile(nms, dynamic=True) + else: + self.model = model + self.nms = nms + + if ori_module is None: + self.ori: nn.Module = PassLAF() + else: + self.ori = ori_module + + if aff_module is None: + self.aff: nn.Module = PassLAF() + else: + self.aff = aff_module + + def remove_borders(self, score_map: torch.Tensor, borders: int = 15) -> torch.Tensor: + """Remove the borders of the image to avoid detections on the corners.""" + mask = torch.zeros_like(score_map) + mask[:, :, borders:-borders, borders:-borders] = 1 + return mask * score_map + + def detect_features_on_single_level( + self, level_img: torch.Tensor, num_kp: int, factor: Tuple[float, float] + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Detect keypoints on one image-pyramid level. + + Args: + level_img: Image tensor for a single pyramid level. + num_kp: Number of keypoints requested from this pyramid level. + factor: Scale factor mapping coordinates from the current pyramid level back to the original image + resolution. + + Returns: + Tuple containing scores and local affine frames detected at the requested pyramid level. + """ + det_map = self.nms(self.remove_borders(self.model(level_img))) + _, _, _h, w = det_map.shape + det_flat = det_map.view(-1) # (H*W,) — B=1, C=1 guaranteed by MultiResolutionDetector + + # Mask out non-maxima (zeroed by NMS) and below-threshold scores, then topk. + # Using masked_fill + topk instead of nonzero: avoids data-dependent output shapes, + # supports score_threshold, and is compatible with torch.compile. + fill = torch.finfo(det_flat.dtype).min / 2 + det_masked = det_flat.masked_fill(det_flat <= self.score_threshold, fill) + k = min(num_kp, det_flat.numel()) + top_scores, top_flat_idx = torch.topk(det_masked, k=k) + + # Convert flat indices to (y, x) pixel coordinates. + yx = torch.stack([top_flat_idx // w, top_flat_idx % w], dim=1) # (k, 2) + + fx = level_img.new_tensor([factor[0], factor[1]]) + xy_projected = yx.view(1, k, 2).flip(2).float() * fx + scale_val = 0.5 * (factor[0] + factor[1]) * self.mr_size + scale = level_img.new_full((1, k, 1, 1), scale_val) + lafs = laf_from_center_scale_ori(xy_projected, scale, level_img.new_zeros(1, k, 1)) + return top_scores, lafs + + def detect(self, img: torch.Tensor, mask: Optional[torch.Tensor] = None) -> Tuple[torch.Tensor, torch.Tensor]: + # Compute points per level + """Detect local features in an image batch. + + Args: + img: Input image tensor with shape `(B, C, H, W)`. + mask: Optional mask tensor used to restrict valid image locations. + + Returns: + Tuple containing detection scores and local affine frames. Local affine frames are usually shaped `(B, N, 2, + 3)`, where `N` is the selected feature count. + """ + num_features_per_level: List[float] = [] + tmp = 0.0 + factor_points = self.scale_factor_levels**2 + levels = self.num_pyramid_levels + self.num_upscale_levels + 1 + for idx_level in range(levels): + tmp += factor_points ** (-1 * (idx_level - self.num_upscale_levels)) + nf = self.num_features * factor_points ** (-1 * (idx_level - self.num_upscale_levels)) + num_features_per_level.append(nf) + num_features_per_level = [int(x / tmp) for x in num_features_per_level] + + _, _, h, w = img.shape + img_up = img + cur_img = img + all_responses: List[torch.Tensor] = [] + all_lafs: List[torch.Tensor] = [] + # Extract features from the upper levels + for idx_level in range(self.num_upscale_levels): + nf = num_features_per_level[len(num_features_per_level) - self.num_pyramid_levels - 1 - (idx_level + 1)] + num_points_level = int(nf) + + # Resize input image + up_factor = self.scale_factor_levels ** (1 + idx_level) + nh, nw = int(h * up_factor), int(w * up_factor) + up_factor_kpts = (float(w) / float(nw), float(h) / float(nh)) + img_up = resize(img_up, (nh, nw), interpolation="bilinear", align_corners=False) + + cur_scores, cur_lafs = self.detect_features_on_single_level(img_up, num_points_level, up_factor_kpts) + + all_responses.append(cur_scores.view(1, -1)) + all_lafs.append(cur_lafs) + + # Extract features from the downsampling pyramid + for idx_level in range(self.num_pyramid_levels + 1): + if idx_level > 0: + cur_img = pyrdown(cur_img, factor=self.scale_factor_levels) + _, _, nh, nw = cur_img.shape + factor = (float(w) / float(nw), float(h) / float(nh)) + else: + factor = (1.0, 1.0) + + num_points_level = int(num_features_per_level[idx_level]) + if idx_level > 0 or (self.num_upscale_levels > 0): + num_points_level = sum(num_features_per_level[: idx_level + 1 + self.num_upscale_levels]) + + cur_scores, cur_lafs = self.detect_features_on_single_level(cur_img, num_points_level, factor) + all_responses.append(cur_scores.view(1, -1)) + all_lafs.append(cur_lafs) + responses = torch.cat(all_responses, 1) + lafs = torch.cat(all_lafs, 1) + if lafs.shape[1] > self.num_features: + responses, idxs = torch.topk(responses, k=self.num_features, dim=1) + lafs = torch.gather(lafs, 1, idxs[..., None, None].expand(-1, -1, 2, 3)) + return responses, lafs + + def forward(self, img: torch.Tensor, mask: Optional[torch.Tensor] = None) -> Tuple[torch.Tensor, torch.Tensor]: + """Three stage local feature detection. + + First the location and scale of interest points are determined by detect function. + Then affine shape and orientation. + + Args: + img: image to extract features with shape [1xCxHxW]. KeyNetDetector does not support batch processing, + because the number of detections is different on each image. + mask: a mask with weights where to apply the response function. The shape must be the same as + the input image. + + Returns: + lafs: shape [1xNx2x3]. Detected local affine frames. + responses: shape [1xNx1]. Response function values for corresponding lafs + + """ + KORNIA_CHECK_SHAPE(img, ["1", "C", "H", "W"]) + responses, lafs = self.detect(img, mask) + lafs = self.aff(lafs, img) + lafs = self.ori(lafs, img) + return lafs, responses + + +_DEFAULT_DETECTOR_CONFIG: Detector_config = { + # Extraction Parameters + "nms_size": 15, + "pyramid_levels": 4, + "up_levels": 1, + "scale_factor_levels": math.sqrt(2), + "s_mult": 22.0, +} diff --git a/kornia/feature/siftdesc.py b/kornia/feature/siftdesc.py new file mode 100644 index 0000000..8314aa7 --- /dev/null +++ b/kornia/feature/siftdesc.py @@ -0,0 +1,381 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math +from typing import Tuple + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK_SHAPE +from kornia.filters import get_gaussian_kernel2d, spatial_gradient +from kornia.geometry.conversions import pi + + +def _get_reshape_kernel(kd: int, ky: int, kx: int) -> torch.Tensor: + """Return neigh2channels conv kernel.""" + numel: int = kd * ky * kx + + # Fast-path: use static _eye_cache if available for small numel + # (to avoid repeated allocations for common kernel sizes) + # The cache size is limited for memory efficiency. + # NOTE: If memory is a concern and large kd/ky/kx are rare, adjust _MAX_CACHED. + _MAX_CACHED = 4096 + if numel <= _MAX_CACHED: + if not hasattr(_get_reshape_kernel, "_eye_cache"): + _get_reshape_kernel._eye_cache = {} # type: ignore[attr-defined] + cache = _get_reshape_kernel._eye_cache # type: ignore[attr-defined] + res = cache.get(numel) + if res is None: + res = torch.eye(numel) + cache[numel] = res + return res.view(numel, kd, ky, kx) + else: + # fallback to normal allocation for big kernels + return torch.eye(numel).view(numel, kd, ky, kx) + + +def get_sift_pooling_kernel(ksize: int = 25) -> torch.Tensor: + r"""Return a weighted pooling kernel for SIFT descriptor. + + Args: + ksize: kernel_size. + + Returns: + the pooling kernel with shape :math:`(ksize, ksize)`. + + """ + ks_2: float = float(ksize) / 2.0 + xc2 = ks_2 - (torch.arange(ksize).float() + 0.5 - ks_2).abs() + kernel = torch.ger(xc2, xc2) / (ks_2**2) + return kernel + + +def get_sift_bin_ksize_stride_pad(patch_size: int, num_spatial_bins: int) -> Tuple[int, int, int]: + r"""Return a tuple with SIFT parameters. + + Args: + patch_size: the given patch size. + num_spatial_bins: the ggiven number of spatial bins. + + Returns: + ksize, stride, pad. + + """ + ksize: int = 2 * int(patch_size / (num_spatial_bins + 1)) + stride: int = patch_size // num_spatial_bins + pad: int = ksize // 4 + out_size: int = (patch_size + 2 * pad - (ksize - 1) - 1) // stride + 1 + if out_size != num_spatial_bins: + raise ValueError( + f"Patch size {patch_size} is incompatible with requested number of spatial bins" + f" {num_spatial_bins} for SIFT descriptor. Usually it happens when patch size is too small " + " for num_spatial_bins specified" + ) + return ksize, stride, pad + + +class SIFTDescriptor(nn.Module): + r"""nn.Module which computes SIFT descriptors of given patches. + + Args: + patch_size: Input patch size in pixels. + num_ang_bins: Number of angular bins. + num_spatial_bins: Number of spatial bins. + clipval: clipping value to reduce single-bin dominance + rootsift: if ``True``, RootSIFT (Arandjelović et. al, 2012) is computed. + + Returns: + SIFT descriptor of the patches with shape. + + Shape: + - Input: :math:`(B, 1, \text{num_spatial_bins}, \text{num_spatial_bins})` + - Output: :math:`(B, \text{num_ang_bins * num_spatial_bins ** 2})` + + Example: + >>> input = torch.rand(23, 1, 32, 32) + >>> SIFT = SIFTDescriptor(32, 8, 4) + >>> descs = SIFT(input) # 23x128 + + """ + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}(" + f"num_ang_bins={self.num_ang_bins}, " + f"num_spatial_bins={self.num_spatial_bins}, " + f"patch_size={self.patch_size}, " + f"rootsift={self.rootsift}, " + f"clipval={self.clipval})" + ) + + def __init__( + self, + patch_size: int = 41, + num_ang_bins: int = 8, + num_spatial_bins: int = 4, + rootsift: bool = True, + clipval: float = 0.2, + ) -> None: + super().__init__() + self.eps = 1e-10 + self.num_ang_bins = num_ang_bins + self.num_spatial_bins = num_spatial_bins + self.clipval = clipval + self.rootsift = rootsift + self.patch_size = patch_size + + ks: int = self.patch_size + sigma: float = float(ks) / math.sqrt(2.0) + self.gk = get_gaussian_kernel2d((ks, ks), (sigma, sigma), True) + + (self.bin_ksize, self.bin_stride, self.pad) = get_sift_bin_ksize_stride_pad(patch_size, num_spatial_bins) + + nw = get_sift_pooling_kernel(ksize=self.bin_ksize).float() + self.pk = nn.Conv2d( + 1, + 1, + kernel_size=(nw.size(0), nw.size(1)), + stride=(self.bin_stride, self.bin_stride), + padding=(self.pad, self.pad), + bias=False, + ) + self.pk.weight.data.copy_(nw.reshape(1, 1, nw.size(0), nw.size(1))) + + def get_pooling_kernel(self) -> torch.Tensor: + """Return the spatial pooling kernel used for histogram accumulation. + + Returns: + Detached convolution kernel tensor from the pooling layer. + """ + return self.pk.weight.detach() + + def get_weighting_kernel(self) -> torch.Tensor: + """Return the Gaussian weighting kernel used before orientation pooling. + + Returns: + Detached Gaussian kernel tensor. + """ + return self.gk.detach() + + def forward(self, input: torch.Tensor) -> torch.Tensor: + r"""Compute SIFT descriptors for square grayscale patches. + + Args: + input: Patch tensor shaped + :math:`(B, 1, \text{patch\_size}, \text{patch\_size})`. + + Returns: + Descriptor tensor of size + :math:`(B, \text{num\_ang\_bins} \times \text{num\_spatial\_bins}^2)`. + """ + KORNIA_CHECK_SHAPE(input, ["B", "1", f"{self.patch_size}", f"{self.patch_size}"]) + B: int = input.shape[0] + self.pk = self.pk.to(input.dtype).to(input.device) + + grads = spatial_gradient(input, "diff") + # unpack the edges + gx = grads[:, :, 0] + gy = grads[:, :, 1] + + mag = torch.sqrt(gx * gx + gy * gy + self.eps) + ori = torch.atan2(gy, gx + self.eps) + 2.0 * pi + mag = mag * self.gk.expand_as(mag).type_as(mag).to(mag.device) + o_big = float(self.num_ang_bins) * ori / (2.0 * pi) + + bo0_big_ = torch.floor(o_big) + wo1_big_ = o_big - bo0_big_ + bo0_big = bo0_big_ % self.num_ang_bins + bo1_big = (bo0_big + 1) % self.num_ang_bins + wo0_big = (1.0 - wo1_big_) * mag + wo1_big = wo1_big_ * mag + + ang_bins = torch.cat( + [ + self.pk((bo0_big == i).to(input.dtype) * wo0_big + (bo1_big == i).to(input.dtype) * wo1_big) + for i in range(0, self.num_ang_bins) + ], + 1, + ) + ang_bins = ang_bins.view(B, -1) + ang_bins = F.normalize(ang_bins, p=2) + ang_bins = torch.clamp(ang_bins, 0.0, float(self.clipval)) + ang_bins = F.normalize(ang_bins, p=2) + if self.rootsift: + ang_bins = torch.sqrt(F.normalize(ang_bins, p=1) + self.eps) + return ang_bins + + +def sift_describe( + input: torch.Tensor, + patch_size: int = 41, + num_ang_bins: int = 8, + num_spatial_bins: int = 4, + rootsift: bool = True, + clipval: float = 0.2, +) -> torch.Tensor: + r"""Compute the sift descriptor. + + See + :class: `~kornia.feature.SIFTDescriptor` for details. + """ + return SIFTDescriptor(patch_size, num_ang_bins, num_spatial_bins, rootsift, clipval)(input) + + +class DenseSIFTDescriptor(nn.Module): + """nn.Module, which computes SIFT descriptor densely over the image. + + Args: + num_ang_bins: Number of angular bins. (8 is default) + num_spatial_bins: Number of spatial bins per descriptor (4 is default). + You might want to set odd number and relevant padding to keep feature map size + spatial_bin_size: Size of a spatial bin in pixels (4 is default) + clipval: clipping value to reduce single-bin dominance + rootsift: (bool) if True, RootSIFT (Arandjelović et. al, 2012) is computed + stride: default 1 + padding: default 0 + + Returns: + torch.Tensor: DenseSIFT descriptor of the image + + Shape: + - Input: (B, 1, H, W) + - Output: (B, num_ang_bins * num_spatial_bins ** 2, (H+padding)/stride, (W+padding)/stride) + + Examples:: + >>> input = torch.rand(2, 1, 200, 300) + >>> SIFT = DenseSIFTDescriptor() + >>> descs = SIFT(input) # 2x128x194x294 + + """ + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}(" + f"num_ang_bins={self.num_ang_bins}, " + f"num_spatial_bins={self.num_spatial_bins}, " + f"spatial_bin_size={self.spatial_bin_size}, " + f"rootsift={self.rootsift}, " + f"stride={self.stride}, " + f"clipval={self.clipval})" + ) + + def __init__( + self, + num_ang_bins: int = 8, + num_spatial_bins: int = 4, + spatial_bin_size: int = 4, + rootsift: bool = True, + clipval: float = 0.2, + stride: int = 1, + padding: int = 1, + ) -> None: + super().__init__() + self.eps = 1e-10 + self.num_ang_bins = num_ang_bins + self.num_spatial_bins = num_spatial_bins + self.spatial_bin_size = spatial_bin_size + self.clipval = clipval + self.rootsift = rootsift + self.stride = stride + self.pad = padding + + # Only allocate pooling kernels once during construction + nw = get_sift_pooling_kernel(ksize=self.spatial_bin_size).float() + self.register_buffer("_bin_pooling_kernel_weight", nw.reshape(1, 1, nw.size(0), nw.size(1))) + bin_pooling_kernel = nn.Conv2d( + 1, + 1, + kernel_size=(nw.size(0), nw.size(1)), + stride=(1, 1), + bias=False, + padding=(nw.size(0) // 2, nw.size(1) // 2), + ) + bin_pooling_kernel.weight.data.copy_(self._bin_pooling_kernel_weight) + self.bin_pooling_kernel = bin_pooling_kernel + + Pw = _get_reshape_kernel(num_ang_bins, num_spatial_bins, num_spatial_bins).float() + self.register_buffer("_poolingconv_weight", Pw) + PoolingConv = nn.Conv2d( + num_ang_bins, + num_ang_bins * num_spatial_bins**2, + kernel_size=(num_spatial_bins, num_spatial_bins), + stride=(self.stride, self.stride), + bias=False, + padding=(self.pad, self.pad), + ) + PoolingConv.weight.data.copy_(self._poolingconv_weight) + self.PoolingConv = PoolingConv + + # Cache pooling kernel torch.Tensor for fast return in get_pooling_kernel + self._pooling_kernel = self._bin_pooling_kernel_weight.detach() + + def get_pooling_kernel(self) -> torch.Tensor: + """Return the cached pooling kernel for dense SIFT binning. + + Returns: + Detached tensor containing pooling weights. + """ + # Return the cached detached pooling kernel directly for optimal speed + return self._pooling_kernel + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Compute dense SIFT descriptors over a full image grid. + + Args: + input: Grayscale image tensor with shape :math:`(B, 1, H, W)`. + + Returns: + Dense descriptor map tensor with channel dimension + ``num_ang_bins * num_spatial_bins**2``. + """ + KORNIA_CHECK_SHAPE(input, ["B", "1", "H", "W"]) + + _B, _CH, _W, _H = input.size() + self.bin_pooling_kernel = self.bin_pooling_kernel.to(input.dtype).to(input.device) + self.PoolingConv = self.PoolingConv.to(input.dtype).to(input.device) + grads = spatial_gradient(input, "diff") + # unpack the edges + gx = grads[:, :, 0] + gy = grads[:, :, 1] + mag = torch.sqrt(gx * gx + gy * gy + self.eps) + ori = torch.atan2(gy, gx + self.eps) + 2.0 * pi + o_big = float(self.num_ang_bins) * ori / (2.0 * pi) + + bo0_big_ = torch.floor(o_big) + wo1_big_ = o_big - bo0_big_ + bo0_big = bo0_big_ % self.num_ang_bins + bo1_big = (bo0_big + 1) % self.num_ang_bins + wo0_big = (1.0 - wo1_big_) * mag + wo1_big = wo1_big_ * mag + ang_bins = torch.cat( + [ + self.bin_pooling_kernel( + (bo0_big == i).to(input.dtype) * wo0_big + (bo1_big == i).to(input.dtype) * wo1_big + ) + for i in range(0, self.num_ang_bins) + ], + 1, + ) + + out_no_norm = self.PoolingConv(ang_bins) + out = F.normalize(out_no_norm, dim=1, p=2).clamp_(0, float(self.clipval)) + out = F.normalize(out, dim=1, p=2) + if self.rootsift: + out = torch.sqrt(F.normalize(out, p=1) + self.eps) + return out diff --git a/kornia/feature/sold2/__init__.py b/kornia/feature/sold2/__init__.py new file mode 100644 index 0000000..63a2db6 --- /dev/null +++ b/kornia/feature/sold2/__init__.py @@ -0,0 +1,24 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Feature SOLD2 — SOLD2 line segment detector for Kornia. + +This subpackage provides the SOLD2 detector and related utilities. +""" + +from .sold2 import SOLD2 +from .sold2_detector import SOLD2_detector diff --git a/kornia/feature/sold2/backbones.py b/kornia/feature/sold2/backbones.py new file mode 100644 index 0000000..aa6d94c --- /dev/null +++ b/kornia/feature/sold2/backbones.py @@ -0,0 +1,583 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Implements several backbone networks.""" + +import functools +import operator +from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Type, Union + +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn.functional import pixel_shuffle, softmax + + +class HourglassConfig(NamedTuple): + """Define the configuration for the Hourglass backbone. + + Attributes: + depth: The number of downsampling levels in the hourglass. + num_stacks: The number of stacked hourglass modules. + """ + + depth: int + num_stacks: int + num_blocks: int + num_classes: int + input_channels: int + head: Type[nn.Module] + + +# [Hourglass backbone classes] +class HourglassBackbone(nn.Module): + """Hourglass network, taken from https://github.com/zhou13/lcnn. + + Args: + input_channel: number of input channels. + depth: number of residual blocks per hourglass module. + num_stacks: number of hourglass modules stacked together. + num_blocks: number of layers in each residual block. + num_classes: number of heads for the output of a hourglass module. + + """ + + def __init__( + self, input_channel: int = 1, depth: int = 4, num_stacks: int = 2, num_blocks: int = 1, num_classes: int = 5 + ) -> None: + super().__init__() + self.head = MultitaskHead + self.net = hg(HourglassConfig(depth, num_stacks, num_blocks, num_classes, input_channel, head=self.head)) + + def forward(self, input_images: torch.Tensor) -> torch.Tensor: + """Run this SOLD2 backbone component forward. + + This method processes image or feature tensors used by the SOLD2 line detector. Tensor layouts follow `(B, C, H, + W)` for feature maps unless the surrounding class documentation states otherwise. + + Args: + input_images: Input image batch with shape `(B, C, H, W)`, where `B` is batch size, `C` is channel count, + and `H`/`W` are spatial dimensions. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + return self.net(input_images) + + +class MultitaskHead(nn.Module): + """Implement a multitask head for the Hourglass backbone.""" + + def __init__(self, input_channels: int) -> None: + super().__init__() + + m = int(input_channels / 4) + head_size = [[2], [1], [2]] + heads = [] + _iter: list[int] = functools.reduce(operator.iconcat, head_size, []) + for output_channels in _iter: + heads.append( + nn.Sequential( + nn.Conv2d(input_channels, m, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(m, output_channels, kernel_size=1), + ) + ) + self.heads = nn.ModuleList(heads) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run this SOLD2 backbone component forward. + + This method processes image or feature tensors used by the SOLD2 line detector. Tensor layouts follow `(B, C, H, + W)` for feature maps unless the surrounding class documentation states otherwise. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + return torch.cat([head(x) for head in self.heads], dim=1) + + +class Bottleneck2D(nn.Module): + """Implement a 2D Bottleneck block for the Hourglass backbone.""" + + def __init__( + self, + inplanes: int, + planes: int, + stride: Union[int, Tuple[int, int]] = 1, + downsample: Optional[nn.Module] = None, + ) -> None: + super().__init__() + + self.bn1 = nn.BatchNorm2d(inplanes) + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1) + self.bn2 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1) + self.bn3 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, planes * 2, kernel_size=1) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run this SOLD2 backbone component forward. + + This method processes image or feature tensors used by the SOLD2 line detector. Tensor layouts follow `(B, C, H, + W)` for feature maps unless the surrounding class documentation states otherwise. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + residual = x + + out = self.bn1(x) + out = self.relu(out) + out = self.conv1(out) + + out = self.bn2(out) + out = self.relu(out) + out = self.conv2(out) + + out = self.bn3(out) + out = self.relu(out) + out = self.conv3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + + return out + + +class Hourglass(nn.Module): + """Implement the Hourglass network for symmetric feature extraction. + + Args: + block: The block type to use (e.g., Bottleneck2D). + num_blocks: The number of blocks per level. + planes: The number of channels in the blocks. + depth: The depth of the hourglass. + expansion: The expansion factor for the bottleneck. Default: 2. + """ + + def __init__(self, block: Type[Bottleneck2D], num_blocks: int, planes: int, depth: int, expansion: int = 2) -> None: + super().__init__() + self.depth = depth + self.block = block + self.expansion = expansion + self.hg = self._make_hour_glass(block, num_blocks, planes, depth) + + def _make_residual(self, block: Type[Bottleneck2D], num_blocks: int, planes: int) -> nn.Module: + layers = [] + for _ in range(0, num_blocks): + layers.append(block(planes * self.expansion, planes)) + return nn.Sequential(*layers) + + def _make_hour_glass(self, block: Type[Bottleneck2D], num_blocks: int, planes: int, depth: int) -> nn.ModuleList: + hgl = [] + for i in range(depth): + res = [] + for _ in range(3): + res.append(self._make_residual(block, num_blocks, planes)) + if i == 0: + res.append(self._make_residual(block, num_blocks, planes)) + hgl.append(nn.ModuleList(res)) + return nn.ModuleList(hgl) + + def _hour_glass_forward(self, n: int, x: torch.Tensor) -> torch.Tensor: + up1 = self.hg[n - 1][0](x) # type: ignore[index] + low1 = F.max_pool2d(x, 2, stride=2) + low1 = self.hg[n - 1][1](low1) # type: ignore[index] + + if n > 1: + low2 = self._hour_glass_forward(n - 1, low1) + else: + low2 = self.hg[n - 1][3](low1) # type: ignore[index] + low3 = self.hg[n - 1][2](low2) # type: ignore[index] + up2 = F.interpolate(low3, size=up1.shape[2:]) + out = up1 + up2 + return out + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run this SOLD2 backbone component forward. + + This method processes image or feature tensors used by the SOLD2 line detector. Tensor layouts follow `(B, C, H, + W)` for feature maps unless the surrounding class documentation states otherwise. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + return self._hour_glass_forward(self.depth, x) + + +class HourglassNet(nn.Module): + """Hourglass model from Newell et al ECCV 2016.""" + + def __init__( + self, + block: Type[Bottleneck2D], + head: Type[nn.Module], + depth: int, + num_stacks: int, + num_blocks: int, + num_classes: int, + input_channels: int, + expansion: int = 2, + ) -> None: + super().__init__() + + self.inplanes = 64 + self.num_feats = 128 + self.num_stacks = num_stacks + self.expansion = expansion + self.conv1 = nn.Conv2d(input_channels, self.inplanes, kernel_size=7, stride=2, padding=3) + self.bn1 = nn.BatchNorm2d(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.layer1 = self._make_residual(block, self.inplanes, 1) + self.layer2 = self._make_residual(block, self.inplanes, 1) + self.layer3 = self._make_residual(block, self.num_feats, 1) + self.maxpool = nn.MaxPool2d(2, stride=2) + + # Build hourglass modules + ch = self.num_feats * self.expansion + hgl, res, fc, score, fc_, score_ = [], [], [], [], [], [] + for i in range(num_stacks): + hgl.append(Hourglass(block, num_blocks, self.num_feats, depth)) + res.append(self._make_residual(block, self.num_feats, num_blocks)) + fc.append(self._make_fc(ch, ch)) + score.append(head(ch)) + if i < num_stacks - 1: + fc_.append(nn.Conv2d(ch, ch, kernel_size=1)) + score_.append(nn.Conv2d(num_classes, ch, kernel_size=1)) + self.hg = nn.ModuleList(hgl) + self.res = nn.ModuleList(res) + self.fc = nn.ModuleList(fc) + self.score = nn.ModuleList(score) + self.fc_ = nn.ModuleList(fc_) + self.score_ = nn.ModuleList(score_) + + def _make_residual( + self, block: Type[Bottleneck2D], planes: int, blocks: int, stride: Union[int, Tuple[int, int]] = 1 + ) -> nn.Module: + downsample = None + if stride != 1 or self.inplanes != planes * self.expansion: + downsample = nn.Sequential(nn.Conv2d(self.inplanes, planes * self.expansion, kernel_size=1, stride=stride)) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * self.expansion + for _ in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return nn.Sequential(*layers) + + def _make_fc(self, inplanes: int, outplanes: int) -> nn.Module: + bn = nn.BatchNorm2d(inplanes) + conv = nn.Conv2d(inplanes, outplanes, kernel_size=1) + return nn.Sequential(conv, bn, self.relu) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run this SOLD2 backbone component forward. + + This method processes image or feature tensors used by the SOLD2 line detector. Tensor layouts follow `(B, C, H, + W)` for feature maps unless the surrounding class documentation states otherwise. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + out = [] + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + + x = self.layer1(x) + x = self.maxpool(x) + x = self.layer2(x) + x = self.layer3(x) + + for i in range(self.num_stacks): + y = self.hg[i](x) + y = self.res[i](y) + y = self.fc[i](y) + score = self.score[i](y) + out.append(score) + if i < self.num_stacks - 1: + fc_ = self.fc_[i](y) + score_ = self.score_[i](score) + x = x + fc_ + score_ + + return y + + +def hg(cfg: HourglassConfig) -> HourglassNet: + """Create HourglassNet.""" + return HourglassNet( + Bottleneck2D, + head=cfg.head, + depth=cfg.depth, + num_stacks=cfg.num_stacks, + num_blocks=cfg.num_blocks, + num_classes=cfg.num_classes, + input_channels=cfg.input_channels, + ) + + +# [Backbone decoders] +class SuperpointDecoder(nn.Module): + """Junction decoder based on the SuperPoint architecture. + + Args: + input_feat_dim: channel size of the input features. + + Returns: + the junction heatmap, with shape (B, H, W). + + """ + + def __init__(self, input_feat_dim: int = 128, grid_size: int = 8) -> None: + super().__init__() + self.relu = nn.ReLU(inplace=True) + # Perform strided convolution when using lcnn backbone. + self.convPa = nn.Conv2d(input_feat_dim, 256, kernel_size=3, stride=2, padding=1) + self.convPb = nn.Conv2d(256, 65, kernel_size=1, stride=1, padding=0) + self.grid_size = grid_size + + def forward(self, input_features: torch.Tensor) -> torch.Tensor: + """Run this SOLD2 backbone component forward. + + This method processes image or feature tensors used by the SOLD2 line detector. Tensor layouts follow `(B, C, H, + W)` for feature maps unless the surrounding class documentation states otherwise. + + Args: + input_features: Input feature map with shape `(B, C, H, W)`. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + feat = self.relu(self.convPa(input_features)) + semi = self.convPb(feat) + + # Convert from semi-dense to dense heatmap + junc_prob = softmax(semi, dim=1) + junc_pred = pixel_shuffle(junc_prob[:, :-1, :, :], self.grid_size)[:, 0] + return junc_pred + + +class PixelShuffleDecoder(nn.Module): + """Pixel shuffle decoder used to predict the line heatmap. + + Args: + input_feat_dim: channel size of the input features. + num_upsample: how many upsamples are performed. + output_channel: number of output channels. + + Returns: + the (B, 1, H, W) line heatmap. + + """ + + def __init__(self, input_feat_dim: int = 128, num_upsample: int = 2, output_channel: int = 2) -> None: + super().__init__() + # Get channel parameters + self.channel_conf = self.get_channel_conf(num_upsample) + + # Define the pixel shuffle + self.pixshuffle = nn.PixelShuffle(2) + + # Process the feature + conv_block_lst = [] + # The input block + conv_block_lst.append( + nn.Sequential( + nn.Conv2d(input_feat_dim, self.channel_conf[0], kernel_size=3, stride=1, padding=1), + nn.BatchNorm2d(self.channel_conf[0]), + nn.ReLU(inplace=True), + ) + ) + + # Intermediate block + for channel in self.channel_conf[1:-1]: + conv_block_lst.append( + nn.Sequential( + nn.Conv2d(channel, channel, kernel_size=3, stride=1, padding=1), + nn.BatchNorm2d(channel), + nn.ReLU(inplace=True), + ) + ) + + # Output block + conv_block_lst.append( + nn.Sequential(nn.Conv2d(self.channel_conf[-1], output_channel, kernel_size=1, stride=1, padding=0)) + ) + self.conv_block_lst = nn.ModuleList(conv_block_lst) + + def get_channel_conf(self, num_upsample: int) -> List[int]: + """Get num of channels based on number of upsampling.""" + if num_upsample == 2: + return [256, 64, 16] + return [256, 64, 16, 4] + + def forward(self, input_features: torch.Tensor) -> torch.Tensor: + # Iterate til output block + """Run this SOLD2 backbone component forward. + + This method processes image or feature tensors used by the SOLD2 line detector. Tensor layouts follow `(B, C, H, + W)` for feature maps unless the surrounding class documentation states otherwise. + + Args: + input_features: Input feature map with shape `(B, C, H, W)`. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + out = input_features + for block in self.conv_block_lst[:-1]: + out = block(out) + out = self.pixshuffle(out) + + # Output layer + out = self.conv_block_lst[-1](out) + heatmap = softmax(out, dim=1)[:, 1, :, :] + + return heatmap + + +class SuperpointDescriptor(nn.Module): + """Descriptor decoder based on the SuperPoint arcihtecture. + + Args: + input_feat_dim: channel size of the input features. + + Returns: + the semi-dense descriptors with shape (B, 128, H/4, W/4). + + """ + + def __init__(self, input_feat_dim: int = 128) -> None: + super().__init__() + self.relu = nn.ReLU(inplace=True) + self.convPa = nn.Conv2d(input_feat_dim, 256, kernel_size=3, stride=1, padding=1) + self.convPb = nn.Conv2d(256, 128, kernel_size=1, stride=1, padding=0) + + def forward(self, input_features: torch.Tensor) -> torch.Tensor: + """Run this SOLD2 backbone component forward. + + This method processes image or feature tensors used by the SOLD2 line detector. Tensor layouts follow `(B, C, H, + W)` for feature maps unless the surrounding class documentation states otherwise. + + Args: + input_features: Input feature map with shape `(B, C, H, W)`. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + feat = self.relu(self.convPa(input_features)) + semi = self.convPb(feat) + + return semi + + +# [Combination of all previous models in one] + + +class SOLD2Net(nn.Module): + """Full network for SOLD². + + Args: + model_cfg: the configuration as a Dict. + + Returns: + a Dict with the following values: + junctions: heatmap of junctions. + heatmap: line heatmap. + descriptors: semi-dense descriptors. + + """ + + def __init__(self, model_cfg: Dict[str, Any]) -> None: + super().__init__() + self.cfg = model_cfg + + # Backbone + self.backbone_net = HourglassBackbone(**self.cfg["backbone_cfg"]) + feat_channel = 256 + + # Junction decoder + self.junction_decoder = SuperpointDecoder(feat_channel, self.cfg["grid_size"]) + + # Line heatmap decoder + self.heatmap_decoder = PixelShuffleDecoder(feat_channel, num_upsample=2) + + # Descriptor decoder + if "use_descriptor" in self.cfg: + self.descriptor_decoder = SuperpointDescriptor(feat_channel) + + def forward(self, input_images: torch.Tensor) -> Dict[str, torch.Tensor]: + # The backbone + """Run this SOLD2 backbone component forward. + + This method processes image or feature tensors used by the SOLD2 line detector. Tensor layouts follow `(B, C, H, + W)` for feature maps unless the surrounding class documentation states otherwise. + + Args: + input_images: Input image batch with shape `(B, C, H, W)`, where `B` is batch size, `C` is channel count, + and `H`/`W` are spatial dimensions. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + features = self.backbone_net(input_images) + + # junction decoder + junctions = self.junction_decoder(features) + + # heatmap decoder + heatmaps = self.heatmap_decoder(features) + + outputs = {"junctions": junctions, "heatmap": heatmaps} + + # Descriptor decoder + if "use_descriptor" in self.cfg: + outputs["descriptors"] = self.descriptor_decoder(features) + + return outputs diff --git a/kornia/feature/sold2/sold2.py b/kornia/feature/sold2/sold2.py new file mode 100644 index 0000000..352fe8d --- /dev/null +++ b/kornia/feature/sold2/sold2.py @@ -0,0 +1,349 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Optional, Tuple, cast + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK_SHAPE +from kornia.core.utils import dataclass_to_dict, dict_to_dataclass +from kornia.feature.sold2.structures import DetectorCfg, LineMatcherCfg +from kornia.geometry.conversions import normalize_pixel_coordinates + +from .backbones import SOLD2Net +from .sold2_detector import LineSegmentDetectionModule, line_map_to_segments, prob_to_junctions + +urls: Dict[str, str] = {} +urls["wireframe"] = "http://cmp.felk.cvut.cz/~mishkdmy/models/sold2_wireframe.pth" + + +class SOLD2(nn.Module): + r"""nn.Module, which detects and describe line segments in an image. + + This is based on the original code from the paper "SOLD²: Self-supervised + Occlusion-aware Line Detector and Descriptor". See :cite:`SOLD22021` for more details. + + Args: + config: Dict specifying parameters. None will load the default parameters, + which are tuned for images in the range 400~800 px. + pretrained: If True, download and set pretrained weights to the model. + + Returns: + The raw junction and line heatmaps, the semi-dense descriptor map, + as well as the list of detected line segments (ij coordinates convention). + + Example: + >>> images = torch.rand(2, 1, 64, 64) + >>> sold2 = SOLD2() + >>> outputs = sold2(images) + >>> line_seg1 = outputs["line_segments"][0] + >>> line_seg2 = outputs["line_segments"][1] + >>> desc1 = outputs["dense_desc"][0] + >>> desc2 = outputs["dense_desc"][1] + >>> matches = sold2.match(line_seg1, line_seg2, desc1[None], desc2[None]) + + """ + + def __init__(self, pretrained: bool = True, config: Optional[DetectorCfg] = None) -> None: + if isinstance(config, dict): + config = dict_to_dataclass(cast(Dict[str, Any], config), DetectorCfg) + super().__init__() + # Initialize some parameters + self.config = config if config is not None else DetectorCfg() + self.config.use_descriptor = True # Only difference to SOLD2_detector DetectorCfg + self.grid_size = self.config.grid_size + self.junc_detect_thresh = self.config.detection_thresh + self.max_num_junctions = self.config.max_num_junctions + + # Load the pre-trained model + self.model = SOLD2Net(dataclass_to_dict(self.config)) + if pretrained: + pretrained_dict = torch.hub.load_state_dict_from_url(urls["wireframe"], map_location=torch.device("cpu")) + state_dict = self.adapt_state_dict(pretrained_dict["model_state_dict"]) + self.model.load_state_dict(state_dict) + self.eval() + + # Initialize the line detector + self.line_detector = LineSegmentDetectionModule(self.config.line_detector_cfg) + + # Initialize the line matcher + self.line_matcher = WunschLineMatcher(self.config.line_matcher_cfg) + + def forward(self, img: torch.Tensor) -> Dict[str, Any]: + """Run forward. + + Args: + img: batched images with shape :math:`(B, 1, H, W)`. + + Returns: + line_segments: list of N line segments in each of the B images :math:`List[(N, 2, 2)]`. + junction_heatmap: raw junction heatmap of shape :math:`(B, H, W)`. + line_heatmap: raw line heatmap of shape :math:`(B, H, W)`. + dense_desc: the semi-dense descriptor map of shape :math:`(B, 128, H/4, W/4)`. + + """ + KORNIA_CHECK_SHAPE(img, ["B", "1", "H", "W"]) + outputs = {} + + # Forward pass of the CNN backbone + net_outputs = self.model(img) + outputs["junction_heatmap"] = net_outputs["junctions"] + outputs["line_heatmap"] = net_outputs["heatmap"] + outputs["dense_desc"] = net_outputs["descriptors"] + + # Loop through all images + lines = [] + for junc_prob, heatmap in zip(net_outputs["junctions"], net_outputs["heatmap"]): + # Get the junctions + junctions = prob_to_junctions(junc_prob, self.grid_size, self.junc_detect_thresh, self.max_num_junctions) + + # Run the line detector + line_map, junctions, _ = self.line_detector.detect(junctions, heatmap) + lines.append(line_map_to_segments(junctions, line_map)) + outputs["line_segments"] = lines + + return outputs + + def match( + self, line_seg1: torch.Tensor, line_seg2: torch.Tensor, desc1: torch.Tensor, desc2: torch.Tensor + ) -> torch.Tensor: + """Find the best matches between two sets of line segments and their corresponding descriptors. + + Args: + line_seg1: list of line segments in image 1, with shape [num_lines, 2, 2]. + line_seg2: list of line segments in image 2, with shape [num_lines, 2, 2]. + desc1: semi-dense descriptor map of image 1, with shape [1, 128, H/4, W/4]. + desc2: semi-dense descriptor map of image 2, with shape [1, 128, H/4, W/4]. + + Returns: + A np.array of size [num_lines1] indicating the index in line_seg2 of the matched line, + for each line in line_seg1. -1 means that the line is not matched. + + """ + return self.line_matcher(line_seg1, line_seg2, desc1, desc2) + + def adapt_state_dict(self, state_dict: Dict[str, Any]) -> Dict[str, Any]: + """Adapt pretrained checkpoint keys to this module implementation. + + Args: + state_dict: Checkpoint state dictionary whose keys are adapted to this module layout. + + Returns: + State dictionary with checkpoint keys renamed or removed so it can be loaded by the current module. + """ + del state_dict["w_junc"] + del state_dict["w_heatmap"] + del state_dict["w_desc"] + state_dict["heatmap_decoder.conv_block_lst.2.0.weight"] = state_dict["heatmap_decoder.conv_block_lst.2.weight"] + state_dict["heatmap_decoder.conv_block_lst.2.0.bias"] = state_dict["heatmap_decoder.conv_block_lst.2.bias"] + del state_dict["heatmap_decoder.conv_block_lst.2.weight"] + del state_dict["heatmap_decoder.conv_block_lst.2.bias"] + return state_dict + + +class WunschLineMatcher(nn.Module): + """Class matching two sets of line segments with the Needleman-Wunsch algorithm. + + TODO: move it later in kornia.feature.matching + """ + + def __init__(self, config: Optional[LineMatcherCfg] = None) -> None: + super().__init__() + # Initialize the parameters + if config is None: + config = LineMatcherCfg() + self.config = config + self.cross_check = self.config.cross_check + self.num_samples = self.config.num_samples + self.min_dist_pts = self.config.min_dist_pts + self.top_k_candidates = self.config.top_k_candidates + self.grid_size = self.config.grid_size + self.line_score = self.config.line_score + + def forward( + self, line_seg1: torch.Tensor, line_seg2: torch.Tensor, desc1: torch.Tensor, desc2: torch.Tensor + ) -> torch.Tensor: + """Find the best matches between two sets of line segments and their corresponding descriptors.""" + KORNIA_CHECK_SHAPE(line_seg1, ["N", "2", "2"]) + KORNIA_CHECK_SHAPE(line_seg2, ["N", "2", "2"]) + KORNIA_CHECK_SHAPE(desc1, ["B", "D", "H", "H"]) + KORNIA_CHECK_SHAPE(desc2, ["B", "D", "H", "H"]) + device = desc1.device + img_size1 = (desc1.shape[2] * self.grid_size, desc1.shape[3] * self.grid_size) + img_size2 = (desc2.shape[2] * self.grid_size, desc2.shape[3] * self.grid_size) + + # Default case when an image has no lines + if len(line_seg1) == 0: + return torch.empty(0, dtype=torch.int, device=device) + if len(line_seg2) == 0: + return -torch.ones(len(line_seg1), dtype=torch.int, device=device) + + # Sample points regularly along each line + line_points1, valid_points1 = self.sample_line_points(line_seg1) + line_points2, valid_points2 = self.sample_line_points(line_seg2) + line_points1 = line_points1.reshape(-1, 2) + line_points2 = line_points2.reshape(-1, 2) + + # Extract the descriptors for each point + grid1 = keypoints_to_grid(line_points1, img_size1) + grid2 = keypoints_to_grid(line_points2, img_size2) + desc1 = F.normalize(F.grid_sample(desc1, grid1, align_corners=False)[0, :, :, 0], dim=0) + desc2 = F.normalize(F.grid_sample(desc2, grid2, align_corners=False)[0, :, :, 0], dim=0) + + # Precompute the distance between line points for every pair of lines + # Assign a score of -1 for invalid points + scores = desc1.t() @ desc2 + scores[~valid_points1.flatten()] = -1 + scores[:, ~valid_points2.flatten()] = -1 + scores = scores.reshape(len(line_seg1), self.num_samples, len(line_seg2), self.num_samples) + scores = scores.permute(0, 2, 1, 3) + # scores.shape = (n_lines1, n_lines2, num_samples, num_samples) + + # Pre-filter the line candidates and find the best match for each line + matches = self.filter_and_match_lines(scores) + + # [Optionally] filter matches with mutual nearest neighbor filtering + if self.cross_check: + matches2 = self.filter_and_match_lines(scores.permute(1, 0, 3, 2)) + mutual = matches2[matches] == torch.arange(len(line_seg1), device=device) + matches[~mutual] = -1 + + return matches + + def sample_line_points(self, line_seg: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Regularly sample points along each line segments, with a minimal distance between each point. + + Pad the remaining points. + + Args: + line_seg: an Nx2x2 torch.Tensor. + + Returns: + line_points: an N x num_samples x 2 torch.Tensor. + valid_points: a boolean N x num_samples torch.Tensor. + """ + _N, _, _ = line_seg.shape + M = self.num_samples + dev = line_seg.device + + lengths = torch.norm(line_seg[:, 0] - line_seg[:, 1], dim=1) + num_pts = torch.clamp((lengths / self.min_dist_pts).floor().int(), min=2, max=M) # (N,) + + orig = line_seg[:, 0].unsqueeze(1) + dirs = (line_seg[:, 1] - line_seg[:, 0]).unsqueeze(1) + idx = torch.arange(M, device=dev).unsqueeze(0) + denom = (num_pts - 1).unsqueeze(1) + alpha = idx / denom + pts = orig + dirs * alpha.unsqueeze(-1) + valid = idx < num_pts.unsqueeze(1) + pts = pts.masked_fill(~valid.unsqueeze(-1), 0.0) + + return pts, valid + + def filter_and_match_lines(self, scores: torch.Tensor) -> torch.Tensor: + """Use scores to keep the top k best lines. + + Compute the Needleman- Wunsch algorithm on each candidate pairs, and keep the highest score. + + Args: + scores: a (N, M, n, n) torch.Tensor containing the pairwise scores + of the elements to match. + + Returns: + matches: a (N) torch.Tensor containing the indices of the best match + """ + KORNIA_CHECK_SHAPE(scores, ["M", "N", "n", "n"]) + + # Pre-filter the pairs and keep the top k best candidate lines + line_scores1 = scores.max(3)[0] + valid_scores1 = line_scores1 != -1 + line_scores1 = (line_scores1 * valid_scores1).sum(2) / valid_scores1.sum(2) + line_scores2 = scores.max(2)[0] + valid_scores2 = line_scores2 != -1 + line_scores2 = (line_scores2 * valid_scores2).sum(2) / valid_scores2.sum(2) + line_scores = (line_scores1 + line_scores2) / 2 + topk_lines = torch.argsort(line_scores, dim=1)[:, -self.top_k_candidates :] + # topk_lines.shape = (n_lines1, top_k_candidates) + + top_scores = torch.take_along_dim(scores, topk_lines[:, :, None, None], dim=1) + + # Consider the reversed line segments as well + top_scores = torch.cat([top_scores, torch.flip(top_scores, dims=[-1])], 1) + + # Compute the line distance matrix with Needleman-Wunsch algo and + # retrieve the closest line neighbor + n_lines1, top2k, n, m = top_scores.shape + top_scores = top_scores.reshape((n_lines1 * top2k, n, m)) + nw_scores = self.needleman_wunsch(top_scores) + nw_scores = nw_scores.reshape(n_lines1, top2k) + matches = torch.remainder(torch.argmax(nw_scores, dim=1), top2k // 2) + matches = topk_lines[torch.arange(n_lines1), matches] + return matches + + def needleman_wunsch(self, scores: torch.Tensor) -> torch.Tensor: + """Batched implementation of the Needleman-Wunsch algorithm. + + The cost of the InDel operation is set to 0 by subtracting the gap + penalty to the scores. + + Args: + scores: a (B, N, M) torch.Tensor containing the pairwise scores + of the elements to match. + """ + KORNIA_CHECK_SHAPE(scores, ["B", "N", "M"]) + # Recalibrate the scores to get a gap score of 0 + gap = 0.1 + B, N, M = scores.shape + dp = torch.zeros(B, N + 1, M + 1, device=scores.device) + S = scores - gap + for k in range(2, N + M + 1): + i_min = max(1, k - M) + i_max = min(N, k - 1) + i = torch.arange(i_min, i_max + 1, device=scores.device) + j = k - i + up = dp[:, i - 1, j] + left = dp[:, i, j - 1] + torch.diag = dp[:, i - 1, j - 1] + S[:, i - 1, j - 1] + dp[:, i, j] = torch.max(torch.max(up, left), torch.diag) + return dp[:, -1, -1] + + +def keypoints_to_grid(keypoints: torch.Tensor, img_size: Tuple[int, int]) -> torch.Tensor: + """Convert a list of keypoints into a grid in [-1, 1]² that can be used in torch.nn.functional.interpolate. + + Args: + keypoints: a torch.Tensor [N, 2] of N keypoints (ij coordinates convention). + img_size: the original image size (H, W) + + """ + KORNIA_CHECK_SHAPE(keypoints, ["N", "2"]) + n_points = len(keypoints) + grid_points = normalize_pixel_coordinates(keypoints[:, [1, 0]], img_size[0], img_size[1]) + grid_points = grid_points.view(-1, n_points, 1, 2) + return grid_points + + +def batched_linspace(start: torch.Tensor, end: torch.Tensor, step: int, dim: int) -> torch.Tensor: + """Batch version of torch.F.normalize (similar to the numpy one).""" + intervals = ((end - start) / (step - 1)).unsqueeze(dim) + broadcast_size = [1] * len(intervals.shape) + broadcast_size[dim] = step + samples = torch.arange(step, dtype=torch.float, device=start.device).reshape(broadcast_size) + samples = start.unsqueeze(dim) + samples * intervals + return samples diff --git a/kornia/feature/sold2/sold2_detector.py b/kornia/feature/sold2/sold2_detector.py new file mode 100644 index 0000000..b2797cb --- /dev/null +++ b/kornia/feature/sold2/sold2_detector.py @@ -0,0 +1,599 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math +import warnings +from typing import Any, Dict, Optional, Tuple, cast + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK_SHAPE +from kornia.core.utils import dataclass_to_dict, dict_to_dataclass +from kornia.feature.sold2.structures import DetectorCfg, HeatMapRefineCfg, JunctionRefineCfg, LineDetectorCfg +from kornia.geometry.bbox import nms + +from .backbones import SOLD2Net + +urls: Dict[str, str] = {} +urls["wireframe"] = "https://cvg-data.inf.ethz.ch/SOLD2/sold2_wireframe.tar" + + +class SOLD2_detector(nn.Module): + r"""nn.Module, which detects line segments in an image. + + This is based on the original code from the paper "SOLD²: Self-supervised + Occlusion-aware Line Detector and Descriptor". See :cite:`SOLD22021` for more details. + + Args: + config (DetectorCfg): Configuration object containing all parameters. None will load the default parameters, + which are tuned for images in the range 400~800 px. Using a dataclass ensures type safety and clearer + parameter management. + pretrained (bool): If True, download and set pretrained weights to the model. + + Returns: + The raw junction and line heatmaps, as well as the list of detected line segments (ij coordinates convention). + + Example: + >>> img = torch.rand(1, 1, 128, 128) + >>> sold2_detector = SOLD2_detector(pretrained=False) + >>> line_segments = sold2_detector(img)["line_segments"] + + """ + + def __init__(self, pretrained: bool = True, config: Optional[DetectorCfg] = None) -> None: + if isinstance(config, dict): + warnings.warn( + "Usage of config as a plain dictionary is deprecated in favor of" + "`kornia.features.sold2.structures.DetectorCfg`. The support of plain" + "dictionaries as config will be removed in kornia v0.8.0 (December 2024).", + category=DeprecationWarning, + stacklevel=2, + ) + config = dict_to_dataclass(cast(Dict[str, Any], config), DetectorCfg) + super().__init__() + # Initialize some parameters + self.config = config if config is not None else DetectorCfg() + self.grid_size = self.config.grid_size + self.junc_detect_thresh = self.config.detection_thresh + self.max_num_junctions = self.config.max_num_junctions + + # Load the pre-trained model + self.model = SOLD2Net(dataclass_to_dict(self.config)) + + if pretrained: + pretrained_dict = torch.hub.load_state_dict_from_url(urls["wireframe"], map_location=torch.device("cpu")) + state_dict = self.adapt_state_dict(pretrained_dict["model_state_dict"]) + self.model.load_state_dict(state_dict) + self.eval() + + # Initialize the line detector with a configuration from the dataclass + self.line_detector = LineSegmentDetectionModule(self.config.line_detector_cfg) + + def adapt_state_dict(self, state_dict: Dict[str, Any]) -> Dict[str, Any]: + """Adapt pretrained checkpoint keys to this module implementation. + + Args: + state_dict: Checkpoint state dictionary whose keys are adapted to this module layout. + + Returns: + State dictionary with checkpoint keys renamed or removed so it can be loaded by the current module. + """ + del state_dict["w_junc"] + del state_dict["w_heatmap"] + del state_dict["w_desc"] + state_dict["heatmap_decoder.conv_block_lst.2.0.weight"] = state_dict["heatmap_decoder.conv_block_lst.2.weight"] + state_dict["heatmap_decoder.conv_block_lst.2.0.bias"] = state_dict["heatmap_decoder.conv_block_lst.2.bias"] + del state_dict["heatmap_decoder.conv_block_lst.2.weight"] + del state_dict["heatmap_decoder.conv_block_lst.2.bias"] + return state_dict + + def forward(self, img: torch.Tensor) -> Dict[str, Any]: + """Run forward. + + Args: + img: batched images with shape :math:`(B, 1, H, W)`. + + Returns: + line_segments: list of N line segments in each of the B images :math:`List[(N, 2, 2)]`. + junction_heatmap: raw junction heatmap of shape :math:`(B, H, W)`. + line_heatmap: raw line heatmap of shape :math:`(B, H, W)`. + + """ + KORNIA_CHECK_SHAPE(img, ["B", "1", "H", "W"]) + outputs = {} + + # Forward pass of the CNN backbone + net_outputs = self.model(img) + outputs["junction_heatmap"] = net_outputs["junctions"] + outputs["line_heatmap"] = net_outputs["heatmap"] + + # Loop through all images + lines = [] + for junc_prob, heatmap in zip(net_outputs["junctions"], net_outputs["heatmap"]): + # Get the junctions + junctions = prob_to_junctions(junc_prob, self.grid_size, self.junc_detect_thresh, self.max_num_junctions) + + # Run the line detector + line_map, junctions, _ = self.line_detector.detect(junctions, heatmap) + lines.append(line_map_to_segments(junctions, line_map)) + outputs["line_segments"] = lines + + return outputs + + +class LineSegmentDetectionModule: + r"""nn.Module extracting line segments from junctions and line heatmaps. + + Args: + config (LineDetectorCfg): Configuration dataclass containing all settings required for line segment detection. + - detect_thresh (float): Probability threshold for mean activation (0. ~ 1.). + - num_samples (int): Number of sampling locations along the line segments. + - inlier_thresh (float): Minimum inlier ratio to satisfy (0. ~ 1.) => 0. means no threshold. + - heatmap_low_thresh (float): Lowest threshold for pixel considered as a candidate in junction recovery. + - heatmap_high_thresh (float): Higher threshold for NMS in junction recovery. + - max_local_patch_radius (float): Maximum patch to be considered in local maximum search. + - lambda_radius (float): Lambda factor in linear local maximum search formulation. + - use_candidate_suppression (bool): Apply candidate suppression to break long segments into sub-segments. + - nms_dist_tolerance (float): Distance tolerance for NMS. Decides whether the junctions are on the line. + - use_heatmap_refinement (bool): Whether to use heatmap refinement methods. + - heatmap_refine_cfg: Configuration for heatmap refinement methods. + - use_junction_refinement (bool): Whether to use junction refinement methods. + - junction_refine_cfg: Configuration for junction refinement methods. + + Example: + >>> config = LineDetectorCfg(detect_thresh=0.6, use_heatmap_refinement=True) + >>> module = LineSegmentDetectionModule(config) + >>> junctions, heatmap = torch.rand(10, 2), torch.rand(256, 256) + >>> line_map, junctions, _ = module.detect(junctions, heatmap) + + """ + + def __init__(self, config: Optional[LineDetectorCfg] = None) -> None: + # Load LineDetectorCfg + if config is None: + config = LineDetectorCfg() + self.config = config + + # Line detection parameters + self.detect_thresh = self.config.detect_thresh + # self.detect_thresh = detect_thresh + + # Line sampling parameters + self.num_samples = self.config.num_samples + self.inlier_thresh = self.config.inlier_thresh + self.local_patch_radius = self.config.max_local_patch_radius + self.lambda_radius = self.config.lambda_radius + + # Detecting junctions on the boundary parameters + self.low_thresh = self.config.heatmap_low_thresh + self.high_thresh = self.config.heatmap_high_thresh + + # Pre-compute the torch.linspace sampler + self.torch_sampler = torch.linspace(0, 1, self.num_samples) + + # Long line segment suppression configuration + self.use_candidate_suppression = self.config.use_candidate_suppression + self.nms_dist_tolerance = self.config.nms_dist_tolerance + + # Heatmap refinement configuration + self.use_heatmap_refinement = self.config.use_heatmap_refinement + self.heatmap_refine_cfg = self.config.heatmap_refine_cfg + if self.use_heatmap_refinement and self.heatmap_refine_cfg is None: + raise ValueError("[Error] Missing heatmap refinement config.") + + # Junction refinement configuration + self.use_junction_refinement = self.config.use_junction_refinement + self.junction_refine_cfg = self.config.junction_refine_cfg + if self.use_junction_refinement and self.junction_refine_cfg is None: + raise ValueError("[Error] Missing junction refinement config.") + + def detect(self, junctions: torch.Tensor, heatmap: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Perform line segment detection.""" + KORNIA_CHECK_SHAPE(heatmap, ["H", "W"]) + H, W = heatmap.shape + device = junctions.device + + # Perform the heatmap refinement + if self.use_heatmap_refinement and isinstance(self.heatmap_refine_cfg, HeatMapRefineCfg): + if self.heatmap_refine_cfg.mode == "global": + heatmap = self.refine_heatmap( + heatmap, self.heatmap_refine_cfg.ratio, self.heatmap_refine_cfg.valid_thresh + ) + elif self.heatmap_refine_cfg.mode == "local": + heatmap = self.refine_heatmap_local( + heatmap, + self.heatmap_refine_cfg.num_blocks, + self.heatmap_refine_cfg.overlap_ratio, + self.heatmap_refine_cfg.ratio, + self.heatmap_refine_cfg.valid_thresh, + ) + + # Initialize empty line map + num_junctions = len(junctions) + line_map_pred = torch.zeros([num_junctions, num_junctions], device=device, dtype=torch.int32) + + # Stop if there are not enough junctions + if num_junctions < 2: + return line_map_pred, junctions, heatmap + + # Generate the candidate map + candidate_map = torch.triu( + torch.ones([num_junctions, num_junctions], device=device, dtype=torch.int32), diagonal=1 + ) + + # Optionally perform candidate filtering + if self.use_candidate_suppression: + candidate_map = self.candidate_suppression(junctions, candidate_map) + + # Fetch the candidates + candidate_indices = torch.where(candidate_map) + candidate_index_map = torch.cat([candidate_indices[0][..., None], candidate_indices[1][..., None]], -1) + + # Get the corresponding start and end junctions + candidate_junc_start = junctions[candidate_index_map[:, 0]] + candidate_junc_end = junctions[candidate_index_map[:, 1]] + + # Get the sampling locations (N x 64) + sampler = self.torch_sampler.to(device)[None] + cand_samples_h = candidate_junc_start[:, 0:1] * sampler + candidate_junc_end[:, 0:1] * (1 - sampler) + cand_samples_w = candidate_junc_start[:, 1:2] * sampler + candidate_junc_end[:, 1:2] * (1 - sampler) + + # Clip to image boundary + cand_h = torch.clamp(cand_samples_h, min=0, max=H - 1) + cand_w = torch.clamp(cand_samples_w, min=0, max=W - 1) + + # [Local maximum search] + # Compute normalized segment lengths + segments_length = torch.sqrt( + torch.sum((candidate_junc_start.to(torch.float32) - candidate_junc_end.to(torch.float32)) ** 2, dim=-1) + ) + normalized_seg_length = segments_length / (((H**2) + (W**2)) ** 0.5) + + # Perform local max search + num_cand = len(cand_h) + group_size = 10000 + if num_cand > group_size: + num_iter = math.ceil(num_cand / group_size) + sampled_feat_lst = [] + for iter_idx in range(num_iter): + if not iter_idx == num_iter - 1: + cand_h_ = cand_h[iter_idx * group_size : (iter_idx + 1) * group_size, :] + cand_w_ = cand_w[iter_idx * group_size : (iter_idx + 1) * group_size, :] + normalized_seg_length_ = normalized_seg_length[iter_idx * group_size : (iter_idx + 1) * group_size] + else: + cand_h_ = cand_h[iter_idx * group_size :, :] + cand_w_ = cand_w[iter_idx * group_size :, :] + normalized_seg_length_ = normalized_seg_length[iter_idx * group_size :] + sampled_feat_ = self.detect_local_max(heatmap, cand_h_, cand_w_, H, W, normalized_seg_length_, device) + sampled_feat_lst.append(sampled_feat_) + sampled_feat = torch.cat(sampled_feat_lst, 0) + else: + sampled_feat = self.detect_local_max(heatmap, cand_h, cand_w, H, W, normalized_seg_length, device) + + # [Simple threshold detection] + # detection_results is a mask over all candidates + detection_results = torch.mean(sampled_feat, dim=-1) > self.detect_thresh + + # [Inlier threshold detection] + if self.inlier_thresh > 0: + inlier_ratio = torch.sum(sampled_feat > self.detect_thresh, dim=-1).to(heatmap.dtype) / self.num_samples + detection_results_inlier = inlier_ratio >= self.inlier_thresh + detection_results = detection_results * detection_results_inlier + + # Convert detection results back to line_map_pred + detected_junc_indexes = candidate_index_map[detection_results] + line_map_pred[detected_junc_indexes[:, 0], detected_junc_indexes[:, 1]] = 1 + line_map_pred[detected_junc_indexes[:, 1], detected_junc_indexes[:, 0]] = 1 + + # [Junction refinement] + if self.use_junction_refinement and len(detected_junc_indexes) > 0: + junctions, line_map_pred = self.refine_junction_perturb(junctions, line_map_pred, heatmap, H, W, device) + + return line_map_pred, junctions, heatmap + + def refine_heatmap(self, heatmap: torch.Tensor, ratio: float = 0.2, valid_thresh: float = 1e-2) -> torch.Tensor: + """Global heatmap refinement method.""" + # Grab the top 10% values + heatmap_values = heatmap[heatmap > valid_thresh] + sorted_values = torch.sort(heatmap_values, descending=True)[0] + top10_len = math.ceil(sorted_values.shape[0] * ratio) + max20 = torch.mean(sorted_values[:top10_len]) + heatmap = torch.clamp(heatmap / max20, min=0.0, max=1.0) + return heatmap + + def refine_heatmap_local( + self, + heatmap: torch.Tensor, + num_blocks: int = 5, + overlap_ratio: float = 0.5, + ratio: float = 0.2, + valid_thresh: float = 2e-3, + ) -> torch.Tensor: + """Local heatmap refinement method.""" + # Get the shape of the heatmap + H, W = heatmap.shape + increase_ratio = 1 - overlap_ratio + h_block = round(H / (1 + (num_blocks - 1) * increase_ratio)) + w_block = round(W / (1 + (num_blocks - 1) * increase_ratio)) + + # Iterate through each block + count_map = torch.zeros(heatmap.shape, dtype=torch.int, device=heatmap.device) + heatmap_output = torch.zeros(heatmap.shape, dtype=torch.float, device=heatmap.device) + for h_idx in range(num_blocks): + for w_idx in range(num_blocks): + # Fetch the heatmap + h_start = round(h_idx * h_block * increase_ratio) + w_start = round(w_idx * w_block * increase_ratio) + h_end = h_start + h_block if h_idx < num_blocks - 1 else H + w_end = w_start + w_block if w_idx < num_blocks - 1 else W + + subheatmap = heatmap[h_start:h_end, w_start:w_end] + if subheatmap.max() > valid_thresh: + subheatmap = self.refine_heatmap(subheatmap, ratio, valid_thresh=valid_thresh) + + # Aggregate it to the final heatmap + heatmap_output[h_start:h_end, w_start:w_end] += subheatmap + count_map[h_start:h_end, w_start:w_end] += 1 + heatmap_output = torch.clamp(heatmap_output / count_map, max=1.0, min=0.0) + + return heatmap_output + + def candidate_suppression(self, junctions: torch.Tensor, candidate_map: torch.Tensor) -> torch.Tensor: + """Suppress overlapping long lines in the candidate segments.""" + # Define the distance tolerance + dist_tolerance = self.nms_dist_tolerance + + # Compute distance between junction pairs + # (num_junc x 1 x 2) - (1 x num_junc x 2) => num_junc x num_junc map + line_dist_map = torch.sum((torch.unsqueeze(junctions, dim=1) - junctions[None, ...]) ** 2, dim=-1) ** 0.5 + + # Fetch all the "detected lines" + seg_indexes = torch.where(torch.triu(candidate_map, diagonal=1)) + start_point_idxs = seg_indexes[0] + end_point_idxs = seg_indexes[1] + start_points = junctions[start_point_idxs, :] + end_points = junctions[end_point_idxs, :] + + # Fetch corresponding entries + line_dists = line_dist_map[start_point_idxs, end_point_idxs] + + # Check whether they are on the line + dir_vecs = (end_points - start_points) / torch.norm(end_points - start_points, dim=-1)[..., None] + # Get the orthogonal distance + cand_vecs = junctions[None, ...] - start_points.unsqueeze(dim=1) + cand_vecs_norm = torch.norm(cand_vecs, dim=-1) + # Check whether they are projected directly onto the segment + proj = torch.einsum("bij,bjk->bik", cand_vecs, dir_vecs[..., None]) / line_dists[..., None, None] + # proj is num_segs x num_junction x 1 + proj_mask = (proj >= 0) * (proj <= 1) + cand_angles = torch.acos( + torch.einsum("bij,bjk->bik", cand_vecs, dir_vecs[..., None]) / cand_vecs_norm[..., None] + ) + cand_dists = cand_vecs_norm[..., None] * torch.sin(cand_angles) + junc_dist_mask = cand_dists <= dist_tolerance + junc_mask = junc_dist_mask * proj_mask + + # Minus starting points + num_segs = len(start_point_idxs) + junc_counts = torch.sum(junc_mask, dim=[1, 2]) + junc_counts -= junc_mask[..., 0][torch.arange(0, num_segs), start_point_idxs].to(torch.int) + junc_counts -= junc_mask[..., 0][torch.arange(0, num_segs), end_point_idxs].to(torch.int) + + # Get the invalid candidate mask + final_mask = junc_counts > 0 + candidate_map[start_point_idxs[final_mask], end_point_idxs[final_mask]] = 0 + + return candidate_map + + def refine_junction_perturb( + self, + junctions: torch.Tensor, + line_map: torch.Tensor, + heatmap: torch.Tensor, + H: int, + W: int, + device: torch.device, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Refine the line endpoints in a similar way as in LSD.""" + # Fetch refinement parameters + if not isinstance(self.junction_refine_cfg, JunctionRefineCfg): + raise TypeError( + "Expected to have dataclass of type JunctionRefineCfg for junction." + f"Gotcha {type(self.junction_refine_cfg)}" + ) + num_perturbs = self.junction_refine_cfg.num_perturbs + perturb_interval = self.junction_refine_cfg.perturb_interval + side_perturbs = (num_perturbs - 1) // 2 + + # Fetch the 2D perturb mat + perturb_vec = torch.arange( + start=-perturb_interval * side_perturbs, + end=perturb_interval * (side_perturbs + 1), + step=perturb_interval, + device=device, + ) + + h1_grid, w1_grid, h2_grid, w2_grid = torch.meshgrid( + [perturb_vec, perturb_vec, perturb_vec, perturb_vec], indexing="ij" + ) + + perturb_tensor = torch.cat([h1_grid[..., None], w1_grid[..., None], h2_grid[..., None], w2_grid[..., None]], -1) + perturb_tensor_flat = perturb_tensor.view(-1, 2, 2) + + # Fetch all the detected lines + detected_seg_indexes = torch.where(torch.triu(line_map, diagonal=1)) + start_points = junctions[detected_seg_indexes[0]] + end_points = junctions[detected_seg_indexes[1]] + line_segments = torch.stack([start_points, end_points], 1) + + line_segment_candidates = line_segments.unsqueeze(dim=1) + perturb_tensor_flat[None] + # Clip the boundaries + line_segment_candidates[..., 0] = torch.clamp(line_segment_candidates[..., 0], min=0, max=H - 1) + line_segment_candidates[..., 1] = torch.clamp(line_segment_candidates[..., 1], min=0, max=W - 1) + + # Iterate through all the segments + refined_segment_lst = [] + num_segments = len(line_segments) + for idx in range(num_segments): + segment = line_segment_candidates[idx] + # Get the corresponding start and end junctions + candidate_junc_start = segment[:, 0] + candidate_junc_end = segment[:, 1] + + # Get the sampling locations (N x 64) + sampler = self.torch_sampler.to(device)[None] + cand_samples_h = candidate_junc_start[:, 0:1] * sampler + candidate_junc_end[:, 0:1] * (1 - sampler) + cand_samples_w = candidate_junc_start[:, 1:2] * sampler + candidate_junc_end[:, 1:2] * (1 - sampler) + + # Clip to image boundary + cand_h = torch.clamp(cand_samples_h, min=0, max=H - 1) + cand_w = torch.clamp(cand_samples_w, min=0, max=W - 1) + + # Perform bilinear sampling + segment_feat = self.detect_bilinear(heatmap, cand_h, cand_w) + segment_results = torch.mean(segment_feat, dim=-1) + max_idx = torch.argmax(segment_results) + refined_segment_lst.append(segment[max_idx][None]) + + # Concatenate back to segments + refined_segments = torch.cat(refined_segment_lst, 0) + + # Convert back to junctions and line_map + junctions_new = torch.cat([refined_segments[:, 0, :], refined_segments[:, 1, :]], 0) + junctions_new = torch.unique(junctions_new, dim=0) + line_map_new = self.segments_to_line_map(junctions_new, refined_segments) + + return junctions_new, line_map_new + + def segments_to_line_map(self, junctions: torch.Tensor, segments: torch.Tensor) -> torch.Tensor: + """Convert the list of segments to line map.""" + # Create empty line map + num_junctions = len(junctions) + line_map = torch.zeros([num_junctions, num_junctions], device=junctions.device) + + # Get the indices of paired junctions + _, idx_junc1 = torch.where(torch.all(junctions[None] == segments[:, None, 0], dim=2)) + _, idx_junc2 = torch.where(torch.all(junctions[None] == segments[:, None, 1], dim=2)) + + # Assign the labels + line_map[idx_junc1, idx_junc2] = 1 + line_map[idx_junc2, idx_junc1] = 1 + + return line_map + + def detect_bilinear(self, heatmap: torch.Tensor, cand_h: torch.Tensor, cand_w: torch.Tensor) -> torch.Tensor: + """Detect by bilinear sampling.""" + # Get the floor and ceiling locations + cand_h_floor = torch.floor(cand_h).to(torch.long) + cand_h_ceil = torch.ceil(cand_h).to(torch.long) + cand_w_floor = torch.floor(cand_w).to(torch.long) + cand_w_ceil = torch.ceil(cand_w).to(torch.long) + + # Perform the bilinear sampling + cand_samples_feat = ( + heatmap[cand_h_floor, cand_w_floor] * (cand_h_ceil - cand_h) * (cand_w_ceil - cand_w) + + heatmap[cand_h_floor, cand_w_ceil] * (cand_h_ceil - cand_h) * (cand_w - cand_w_floor) + + heatmap[cand_h_ceil, cand_w_floor] * (cand_h - cand_h_floor) * (cand_w_ceil - cand_w) + + heatmap[cand_h_ceil, cand_w_ceil] * (cand_h - cand_h_floor) * (cand_w - cand_w_floor) + ) + + return cand_samples_feat + + def detect_local_max( + self, + heatmap: torch.Tensor, + cand_h: torch.Tensor, + cand_w: torch.Tensor, + H: int, + W: int, + normalized_seg_length: torch.Tensor, + device: torch.device, + ) -> torch.Tensor: + """Detect by local maximum search.""" + # Compute the distance threshold + dist_thresh = 0.5 * (2**0.5) + self.lambda_radius * normalized_seg_length + # Make it N x 64 + dist_thresh = torch.repeat_interleave(dist_thresh[..., None], self.num_samples, dim=-1) + + # Compute the candidate points + cand_points = torch.cat([cand_h[..., None], cand_w[..., None]], -1) + cand_points_round = torch.round(cand_points) # N x 64 x 2 + + # Construct local patches 9x9 = 81 + patch_mask = torch.zeros( + [int(2 * self.local_patch_radius + 1), int(2 * self.local_patch_radius + 1)], device=device + ) + patch_center = torch.tensor( + [[self.local_patch_radius, self.local_patch_radius]], device=device, dtype=torch.float32 + ) + H_patch_points, W_patch_points = torch.where(patch_mask >= 0) + patch_points = torch.cat([H_patch_points[..., None], W_patch_points[..., None]], -1) + # Fetch the circle region + patch_center_dist = torch.sqrt(torch.sum((patch_points - patch_center) ** 2, dim=-1)) + patch_points = patch_points[patch_center_dist <= self.local_patch_radius, :] + # Shift [0, 0] to the center + patch_points = patch_points - self.local_patch_radius + + # Construct local patch mask + patch_points_shifted = torch.unsqueeze(cand_points_round, dim=2) + patch_points[None, None] + patch_dist = torch.sqrt(torch.sum((torch.unsqueeze(cand_points, dim=2) - patch_points_shifted) ** 2, dim=-1)) + patch_dist_mask = patch_dist < dist_thresh[..., None] + + # Get all points => num_points_center x num_patch_points x 2 + points_H = torch.clamp(patch_points_shifted[:, :, :, 0], min=0, max=H - 1).to(torch.long) + points_W = torch.clamp(patch_points_shifted[:, :, :, 1], min=0, max=W - 1).to(torch.long) + points = torch.cat([points_H[..., None], points_W[..., None]], -1) + + # Sample the feature (N x 64 x 81) + sampled_feat = heatmap[points[:, :, :, 0], points[:, :, :, 1]] + # Filtering using the valid mask + sampled_feat = sampled_feat * patch_dist_mask.to(torch.float32) + if len(sampled_feat) == 0: + sampled_feat_lmax = torch.empty(0, self.num_samples) + else: + sampled_feat_lmax = torch.max(sampled_feat, dim=-1)[0] + + return sampled_feat_lmax + + +def line_map_to_segments(junctions: torch.Tensor, line_map: torch.Tensor) -> torch.Tensor: + """Convert a junction connectivity map to a Nx2x2 torch.Tensor of segments.""" + junc_loc1, junc_loc2 = torch.where(torch.triu(line_map)) + segments = torch.stack([junctions[junc_loc1], junctions[junc_loc2]], 1) + return segments + + +def prob_to_junctions(prob: torch.Tensor, dist: float, prob_thresh: float = 0.01, top_k: int = 0) -> torch.Tensor: + """Extract junctions from a probability map, apply NMS, and extract the top k candidates.""" + # Extract the junctions + junctions = torch.stack(torch.where(prob >= prob_thresh), -1).float() + if len(junctions) == 0: + return junctions + + # Perform NMS + boxes = torch.cat([junctions - dist / 2, junctions + dist / 2], 1) + scores = prob[prob >= prob_thresh] + remainings = nms(boxes, scores, 0.001) + junctions = junctions[remainings] + + # Keep only the topk values + if top_k > 0: + k = min(len(junctions), top_k) + junctions = junctions[:k] + + return junctions diff --git a/kornia/feature/sold2/structures.py b/kornia/feature/sold2/structures.py new file mode 100644 index 0000000..6774ab1 --- /dev/null +++ b/kornia/feature/sold2/structures.py @@ -0,0 +1,93 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from dataclasses import dataclass, field + + +@dataclass +class HeatMapRefineCfg: + """Configuration for heatmap refinement in the SOLD2 pipeline.""" + + mode: str = "local" + ratio: float = 0.2 + valid_thresh: float = 0.001 + num_blocks: int = 20 + overlap_ratio: float = 0.5 + + +@dataclass +class JunctionRefineCfg: + """Configuration for junction refinement in the SOLD2 pipeline.""" + + num_perturbs: int = 9 + perturb_interval: float = 0.25 + + +@dataclass +class LineDetectorCfg: + """Configuration for line detection in the SOLD2 pipeline.""" + + detect_thresh: float = 0.5 + num_samples: int = 64 + inlier_thresh: float = 0.99 + use_candidate_suppression: bool = True + nms_dist_tolerance: float = 3.0 + heatmap_low_thresh: float = 0.15 + heatmap_high_thresh: float = 0.2 + max_local_patch_radius: float = 3 + lambda_radius: float = 2.0 + use_heatmap_refinement: bool = True + heatmap_refine_cfg: HeatMapRefineCfg = field(default_factory=HeatMapRefineCfg) + use_junction_refinement: bool = True + junction_refine_cfg: JunctionRefineCfg = field(default_factory=JunctionRefineCfg) + + +@dataclass +class LineMatcherCfg: + """Configuration for line matching in the SOLD2 pipeline.""" + + cross_check: bool = True + num_samples: int = 5 + min_dist_pts: int = 8 + top_k_candidates: int = 10 + grid_size: int = 4 + line_score: bool = False # True to compute saliency on a line + + +@dataclass +class BackboneCfg: + """Configuration for the backbone network in the SOLD2 pipeline.""" + + input_channel: int = 1 + depth: int = 4 + num_stacks: int = 2 + num_blocks: int = 1 + num_classes: int = 5 + + +@dataclass +class DetectorCfg: + """Configuration for the SOLD2 detector.""" + + backbone_cfg: BackboneCfg = field(default_factory=BackboneCfg) + use_descriptor: bool = False + grid_size: int = 8 + keep_border_valid: bool = True + detection_thresh: float = 0.0153846 # = 1/65: threshold of junction detection + max_num_junctions: int = 500 # maximum number of junctions per image + line_detector_cfg: LineDetectorCfg = field(default_factory=LineDetectorCfg) + line_matcher_cfg: LineMatcherCfg = field(default_factory=LineMatcherCfg) diff --git a/kornia/feature/sosnet.py b/kornia/feature/sosnet.py new file mode 100644 index 0000000..c6bb9d0 --- /dev/null +++ b/kornia/feature/sosnet.py @@ -0,0 +1,98 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK_SHAPE + +urls: Dict[str, str] = {} +urls["lib"] = "https://github.com/yuruntian/SOSNet/raw/master/sosnet-weights/sosnet_32x32_liberty.pth" +urls["hp_a"] = "https://github.com/yuruntian/SOSNet/raw/master/sosnet-weights/sosnet_32x32_hpatches_a.pth" + + +class SOSNet(nn.Module): + r"""128-dimensional SOSNet model definition for 32x32 patches. + + This is based on the original code from paper + "SOSNet:Second Order Similarity Regularization for Local Descriptor Learning". + + Args: + pretrained: Download and set pretrained weights to the model. + + Shape: + - Input: :math:`(B, 1, 32, 32)` + - Output: :math:`(B, 128)` + + Examples: + >>> input = torch.rand(8, 1, 32, 32) + >>> sosnet = SOSNet() + >>> descs = sosnet(input) # 8x128 + + """ + + patch_size = 32 + + def __init__(self, pretrained: bool = False) -> None: + super().__init__() + self.layers = nn.Sequential( + nn.InstanceNorm2d(1, affine=False), + nn.Conv2d(1, 32, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(32, affine=False), + nn.ReLU(), + nn.Conv2d(32, 32, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(32, affine=False), + nn.ReLU(), + nn.Conv2d(32, 64, kernel_size=3, stride=2, padding=1, bias=False), + nn.BatchNorm2d(64, affine=False), + nn.ReLU(), + nn.Conv2d(64, 64, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(64, affine=False), + nn.ReLU(), + nn.Conv2d(64, 128, kernel_size=3, stride=2, padding=1, bias=False), + nn.BatchNorm2d(128, affine=False), + nn.ReLU(), + nn.Conv2d(128, 128, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(128, affine=False), + nn.ReLU(), + nn.Dropout(0.1), + nn.Conv2d(128, 128, kernel_size=8, bias=False), + nn.BatchNorm2d(128, affine=False), + ) + self.desc_norm = nn.Sequential(nn.LocalResponseNorm(256, alpha=256.0, beta=0.5, k=0.0)) + # load pretrained model + if pretrained: + pretrained_dict = torch.hub.load_state_dict_from_url(urls["lib"], map_location=torch.device("cpu")) + self.load_state_dict(pretrained_dict, strict=True) + self.eval() + + def forward(self, input: torch.Tensor, eps: float = 1e-10) -> torch.Tensor: + """Compute SOSNet descriptors for grayscale 32x32 patches. + + Args: + input: Input tensor with shape :math:`(B, 1, 32, 32)`. + eps: Small constant added before normalization for stability. + + Returns: + Descriptor tensor with shape :math:`(B, 128)`. + """ + KORNIA_CHECK_SHAPE(input, ["B", "1", "32", "32"]) + descr = self.desc_norm(self.layers(input) + eps) + descr = descr.view(descr.size(0), -1) + return descr diff --git a/kornia/feature/steerers.py b/kornia/feature/steerers.py new file mode 100644 index 0000000..f354190 --- /dev/null +++ b/kornia/feature/steerers.py @@ -0,0 +1,132 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math + +import torch +import torch.nn.functional as F +from torch import nn + + +class DiscreteSteerer(nn.Module): + """nn.Module for discrete rotation steerers. + + A steerer rotates keypoint descriptions in latent space as if they were obtained from rotated images. + + Args: + generator: [N, N] torch.Tensor where N is the descriptor dimension. + + Example: + >>> desc = torch.randn(512, 128) + >>> generator = torch.randn(128, 128) + >>> steerer = DiscreteSteerer(generator) + >>> # steer 3 times: + >>> steered_desc = steerer.steer_descriptions(desc, steerer_power=3, normalize=True) + + """ + + def __init__(self, generator: torch.Tensor) -> None: + super().__init__() + self.generator = torch.nn.Parameter(generator) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply one steering step in descriptor space. + + Args: + x: Input descriptor tensor whose last dimension matches the + steerer generator size. + + Returns: + Steered descriptors after one linear transformation. + """ + return torch.nn.functional.linear(x, self.generator) + + def steer_descriptions( + self, + descriptions: torch.Tensor, + steerer_power: int = 1, + normalize: bool = False, + ) -> torch.Tensor: + """Apply the steerer repeatedly to descriptor vectors. + + Args: + descriptions: Descriptor tensor to steer. + steerer_power: Number of repeated steering applications. + normalize: If ``True``, L2-normalize descriptors after steering. + + Returns: + Steered descriptors with optional normalization. + """ + for _ in range(steerer_power): + descriptions = self.forward(descriptions) + if normalize: + descriptions = F.normalize(descriptions, dim=-1) + return descriptions + + @classmethod + def create_dedode_default( + cls, + generator_type: str = "C4", + steerer_order: int = 8, + ) -> nn.Module: + r"""Create a steerer for pretrained DeDoDe descriptors int the "C-setting" + from the paper https://arxiv.org/abs/2312.02152, where descriptors were + trained for fixed steerers. + + Args: + generator_type: The type of steerer generator. + One of 'C4', 'SO2', default is 'C4'. + These can be used with the DeDoDe descriptors in Kornia + with C4 or SO2 in the name respectively (so called C-setting steerers). + steerer_order: The discretisation order for SO2-steerers (NOT used for C4-steerers). + + Returns: + The pretrained model. + + """ # noqa: D205 + descriptor_dim = 256 + if generator_type == "C4": + c4_block = torch.tensor([[0.0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1], [1, 0, 0, 0]]) + generator = torch.block_diag(*([c4_block] * (descriptor_dim // 4))) + return cls(generator).eval() + + elif generator_type == "SO2": + num_rot_blocks_per_freq = descriptor_dim // 14 + dim_rot = 12 * num_rot_blocks_per_freq + dim_trivial = descriptor_dim - dim_rot + + blocks = [] + if dim_trivial > 0: + blocks.append(torch.eye(dim_trivial)) + + angle_step = 2 * math.pi / steerer_order + for j in range(1, 7): + theta = j * angle_step + cos_theta = math.cos(theta) + sin_theta = math.sin(theta) + rot_matrix = torch.tensor( + # The matrix exponential of a 2x2 skew-symmetric matrix is a rotation matrix + # exp(alpha * [[0, j], [-j, 0]]) -> R(j * alpha) + [[cos_theta, sin_theta], [-sin_theta, cos_theta]], + dtype=torch.float32, + ) + blocks.extend([rot_matrix] * num_rot_blocks_per_freq) + + generator = torch.block_diag(*blocks) + return cls(generator).eval() + else: + raise ValueError(f"Unknown generator_type: {generator_type}") diff --git a/kornia/feature/tfeat.py b/kornia/feature/tfeat.py new file mode 100644 index 0000000..31fc77e --- /dev/null +++ b/kornia/feature/tfeat.py @@ -0,0 +1,87 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK_SHAPE + +urls: Dict[str, str] = {} +urls["liberty"] = "https://github.com/vbalnt/tfeat/raw/master/pretrained-models/tfeat-liberty.params" # pylint: disable +urls["notredame"] = "https://github.com/vbalnt/tfeat/raw/master/pretrained-models/tfeat-notredame.params" # pylint: disable +urls["yosemite"] = "https://github.com/vbalnt/tfeat/raw/master/pretrained-models/tfeat-yosemite.params" # pylint: disable + + +class TFeat(nn.Module): + r"""Module, which computes TFeat descriptors of given grayscale patches of 32x32. + + This is based on the original code from paper "Learning local feature descriptors + with triplets and shallow convolutional neural networks". + See :cite:`TFeat2016` for more details + + Args: + pretrained: Download and set pretrained weights to the model. + + Returns: + torch.Tensor: TFeat descriptor of the patches. + + Shape: + - Input: :math:`(B, 1, 32, 32)` + - Output: :math:`(B, 128)` + + Examples: + >>> input = torch.rand(16, 1, 32, 32) + >>> tfeat = TFeat() + >>> descs = tfeat(input) # 16x128 + + """ + + patch_size = 32 + + def __init__(self, pretrained: bool = False) -> None: + super().__init__() + self.features = nn.Sequential( + nn.InstanceNorm2d(1, affine=False), + nn.Conv2d(1, 32, kernel_size=7), + nn.Tanh(), + nn.MaxPool2d(kernel_size=2, stride=2), + nn.Conv2d(32, 64, kernel_size=6), + nn.Tanh(), + ) + self.descr = nn.Sequential(nn.Linear(64 * 8 * 8, 128), nn.Tanh()) + # use torch.hub to load pretrained model + if pretrained: + pretrained_dict = torch.hub.load_state_dict_from_url(urls["liberty"], map_location=torch.device("cpu")) + self.load_state_dict(pretrained_dict, strict=True) + self.eval() + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Compute TFeat descriptors for grayscale 32x32 patches. + + Args: + input: Input tensor with shape :math:`(B, 1, 32, 32)`. + + Returns: + Descriptor tensor with shape :math:`(B, 128)`. + """ + KORNIA_CHECK_SHAPE(input, ["B", "1", "32", "32"]) + x = self.features(input) + x = x.view(x.size(0), -1) + x = self.descr(x) + return x diff --git a/kornia/feature/xfeat.py b/kornia/feature/xfeat.py new file mode 100644 index 0000000..de18c10 --- /dev/null +++ b/kornia/feature/xfeat.py @@ -0,0 +1,619 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""XFeat: Accelerated Features for Lightweight Image Matching. + +This module is adapted from the original XFeat implementation: + https://github.com/verlab/accelerated_features + +Original work copyright (c) 2024 Verlab / UFMG, licensed under the Apache License 2.0. +Modifications copyright (c) 2024 Kornia Team, licensed under the Apache License 2.0. + +Reference: + Guilherme Potje, Felipe Cadar, Andre Araujo, Renato Martins, Erickson R. Nascimento. + "XFeat: Accelerated Features for Lightweight Image Matching", CVPR 2024. + https://www.verlab.dcc.ufmg.br/descriptors/xfeat_cvpr24/ +""" + +from __future__ import annotations + +from typing import Dict, List, Optional, Tuple + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK +from kornia.geometry.subpix import nms2d + + +class BasicLayer(nn.Module): + """Basic convolutional layer: Conv2d -> BatchNorm2d (no affine) -> ReLU. + + Args: + in_channels: number of input channels. + out_channels: number of output channels. + kernel_size: convolution kernel size. Default: ``3``. + stride: convolution stride. Default: ``1``. + padding: convolution padding. Default: ``1``. + dilation: convolution dilation. Default: ``1``. + bias: whether to use bias. Default: ``False``. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int = 3, + stride: int = 1, + padding: int = 1, + dilation: int = 1, + bias: bool = False, + ) -> None: + super().__init__() + self.layer = nn.Sequential( + nn.Conv2d( + in_channels, out_channels, kernel_size, padding=padding, stride=stride, dilation=dilation, bias=bias + ), + nn.BatchNorm2d(out_channels, affine=False), + nn.ReLU(inplace=True), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run the module forward pass. + + Args: + x: Input tensor processed by this module. For image-like features this usually follows the `(B, C, H, W)` + layout, where `B` is batch size, `C` is channels, and `H`/`W` are height and width. + + Returns: + Output tensor or dictionary produced by the module while preserving the shape contract documented by the + surrounding class. + """ + return self.layer(x) + + +class XFeatModel(nn.Module): + """XFeat backbone: CNN feature extractor, keypoint and reliability heads. + + Implements the architecture from + "XFeat: Accelerated Features for Lightweight Image Matching", CVPR 2024. + + Input: float image tensor :math:`(B, C, H, W)` (grayscale or RGB, any channel count). + Output: + + - ``feats``: dense descriptors :math:`(B, 64, H/8, W/8)`. + - ``keypoints``: keypoint logits :math:`(B, 65, H/8, W/8)`. + - ``heatmap``: reliability map :math:`(B, 1, H/8, W/8)`. + + Note: + Image normalisation (``InstanceNorm2d``) is wrapped in ``torch.no_grad()`` + following the original design; backpropagating through it is not supported. + """ + + def __init__(self) -> None: + super().__init__() + self.norm = nn.InstanceNorm2d(1) + + self.skip1 = nn.Sequential( + nn.AvgPool2d(4, stride=4), + nn.Conv2d(1, 24, 1, stride=1, padding=0), + ) + + self.block1 = nn.Sequential( + BasicLayer(1, 4, stride=1), + BasicLayer(4, 8, stride=2), + BasicLayer(8, 8, stride=1), + BasicLayer(8, 24, stride=2), + ) + + self.block2 = nn.Sequential( + BasicLayer(24, 24, stride=1), + BasicLayer(24, 24, stride=1), + ) + + self.block3 = nn.Sequential( + BasicLayer(24, 64, stride=2), + BasicLayer(64, 64, stride=1), + BasicLayer(64, 64, 1, padding=0), + ) + + self.block4 = nn.Sequential( + BasicLayer(64, 64, stride=2), + BasicLayer(64, 64, stride=1), + BasicLayer(64, 64, stride=1), + ) + + self.block5 = nn.Sequential( + BasicLayer(64, 128, stride=2), + BasicLayer(128, 128, stride=1), + BasicLayer(128, 128, stride=1), + BasicLayer(128, 64, 1, padding=0), + ) + + self.block_fusion = nn.Sequential( + BasicLayer(64, 64, stride=1), + BasicLayer(64, 64, stride=1), + nn.Conv2d(64, 64, 1, padding=0), + ) + + self.heatmap_head = nn.Sequential( + BasicLayer(64, 64, 1, padding=0), + BasicLayer(64, 64, 1, padding=0), + nn.Conv2d(64, 1, 1), + nn.Sigmoid(), + ) + + self.keypoint_head = nn.Sequential( + BasicLayer(64, 64, 1, padding=0), + BasicLayer(64, 64, 1, padding=0), + BasicLayer(64, 64, 1, padding=0), + nn.Conv2d(64, 65, 1), + ) + + self.fine_matcher = nn.Sequential( + nn.Linear(128, 512), + nn.BatchNorm1d(512, affine=False), + nn.ReLU(inplace=True), + nn.Linear(512, 512), + nn.BatchNorm1d(512, affine=False), + nn.ReLU(inplace=True), + nn.Linear(512, 512), + nn.BatchNorm1d(512, affine=False), + nn.ReLU(inplace=True), + nn.Linear(512, 512), + nn.BatchNorm1d(512, affine=False), + nn.ReLU(inplace=True), + nn.Linear(512, 64), + ) + + def _unfold2d(self, x: torch.Tensor, ws: int = 2) -> torch.Tensor: + """Unfold spatial dims by window size ``ws`` and concatenate into channels.""" + B, C, H, W = x.shape + x = x.unfold(2, ws, ws).unfold(3, ws, ws).reshape(B, C, H // ws, W // ws, ws**2) + return x.permute(0, 1, 4, 2, 3).reshape(B, -1, H // ws, W // ws) + + def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Run the XFeat backbone. + + Args: + x: image tensor of shape :math:`(B, C, H, W)`. + + Returns: + Tuple of: + - feats: dense descriptors :math:`(B, 64, H/8, W/8)`. + - keypoints: keypoint logits :math:`(B, 65, H/8, W/8)`. + - heatmap: reliability map :math:`(B, 1, H/8, W/8)`. + """ + with torch.no_grad(): + x = x.mean(dim=1, keepdim=True) + x = self.norm(x) + + x1 = self.block1(x) + x2 = self.block2(x1 + self.skip1(x)) + x3 = self.block3(x2) + x4 = self.block4(x3) + x5 = self.block5(x4) + + x4 = F.interpolate(x4, (x3.shape[-2], x3.shape[-1]), mode="bilinear", align_corners=False) + x5 = F.interpolate(x5, (x3.shape[-2], x3.shape[-1]), mode="bilinear", align_corners=False) + feats = self.block_fusion(x3 + x4 + x5) + + heatmap = self.heatmap_head(feats) + keypoints = self.keypoint_head(self._unfold2d(x, ws=8)) + return feats, keypoints, heatmap + + +class InterpolateSparse2d(nn.Module): + """Bilinearly or bicubically sample a dense feature map at sparse 2-D positions. + + Args: + mode: interpolation mode for :func:`torch.nn.functional.grid_sample`. + Default: ``'bicubic'``. + align_corners: passed to ``grid_sample``. Default: ``False``. + + Shape: + - Input ``x``: :math:`(B, C, H, W)`. + - Input ``pos``: :math:`(B, N, 2)` integer or float (x, y) coordinates. + - Output: :math:`(B, N, C)`. + """ + + def __init__(self, mode: str = "bicubic", align_corners: bool = False) -> None: + super().__init__() + self.mode = mode + self.align_corners = align_corners + + def normgrid(self, x: torch.Tensor, H: int, W: int) -> torch.Tensor: + """Normalise pixel coordinates to the ``[-1, 1]`` range expected by ``grid_sample``. + + Uses the formula from the original XFeat implementation: + ``2 * x / [W-1, H-1] - 1``. Note that ``grid_sample`` is always + called with ``align_corners=False``; this asymmetry matches the + convention the pretrained weights were trained with. + """ + return 2.0 * (x / torch.tensor([W - 1, H - 1], device=x.device, dtype=x.dtype)) - 1.0 + + def forward(self, x: torch.Tensor, pos: torch.Tensor, H: int, W: int) -> torch.Tensor: + """Sample ``x`` at positions ``pos``. + + Args: + x: feature map :math:`(B, C, H, W)`. + pos: sampling positions :math:`(B, N, 2)` in pixel coordinates. + H: height used for coordinate normalisation. + W: width used for coordinate normalisation. + + Returns: + Sampled features :math:`(B, N, C)`. + """ + grid = self.normgrid(pos, H, W).unsqueeze(-2).to(x.dtype) + # align_corners=False is intentional: the pretrained XFeat weights were trained + # with normgrid using the (W-1) denominator but grid_sample using align_corners=False. + # Changing either side would break compatibility with the pretrained weights. + x = F.grid_sample(x, grid, mode=self.mode, align_corners=False) + return x.permute(0, 2, 3, 1).squeeze(-2) + + +class XFeat(nn.Module): + """XFeat sparse and semi-dense local feature extractor and matcher. + + Wraps :class:`XFeatModel` with NMS keypoint detection, descriptor + interpolation, and mutual nearest-neighbour matching helpers. + + Reference: + "XFeat: Accelerated Features for Lightweight Image Matching", CVPR 2024. + https://www.verlab.dcc.ufmg.br/descriptors/xfeat_cvpr24/ + + .. image:: _static/img/XFeat.png + + Args: + top_k: maximum number of keypoints to keep per image. Default: ``4096``. + detection_threshold: minimum keypoint score. Default: ``0.05``. + + Example: + >>> model = XFeat() + >>> img = torch.rand(1, 3, 256, 256) + >>> out = model.detectAndCompute(img) + >>> out[0]['keypoints'].shape + torch.Size([..., 2]) + """ + + weights_url: str = "https://github.com/verlab/accelerated_features/raw/main/weights/xfeat.pt" + + def __init__(self, top_k: int = 4096, detection_threshold: float = 0.05) -> None: + super().__init__() + self.top_k = top_k + self.detection_threshold = detection_threshold + self.net = XFeatModel() + self.interpolator = InterpolateSparse2d("bicubic") + self._nearest = InterpolateSparse2d("nearest") + self._bilinear = InterpolateSparse2d("bilinear") + + @classmethod + def from_pretrained(cls, top_k: int = 4096, detection_threshold: float = 0.05) -> XFeat: + """Instantiate XFeat with pretrained weights downloaded from the official release. + + Args: + top_k: maximum number of keypoints to keep. Default: ``4096``. + detection_threshold: minimum keypoint score. Default: ``0.05``. + + Returns: + XFeat model with pretrained weights loaded, set to eval mode. + """ + model = cls(top_k=top_k, detection_threshold=detection_threshold) + state_dict = torch.hub.load_state_dict_from_url(cls.weights_url, file_name="xfeat.pt") + model.net.load_state_dict(state_dict) + model.eval() + return model + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _preprocess_tensor(self, x: torch.Tensor) -> Tuple[torch.Tensor, float, float]: + """Resize to the largest multiple of 32 not exceeding the input size (min 32); return image and scale ratios.""" + KORNIA_CHECK(x.dim() == 4, "Input must be a 4-D tensor (B, C, H, W)") + H, W = x.shape[-2:] + _H, _W = max(32, (H // 32) * 32), max(32, (W // 32) * 32) + rh, rw = H / _H, W / _W + x = F.interpolate(x.float(), (_H, _W), mode="bilinear", align_corners=False) + return x, rh, rw + + @staticmethod + def _get_kpts_heatmap(kpts: torch.Tensor, softmax_temp: float = 1.0) -> torch.Tensor: + """Convert 65-channel keypoint logits to a full-resolution heatmap.""" + scores = F.softmax(kpts * softmax_temp, 1)[:, :64] + B, _, H, W = scores.shape + heatmap = scores.permute(0, 2, 3, 1).reshape(B, H, W, 8, 8) + return heatmap.permute(0, 1, 3, 2, 4).reshape(B, 1, H * 8, W * 8) + + @staticmethod + def _nms(x: torch.Tensor, threshold: float = 0.05, kernel_size: int = 5) -> torch.Tensor: + """Non-maximum suppression on a heatmap; returns (B, N, 2) integer keypoint positions.""" + B = x.shape[0] + pos = nms2d(x, (kernel_size, kernel_size), mask_only=True) & (x > threshold) + pos_batched = [k.nonzero()[..., 1:].flip(-1) for k in pos] + pad_val = max(len(p) for p in pos_batched) + pos_out = torch.full((B, pad_val, 2), -1, dtype=torch.long, device=x.device) + for b in range(B): + pos_out[b, : len(pos_batched[b])] = pos_batched[b] + return pos_out + + @staticmethod + def _create_xy(h: int, w: int, device: torch.device) -> torch.Tensor: + """Create a grid of (x, y) pixel coordinates of shape :math:`(H*W, 2)`.""" + y, x = torch.meshgrid(torch.arange(h, device=device), torch.arange(w, device=device), indexing="ij") + return torch.cat([x[..., None], y[..., None]], -1).reshape(-1, 2) + + @staticmethod + def _subpix_softmax2d(heatmaps: torch.Tensor, temp: float = 3.0) -> torch.Tensor: + """Compute soft-argmax offsets from an (N, H, W) heatmap.""" + N, H, W = heatmaps.shape + heatmaps = torch.softmax(temp * heatmaps.view(N, H * W), -1).view(N, H, W) + x, y = torch.meshgrid( + torch.arange(W, device=heatmaps.device), torch.arange(H, device=heatmaps.device), indexing="xy" + ) + x = x - (W // 2) + y = y - (H // 2) + coords = torch.cat([(x[None] * heatmaps)[..., None], (y[None] * heatmaps)[..., None]], -1) + return coords.view(N, H * W, 2).sum(1) + + def _match_mnn( + self, feats1: torch.Tensor, feats2: torch.Tensor, min_cossim: float = 0.82 + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Mutual nearest-neighbour matching on L2-normalised descriptor pairs.""" + # If either descriptor set is empty there are no possible matches; return early + # with empty index tensors. + if feats1.shape[0] == 0 or feats2.shape[0] == 0: + empty = torch.empty(0, dtype=torch.long, device=feats1.device) + return empty, empty.clone() + + cossim = feats1 @ feats2.t() + cossim_t = feats2 @ feats1.t() + _, match12 = cossim.max(dim=1) + _, match21 = cossim_t.max(dim=1) + idx0 = torch.arange(len(match12), device=match12.device) + mutual = match21[match12] == idx0 + if min_cossim > 0: + cossim_max, _ = cossim.max(dim=1) + good = cossim_max > min_cossim + idx0 = idx0[mutual & good] + idx1 = match12[mutual & good] + else: + idx0 = idx0[mutual] + idx1 = match12[mutual] + return idx0, idx1 + + def _batch_match( + self, feats1: torch.Tensor, feats2: torch.Tensor, min_cossim: float = -1 + ) -> List[Tuple[torch.Tensor, torch.Tensor]]: + """Batched MNN matching; returns a list of ``(idx0, idx1)`` tuples.""" + B = feats1.shape[0] + cossim = torch.bmm(feats1, feats2.permute(0, 2, 1)) + match12 = torch.argmax(cossim, dim=-1) + match21 = torch.argmax(cossim.permute(0, 2, 1), dim=-1) + idx0 = torch.arange(match12.shape[1], device=match12.device) + batched_matches = [] + for b in range(B): + mutual = match21[b][match12[b]] == idx0 + if min_cossim > 0: + cossim_max, _ = cossim[b].max(dim=1) + good = cossim_max > min_cossim + idx0_b = idx0[mutual & good] + idx1_b = match12[b][mutual & good] + else: + idx0_b = idx0[mutual] + idx1_b = match12[b][mutual] + batched_matches.append((idx0_b, idx1_b)) + return batched_matches + + def _extract_dense(self, x: torch.Tensor, top_k: int = 8000) -> Tuple[torch.Tensor, torch.Tensor]: + """Extract coarse descriptors from an FPN feature map, top-k by reliability.""" + if top_k < 1: + top_k = 100_000_000 + x, rh1, rw1 = self._preprocess_tensor(x) + M1, _K1, H1 = self.net(x) + B, C, _H1, _W1 = M1.shape + xy1 = (self._create_xy(_H1, _W1, M1.device) * 8).expand(B, -1, -1) + M1 = M1.permute(0, 2, 3, 1).reshape(B, -1, C) + H1 = H1.permute(0, 2, 3, 1).reshape(B, -1) + _, top_k_idx = torch.topk(H1, k=min(H1.shape[1], top_k), dim=-1) + feats = torch.gather(M1, 1, top_k_idx[..., None].expand(-1, -1, C)) + mkpts = torch.gather(xy1, 1, top_k_idx[..., None].expand(-1, -1, 2)) + mkpts = mkpts * torch.tensor([rw1, rh1], device=mkpts.device).view(1, -1) + return mkpts, feats + + def _extract_dualscale( + self, x: torch.Tensor, top_k: int, s1: float = 0.6, s2: float = 1.3 + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Extract dense features at two scales and merge.""" + x1 = F.interpolate(x, scale_factor=s1, align_corners=False, mode="bilinear") + x2 = F.interpolate(x, scale_factor=s2, align_corners=False, mode="bilinear") + mkpts_1, feats_1 = self._extract_dense(x1, int(top_k * 0.20)) + mkpts_2, feats_2 = self._extract_dense(x2, int(top_k * 0.80)) + mkpts = torch.cat([mkpts_1 / s1, mkpts_2 / s2], dim=1) + sc1 = torch.ones(mkpts_1.shape[:2], device=mkpts_1.device) * (1 / s1) + sc2 = torch.ones(mkpts_2.shape[:2], device=mkpts_2.device) * (1 / s2) + return mkpts, torch.cat([sc1, sc2], dim=1), torch.cat([feats_1, feats_2], dim=1) + + def _refine_matches( + self, + d0: Dict[str, torch.Tensor], + d1: Dict[str, torch.Tensor], + matches: List[Tuple[torch.Tensor, torch.Tensor]], + batch_idx: int, + fine_conf: float = 0.25, + ) -> torch.Tensor: + """Refine coarse match positions using the fine-matcher MLP with subpixel softmax.""" + idx0, idx1 = matches[batch_idx] + feats1 = d0["descriptors"][batch_idx][idx0] + feats2 = d1["descriptors"][batch_idx][idx1] + mkpts_0 = d0["keypoints"][batch_idx][idx0].clone() + mkpts_1 = d1["keypoints"][batch_idx][idx1] + sc0 = d0["scales"][batch_idx][idx0] + offsets = self.net.fine_matcher(torch.cat([feats1, feats2], dim=-1)) + conf = F.softmax(offsets * 3, dim=-1).max(dim=-1)[0] + offsets = self._subpix_softmax2d(offsets.view(-1, 8, 8)) + mkpts_0 = mkpts_0 + offsets * sc0[:, None] + mask_good = conf > fine_conf + return torch.cat([mkpts_0[mask_good], mkpts_1[mask_good]], dim=-1) + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + @torch.inference_mode() + def detectAndCompute( + self, + x: torch.Tensor, + top_k: Optional[int] = None, + detection_threshold: Optional[float] = None, + ) -> List[Dict[str, torch.Tensor]]: + """Detect sparse keypoints and compute descriptors. + + Args: + x: image tensor of shape :math:`(B, C, H, W)`. + top_k: number of keypoints to keep (overrides ``self.top_k``). + detection_threshold: minimum score (overrides ``self.detection_threshold``). + + Returns: + List of length ``B``. Each element is a dict with: + + - ``'keypoints'``: :math:`(N, 2)` keypoints in (x, y) pixel coordinates. + - ``'scores'``: :math:`(N,)` reliability scores. + - ``'descriptors'``: :math:`(N, 64)` L2-normalised descriptors. + """ + if top_k is None: + top_k = self.top_k + if detection_threshold is None: + detection_threshold = self.detection_threshold + x, rh1, rw1 = self._preprocess_tensor(x) + B, _, _H1, _W1 = x.shape + + M1, K1, H1 = self.net(x) + M1 = F.normalize(M1, dim=1) + + K1h = self._get_kpts_heatmap(K1) + mkpts = self._nms(K1h, threshold=detection_threshold, kernel_size=5) + + scores = (self._nearest(K1h, mkpts, _H1, _W1) * self._bilinear(H1, mkpts, _H1, _W1)).squeeze(-1) + scores[torch.all(mkpts == -1, dim=-1)] = -1 + + k = min(top_k, scores.shape[-1]) + scores, idxs = torch.topk(scores, k=k, dim=-1, largest=True, sorted=True) + mkpts_x = torch.gather(mkpts[..., 0], -1, idxs) + mkpts_y = torch.gather(mkpts[..., 1], -1, idxs) + mkpts = torch.cat([mkpts_x[..., None], mkpts_y[..., None]], dim=-1) + + feats = self.interpolator(M1, mkpts, H=_H1, W=_W1) + feats = F.normalize(feats, dim=-1) + + mkpts = mkpts * torch.tensor([rw1, rh1], device=mkpts.device).view(1, 1, -1) + + valid = scores > 0 + return [ + { + "keypoints": mkpts[b][valid[b]], + "scores": scores[b][valid[b]], + "descriptors": feats[b][valid[b]], + } + for b in range(B) + ] + + @torch.inference_mode() + def detectAndComputeDense( + self, + x: torch.Tensor, + top_k: Optional[int] = None, + multiscale: bool = True, + ) -> Dict[str, torch.Tensor]: + """Detect keypoints and compute dense coarse descriptors. + + Args: + x: image tensor of shape :math:`(B, C, H, W)`. + top_k: number of features to keep (overrides ``self.top_k``). + multiscale: use dual-scale (0.6x and 1.3x) extraction. Default: ``True``. + + Returns: + Dict with: + + - ``'keypoints'``: :math:`(B, K, 2)` coarse keypoints. + - ``'descriptors'``: :math:`(B, K, 64)` coarse descriptors. + - ``'scales'``: :math:`(B, K)` extraction scale per keypoint. + """ + if top_k is None: + top_k = self.top_k + if multiscale: + mkpts, sc, feats = self._extract_dualscale(x, top_k) + else: + mkpts, feats = self._extract_dense(x, top_k) + sc = torch.ones(mkpts.shape[:2], device=mkpts.device) + return {"keypoints": mkpts, "descriptors": feats, "scales": sc} + + @torch.inference_mode() + def match_xfeat( + self, + img1: torch.Tensor, + img2: torch.Tensor, + top_k: Optional[int] = None, + min_cossim: float = -1, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Detect, describe and mutually match keypoints from two images. + + Args: + img1: first image tensor of shape :math:`(1, C, H, W)`. + img2: second image tensor of shape :math:`(1, C, H, W)`. + top_k: number of top keypoints to use. + min_cossim: minimum cosine similarity threshold. Use ``-1`` to disable. + + Returns: + Tuple ``(mkpts0, mkpts1)`` of matched keypoints, each :math:`(N, 2)`. + """ + if top_k is None: + top_k = self.top_k + out1 = self.detectAndCompute(img1, top_k=top_k)[0] + out2 = self.detectAndCompute(img2, top_k=top_k)[0] + idx0, idx1 = self._match_mnn(out1["descriptors"], out2["descriptors"], min_cossim=min_cossim) + return out1["keypoints"][idx0], out2["keypoints"][idx1] + + @torch.inference_mode() + def match_xfeat_star( + self, + im_set1: torch.Tensor, + im_set2: torch.Tensor, + top_k: Optional[int] = None, + ) -> List[torch.Tensor] | Tuple[torch.Tensor, torch.Tensor]: + """Extract coarse features, match pairs and refine matches (XFeat*). + + Args: + im_set1: batch of images :math:`(B, C, H, W)`. + im_set2: batch of images :math:`(B, C, H, W)`. + top_k: number of top features to use. + + Returns: + If ``B > 1``: list of :math:`(N, 4)` tensors with ``(x1, y1, x2, y2)`` matches. + If ``B == 1``: tuple of :math:`(N, 2)` matched keypoint tensors. + """ + if top_k is None: + top_k = self.top_k + out1 = self.detectAndComputeDense(im_set1, top_k=top_k) + out2 = self.detectAndComputeDense(im_set2, top_k=top_k) + idxs_list = self._batch_match(out1["descriptors"], out2["descriptors"]) + B = im_set1.shape[0] + matches = [self._refine_matches(out1, out2, matches=idxs_list, batch_idx=b) for b in range(B)] + if B > 1: + return matches + return matches[0][:, :2], matches[0][:, 2:] diff --git a/kornia/filters/__init__.py b/kornia/filters/__init__.py new file mode 100644 index 0000000..112daae --- /dev/null +++ b/kornia/filters/__init__.py @@ -0,0 +1,140 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Filters — Image filtering operations for Kornia. + +This subpackage provides modules for blurring, sharpening, and other image filters. +""" + +from __future__ import annotations + +from .bilateral import BilateralBlur, JointBilateralBlur, bilateral_blur, joint_bilateral_blur +from .blur import BoxBlur, box_blur +from .blur_pool import ( + BlurPool2D, + EdgeAwareBlurPool2D, + MaxBlurPool2D, + blur_pool2d, + edge_aware_blur_pool2d, + max_blur_pool2d, +) +from .canny import Canny, canny +from .dissolving import StableDiffusionDissolving +from .filter import convolve2d, convolve3d, correlate2d, correlate3d, fft_conv, filter2d, filter2d_separable, filter3d +from .gaussian import GaussianBlur2d, gaussian_blur2d, gaussian_blur2d_t +from .guided import GuidedBlur, guided_blur +from .in_range import InRange, in_range +from .kernels import ( + gaussian, + get_binary_kernel2d, + get_box_kernel1d, + get_box_kernel2d, + get_diff_kernel2d, + get_gaussian_discrete_kernel1d, + get_gaussian_erf_kernel1d, + get_gaussian_kernel1d, + get_gaussian_kernel1d_t, + get_gaussian_kernel2d, + get_gaussian_kernel2d_t, + get_gaussian_kernel3d, + get_gaussian_kernel3d_t, + get_hanning_kernel1d, + get_hanning_kernel2d, + get_laplacian_kernel1d, + get_laplacian_kernel2d, + get_sobel_kernel2d, + get_spatial_gradient_kernel2d, + get_spatial_gradient_kernel3d, + laplacian_1d, +) +from .kernels_geometry import get_motion_kernel2d, get_motion_kernel3d +from .laplacian import Laplacian, laplacian +from .median import MedianBlur, median_blur +from .motion import MotionBlur, MotionBlur3D, motion_blur, motion_blur3d +from .otsu_thresholding import OtsuThreshold, otsu_threshold +from .sobel import Sobel, SpatialGradient, SpatialGradient3d, sobel, spatial_gradient, spatial_gradient3d +from .unsharp import UnsharpMask, unsharp_mask + +__all__ = [ + "BilateralBlur", + "BlurPool2D", + "BoxBlur", + "Canny", + "EdgeAwareBlurPool2D", + "GaussianBlur2d", + "GuidedBlur", + "InRange", + "JointBilateralBlur", + "Laplacian", + "MaxBlurPool2D", + "MedianBlur", + "MotionBlur", + "MotionBlur3D", + "OtsuThreshold", + "Sobel", + "SpatialGradient", + "SpatialGradient3d", + "StableDiffusionDissolving", + "UnsharpMask", + "bilateral_blur", + "blur_pool2d", + "box_blur", + "canny", + "edge_aware_blur_pool2d", + "fft_conv", + "filter2d", + "filter2d_separable", + "filter3d", + "gaussian", + "gaussian_blur2d", + "gaussian_blur2d_t", + "get_binary_kernel2d", + "get_box_kernel1d", + "get_box_kernel2d", + "get_diff_kernel2d", + "get_gaussian_discrete_kernel1d", + "get_gaussian_erf_kernel1d", + "get_gaussian_kernel1d", + "get_gaussian_kernel1d_t", + "get_gaussian_kernel2d", + "get_gaussian_kernel2d_t", + "get_gaussian_kernel3d", + "get_gaussian_kernel3d_t", + "get_hanning_kernel1d", + "get_hanning_kernel2d", + "get_laplacian_kernel1d", + "get_laplacian_kernel2d", + "get_motion_kernel2d", + "get_motion_kernel3d", + "get_sobel_kernel2d", + "get_spatial_gradient_kernel2d", + "get_spatial_gradient_kernel3d", + "guided_blur", + "in_range", + "joint_bilateral_blur", + "laplacian", + "laplacian_1d", + "max_blur_pool2d", + "median_blur", + "motion_blur", + "motion_blur3d", + "otsu_threshold", + "sobel", + "spatial_gradient", + "spatial_gradient3d", + "unsharp_mask", +] diff --git a/kornia/filters/bilateral.py b/kornia/filters/bilateral.py new file mode 100644 index 0000000..a88707a --- /dev/null +++ b/kornia/filters/bilateral.py @@ -0,0 +1,338 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE + +from .kernels import _unpack_2d_ks, get_gaussian_kernel2d +from .median import _compute_zero_padding + + +def _bilateral_blur( + input: torch.Tensor, + guidance: Optional[torch.Tensor], + kernel_size: tuple[int, int] | int, + sigma_color: float | torch.Tensor, + sigma_space: tuple[float, float] | torch.Tensor, + border_type: str = "reflect", + color_distance_type: str = "l1", +) -> torch.Tensor: + """Single implementation for both Bilateral Filter and Joint Bilateral Filter.""" + KORNIA_CHECK_IS_TENSOR(input) + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + if guidance is not None: + # NOTE: allow guidance and input having different number of channels + KORNIA_CHECK_IS_TENSOR(guidance) + KORNIA_CHECK_SHAPE(guidance, ["B", "C", "H", "W"]) + KORNIA_CHECK( + (guidance.shape[0] == input.shape[0]) and (guidance.shape[-2:] == input.shape[-2:]), + "guidance and input should have the same batch size and spatial dimensions", + ) + + if isinstance(sigma_color, torch.Tensor): + KORNIA_CHECK_SHAPE(sigma_color, ["B"]) + sigma_color = sigma_color.to(device=input.device, dtype=input.dtype).view(-1, 1, 1, 1, 1) + + ky, kx = _unpack_2d_ks(kernel_size) + pad_y, pad_x = _compute_zero_padding(kernel_size) + + padded_input = F.pad(input, (pad_x, pad_x, pad_y, pad_y), mode=border_type) + unfolded_input = padded_input.unfold(2, ky, 1).unfold(3, kx, 1).flatten(-2) # (B, C, H, W, Ky x Kx) + + if guidance is None: + guidance = input + unfolded_guidance = unfolded_input + else: + padded_guidance = F.pad(guidance, (pad_x, pad_x, pad_y, pad_y), mode=border_type) + unfolded_guidance = padded_guidance.unfold(2, ky, 1).unfold(3, kx, 1).flatten(-2) # (B, C, H, W, Ky x Kx) + + diff = unfolded_guidance - guidance.unsqueeze(-1) + if color_distance_type == "l1": + color_distance_sq = diff.abs().sum(1, keepdim=True).square() + elif color_distance_type == "l2": + color_distance_sq = diff.square().sum(1, keepdim=True) + else: + raise ValueError("color_distance_type only accepts l1 or l2") + color_kernel = (-0.5 / sigma_color**2 * color_distance_sq).exp() # (B, 1, H, W, Ky x Kx) + + space_kernel = get_gaussian_kernel2d(kernel_size, sigma_space, device=input.device, dtype=input.dtype) + space_kernel = space_kernel.view(-1, 1, 1, 1, kx * ky) + + kernel = space_kernel * color_kernel + out = (unfolded_input * kernel).sum(-1) / kernel.sum(-1) + return out + + +def bilateral_blur( + input: torch.Tensor, + kernel_size: tuple[int, int] | int, + sigma_color: float | torch.Tensor, + sigma_space: tuple[float, float] | torch.Tensor, + border_type: str = "reflect", + color_distance_type: str = "l1", +) -> torch.Tensor: + r"""Blur a torch.Tensor using a Bilateral filter. + + .. image:: _static/img/bilateral_blur.png + + The operator is an edge-preserving image smoothing filter. The weight + for each pixel in a neighborhood is determined not only by its distance + to the center pixel, but also the difference in intensity or color. + + Arguments: + input: the input torch.Tensor with shape :math:`(B,C,H,W)`. + kernel_size: the size of the kernel. + sigma_color: the standard deviation for intensity/color Gaussian kernel. + Smaller values preserve more edges. + sigma_space: the standard deviation for spatial Gaussian kernel. + This is similar to ``sigma`` in :func:`gaussian_blur2d()`. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. Default: ``'reflect'``. + color_distance_type: the type of distance to calculate intensity/color + difference. Only ``'l1'`` or ``'l2'`` is allowed. Use ``'l1'`` to + match OpenCV implementation. Use ``'l2'`` to match Matlab implementation. + Default: ``'l1'``. + + Returns: + the blurred torch.Tensor with shape :math:`(B, C, H, W)`. + + Examples: + >>> input = torch.rand(2, 4, 5, 5) + >>> output = bilateral_blur(input, (3, 3), 0.1, (1.5, 1.5)) + >>> output.shape + torch.Size([2, 4, 5, 5]) + + """ + return _bilateral_blur(input, None, kernel_size, sigma_color, sigma_space, border_type, color_distance_type) + + +def joint_bilateral_blur( + input: torch.Tensor, + guidance: torch.Tensor, + kernel_size: tuple[int, int] | int, + sigma_color: float | torch.Tensor, + sigma_space: tuple[float, float] | torch.Tensor, + border_type: str = "reflect", + color_distance_type: str = "l1", +) -> torch.Tensor: + r"""Blur a torch.Tensor using a Joint Bilateral filter. + + .. image:: _static/img/joint_bilateral_blur.png + + This operator is almost identical to a Bilateral filter. The only difference + is that the color Gaussian kernel is computed based on another image called + a guidance image. See :func:`bilateral_blur()` for more information. + + Arguments: + input: the input torch.Tensor with shape :math:`(B,C,H,W)`. + guidance: the guidance torch.Tensor with shape :math:`(B,C,H,W)`. + kernel_size: the size of the kernel. + sigma_color: the standard deviation for intensity/color Gaussian kernel. + Smaller values preserve more edges. + sigma_space: the standard deviation for spatial Gaussian kernel. + This is similar to ``sigma`` in :func:`gaussian_blur2d()`. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. Default: ``'reflect'``. + color_distance_type: the type of distance to calculate intensity/color + difference. Only ``'l1'`` or ``'l2'`` is allowed. Use ``'l1'`` to + match OpenCV implementation. + + Returns: + the blurred torch.Tensor with shape :math:`(B, C, H, W)`. + + Examples: + >>> input = torch.rand(2, 4, 5, 5) + >>> guidance = torch.rand(2, 4, 5, 5) + >>> output = joint_bilateral_blur(input, guidance, (3, 3), 0.1, (1.5, 1.5)) + >>> output.shape + torch.Size([2, 4, 5, 5]) + + """ + return _bilateral_blur(input, guidance, kernel_size, sigma_color, sigma_space, border_type, color_distance_type) + + +# trick to make mypy not throw errors about difference in .forward() signatures of subclass and superclass +class _BilateralBlur(nn.Module): + def __init__( + self, + kernel_size: tuple[int, int] | int, + sigma_color: float | torch.Tensor, + sigma_space: tuple[float, float] | torch.Tensor, + border_type: str = "reflect", + color_distance_type: str = "l1", + ) -> None: + super().__init__() + self.kernel_size = kernel_size + self.sigma_color = sigma_color + self.sigma_space = sigma_space + self.border_type = border_type + self.color_distance_type = color_distance_type + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}" + f"(kernel_size={self.kernel_size}, " + f"sigma_color={self.sigma_color}, " + f"sigma_space={self.sigma_space}, " + f"border_type={self.border_type}, " + f"color_distance_type={self.color_distance_type})" + ) + + +class BilateralBlur(_BilateralBlur): + r"""Blur a torch.Tensor using a Bilateral filter. + + The operator is an edge-preserving image smoothing filter. The weight + for each pixel in a neighborhood is determined not only by its distance + to the center pixel, but also the difference in intensity or color. + + Arguments: + kernel_size: the size of the kernel. + sigma_color: the standard deviation for intensity/color Gaussian kernel. + Smaller values preserve more edges. + sigma_space: the standard deviation for spatial Gaussian kernel. + This is similar to ``sigma`` in :func:`gaussian_blur2d()`. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. Default: ``'reflect'``. + color_distance_type: the type of distance to calculate intensity/color + difference. Only ``'l1'`` or ``'l2'`` is allowed. Use ``'l1'`` to + match OpenCV implementation. Use ``'l2'`` to match Matlab implementation. + Default: ``'l1'``. + + Returns: + the blurred input torch.Tensor. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + Examples: + >>> input = torch.rand(2, 4, 5, 5) + >>> blur = BilateralBlur((3, 3), 0.1, (1.5, 1.5)) + >>> output = blur(input) + >>> output.shape + torch.Size([2, 4, 5, 5]) + + """ + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Smooth an image while keeping strong intensity edges visible. + + Bilateral filtering combines two weights for every pixel in the local + window: a spatial weight based on distance in the image plane and a + range weight based on color or intensity similarity. Nearby pixels with + similar values contribute strongly, while pixels across an edge are + suppressed even if they are spatially close. + + Args: + input: Input image tensor with shape :math:`(B, C, H, W)`, + where :math:`B` is the batch size, :math:`C` is the number of + channels, :math:`H` is the image height, and :math:`W` is the + image width. + + Returns: + Tensor with shape :math:`(B, C, H, W)` containing the + edge-preserving smoothed image. The output keeps the same layout, + dtype, and device as ``input`` while reducing small local + variations according to the configured kernel size and sigma + values. + """ + return bilateral_blur( + input, self.kernel_size, self.sigma_color, self.sigma_space, self.border_type, self.color_distance_type + ) + + +class JointBilateralBlur(_BilateralBlur): + r"""Blur a torch.Tensor using a Joint Bilateral filter. + + This operator is almost identical to a Bilateral filter. The only difference + is that the color Gaussian kernel is computed based on another image called + a guidance image. See :class:`BilateralBlur` for more information. + + Arguments: + kernel_size: the size of the kernel. + sigma_color: the standard deviation for intensity/color Gaussian kernel. + Smaller values preserve more edges. + sigma_space: the standard deviation for spatial Gaussian kernel. + This is similar to ``sigma`` in :func:`gaussian_blur2d()`. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. Default: ``'reflect'``. + color_distance_type: the type of distance to calculate intensity/color + difference. Only ``'l1'`` or ``'l2'`` is allowed. Use ``'l1'`` to + match OpenCV implementation. + + Returns: + the blurred input torch.Tensor. + + Shape: + - Input: :math:`(B, C, H, W)`, :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + Examples: + >>> input = torch.rand(2, 4, 5, 5) + >>> guidance = torch.rand(2, 4, 5, 5) + >>> blur = JointBilateralBlur((3, 3), 0.1, (1.5, 1.5)) + >>> output = blur(input, guidance) + >>> output.shape + torch.Size([2, 4, 5, 5]) + + """ + + def forward(self, input: torch.Tensor, guidance: torch.Tensor) -> torch.Tensor: + """Smooth an input image using edges measured from a guidance image. + + Joint bilateral filtering is useful when the image being smoothed and + the image that should define the edges are different tensors. The + spatial kernel is applied around each location of ``input``, but the + range similarity term is computed from ``guidance``. This lets a clean + or higher-quality reference image preserve boundaries in another + signal, such as a depth map, mask, or noisy feature image. + + Args: + input: Tensor to smooth with shape :math:`(B, C, H, W)`, where + :math:`B` is the batch size, :math:`C` is the number of input + channels, :math:`H` is the height, and :math:`W` is the width. + guidance: Tensor used to compute range weights. Its batch and + spatial dimensions must be compatible with ``input``; its + channel count may differ when the guidance signal uses a + different representation. + + Returns: + Tensor with shape :math:`(B, C, H, W)` containing the filtered + ``input`` values. Edges present in ``guidance`` reduce mixing + across boundaries, while smooth regions are averaged more strongly. + """ + return joint_bilateral_blur( + input, + guidance, + self.kernel_size, + self.sigma_color, + self.sigma_space, + self.border_type, + self.color_distance_type, + ) diff --git a/kornia/filters/blur.py b/kornia/filters/blur.py new file mode 100644 index 0000000..f967d67 --- /dev/null +++ b/kornia/filters/blur.py @@ -0,0 +1,168 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK_IS_TENSOR + +from .filter import filter2d, filter2d_separable +from .kernels import _unpack_2d_ks, get_box_kernel1d, get_box_kernel2d + + +def box_blur( + input: torch.Tensor, kernel_size: tuple[int, int] | int, border_type: str = "reflect", separable: bool = False +) -> torch.Tensor: + r"""Blur an image using the box filter. + + .. image:: _static/img/box_blur.png + + The function smooths an image using the kernel: + + .. math:: + K = \frac{1}{\text{kernel_size}_x * \text{kernel_size}_y} + \begin{bmatrix} + 1 & 1 & 1 & \cdots & 1 & 1 \\ + 1 & 1 & 1 & \cdots & 1 & 1 \\ + \vdots & \vdots & \vdots & \ddots & \vdots & \vdots \\ + 1 & 1 & 1 & \cdots & 1 & 1 \\ + \end{bmatrix} + + Args: + input: the image to blur with shape :math:`(B,C,H,W)`. + kernel_size: the blurring kernel size. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, ``'replicate'`` or ``'circular'``. + separable: run as composition of two 1d-convolutions. + + Returns: + the blurred torch.Tensor with shape :math:`(B,C,H,W)`. + + .. note:: + See a working example `here `__. + + Example: + >>> input = torch.rand(2, 4, 5, 7) + >>> output = box_blur(input, (3, 3)) # 2x4x5x7 + >>> output.shape + torch.Size([2, 4, 5, 7]) + + """ + KORNIA_CHECK_IS_TENSOR(input) + + if separable: + ky, kx = _unpack_2d_ks(kernel_size) + kernel_y = get_box_kernel1d(ky, device=input.device, dtype=input.dtype) + kernel_x = get_box_kernel1d(kx, device=input.device, dtype=input.dtype) + out = filter2d_separable(input, kernel_x, kernel_y, border_type) + else: + kernel = get_box_kernel2d(kernel_size, device=input.device, dtype=input.dtype) + out = filter2d(input, kernel, border_type) + + return out + + +class BoxBlur(nn.Module): + r"""Blur an image using the box filter. + + The function smooths an image using the kernel: + + .. math:: + K = \frac{1}{\text{kernel_size}_x * \text{kernel_size}_y} + \begin{bmatrix} + 1 & 1 & 1 & \cdots & 1 & 1 \\ + 1 & 1 & 1 & \cdots & 1 & 1 \\ + \vdots & \vdots & \vdots & \ddots & \vdots & \vdots \\ + 1 & 1 & 1 & \cdots & 1 & 1 \\ + \end{bmatrix} + + Args: + kernel_size: the blurring kernel size. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. Default: ``'reflect'``. + separable: run as composition of two 1d-convolutions. + + Returns: + the blurred input torch.Tensor. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + Example: + >>> input = torch.rand(2, 4, 5, 7) + >>> blur = BoxBlur((3, 3)) + >>> output = blur(input) # 2x4x5x7 + >>> output.shape + torch.Size([2, 4, 5, 7]) + + """ + + def __init__( + self, kernel_size: tuple[int, int] | int, border_type: str = "reflect", separable: bool = False + ) -> None: + super().__init__() + self.kernel_size = kernel_size + self.border_type = border_type + self.separable = separable + + if separable: + ky, kx = _unpack_2d_ks(self.kernel_size) + self.register_buffer("kernel_y", get_box_kernel1d(ky)) + self.register_buffer("kernel_x", get_box_kernel1d(kx)) + self.kernel_y: torch.Tensor + self.kernel_x: torch.Tensor + else: + self.register_buffer("kernel", get_box_kernel2d(kernel_size)) + self.kernel: torch.Tensor + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}" + f"(kernel_size={self.kernel_size}, " + f"border_type={self.border_type}, " + f"separable={self.separable})" + ) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Average each pixel with its local neighborhood. + + The box blur uses a rectangular averaging kernel where every location + inside the window has the same weight. It is a simple low-pass filter: + high-frequency details are reduced, while broader image structures are + preserved. Depending on the module configuration, the kernel is applied + either as a full two-dimensional filter or as two separable one- + dimensional passes. + + Args: + input: Image or feature tensor with shape :math:`(B, C, H, W)`, + where :math:`B` is the batch size, :math:`C` is the number of + channels, :math:`H` is the height, and :math:`W` is the width. + + Returns: + Tensor with shape :math:`(B, C, H, W)` containing the locally + averaged result. The output keeps the same batch, channel, and + spatial layout as ``input``; border pixels are handled according to + the configured border mode. + """ + KORNIA_CHECK_IS_TENSOR(input) + if self.separable: + return filter2d_separable(input, self.kernel_x, self.kernel_y, self.border_type) + return filter2d(input, self.kernel, self.border_type) diff --git a/kornia/filters/blur_pool.py b/kornia/filters/blur_pool.py new file mode 100644 index 0000000..cfd770e --- /dev/null +++ b/kornia/filters/blur_pool.py @@ -0,0 +1,377 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SHAPE + +from .kernels import get_pascal_kernel_2d +from .median import _compute_zero_padding # TODO: Move to proper place + +__all__ = [ + "BlurPool2D", + "EdgeAwareBlurPool2D", + "MaxBlurPool2D", + "blur_pool2d", + "edge_aware_blur_pool2d", + "max_blur_pool2d", +] + + +class BlurPool2D(nn.Module): + r"""Compute blur (anti-aliasing) and downsample a given feature map. + + See :cite:`zhang2019shiftinvar` for more details. + + Args: + kernel_size: the kernel size for max pooling. + stride: stride for pooling. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output: :math:`(N, C, H_{out}, W_{out})`, where + + .. math:: + H_{out} = \left\lfloor\frac{H_{in} + 2 \times \text{kernel\_size//2}[0] - + \text{kernel\_size}[0]}{\text{stride}[0]} + 1\right\rfloor + + .. math:: + W_{out} = \left\lfloor\frac{W_{in} + 2 \times \text{kernel\_size//2}[1] - + \text{kernel\_size}[1]}{\text{stride}[1]} + 1\right\rfloor + + Examples: + >>> from kornia.filters.blur_pool import BlurPool2D + >>> input = torch.eye(5)[None, None] + >>> bp = BlurPool2D(kernel_size=3, stride=2) + >>> bp(input) + tensor([[[[0.3125, 0.0625, 0.0000], + [0.0625, 0.3750, 0.0625], + [0.0000, 0.0625, 0.3125]]]]) + + """ + + def __init__(self, kernel_size: tuple[int, int] | int, stride: int = 2) -> None: + super().__init__() + self.kernel_size = kernel_size + self.stride = stride + self.kernel = get_pascal_kernel_2d(kernel_size, norm=True) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Downsample a feature map after applying an anti-aliasing blur. + + Blur pooling first smooths the input with a normalized Pascal kernel + and then samples it with the configured stride. The blur step reduces + aliasing artifacts that can appear when high-frequency image or feature + content is subsampled directly. + + Args: + input: Feature map tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is the batch size, :math:`C` is the number of + channels, :math:`H` is the input height, and :math:`W` is the + input width. + + Returns: + Downsampled tensor with shape :math:`(B, C, H_{out}, W_{out})`. + :math:`H_{out}` and :math:`W_{out}` are determined by the kernel + size, padding used inside the helper, and ``self.stride``. + """ + self.kernel = torch.as_tensor(self.kernel, device=input.device, dtype=input.dtype) + return _blur_pool_by_kernel2d(input, self.kernel.repeat((input.shape[1], 1, 1, 1)), self.stride) + + +class MaxBlurPool2D(nn.Module): + r"""Compute pools and blurs and downsample a given feature map. + + Equivalent to ```nn.Sequential(nn.MaxPool2d(...), BlurPool2D(...))``` + + See :cite:`zhang2019shiftinvar` for more details. + + Args: + kernel_size: the kernel size for max pooling. + stride: stride for pooling. + max_pool_size: the kernel size for max pooling. + ceil_mode: should be true to match output size of conv2d with same kernel size. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output: :math:`(B, C, H / stride, W / stride)` + + Returns: + torch.Tensor: the transformed torch.tensor. + + Examples: + >>> import torch.nn as nn + >>> from kornia.filters.blur_pool import BlurPool2D + >>> input = torch.eye(5)[None, None] + >>> mbp = MaxBlurPool2D(kernel_size=3, stride=2, max_pool_size=2, ceil_mode=False) + >>> mbp(input) + tensor([[[[0.5625, 0.3125], + [0.3125, 0.8750]]]]) + >>> seq = nn.Sequential(nn.MaxPool2d(kernel_size=2, stride=1), BlurPool2D(kernel_size=3, stride=2)) + >>> seq(input) + tensor([[[[0.5625, 0.3125], + [0.3125, 0.8750]]]]) + + """ + + def __init__( + self, kernel_size: tuple[int, int] | int, stride: int = 2, max_pool_size: int = 2, ceil_mode: bool = False + ) -> None: + super().__init__() + self.kernel_size = kernel_size + self.stride = stride + self.max_pool_size = max_pool_size + self.ceil_mode = ceil_mode + self.kernel = get_pascal_kernel_2d(kernel_size, norm=True) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Apply max pooling and then anti-aliased blur downsampling. + + This layer keeps the local peak response with max pooling before + applying the blur-pool downsampling step. It is useful in convolutional + networks where pooling should remain less sensitive to small spatial + shifts while still reducing the feature resolution. + + Args: + input: Feature map tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is the batch size, :math:`C` is the channel count, + :math:`H` is the height, and :math:`W` is the width. + + Returns: + Tensor with shape :math:`(B, C, H_{out}, W_{out})` after max + pooling and blur pooling. The spatial output sizes depend on the + configured max-pool size, stride, blur kernel, and ``ceil_mode``. + """ + self.kernel = torch.as_tensor(self.kernel, device=input.device, dtype=input.dtype) + return _max_blur_pool_by_kernel2d( + input, self.kernel.repeat((input.size(1), 1, 1, 1)), self.stride, self.max_pool_size, self.ceil_mode + ) + + +class EdgeAwareBlurPool2D(nn.Module): + """Apply an edge-aware anti-aliasing filter during downsampling. + + This module performs blur pooling while preserving edges by using an + edge-intensity threshold. + + Args: + kernel_size: The size of the Gaussian blur kernel. + edge_threshold: The threshold for detecting edges. Default: 1.25. + edge_dilation_kernel_size: The kernel size for dilating the edge map. Default: 3. + """ + + def __init__( + self, kernel_size: tuple[int, int] | int, edge_threshold: float = 1.25, edge_dilation_kernel_size: int = 3 + ) -> None: + super().__init__() + self.kernel_size = kernel_size + self.edge_threshold = edge_threshold + self.edge_dilation_kernel_size = edge_dilation_kernel_size + + def forward(self, input: torch.Tensor, epsilon: float = 1e-6) -> torch.Tensor: + """Downsample while adapting the blur near image or feature edges. + + Edge-aware blur pooling estimates where strong local changes occur and + uses that information to avoid over-smoothing across boundaries before + reducing the spatial resolution. The operation is designed for feature + maps where edges should remain sharp after downsampling. + + Args: + input: Feature map tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is the batch size, :math:`C` is the number of + channels, :math:`H` is the input height, and :math:`W` is the + input width. + epsilon: Small positive value used to keep edge-aware divisions + numerically stable when local normalization terms are close to + zero. + + Returns: + Edge-aware downsampled tensor with shape + :math:`(B, C, H_{out}, W_{out})`. The output preserves channel + order while reducing the spatial dimensions according to the + configured pooling parameters. + """ + return edge_aware_blur_pool2d( + input, self.kernel_size, self.edge_threshold, self.edge_dilation_kernel_size, epsilon + ) + + +def blur_pool2d(input: torch.Tensor, kernel_size: tuple[int, int] | int, stride: int = 2) -> torch.Tensor: + r"""Compute blurs and downsample a given feature map. + + .. image:: _static/img/blur_pool2d.png + + See :class:`~kornia.filters.BlurPool2D` for details. + + See :cite:`zhang2019shiftinvar` for more details. + + Args: + input: torch.Tensor to apply operation to. + kernel_size: the kernel size for max pooling. + stride: stride for pooling. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output: :math:`(N, C, H_{out}, W_{out})`, where + + .. math:: + H_{out} = \left\lfloor\frac{H_{in} + 2 \times \text{kernel\_size//2}[0] - + \text{kernel\_size}[0]}{\text{stride}[0]} + 1\right\rfloor + + .. math:: + W_{out} = \left\lfloor\frac{W_{in} + 2 \times \text{kernel\_size//2}[1] - + \text{kernel\_size}[1]}{\text{stride}[1]} + 1\right\rfloor + + Returns: + the transformed torch.Tensor. + + .. note:: + This function is tested against https://github.com/adobe/antialiased-cnns. + + .. note:: + See a working example `here `__. + + Examples: + >>> input = torch.eye(5)[None, None] + >>> blur_pool2d(input, 3) + tensor([[[[0.3125, 0.0625, 0.0000], + [0.0625, 0.3750, 0.0625], + [0.0000, 0.0625, 0.3125]]]]) + + """ + kernel = get_pascal_kernel_2d(kernel_size, norm=True, device=input.device, dtype=input.dtype).repeat( + (input.size(1), 1, 1, 1) + ) + return _blur_pool_by_kernel2d(input, kernel, stride) + + +def max_blur_pool2d( + input: torch.Tensor, + kernel_size: tuple[int, int] | int, + stride: int = 2, + max_pool_size: int = 2, + ceil_mode: bool = False, +) -> torch.Tensor: + r"""Compute pools and blurs and downsample a given feature map. + + .. image:: _static/img/max_blur_pool2d.png + + See :class:`~kornia.filters.MaxBlurPool2D` for details. + + Args: + input: torch.Tensor to apply operation to. + kernel_size: the kernel size for max pooling. + stride: stride for pooling. + max_pool_size: the kernel size for max pooling. + ceil_mode: should be true to match output size of conv2d with same kernel size. + + .. note:: + This function is tested against https://github.com/adobe/antialiased-cnns. + + .. note:: + See a working example `here `__. + + Examples: + >>> input = torch.eye(5)[None, None] + >>> max_blur_pool2d(input, 3) + tensor([[[[0.5625, 0.3125], + [0.3125, 0.8750]]]]) + + """ + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + + kernel = get_pascal_kernel_2d(kernel_size, norm=True, device=input.device, dtype=input.dtype).repeat( + (input.shape[1], 1, 1, 1) + ) + return _max_blur_pool_by_kernel2d(input, kernel, stride, max_pool_size, ceil_mode) + + +def _blur_pool_by_kernel2d(input: torch.Tensor, kernel: torch.Tensor, stride: int) -> torch.Tensor: + """Compute blur_pool by a given :math:`CxC_{out}xNxN` kernel.""" + KORNIA_CHECK( + len(kernel.shape) == 4 and kernel.shape[-2] == kernel.shape[-1], + f"Invalid kernel shape. Expect CxC_(out, None)xNxN, Got {kernel.shape}", + ) + + padding = _compute_zero_padding((kernel.shape[-2], kernel.shape[-1])) + return F.conv2d(input, kernel, padding=padding, stride=stride, groups=input.shape[1]) + + +def _max_blur_pool_by_kernel2d( + input: torch.Tensor, kernel: torch.Tensor, stride: int, max_pool_size: int, ceil_mode: bool +) -> torch.Tensor: + """Compute max_blur_pool by a given :math:`CxC_(out, None)xNxN` kernel.""" + KORNIA_CHECK( + len(kernel.shape) == 4 and kernel.shape[-2] == kernel.shape[-1], + f"Invalid kernel shape. Expect CxC_outxNxN, Got {kernel.shape}", + ) + # compute local maxima + input = F.max_pool2d(input, kernel_size=max_pool_size, padding=0, stride=1, ceil_mode=ceil_mode) + # blur and downsample + padding = _compute_zero_padding((kernel.shape[-2], kernel.shape[-1])) + return F.conv2d(input, kernel, padding=padding, stride=stride, groups=input.size(1)) + + +def edge_aware_blur_pool2d( + input: torch.Tensor, + kernel_size: tuple[int, int] | int, + edge_threshold: float = 1.25, + edge_dilation_kernel_size: int = 3, + epsilon: float = 1e-6, +) -> torch.Tensor: + r"""Blur the input torch.Tensor while maintaining its edges. + + Args: + input: the input image to blur with shape :math:`(B, C, H, W)`. + kernel_size: the kernel size for max pooling. + edge_threshold: positive threshold for the edge decision rule; edge/non-edge. + edge_dilation_kernel_size: the kernel size for dilating the edges. + epsilon: for numerical stability. + + Returns: + The blurred torch.Tensor of shape :math:`(B, C, H, W)`. + + """ + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + KORNIA_CHECK(edge_threshold > 0.0, f"edge threshold should be positive, but got '{edge_threshold}'") + + input = F.pad(input, (2, 2, 2, 2), mode="reflect") # F.pad to avoid artifacts near physical edges + blurred_input = blur_pool2d(input, kernel_size=kernel_size, stride=1) # blurry version of the input + + # calculate the edges (add epsilon to avoid taking the log of 0) + log_input, log_thresh = (input + epsilon).log2(), (torch.tensor(edge_threshold)).log2() + edges_x = log_input[..., :, 4:] - log_input[..., :, :-4] + edges_y = log_input[..., 4:, :] - log_input[..., :-4, :] + edges_x, edges_y = edges_x.mean(dim=-3, keepdim=True), edges_y.mean(dim=-3, keepdim=True) + edges_x_mask, edges_y_mask = edges_x.abs() > log_thresh.to(edges_x), edges_y.abs() > log_thresh.to(edges_y) + edges_xy_mask = (edges_x_mask[..., 2:-2, :] + edges_y_mask[..., :, 2:-2]).type_as(input) + + # dilate the content edges to have a soft mask of edges + dilated_edges = F.max_pool3d(edges_xy_mask, edge_dilation_kernel_size, 1, edge_dilation_kernel_size // 2) + + # slice the padded regions + input = input[..., 2:-2, 2:-2] + blurred_input = blurred_input[..., 2:-2, 2:-2] + + # fuse the input image on edges and blurry input everywhere else + blurred = dilated_edges * input + (1.0 - dilated_edges) * blurred_input + + return blurred diff --git a/kornia/filters/canny.py b/kornia/filters/canny.py new file mode 100644 index 0000000..1c9877f --- /dev/null +++ b/kornia/filters/canny.py @@ -0,0 +1,264 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import math + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.color import rgb_to_grayscale +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE + +from .gaussian import gaussian_blur2d +from .kernels import get_canny_nms_kernel, get_hysteresis_kernel +from .sobel import spatial_gradient + + +def canny( + input: torch.Tensor, + low_threshold: float = 0.1, + high_threshold: float = 0.2, + kernel_size: tuple[int, int] | int = (5, 5), + sigma: tuple[float, float] | torch.Tensor = (1, 1), + hysteresis: bool = True, + eps: float = 1e-6, +) -> tuple[torch.Tensor, torch.Tensor]: + r"""Find edges of the input image and filters them using the Canny algorithm. + + .. image:: _static/img/canny.png + + Args: + input: input image torch.Tensor with shape :math:`(B,C,H,W)`. + low_threshold: lower threshold for the hysteresis procedure. + high_threshold: upper threshold for the hysteresis procedure. + kernel_size: the size of the kernel for the gaussian blur. + sigma: the standard deviation of the kernel for the gaussian blur. + hysteresis: if True, applies the hysteresis edge tracking. + Otherwise, the edges are divided between weak (0.5) and strong (1) edges. + eps: regularization number to avoid NaN during backprop. + + Returns: + - the canny edge magnitudes map, shape of :math:`(B,1,H,W)`. + - the canny edge detection filtered by thresholds and hysteresis, shape of :math:`(B,1,H,W)`. + + .. note:: + See a working example `here `__. + + Example: + >>> input = torch.rand(5, 3, 4, 4) + >>> magnitude, edges = canny(input) # 5x3x4x4 + >>> magnitude.shape + torch.Size([5, 1, 4, 4]) + >>> edges.shape + torch.Size([5, 1, 4, 4]) + + """ + KORNIA_CHECK_IS_TENSOR(input) + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + KORNIA_CHECK( + low_threshold <= high_threshold, + "Invalid input thresholds. low_threshold should be smaller than the high_threshold. Got: " + f"{low_threshold}>{high_threshold}", + ) + KORNIA_CHECK(0 < low_threshold < 1, f"Invalid low threshold. Should be in range (0, 1). Got: {low_threshold}") + KORNIA_CHECK(0 < high_threshold < 1, f"Invalid high threshold. Should be in range (0, 1). Got: {high_threshold}") + + device = input.device + dtype = input.dtype + + # To Grayscale + if input.shape[1] == 3: + input = rgb_to_grayscale(input) + + # Gaussian filter + blurred: torch.Tensor = gaussian_blur2d(input, kernel_size, sigma) + + # Compute the gradients + gradients: torch.Tensor = spatial_gradient(blurred, normalized=False) + + # Unpack the edges + gx: torch.Tensor = gradients[:, :, 0] + gy: torch.Tensor = gradients[:, :, 1] + + # Compute gradient magnitude and angle + magnitude: torch.Tensor = torch.sqrt(gx * gx + gy * gy + eps) + angle: torch.Tensor = torch.atan2(gy, gx) + + # Radians to degrees and round to nearest 45 degree + # degrees = angle * (180.0 / math.pi) + # angle = torch.round(degrees / 45) * 45 + angle_45 = (angle * (4 / math.pi)).round() + + # Non-maximal suppression + nms_kernels: torch.Tensor = get_canny_nms_kernel(device, dtype) + nms_magnitude: torch.Tensor = F.conv2d(magnitude, nms_kernels, padding=nms_kernels.shape[-1] // 2) + + # Get the indices for both directions + positive_idx: torch.Tensor = angle_45 % 8 + positive_idx = positive_idx.long() + + negative_idx: torch.Tensor = (angle_45 + 4) % 8 + negative_idx = negative_idx.long() + + # Apply the non-maximum suppression to the different directions + channel_select_filtered_positive: torch.Tensor = torch.gather(nms_magnitude, 1, positive_idx) + channel_select_filtered_negative: torch.Tensor = torch.gather(nms_magnitude, 1, negative_idx) + + channel_select_filtered: torch.Tensor = torch.stack( + [channel_select_filtered_positive, channel_select_filtered_negative], 1 + ) + + is_max: torch.Tensor = channel_select_filtered.min(dim=1)[0] > 0.0 + + magnitude = magnitude * is_max + + # Threshold + edges: torch.Tensor = F.threshold(magnitude, low_threshold, 0.0) + + low: torch.Tensor = magnitude > low_threshold + high: torch.Tensor = magnitude > high_threshold + + edges = low * 0.5 + high * 0.5 + edges = edges.to(dtype) + + # Hysteresis + if hysteresis: + edges_old: torch.Tensor = -torch.ones(edges.shape, device=edges.device, dtype=dtype) + hysteresis_kernels: torch.Tensor = get_hysteresis_kernel(device, dtype) + + while ((edges_old - edges).abs() != 0).any(): + weak: torch.Tensor = (edges == 0.5).float() + strong: torch.Tensor = (edges == 1).float() + + hysteresis_magnitude: torch.Tensor = F.conv2d( + edges, hysteresis_kernels, padding=hysteresis_kernels.shape[-1] // 2 + ) + hysteresis_magnitude = (hysteresis_magnitude == 1).any(1, keepdim=True).to(dtype) + hysteresis_magnitude = hysteresis_magnitude * weak + strong + + edges_old = edges.clone() + edges = hysteresis_magnitude + (hysteresis_magnitude == 0) * weak * 0.5 + + edges = hysteresis_magnitude + + return magnitude, edges + + +class Canny(nn.Module): + r"""nn.Module that finds edges of the input image and filters them using the Canny algorithm. + + Args: + input: input image torch.Tensor with shape :math:`(B,C,H,W)`. + low_threshold: lower threshold for the hysteresis procedure. + high_threshold: upper threshold for the hysteresis procedure. + kernel_size: the size of the kernel for the gaussian blur. + sigma: the standard deviation of the kernel for the gaussian blur. + hysteresis: if True, applies the hysteresis edge tracking. + Otherwise, the edges are divided between weak (0.5) and strong (1) edges. + eps: regularization number to avoid NaN during backprop. + + Returns: + - the canny edge magnitudes map, shape of :math:`(B,1,H,W)`. + - the canny edge detection filtered by thresholds and hysteresis, shape of :math:`(B,1,H,W)`. + + Example: + >>> input = torch.rand(5, 3, 4, 4) + >>> magnitude, edges = Canny()(input) # 5x3x4x4 + >>> magnitude.shape + torch.Size([5, 1, 4, 4]) + >>> edges.shape + torch.Size([5, 1, 4, 4]) + + """ + + # TODO: Handle multiple inputs and outputs models later + ONNX_EXPORTABLE = False + + def __init__( + self, + low_threshold: float = 0.1, + high_threshold: float = 0.2, + kernel_size: tuple[int, int] | int = (5, 5), + sigma: tuple[float, float] | torch.Tensor = (1, 1), + hysteresis: bool = True, + eps: float = 1e-6, + ) -> None: + super().__init__() + + KORNIA_CHECK( + low_threshold <= high_threshold, + "Invalid input thresholds. low_threshold should be smaller than the high_threshold. Got: " + f"{low_threshold}>{high_threshold}", + ) + KORNIA_CHECK(0 < low_threshold < 1, f"Invalid low threshold. Should be in range (0, 1). Got: {low_threshold}") + KORNIA_CHECK( + 0 < high_threshold < 1, f"Invalid high threshold. Should be in range (0, 1). Got: {high_threshold}" + ) + + # Gaussian blur parameters + self.kernel_size = kernel_size + self.sigma = sigma + + # Double threshold + self.low_threshold = low_threshold + self.high_threshold = high_threshold + + # Hysteresis + self.hysteresis = hysteresis + + self.eps: float = eps + + def __repr__(self) -> str: + return "".join( + ( + f"{type(self).__name__}(", + ", ".join( + f"{name}={getattr(self, name)}" for name in sorted(self.__dict__) if not name.startswith("_") + ), + ")", + ) + ) + + def forward(self, input: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Detect image edges with the Canny pipeline. + + The Canny detector smooths the image, estimates spatial gradients, + performs non-maximum suppression, and thresholds candidate edges using + the configured low and high thresholds. When hysteresis is enabled, + weak edge pixels are kept only when they are connected to strong edge + pixels. + + Args: + input: Image tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is the batch size, :math:`C` is the number of + channels, :math:`H` is the image height, and :math:`W` is the + image width. Multi-channel inputs are handled by the underlying + functional implementation. + + Returns: + Tuple ``(magnitude, edges)``. ``magnitude`` contains the gradient + magnitude response for each pixel, and ``edges`` contains the final + thresholded edge map. Both tensors follow the layout returned by + :func:`canny` and keep the same batch and spatial dimensions as the + input. + """ + return canny( + input, self.low_threshold, self.high_threshold, self.kernel_size, self.sigma, self.hysteresis, self.eps + ) diff --git a/kornia/filters/dexined.py b/kornia/filters/dexined.py new file mode 100644 index 0000000..29d4886 --- /dev/null +++ b/kornia/filters/dexined.py @@ -0,0 +1,461 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# adapted from: https://github.com/xavysp/DexiNed/blob/d944b70eb6eaf40e22f8467c1e12919aa600d8e4/model.py + +from __future__ import annotations + +from collections import OrderedDict +from typing import ClassVar, Optional + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK + +url: str = "http://cmp.felk.cvut.cz/~mishkdmy/models/DexiNed_BIPED_10.pth" + + +def weight_init(m: nn.Module) -> None: + """Initialize weights.""" + if isinstance(m, (nn.Conv2d,)): + # torch.nn.init.xavier_uniform_(m.weight, gain=1.0) + torch.nn.init.xavier_normal_(m.weight, gain=1.0) + # torch.nn.init.normal_(m.weight, mean=0.0, std=0.01) + if m.weight.data.shape[1] == torch.Size([1]): + torch.nn.init.normal_(m.weight, mean=0.0) + + if m.bias is not None: + torch.nn.init.zeros_(m.bias) + + # for fusion layer + if isinstance(m, (nn.ConvTranspose2d,)): + # torch.nn.init.xavier_uniform_(m.weight, gain=1.0) + torch.nn.init.xavier_normal_(m.weight, gain=1.0) + # torch.nn.init.normal_(m.weight, mean=0.0, std=0.01) + + if m.weight.data.shape[1] == torch.Size([1]): + torch.nn.init.normal_(m.weight, std=0.1) + if m.bias is not None: + torch.nn.init.zeros_(m.bias) + + +class CoFusion(nn.Module): + """Implement the weight-fusion layer for multi-scale edge maps. + + Args: + in_ch: The number of input channels. + out_ch: The number of output channels. + """ + + def __init__(self, in_ch: int, out_ch: int) -> None: + super().__init__() + self.conv1 = nn.Conv2d(in_ch, 64, kernel_size=3, stride=1, padding=1) + self.conv2 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1) + self.conv3 = nn.Conv2d(64, out_ch, kernel_size=3, stride=1, padding=1) + self.relu = nn.ReLU() + self.norm_layer1 = nn.GroupNorm(4, 64) + self.norm_layer2 = nn.GroupNorm(4, 64) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Fuse side-output edge maps into a single edge response. + + The input contains edge predictions from several depths of the DexiNed + network concatenated along the channel dimension. This block predicts + attention weights from those stacked responses and uses them to combine + the side outputs into one refined edge map. + + Args: + x: Concatenated side-output tensor with shape + :math:`(B, C_{side}, H, W)`, where :math:`B` is the batch size, + :math:`C_{side}` is the number of stacked side-output channels, + :math:`H` is the height, and :math:`W` is the width. + + Returns: + Tensor with shape :math:`(B, 1, H, W)` containing the fused edge + response. Larger values indicate locations that the network + considers more likely to be image boundaries. + """ + # fusecat = torch.cat(x, dim=1) + attn = self.relu(self.norm_layer1(self.conv1(x))) + attn = self.relu(self.norm_layer2(self.conv2(attn))) + attn = F.softmax(self.conv3(attn), dim=1) + + # return ((fusecat * attn).sum(1)).unsqueeze(1) + return ((x * attn).sum(1))[:, None, ...] + + +class _DenseLayer(nn.Sequential): + def __init__(self, input_features: int, out_features: int) -> None: + super().__init__( + OrderedDict( + [ + ("relu1", nn.ReLU(inplace=True)), + ("conv1", nn.Conv2d(input_features, out_features, kernel_size=3, stride=1, padding=2, bias=True)), + ("norm1", nn.BatchNorm2d(out_features)), + ("relu2", nn.ReLU(inplace=True)), + ("conv2", nn.Conv2d(out_features, out_features, kernel_size=3, stride=1, bias=True)), + ("norm2", nn.BatchNorm2d(out_features)), + ] + ) + ) + + def forward(self, x: list[torch.Tensor]) -> list[torch.Tensor]: + x1, x2 = x[0], x[1] + x3: torch.Tensor = x1 + for mod in self: + x3 = mod(x3) + return [0.5 * (x3 + x2), x2] + + +class _DenseBlock(nn.Sequential): + def __init__(self, num_layers: int, input_features: int, out_features: int) -> None: + super().__init__() + for i in range(num_layers): + layer = _DenseLayer(input_features, out_features) + self.add_module(f"denselayer{(i + 1)}", layer) + input_features = out_features + + def forward(self, x: list[torch.Tensor]) -> list[torch.Tensor]: + x_out = x + for mod in self: + x_out = mod(x_out) + return x_out + + +class UpConvBlock(nn.Module): + """Implement an upsampling convolutional block for edge refinement. + + Args: + in_features: The number of input channels. + up_scale: The scale factor for upsampling. + """ + + def __init__(self, in_features: int, up_scale: int) -> None: + super().__init__() + self.up_factor = 2 + self.constant_features = 16 + + layers = self.make_deconv_layers(in_features, up_scale) + KORNIA_CHECK(layers is not None, "layers cannot be none") + self.features = nn.Sequential(*layers) + + def make_deconv_layers(self, in_features: int, up_scale: int) -> nn.ModuleList: + """Build the transposed-convolution stages used by an upsampling branch. + + Each stage reduces or preserves the feature-channel count and increases + the spatial resolution by a factor of two. The final stage emits a + single-channel side-output edge map, while intermediate stages use the + block's constant feature width. + + Args: + in_features: Number of channels in the input feature map entering + the first upsampling stage. + up_scale: Number of progressive upsampling stages to create. A + value of ``0`` means no transposed-convolution stages are added. + + Returns: + ``nn.ModuleList`` containing the convolution, activation, and + transposed-convolution layers that are later wrapped by + ``nn.Sequential`` in ``self.features``. + """ + layers = nn.ModuleList([]) + all_pads = [0, 0, 1, 3, 7] + for i in range(up_scale): + kernel_size = 2**up_scale + pad = all_pads[up_scale] # kernel_size-1 + out_features = self.compute_out_features(i, up_scale) + layers.append(nn.Conv2d(in_features, out_features, 1)) + layers.append(nn.ReLU(inplace=True)) + layers.append(nn.ConvTranspose2d(out_features, out_features, kernel_size, stride=2, padding=pad)) + in_features = out_features + return layers + + def compute_out_features(self, idx: int, up_scale: int) -> int: + """Return the channel count produced by one upsampling stage. + + DexiNed side branches keep a fixed number of channels in intermediate + upsampling stages and collapse to a single edge channel at the end. + This helper centralizes that rule while ``make_deconv_layers`` builds + the branch. + + Args: + idx: Zero-based stage index. ``0`` is the first stage and + ``up_scale - 1`` is the final stage. + up_scale: Total number of upsampling stages in the branch. + + Returns: + ``1`` for the final stage, because the side output is an edge map, + otherwise ``self.constant_features`` for intermediate feature maps. + """ + return 1 if idx == up_scale - 1 else self.constant_features + + def forward(self, x: torch.Tensor, out_shape: list[int]) -> torch.Tensor: + """Convert a backbone feature map into an aligned side-output edge map. + + The stored upsampling layers progressively increase the resolution of + ``x``. A final bilinear interpolation aligns the branch output exactly + to ``out_shape`` so that side outputs from different backbone depths can + be concatenated later. + + Args: + x: Feature tensor with shape :math:`(B, C, H_{in}, W_{in})`, where + :math:`B` is the batch size, :math:`C` is the number of input + feature channels, and :math:`H_{in}`, :math:`W_{in}` are the + current feature-map height and width. + out_shape: Target spatial size ``[H, W]`` used for the final + interpolation, where :math:`H` is the desired height and + :math:`W` is the desired width. + + Returns: + Side-output tensor resized to ``out_shape``, typically with shape + :math:`(B, 1, H, W)` for an edge prediction branch. + """ + out = self.features(x) + out = F.interpolate(out, out_shape, mode="bilinear") + return out + + +class SingleConvBlock(nn.Module): + """Implement a single convolutional layer with optional Batch Normalization. + + Args: + in_features: The number of input channels. + out_features: The number of output channels. + stride: The stride of the convolution. + use_bs: Whether to use Batch Normalization. Default: True. + """ + + def __init__(self, in_features: int, out_features: int, stride: int, use_bs: bool = True) -> None: + super().__init__() + self.use_bn = use_bs + self.conv = nn.Conv2d(in_features, out_features, 1, stride=stride, bias=True) + self.bn = nn.BatchNorm2d(out_features) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Project feature channels with a pointwise convolution. + + A ``1x1`` convolution mixes information across channels without + changing the spatial resolution. When ``use_bn`` was enabled during + construction, batch normalization is applied after the projection. + + Args: + x: Feature map tensor with shape :math:`(B, C_{in}, H, W)`, where + :math:`B` is the batch size, :math:`C_{in}` is the number of + input channels, :math:`H` is the height, and :math:`W` is the + width. + + Returns: + Tensor with shape :math:`(B, C_{out}, H, W)`, where + :math:`C_{out}` is the ``out_features`` value configured for this + block. + """ + x = self.conv(x) + if self.use_bn: + x = self.bn(x) + return x + + +class DoubleConvBlock(nn.Sequential): + """Apply two consecutive convolutional blocks for edge feature extraction.""" + + def __init__( + self, + in_features: int, + mid_features: int, + out_features: Optional[int] = None, + stride: int = 1, + use_act: bool = True, + ) -> None: + super().__init__() + if out_features is None: + out_features = mid_features + self.add_module("conv1", nn.Conv2d(in_features, mid_features, 3, padding=1, stride=stride)) + self.add_module("bn1", nn.BatchNorm2d(mid_features)) + self.add_module("relu1", nn.ReLU(inplace=True)) + self.add_module("conv2", nn.Conv2d(mid_features, out_features, 3, padding=1)) + self.add_module("bn2", nn.BatchNorm2d(out_features)) + if use_act: + self.add_module("relu2", nn.ReLU(inplace=True)) + + +class DexiNed(nn.Module): + r"""Definition of the DXtrem network from :cite:`xsoria2020dexined`. + + Return: + A list of torch.Tensor with the intermediate features which the last element + is the edges map with shape :math:`(B,1,H,W)`. + + Example: + >>> img = torch.rand(1, 3, 320, 320) + >>> net = DexiNed(pretrained=False) + >>> out = net(img) + >>> out.shape + torch.Size([1, 1, 320, 320]) + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 1, -1, -1] + + def __init__(self, pretrained: bool) -> None: + super().__init__() + self.block_1 = DoubleConvBlock(3, 32, 64, stride=2) + self.block_2 = DoubleConvBlock(64, 128, use_act=False) + self.dblock_3 = _DenseBlock(2, 128, 256) # [128,256,100,100] + self.dblock_4 = _DenseBlock(3, 256, 512) + self.dblock_5 = _DenseBlock(3, 512, 512) + self.dblock_6 = _DenseBlock(3, 512, 256) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + # left skip connections, figure in Journal + self.side_1 = SingleConvBlock(64, 128, 2) + self.side_2 = SingleConvBlock(128, 256, 2) + self.side_3 = SingleConvBlock(256, 512, 2) + self.side_4 = SingleConvBlock(512, 512, 1) + self.side_5 = SingleConvBlock(512, 256, 1) + + # right skip connections, figure in Journal paper + self.pre_dense_2 = SingleConvBlock(128, 256, 2) + self.pre_dense_3 = SingleConvBlock(128, 256, 1) + self.pre_dense_4 = SingleConvBlock(256, 512, 1) + self.pre_dense_5 = SingleConvBlock(512, 512, 1) + self.pre_dense_6 = SingleConvBlock(512, 256, 1) + + # USNet + self.up_block_1 = UpConvBlock(64, 1) + self.up_block_2 = UpConvBlock(128, 1) + self.up_block_3 = UpConvBlock(256, 2) + self.up_block_4 = UpConvBlock(512, 3) + self.up_block_5 = UpConvBlock(512, 4) + self.up_block_6 = UpConvBlock(256, 4) + self.block_cat = SingleConvBlock(6, 1, stride=1, use_bs=False) # hed fusion method + # self.block_cat = CoFusion(6,6)# cats fusion method + + if pretrained: + self.load_from_file(url) + else: + self.apply(weight_init) + + def load_from_file(self, path_file: str) -> None: + """Load pretrained DexiNed weights and switch the module to eval mode. + + The checkpoint is loaded on CPU through ``torch.hub`` and then applied + with strict key matching. After loading, ``eval()`` is called so layers + such as batch normalization use inference behavior. + + Args: + path_file: URL or local checkpoint identifier accepted by + :func:`torch.hub.load_state_dict_from_url`. + """ + # use torch.hub to load pretrained model + pretrained_dict = torch.hub.load_state_dict_from_url(path_file, map_location=torch.device("cpu")) + self.load_state_dict(pretrained_dict, strict=True) + self.eval() + + def get_features(self, x: torch.Tensor) -> list[torch.Tensor]: + """Compute the DexiNed side-output edge maps before final fusion. + + The image is processed through the stacked backbone blocks. Each block + produces a feature map at a different semantic depth and spatial scale; + the corresponding side branch upsamples it back to the original image + resolution. The returned list is later concatenated and fused by + ``CoFusion``. + + Args: + x: Input image tensor with shape :math:`(B, 3, H, W)`, where + :math:`B` is the batch size, ``3`` is the RGB channel count, + :math:`H` is the image height, and :math:`W` is the image + width. + + Returns: + List of six tensors, each aligned to the input spatial size. Every + tensor represents a side-output edge prediction with batch + dimension :math:`B` and spatial dimensions :math:`H` and + :math:`W`. + """ + # Block 1 + block_1 = self.block_1(x) + block_1_side = self.side_1(block_1) + + # Block 2 + block_2 = self.block_2(block_1) + block_2_down = self.maxpool(block_2) + block_2_add = block_2_down + block_1_side + block_2_side = self.side_2(block_2_add) + + # Block 3 + block_3_pre_dense = self.pre_dense_3(block_2_down) + block_3, _ = self.dblock_3([block_2_add, block_3_pre_dense]) + block_3_down = self.maxpool(block_3) # [128,256,50,50] + block_3_add = block_3_down + block_2_side + block_3_side = self.side_3(block_3_add) + + # Block 4 + block_2_resize_half = self.pre_dense_2(block_2_down) + block_4_pre_dense = self.pre_dense_4(block_3_down + block_2_resize_half) + block_4, _ = self.dblock_4([block_3_add, block_4_pre_dense]) + block_4_down = self.maxpool(block_4) + block_4_add = block_4_down + block_3_side + block_4_side = self.side_4(block_4_add) + + # Block 5 + block_5_pre_dense = self.pre_dense_5(block_4_down) # block_5_pre_dense_512 +block_4_down + block_5, _ = self.dblock_5([block_4_add, block_5_pre_dense]) + block_5_add = block_5 + block_4_side + + # Block 6 + block_6_pre_dense = self.pre_dense_6(block_5) + block_6, _ = self.dblock_6([block_5_add, block_6_pre_dense]) + + # upsampling blocks + out_shape = x.shape[-2:] + out_1 = self.up_block_1(block_1, out_shape) + out_2 = self.up_block_2(block_2, out_shape) + out_3 = self.up_block_3(block_3, out_shape) + out_4 = self.up_block_4(block_4, out_shape) + out_5 = self.up_block_5(block_5, out_shape) + out_6 = self.up_block_6(block_6, out_shape) + results = [out_1, out_2, out_3, out_4, out_5, out_6] + return results + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Predict a fused edge map from an RGB image batch. + + The forward pass collects six multi-scale side outputs, concatenates + them along the channel dimension, and applies the fusion block to + produce the final edge response. This keeps both low-level edge detail + and deeper semantic boundary information in the prediction. + + Args: + x: Input image tensor with shape :math:`(B, 3, H, W)`, where + :math:`B` is the batch size, ``3`` is the RGB channel count, + :math:`H` is the image height, and :math:`W` is the image + width. + + Returns: + Tensor with shape :math:`(B, 1, H, W)` containing the fused edge + probability-style response for each image in the batch. + """ + features = self.get_features(x) + + # torch.cat multiscale outputs + block_cat = torch.cat(features, 1) # Bx6xHxW + block_cat = self.block_cat(block_cat) # Bx1xHxW + + return block_cat diff --git a/kornia/filters/dissolving.py b/kornia/filters/dissolving.py new file mode 100644 index 0000000..cafc420 --- /dev/null +++ b/kornia/filters/dissolving.py @@ -0,0 +1,298 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from typing import Any + +import torch +from torch import nn + +from kornia.core import ImageModule +from kornia.core.external import diffusers + + +class _DissolvingWraper_HF: + def __init__(self, model: nn.Module, num_ddim_steps: int = 50, is_sdxl: bool = False) -> None: + self.model = model + self.num_ddim_steps = num_ddim_steps + self.is_sdxl = is_sdxl + self.tokenizer = self.model.tokenizer + self.model.scheduler.set_timesteps(self.num_ddim_steps) + self.total_steps = len(self.model.scheduler.timesteps) # Total number of sampling steps. + self.prompt: str + self.context: torch.Tensor + self.pooled_embeddings: torch.Tensor | None = None + self.add_time_ids: torch.Tensor | None = None + + def predict_start_from_noise(self, noise_pred: torch.Tensor, timestep: int, latent: torch.Tensor) -> torch.Tensor: + return ( + torch.sqrt(1.0 / self.model.scheduler.alphas_cumprod[timestep]) * latent + - torch.sqrt(1.0 / self.model.scheduler.alphas_cumprod[timestep] - 1) * noise_pred + ) + + @torch.no_grad() + def init_prompt(self, prompt: str) -> None: + if self.is_sdxl: + # Handle models with dual text encoders + tokenizers = ( + [self.model.tokenizer, self.model.tokenizer_2] + if hasattr(self.model, "tokenizer_2") + else [self.model.tokenizer] + ) + text_encoders = ( + [self.model.text_encoder, self.model.text_encoder_2] + if hasattr(self.model, "text_encoder_2") + else [self.model.text_encoder] + ) + + prompt_embeds_list = [] + pooled_prompt_embeds = None + + for i, (tokenizer, text_encoder) in enumerate(zip(tokenizers, text_encoders)): + text_inputs = tokenizer( + [prompt], + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids.to(self.model.device) + + outputs = text_encoder(text_input_ids, output_hidden_states=True) + # Extract pooled embeddings from the last text encoder + if i == len(tokenizers) - 1: + # Get pooled output from text encoder + pooled_prompt_embeds = ( + outputs.pooler_output if hasattr(outputs, "pooler_output") else outputs.text_embeds + ) + # Get the penultimate hidden state + prompt_embeds = outputs.hidden_states[-2] + prompt_embeds_list.append(prompt_embeds) + + prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) + + # Get negative (unconditional) embeddings + negative_prompt_embeds_list = [] + negative_pooled_prompt_embeds = None + + for i, (tokenizer, text_encoder) in enumerate(zip(tokenizers, text_encoders)): + uncond_inputs = tokenizer( + [""], + padding="max_length", + max_length=tokenizer.model_max_length, + return_tensors="pt", + ) + uncond_input_ids = uncond_inputs.input_ids.to(self.model.device) + + outputs = text_encoder(uncond_input_ids, output_hidden_states=True) + # Extract pooled embeddings from the last text encoder + if i == len(tokenizers) - 1: + negative_pooled_prompt_embeds = ( + outputs.pooler_output if hasattr(outputs, "pooler_output") else outputs.text_embeds + ) + negative_prompt_embeds = outputs.hidden_states[-2] + negative_prompt_embeds_list.append(negative_prompt_embeds) + + negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1) + + self.context = torch.cat([negative_prompt_embeds, prompt_embeds]) + # Type narrowing: these are guaranteed to be set in the loops above when is_sdxl=True + if pooled_prompt_embeds is not None and negative_pooled_prompt_embeds is not None: + self.pooled_embeddings = torch.cat([negative_pooled_prompt_embeds, pooled_prompt_embeds]) + + # Create time_ids for models that require additional conditioning + # Format: (original_size, crops_coords_top_left, target_size) + add_time_ids = torch.tensor([[1024, 1024, 0, 0, 1024, 1024]], device=self.model.device) + self.add_time_ids = torch.cat([add_time_ids, add_time_ids]) + else: + # SD 1.x + uncond_input = self.model.tokenizer( + [""], padding="max_length", max_length=self.model.tokenizer.model_max_length, return_tensors="pt" + ) + uncond_embeddings = self.model.text_encoder(uncond_input.input_ids.to(self.model.device))[0] + text_input = self.model.tokenizer( + [prompt], + padding="max_length", + max_length=self.model.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_embeddings = self.model.text_encoder(text_input.input_ids.to(self.model.device))[0] + self.context = torch.cat([uncond_embeddings, text_embeddings]) + + self.prompt = prompt + + # Encode the image to latent using the VAE. + @torch.no_grad() + def encode_tensor_to_latent(self, image: torch.Tensor) -> torch.Tensor: + with torch.no_grad(): + image = (image / 0.5 - 1).to(self.model.device) + latents = self.model.vae.encode(image)["latent_dist"].sample() + latents = latents * 0.18215 + return latents + + @torch.no_grad() + def decode_tensor_to_latent(self, latents: torch.Tensor) -> torch.Tensor: + # Perform in-place detach to reduce memory usage and copies + latents = latents.detach() + latents = latents * (1.0 / 0.18215) # Fused division as multiplication (faster) + # Reduce attribute lookups by localizing frequently used attributes + vae_decode = self.model.vae.decode + image = vae_decode(latents)["sample"] + # Use in-place arithmetic/clamp for throughput + image = image.div_(2).add_(0.5) + image.clamp_(0, 1) + return image + + @torch.no_grad() + def one_step_dissolve(self, latent: torch.Tensor, i: int) -> torch.Tensor: + _, cond_embeddings = self.context.chunk(2) + latent = latent.clone().detach() + # NOTE: This implementation use a reversed timesteps but can reach to + # a stable dissolving effect. + t = self.num_ddim_steps - self.model.scheduler.timesteps[i] + latent = self.model.scheduler.scale_model_input(latent, t) + cond_embeddings = cond_embeddings.repeat(latent.size(0), 1, 1) + + if self.is_sdxl: + # Pass additional conditioning to models that require it + # Type narrowing: these are guaranteed to be set when is_sdxl=True + if self.pooled_embeddings is not None and self.add_time_ids is not None: + _, pooled_embeds = self.pooled_embeddings.chunk(2) + _, add_time_ids = self.add_time_ids.chunk(2) + + # Expand embeddings to match batch size if needed + batch_size = latent.size(0) + if pooled_embeds.size(0) != batch_size: + pooled_embeds = pooled_embeds.expand(batch_size, -1) + if add_time_ids.size(0) != batch_size: + add_time_ids = add_time_ids.expand(batch_size, -1) + + added_cond_kwargs = { + "text_embeds": pooled_embeds, + "time_ids": add_time_ids, + } + noise_pred = self.model.unet(latent, t, cond_embeddings, added_cond_kwargs=added_cond_kwargs).sample + else: + noise_pred = self.model.unet(latent, t, cond_embeddings).sample + + pred_x0 = self.predict_start_from_noise(noise_pred, t, latent) + return pred_x0 + + @torch.no_grad() + def dissolve(self, image: torch.Tensor, t: int) -> torch.Tensor: + self.init_prompt("") + latent = self.encode_tensor_to_latent(image) + ddim_latents = self.one_step_dissolve(latent, t) + dissolved = self.decode_tensor_to_latent(ddim_latents) + return dissolved + + +class StableDiffusionDissolving(ImageModule): + r"""Perform dissolving transformation using StableDiffusion models. + + Based on :cite:`shi2024dissolving`, the dissolving transformation is essentially applying one-step + reverse diffusion. Our implementation currently supports HuggingFace implementations of SD 1.4, 1.5 + and SD XL (replacing the discontinued SD 2.1). SD 1.X tends to remove more details than SD-XL. + + .. list-table:: Title + :widths: 32 32 32 + :header-rows: 1 + + * - SD 1.4 + - SD 1.5 + - SD XL + * - figure:: https://raw.githubusercontent.com/kornia/data/main/dslv-sd-1.4.png + - figure:: https://raw.githubusercontent.com/kornia/data/main/dslv-sd-1.5.png + - figure:: https://raw.githubusercontent.com/kornia/data/main/dslv-sd-2.1.png + + Args: + version: the version of the stable diffusion model. Options: "1.4", "1.5", "xl". + **kwargs: additional arguments for `.from_pretrained`. + + """ + + def __init__(self, version: str = "1.5", **kwargs: Any): + super().__init__() + DDIMScheduler = diffusers.DDIMScheduler + + # Load the scheduler and model pipeline from diffusers library + scheduler = DDIMScheduler( # type:ignore + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + steps_offset=1, + ) + + # Filter out arguments that are not supported by all component models + kwargs.pop("offload_state_dict", None) + + # Get HF token from environment if not explicitly provided + if "token" not in kwargs: + hf_token = os.environ.get("HF_TOKEN") + if hf_token: + kwargs["token"] = hf_token + + is_sdxl = False + + if version == "1.4": + StableDiffusionPipeline = diffusers.StableDiffusionPipeline + self._sdm_model = StableDiffusionPipeline.from_pretrained( # type:ignore + "CompVis/stable-diffusion-v1-4", scheduler=scheduler, **kwargs + ) + elif version == "1.5": + StableDiffusionPipeline = diffusers.StableDiffusionPipeline + self._sdm_model = StableDiffusionPipeline.from_pretrained( # type:ignore + "runwayml/stable-diffusion-v1-5", scheduler=scheduler, **kwargs + ) + elif version == "xl": + StableDiffusionXLPipeline = diffusers.StableDiffusionXLPipeline + self._sdm_model = StableDiffusionXLPipeline.from_pretrained( # type:ignore + "stabilityai/stable-diffusion-xl-base-1.0", scheduler=scheduler, **kwargs + ) + is_sdxl = True + else: + raise NotImplementedError + + self.model = _DissolvingWraper_HF(self._sdm_model, num_ddim_steps=1000, is_sdxl=is_sdxl) + + def forward(self, input: torch.Tensor, step_number: int) -> torch.Tensor: + """Apply one Stable Diffusion dissolving step to an image tensor. + + The wrapped diffusion model progressively removes or weakens image + content according to the requested diffusion timestep. Lower and higher + ``step_number`` values correspond to different points on the internal + denoising schedule configured by the wrapper. + + Args: + input: Image tensor passed to the wrapped dissolving model. The + expected shape follows the model wrapper, typically + :math:`(B, C, H, W)`, where :math:`B` is the batch size, + :math:`C` is the channel count, :math:`H` is the image height, + and :math:`W` is the image width. + step_number: Integer diffusion timestep used by the dissolving + process. It selects how far along the configured DDIM schedule + the image is processed. + + Returns: + Tensor produced by the diffusion model at ``step_number``. The + tensor follows the wrapper's image layout and represents the + dissolved version of ``input``. + """ + return self.model.dissolve(input, step_number) diff --git a/kornia/filters/filter.py b/kornia/filters/filter.py new file mode 100644 index 0000000..58e3d16 --- /dev/null +++ b/kornia/filters/filter.py @@ -0,0 +1,603 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +import torch.nn.functional as F + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE +from kornia.filters.kernels import normalize_kernel2d + +_VALID_BORDERS = {"constant", "reflect", "replicate", "circular"} +_VALID_PADDING = {"valid", "same"} +_VALID_BEHAVIOUR = {"conv", "corr"} + + +def _compute_padding(kernel_size: list[int]) -> list[int]: + """Compute padding tuple.""" + # 4 or 6 ints: (padding_left, padding_right,padding_top,padding_bottom) + # https://pytorch.org/docs/stable/nn.html#torch.nn.functional.pad + if len(kernel_size) < 2: + raise AssertionError(kernel_size) + computed = [k - 1 for k in kernel_size] + + # for even kernels we need to do asymmetric padding :( + out_padding = 2 * len(kernel_size) * [0] + + for i in range(len(kernel_size)): + computed_tmp = computed[-(i + 1)] + + pad_front = computed_tmp // 2 + pad_rear = computed_tmp - pad_front + + out_padding[2 * i + 0] = pad_front + out_padding[2 * i + 1] = pad_rear + + return out_padding + + +def filter2d( + input: torch.Tensor, + kernel: torch.Tensor, + border_type: str = "reflect", + normalized: bool = False, + padding: str = "same", + behaviour: str = "corr", +) -> torch.Tensor: + r"""Convolve a tensor with a 2d kernel. + + The function applies a given kernel to a tensor. The kernel is applied + independently at each depth channel of the tensor. Before applying the + kernel, the function applies padding according to the specified mode so + that the output remains in the same shape. + + Args: + input: the input tensor with shape of + :math:`(B, C, H, W)`. + kernel: the kernel to be convolved with the input + tensor. The kernel shape must be :math:`(1, kH, kW)` or :math:`(B, kH, kW)`. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. + normalized: If True, kernel will be L1 normalized. + padding: This defines the type of padding. + 2 modes available ``'same'`` or ``'valid'``. + behaviour: defines the convolution mode -- correlation (default), using pytorch conv2d, + or true convolution (kernel is flipped). 2 modes available ``'corr'`` or ``'conv'``. + + + Return: + Tensor: the convolved tensor of same size and numbers of channels + as the input with shape :math:`(B, C, H, W)`. + + Example: + >>> input = torch.tensor([[[ + ... [0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... [0., 0., 5., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.],]]]) + >>> kernel = torch.ones(1, 3, 3) + >>> filter2d(input, kernel, padding='same') + tensor([[[[0., 0., 0., 0., 0.], + [0., 5., 5., 5., 0.], + [0., 5., 5., 5., 0.], + [0., 5., 5., 5., 0.], + [0., 0., 0., 0., 0.]]]]) + + """ + KORNIA_CHECK_IS_TENSOR(input) + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + KORNIA_CHECK_IS_TENSOR(kernel) + KORNIA_CHECK_SHAPE(kernel, ["B", "H", "W"]) + + KORNIA_CHECK( + str(border_type).lower() in _VALID_BORDERS, + f"Invalid border, {border_type}. Expected one of {_VALID_BORDERS}", + ) + KORNIA_CHECK( + str(padding).lower() in _VALID_PADDING, + f"Invalid padding mode, {padding}. Expected one of {_VALID_PADDING}", + ) + KORNIA_CHECK( + str(behaviour).lower() in _VALID_BEHAVIOUR, + f"Invalid padding mode, {behaviour}. Expected one of {_VALID_BEHAVIOUR}", + ) + # prepare kernel + b, c, h, w = input.shape + if str(behaviour).lower() == "conv": + tmp_kernel = kernel.flip((-2, -1))[:, None, ...].to(device=input.device, dtype=input.dtype) + else: + tmp_kernel = kernel[:, None, ...].to(device=input.device, dtype=input.dtype) + + if normalized: + tmp_kernel = normalize_kernel2d(tmp_kernel) + + tmp_kernel = tmp_kernel.expand(-1, c, -1, -1) + + height, width = tmp_kernel.shape[-2:] + + # pad the input tensor + if padding == "same": + padding_shape: list[int] = _compute_padding([height, width]) + input = F.pad(input, padding_shape, mode=border_type) + + # kernel and input tensor reshape to align element-wise or batch-wise params + tmp_kernel = tmp_kernel.reshape(-1, 1, height, width) + input = input.view(-1, tmp_kernel.size(0), input.size(-2), input.size(-1)) + + # convolve the tensor with the kernel. + output = F.conv2d(input, tmp_kernel, groups=tmp_kernel.size(0), padding=0, stride=1) + + if padding == "same": + out = output.view(b, c, h, w) + else: + out = output.view(b, c, h - height + 1, w - width + 1) + + return out + + +def filter2d_separable( + input: torch.Tensor, + kernel_x: torch.Tensor, + kernel_y: torch.Tensor, + border_type: str = "reflect", + normalized: bool = False, + padding: str = "same", +) -> torch.Tensor: + r"""Convolve a tensor with two 1d kernels, in x and y directions. + + The function applies a given kernel to a tensor. The kernel is applied + independently at each depth channel of the tensor. Before applying the + kernel, the function applies padding according to the specified mode so + that the output remains in the same shape. + + Args: + input: the input tensor with shape of + :math:`(B, C, H, W)`. + kernel_x: the kernel to be convolved with the input + tensor. The kernel shape must be :math:`(1, kW)` or :math:`(B, kW)`. + kernel_y: the kernel to be convolved with the input + tensor. The kernel shape must be :math:`(1, kH)` or :math:`(B, kH)`. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. + normalized: If True, kernel will be L1 normalized. + padding: This defines the type of padding. + 2 modes available ``'same'`` or ``'valid'``. + + Return: + Tensor: the convolved tensor of same size and numbers of channels + as the input with shape :math:`(B, C, H, W)`. + + Example: + >>> input = torch.tensor([[[ + ... [0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... [0., 0., 5., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.],]]]) + >>> kernel = torch.ones(1, 3) + + >>> filter2d_separable(input, kernel, kernel, padding='same') + tensor([[[[0., 0., 0., 0., 0.], + [0., 5., 5., 5., 0.], + [0., 5., 5., 5., 0.], + [0., 5., 5., 5., 0.], + [0., 0., 0., 0., 0.]]]]) + + """ + out_x = filter2d(input, kernel_x[..., None, :], border_type, normalized, padding) + out = filter2d(out_x, kernel_y[..., None], border_type, normalized, padding) + return out + + +def filter3d( + input: torch.Tensor, + kernel: torch.Tensor, + border_type: str = "replicate", + normalized: bool = False, + behaviour: str = "corr", +) -> torch.Tensor: + r"""Convolve a tensor with a 3d kernel. + + The function applies a given kernel to a tensor. The kernel is applied + independently at each depth channel of the tensor. Before applying the + kernel, the function applies padding according to the specified mode so + that the output remains in the same shape. + + Args: + input: the input tensor with shape of + :math:`(B, C, D, H, W)`. + kernel: the kernel to be convolved with the input + tensor. The kernel shape must be :math:`(1, kD, kH, kW)` or :math:`(B, kD, kH, kW)`. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, + ``'replicate'`` or ``'circular'``. + normalized: If True, kernel will be L1 normalized. + behaviour: defines the convolution mode -- correlation (default), using pytorch conv3d, + or true convolution (kernel is flipped). The expected values are: ``'corr'``, ``'conv'``. + + Return: + the convolved tensor of same size and numbers of channels + as the input with shape :math:`(B, C, D, H, W)`. + + Example: + >>> input = torch.tensor([[[ + ... [[0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.]], + ... [[0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... [0., 0., 5., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.]], + ... [[0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.]] + ... ]]]) + >>> kernel = torch.ones(1, 3, 3, 3) + >>> filter3d(input, kernel) + tensor([[[[[0., 0., 0., 0., 0.], + [0., 5., 5., 5., 0.], + [0., 5., 5., 5., 0.], + [0., 5., 5., 5., 0.], + [0., 0., 0., 0., 0.]], + + [[0., 0., 0., 0., 0.], + [0., 5., 5., 5., 0.], + [0., 5., 5., 5., 0.], + [0., 5., 5., 5., 0.], + [0., 0., 0., 0., 0.]], + + [[0., 0., 0., 0., 0.], + [0., 5., 5., 5., 0.], + [0., 5., 5., 5., 0.], + [0., 5., 5., 5., 0.], + [0., 0., 0., 0., 0.]]]]]) + + """ + KORNIA_CHECK_IS_TENSOR(input) + KORNIA_CHECK_SHAPE(input, ["B", "C", "D", "H", "W"]) + KORNIA_CHECK_IS_TENSOR(kernel) + KORNIA_CHECK_SHAPE(kernel, ["B", "D", "H", "W"]) + + KORNIA_CHECK( + str(border_type).lower() in _VALID_BORDERS, + f"Invalid border, gotcha {border_type}. Expected one of {_VALID_BORDERS}", + ) + + KORNIA_CHECK( + str(behaviour).lower() in _VALID_BEHAVIOUR, + f"Invalid behaviour mode, gotcha {behaviour}. Expected one of {_VALID_BEHAVIOUR}", + ) + + # prepare kernel + b, c, d, h, w = input.shape + if str(behaviour).lower() == "conv": + tmp_kernel = kernel.flip((-3, -2, -1))[:, None, ...].to(device=input.device, dtype=input.dtype) + else: + tmp_kernel = kernel[:, None, ...].to(device=input.device, dtype=input.dtype) + + if normalized: + bk, dk, hk, wk = kernel.shape + tmp_kernel = normalize_kernel2d(tmp_kernel.view(bk, dk, hk * wk)).view_as(tmp_kernel) + + tmp_kernel = tmp_kernel.expand(-1, c, -1, -1, -1) + + # pad the input tensor + depth, height, width = tmp_kernel.shape[-3:] + padding_shape: list[int] = _compute_padding([depth, height, width]) + input_pad = F.pad(input, padding_shape, mode=border_type) + + # kernel and input tensor reshape to align element-wise or batch-wise params + tmp_kernel = tmp_kernel.reshape(-1, 1, depth, height, width) + input_pad = input_pad.view(-1, tmp_kernel.size(0), input_pad.size(-3), input_pad.size(-2), input_pad.size(-1)) + + # convolve the tensor with the kernel. + output = F.conv3d(input_pad, tmp_kernel, groups=tmp_kernel.size(0), padding=0, stride=1) + + return output.view(b, c, d, h, w) + + +def fft_conv( + input: torch.Tensor, + kernel: torch.Tensor, + border_type: str = "reflect", + normalized: bool = False, + padding: str = "same", + behaviour: str = "corr", +) -> torch.Tensor: + r"""Apply a 2D convolution (or correlation) using an FFT-based backend. + + This function applies a spatial kernel to a batched tensor using the + convolution theorem, i.e., convolution in the spatial domain is performed + as element-wise multiplication in the frequency domain. + + The kernel is applied independently to each channel of the input tensor. + Depending on the selected padding mode, the output can either preserve + the input spatial resolution (`'same'`) or return only the valid region + (`'valid'`). Boundary handling is performed in the spatial domain prior + to the FFT. + + This function is recommended when the kernel size is larger than + approximately (20 x 20). For large kernels, FFT-based convolution is + computationally more efficient than direct spatial convolution, + reducing complexity from O(H * W * kH * kW) to approximately + O(H * W log(H * W)). For small kernels, however, direct convolution + is usually faster due to lower constant overhead. + + Args: + input: Input tensor of shape :math:`(B, C, H, W)`. + kernel: Convolution kernel of shape :math:`(B, kH, kW)`. Each batch + element provides one kernel, which is shared across all channels + of the corresponding input batch. + border_type: Padding mode applied to the input before convolution. + Supported values are ``'constant'``, ``'reflect'``, + ``'replicate'``, and ``'circular'``. + normalized: If ``True``, the kernel is L1-normalized before applying + the convolution. + padding: Padding strategy to use. Supported values are: + ``'same'`` (output has the same spatial size as the input) or + ``'valid'`` (no implicit padding). + behaviour: Convolution mode. If ``'corr'`` (default), performs + cross-correlation. If ``'conv'``, performs true convolution + by flipping the kernel spatially. + + Returns: + Tensor: The filtered tensor. If ``padding='same'``, the output shape + is :math:`(B, C, H, W)`. If ``padding='valid'``, the output shape is + :math:`(B, C, H - kH + 1, W - kW + 1)`. + + Note: + - Internally, the function performs zero-padding of the kernel to + match the input size and uses real-valued FFTs (`rfftn` / `irfftn`). + - This implementation computes linear convolution via FFT by + appropriate spatial padding and cropping, avoiding circular + convolution artifacts. + - No stride or dilation is supported. + + Example: + >>> input = torch.tensor([[[[ + ... 0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... [0., 0., 5., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... ]]]) + >>> kernel = torch.ones(1, 3, 3) + >>> fft_conv(input, kernel, padding="same") # doctest: +SKIP + tensor([[[[0., 0., 0., 0., 0.], + [0., 5., 5., 5., 0.], + [0., 5., 5., 5., 0.], + [0., 5., 5., 5., 0.], + [0., 0., 0., 0., 0.]]]]) + """ + KORNIA_CHECK_IS_TENSOR(input) + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + + KORNIA_CHECK_IS_TENSOR(kernel) + KORNIA_CHECK_SHAPE(kernel, ["B", "H", "W"]) + + KORNIA_CHECK( + str(border_type).lower() in _VALID_BORDERS, + f"Invalid border, {border_type}. Expected one of {_VALID_BORDERS}", + ) + + KORNIA_CHECK( + str(padding).lower() in _VALID_PADDING, + f"Invalid padding mode, {padding}. Expected one of {_VALID_PADDING}", + ) + + KORNIA_CHECK( + str(behaviour).lower() in _VALID_BEHAVIOUR, + f"Invalid behaviour mode, {behaviour}. Expected one of {_VALID_BEHAVIOUR}", + ) + + _, c, _, _ = input.shape + kh, kw = kernel.shape[-2:] + + if str(behaviour).lower() == "conv": + tmp_kernel = kernel.flip((-2, -1))[:, None, ...].to(device=input.device, dtype=input.dtype) + else: + tmp_kernel = kernel[:, None, ...].to(device=input.device, dtype=input.dtype) + + if normalized: + tmp_kernel = normalize_kernel2d(tmp_kernel) + + # Expand kernel across channels + tmp_kernel = tmp_kernel.expand(-1, c, -1, -1) + + # Padding (spatial domain) + if padding == "same": + padding_shape = _compute_padding([kh, kw]) + input_padded = F.pad(input, padding_shape, mode=border_type) + else: + input_padded = input + + padded_h, padded_w = input_padded.shape[-2:] + + input_padded = input_padded.contiguous() + tmp_kernel = tmp_kernel.contiguous() + + # FFT + input_fr = torch.fft.rfftn(input_padded, dim=(-2, -1)) + kernel_fr = torch.fft.rfftn(tmp_kernel, s=(padded_h, padded_w), dim=(-2, -1)) + + # Correlation via conjugation + output_fr = input_fr * torch.conj(kernel_fr) + + # Inverse FFT + output = torch.fft.irfftn(output_fr, s=(padded_h, padded_w), dim=(-2, -1)) + + # Crop to valid region + crop_h = padded_h - kh + 1 + crop_w = padded_w - kw + 1 + output = output[..., :crop_h, :crop_w].contiguous() + + return output + + +def correlate2d( + input: torch.Tensor, + kernel: torch.Tensor, + border_type: str = "reflect", + normalized: bool = False, + padding: str = "same", +) -> torch.Tensor: + r"""Correlate a tensor with a 2d kernel. + + Convenience alias for :func:`filter2d` with ``behaviour='corr'`` (cross-correlation). + See :func:`filter2d` for full documentation. + + .. seealso:: :func:`convolve2d`, :func:`filter2d` + + Args: + input: the input tensor with shape of :math:`(B, C, H, W)`. + kernel: the kernel to be correlated with the input tensor. + The kernel shape must be :math:`(1, kH, kW)` or :math:`(B, kH, kW)`. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, ``'replicate'`` or ``'circular'``. + normalized: If True, kernel will be L1 normalized. + padding: This defines the type of padding. 2 modes available ``'same'`` or ``'valid'``. + + Return: + Tensor: the correlated tensor of same size and numbers of channels as the input + with shape :math:`(B, C, H, W)`. + + Example: + >>> input = torch.tensor([[[ + ... [0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... [0., 0., 5., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.],]]]) + >>> kernel = torch.ones(1, 3, 3) + >>> correlate2d(input, kernel, padding='same') + tensor([[[[0., 0., 0., 0., 0.], + [0., 5., 5., 5., 0.], + [0., 5., 5., 5., 0.], + [0., 5., 5., 5., 0.], + [0., 0., 0., 0., 0.]]]]) + """ + return filter2d(input, kernel, border_type=border_type, normalized=normalized, padding=padding, behaviour="corr") + + +def convolve2d( + input: torch.Tensor, + kernel: torch.Tensor, + border_type: str = "reflect", + normalized: bool = False, + padding: str = "same", +) -> torch.Tensor: + r"""Convolve a tensor with a 2d kernel using true convolution. + + Convenience alias for :func:`filter2d` with ``behaviour='conv'`` (true convolution, + where the kernel is flipped before applying). + See :func:`filter2d` for full documentation. + + .. seealso:: :func:`correlate2d`, :func:`filter2d` + + Args: + input: the input tensor with shape of :math:`(B, C, H, W)`. + kernel: the kernel to be convolved with the input tensor. + The kernel shape must be :math:`(1, kH, kW)` or :math:`(B, kH, kW)`. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, ``'replicate'`` or ``'circular'``. + normalized: If True, kernel will be L1 normalized. + padding: This defines the type of padding. 2 modes available ``'same'`` or ``'valid'``. + + Return: + Tensor: the convolved tensor of same size and numbers of channels as the input + with shape :math:`(B, C, H, W)`. + + Example: + >>> input = torch.tensor([[[ + ... [0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... [0., 0., 5., 0., 0.], + ... [0., 0., 0., 0., 0.], + ... [0., 0., 0., 0., 0.],]]]) + >>> kernel = torch.ones(1, 3, 3) + >>> convolve2d(input, kernel, padding='same') + tensor([[[[0., 0., 0., 0., 0.], + [0., 5., 5., 5., 0.], + [0., 5., 5., 5., 0.], + [0., 5., 5., 5., 0.], + [0., 0., 0., 0., 0.]]]]) + """ + return filter2d(input, kernel, border_type=border_type, normalized=normalized, padding=padding, behaviour="conv") + + +def correlate3d( + input: torch.Tensor, + kernel: torch.Tensor, + border_type: str = "replicate", + normalized: bool = False, +) -> torch.Tensor: + r"""Correlate a tensor with a 3d kernel. + + Convenience alias for :func:`filter3d` with ``behaviour='corr'`` (cross-correlation). + See :func:`filter3d` for full documentation. + + .. seealso:: :func:`convolve3d`, :func:`filter3d` + + Args: + input: the input tensor with shape of :math:`(B, C, D, H, W)`. + kernel: the kernel to be correlated with the input tensor. + The kernel shape must be :math:`(1, kD, kH, kW)` or :math:`(B, kD, kH, kW)`. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, ``'replicate'`` or ``'circular'``. + normalized: If True, kernel will be L1 normalized. + + Return: + Tensor: the correlated tensor of same size and numbers of channels as the input. + """ + return filter3d(input, kernel, border_type=border_type, normalized=normalized, behaviour="corr") + + +def convolve3d( + input: torch.Tensor, + kernel: torch.Tensor, + border_type: str = "replicate", + normalized: bool = False, +) -> torch.Tensor: + r"""Convolve a tensor with a 3d kernel using true convolution. + + Convenience alias for :func:`filter3d` with ``behaviour='conv'`` (true convolution, + where the kernel is flipped before applying). + See :func:`filter3d` for full documentation. + + .. seealso:: :func:`correlate3d`, :func:`filter3d` + + Args: + input: the input tensor with shape of :math:`(B, C, D, H, W)`. + kernel: the kernel to be convolved with the input tensor. + The kernel shape must be :math:`(1, kD, kH, kW)` or :math:`(B, kD, kH, kW)`. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, ``'replicate'`` or ``'circular'``. + normalized: If True, kernel will be L1 normalized. + + Return: + Tensor: the convolved tensor of same size and numbers of channels as the input. + """ + return filter3d(input, kernel, border_type=border_type, normalized=normalized, behaviour="conv") diff --git a/kornia/filters/gaussian.py b/kornia/filters/gaussian.py new file mode 100644 index 0000000..56fc9d0 --- /dev/null +++ b/kornia/filters/gaussian.py @@ -0,0 +1,195 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Any + +import torch +from torch import nn + +from kornia.core._compat import deprecated +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE + +from .filter import filter2d, filter2d_separable +from .kernels import _check_kernel_size, _unpack_2d_ks, get_gaussian_kernel1d, get_gaussian_kernel2d + + +def gaussian_blur2d( + input: torch.Tensor, + kernel_size: tuple[int, int] | int, + sigma: tuple[float, float] | torch.Tensor, + border_type: str = "reflect", + separable: bool = True, +) -> torch.Tensor: + r"""Create an operator that blurs a torch.Tensor using a Gaussian filter. + + .. image:: _static/img/gaussian_blur2d.png + + The operator smooths the given torch.Tensor with a gaussian kernel by convolving + it to each channel. It supports batched operation. + + Arguments: + input: the input torch.Tensor with shape :math:`(B,C,H,W)`. + kernel_size: the size of the kernel. Can be an integer or tuple of two integers (height, width). + sigma: the standard deviation of the kernel. Can be a tuple of two floats or a torch.Tensor + with shape :math:`(B, 2)`. Values must be positive. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. Default: ``'reflect'``. + separable: run as composition of two 1d-convolutions. Default: ``True``. + + Returns: + the blurred torch.Tensor with shape :math:`(B, C, H, W)`. + + Raises: + RuntimeError: if input is not a 4D torch.Tensor. + RuntimeError: if sigma values are not positive. + RuntimeError: if kernel_size is not a positive odd integer. + + .. note:: + See a working example `here `__. + + Examples: + >>> import torch + >>> input = torch.rand(2, 4, 5, 5) + >>> output = gaussian_blur2d(input, (3, 3), (1.5, 1.5)) + >>> output.shape + torch.Size([2, 4, 5, 5]) + + >>> # Single kernel size applies to both dimensions + >>> output = gaussian_blur2d(input, 3, (1.5, 1.5)) + >>> output.shape + torch.Size([2, 4, 5, 5]) + + >>> # Using batched sigma (different sigma per batch element) + >>> sigma_batch = torch.tensor([[1.5, 1.5], [2.0, 2.0]]) + >>> output = gaussian_blur2d(input[:2], (3, 3), sigma_batch) + >>> output.shape + torch.Size([2, 4, 5, 5]) + + >>> # Using torch.tensor sigma + >>> output = gaussian_blur2d(input, (3, 3), torch.tensor([[1.5, 1.5]])) + >>> output.shape + torch.Size([2, 4, 5, 5]) + + """ + KORNIA_CHECK_IS_TENSOR(input) + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + _check_kernel_size(kernel_size, min_value=0) + + if isinstance(sigma, tuple): + sigma = torch.tensor([sigma], device=input.device, dtype=input.dtype) + else: + KORNIA_CHECK_IS_TENSOR(sigma) + sigma = sigma.to(device=input.device, dtype=input.dtype) + + # Validate sigma values are positive + KORNIA_CHECK_SHAPE(sigma, ["B", "2"]) + KORNIA_CHECK(bool((sigma > 0).all()), f"sigma must be positive, got {sigma}") + + if separable: + ky, kx = _unpack_2d_ks(kernel_size) + bs = sigma.shape[0] + kernel_x = get_gaussian_kernel1d(kx, sigma[:, 1].view(bs, 1)) + kernel_y = get_gaussian_kernel1d(ky, sigma[:, 0].view(bs, 1)) + out = filter2d_separable(input, kernel_x, kernel_y, border_type) + else: + kernel = get_gaussian_kernel2d(kernel_size, sigma) + out = filter2d(input, kernel, border_type) + + return out + + +class GaussianBlur2d(nn.Module): + r"""Create an operator that blurs a torch.Tensor using a Gaussian filter. + + The operator smooths the given torch.Tensor with a gaussian kernel by convolving + it to each channel. It supports batched operation. + + Arguments: + kernel_size: the size of the kernel. + sigma: the standard deviation of the kernel. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. Default: ``'reflect'``. + separable: run as composition of two 1d-convolutions. + + Returns: + the blurred torch.Tensor. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + Examples:: + + >>> input = torch.rand(2, 4, 5, 5) + >>> gauss = GaussianBlur2d((3, 3), (1.5, 1.5)) + >>> output = gauss(input) # 2x4x5x5 + >>> output.shape + torch.Size([2, 4, 5, 5]) + + """ + + def __init__( + self, + kernel_size: tuple[int, int] | int, + sigma: tuple[float, float] | torch.Tensor, + border_type: str = "reflect", + separable: bool = True, + ) -> None: + super().__init__() + self.kernel_size = kernel_size + self.sigma = sigma + self.border_type = border_type + self.separable = separable + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}" + f"(kernel_size={self.kernel_size}, " + f"sigma={self.sigma}, " + f"border_type={self.border_type}, " + f"separable={self.separable})" + ) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Blur an image with a Gaussian low-pass filter. + + Gaussian blur computes a weighted local average where pixels near the + center of the kernel have larger weights than pixels farther away. The + configured standard deviation, ``sigma``, controls how quickly those + weights decay with distance, and the kernel size controls the support + of the approximation. + + Args: + input: Image or feature tensor with shape :math:`(B, C, H, W)`, + where :math:`B` is the batch size, :math:`C` is the number of + channels, :math:`H` is the height, and :math:`W` is the width. + + Returns: + Tensor with shape :math:`(B, C, H, W)` containing the smoothed + result. The output keeps the same channel layout as ``input`` while + reducing high-frequency noise and fine texture. + """ + return gaussian_blur2d(input, self.kernel_size, self.sigma, self.border_type, self.separable) + + +@deprecated(replace_with="gaussian_blur2d", version="6.9.10") +def gaussian_blur2d_t(*args: Any, **kwargs: Any) -> torch.Tensor: # noqa: D103 + return gaussian_blur2d(*args, **kwargs) diff --git a/kornia/filters/guided.py b/kornia/filters/guided.py new file mode 100644 index 0000000..3bd9bd5 --- /dev/null +++ b/kornia/filters/guided.py @@ -0,0 +1,253 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +from torch import nn +from torch.nn.functional import interpolate + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE + +from .blur import box_blur +from .kernels import _unpack_2d_ks + + +def _preprocess_fast_guided_blur( + guidance: torch.Tensor, input: torch.Tensor, kernel_size: tuple[int, int] | int, subsample: int = 1 +) -> tuple[torch.Tensor, torch.Tensor, tuple[int, int]]: + ky, kx = _unpack_2d_ks(kernel_size) + if subsample > 1: + s = 1 / subsample + guidance_sub = interpolate(guidance, scale_factor=s, mode="nearest") + input_sub = guidance_sub if input is guidance else interpolate(input, scale_factor=s, mode="nearest") + ky, kx = ((k - 1) // subsample + 1 for k in (ky, kx)) + else: + guidance_sub = guidance + input_sub = input + return guidance_sub, input_sub, (ky, kx) + + +def _guided_blur_grayscale_guidance( + guidance: torch.Tensor, + input: torch.Tensor, + kernel_size: tuple[int, int] | int, + eps: float | torch.Tensor, + border_type: str = "reflect", + subsample: int = 1, +) -> torch.Tensor: + guidance_sub, input_sub, kernel_size = _preprocess_fast_guided_blur(guidance, input, kernel_size, subsample) + + mean_I = box_blur(guidance_sub, kernel_size, border_type) + corr_I = box_blur(guidance_sub.square(), kernel_size, border_type) + var_I = corr_I - mean_I.square() + + if input is guidance: + mean_p = mean_I + cov_Ip = var_I + + else: + mean_p = box_blur(input_sub, kernel_size, border_type) + corr_Ip = box_blur(guidance_sub * input_sub, kernel_size, border_type) + cov_Ip = corr_Ip - mean_I * mean_p + + if isinstance(eps, torch.Tensor): + eps = eps.view(-1, 1, 1, 1) # N -> NCHW + + a = cov_Ip / (var_I + eps) + b = mean_p - a * mean_I + + mean_a = box_blur(a, kernel_size, border_type) + mean_b = box_blur(b, kernel_size, border_type) + + if subsample > 1: + mean_a = interpolate(mean_a, scale_factor=subsample, mode="bilinear") + mean_b = interpolate(mean_b, scale_factor=subsample, mode="bilinear") + + return mean_a * guidance + mean_b + + +def _guided_blur_multichannel_guidance( + guidance: torch.Tensor, + input: torch.Tensor, + kernel_size: tuple[int, int] | int, + eps: float | torch.Tensor, + border_type: str = "reflect", + subsample: int = 1, +) -> torch.Tensor: + guidance_sub, input_sub, kernel_size = _preprocess_fast_guided_blur(guidance, input, kernel_size, subsample) + B, C, H, W = guidance_sub.shape + + mean_I = box_blur(guidance_sub, kernel_size, border_type).permute(0, 2, 3, 1) + II = (guidance_sub.unsqueeze(1) * guidance_sub.unsqueeze(2)).flatten(1, 2) + corr_I = box_blur(II, kernel_size, border_type).permute(0, 2, 3, 1) + var_I = corr_I.reshape(B, H, W, C, C) - mean_I.unsqueeze(-2) * mean_I.unsqueeze(-1) + + if guidance is input: + mean_p = mean_I + cov_Ip = var_I + + else: + mean_p = box_blur(input_sub, kernel_size, border_type).permute(0, 2, 3, 1) + Ip = (input_sub.unsqueeze(1) * guidance_sub.unsqueeze(2)).flatten(1, 2) + corr_Ip = box_blur(Ip, kernel_size, border_type).permute(0, 2, 3, 1) + cov_Ip = corr_Ip.reshape(B, H, W, C, -1) - mean_p.unsqueeze(-2) * mean_I.unsqueeze(-1) + + if isinstance(eps, torch.Tensor): + _eps = torch.eye(C, device=guidance.device, dtype=guidance.dtype).view(1, 1, 1, C, C) * eps.view(-1, 1, 1, 1, 1) + else: + _eps = guidance.new_full((C,), eps).diag().view(1, 1, 1, C, C) + a = torch.linalg.solve(var_I + _eps, cov_Ip) # B, H, W, C_guidance, C_input + b = mean_p - (mean_I.unsqueeze(-2) @ a).squeeze(-2) # B, H, W, C_input + + mean_a = box_blur(a.flatten(-2).permute(0, 3, 1, 2), kernel_size, border_type) + mean_b = box_blur(b.permute(0, 3, 1, 2), kernel_size, border_type) + + if subsample > 1: + mean_a = interpolate(mean_a, scale_factor=subsample, mode="bilinear") + mean_b = interpolate(mean_b, scale_factor=subsample, mode="bilinear") + mean_a = mean_a.view(B, C, -1, H * subsample, W * subsample) + + # torch.einsum might not be contiguous, thus mean_b is the first argument + return mean_b + torch.einsum("BCHW,BCcHW->BcHW", guidance, mean_a) + + +def guided_blur( + guidance: torch.Tensor, + input: torch.Tensor, + kernel_size: tuple[int, int] | int, + eps: float | torch.Tensor, + border_type: str = "reflect", + subsample: int = 1, +) -> torch.Tensor: + r"""Blur a torch.Tensor using a Guided filter. + + .. image:: _static/img/guided_blur.png + + The operator is an edge-preserving image smoothing filter. See :cite:`he2010guided` + and :cite:`he2015fast` for details. Guidance and input can have different number of channels. + + Arguments: + guidance: the guidance torch.Tensor with shape :math:`(B,C,H,W)`. + input: the input torch.Tensor with shape :math:`(B,C,H,W)`. + kernel_size: the size of the kernel. + eps: regularization parameter. Smaller values preserve more edges. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. Default: ``'reflect'``. + subsample: subsampling factor for Fast Guided filtering. Default: 1 (no subsampling) + + Returns: + the blurred torch.Tensor with same shape as `input` :math:`(B, C, H, W)`. + + Examples: + >>> guidance = torch.rand(2, 3, 5, 5) + >>> input = torch.rand(2, 4, 5, 5) + >>> output = guided_blur(guidance, input, 3, 0.1) + >>> output.shape + torch.Size([2, 4, 5, 5]) + + """ + KORNIA_CHECK_IS_TENSOR(guidance) + KORNIA_CHECK_SHAPE(guidance, ["B", "C", "H", "W"]) + if input is not guidance: + KORNIA_CHECK_IS_TENSOR(input) + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + KORNIA_CHECK( + (guidance.shape[0] == input.shape[0]) and (guidance.shape[-2:] == input.shape[-2:]), + "guidance and input should have the same batch size and spatial dimensions", + ) + + if guidance.shape[1] == 1: + return _guided_blur_grayscale_guidance(guidance, input, kernel_size, eps, border_type, subsample) + else: + return _guided_blur_multichannel_guidance(guidance, input, kernel_size, eps, border_type, subsample) + + +class GuidedBlur(nn.Module): + r"""Blur a torch.Tensor using a Guided filter. + + The operator is an edge-preserving image smoothing filter. See :cite:`he2010guided` + and :cite:`he2015fast` for details. Guidance and input can have different number of channels. + + Arguments: + kernel_size: the size of the kernel. + eps: regularization parameter. Smaller values preserve more edges. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. Default: ``'reflect'``. + subsample: subsampling factor for Fast Guided filtering. Default: 1 (no subsampling) + + Returns: + the blurred input torch.Tensor. + + Shape: + - Input: :math:`(B, C, H, W)`, :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + Examples: + >>> guidance = torch.rand(2, 3, 5, 5) + >>> input = torch.rand(2, 4, 5, 5) + >>> blur = GuidedBlur(3, 0.1) + >>> output = blur(guidance, input) + >>> output.shape + torch.Size([2, 4, 5, 5]) + + """ + + def __init__( + self, kernel_size: tuple[int, int] | int, eps: float, border_type: str = "reflect", subsample: int = 1 + ) -> None: + super().__init__() + self.kernel_size = kernel_size + self.eps = eps + self.border_type = border_type + self.subsample = subsample + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}" + f"(kernel_size={self.kernel_size}, " + f"eps={self.eps}, " + f"border_type={self.border_type}, " + f"subsample={self.subsample})" + ) + + def forward(self, guidance: torch.Tensor, input: torch.Tensor) -> torch.Tensor: + """Filter an input tensor with local linear guidance. + + Guided filtering assumes that, inside each local window, the output can + be represented as a linear transform of the ``guidance`` image. This + makes the filter useful for edge-aware smoothing: flat regions are + averaged, while strong structures in the guidance tensor are preserved + in the filtered result. + + Args: + guidance: Guidance tensor with shape :math:`(B, C_g, H, W)`, where + :math:`B` is the batch size, :math:`C_g` is the number of + guidance channels, :math:`H` is the height, and :math:`W` is + the width. + input: Tensor to filter with shape :math:`(B, C_i, H, W)`, where + :math:`C_i` is the number of channels in the signal being + smoothed. Its batch and spatial dimensions must be compatible + with ``guidance``. + + Returns: + Tensor with shape :math:`(B, C_i, H, W)` containing the + edge-aware filtered version of ``input``. + """ + return guided_blur(guidance, input, self.kernel_size, self.eps, self.border_type, self.subsample) diff --git a/kornia/filters/in_range.py b/kornia/filters/in_range.py new file mode 100644 index 0000000..0bfed8d --- /dev/null +++ b/kornia/filters/in_range.py @@ -0,0 +1,210 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Any, Union + +import torch +from torch import nn + +from kornia.image.utils import perform_keep_shape_image + + +@perform_keep_shape_image +def in_range( + input: torch.Tensor, + lower: Union[tuple[Any, ...], torch.Tensor], + upper: Union[tuple[Any, ...], torch.Tensor], + return_mask: bool = False, +) -> torch.Tensor: + r"""Create a mask indicating whether elements of the input torch.Tensor are within the specified range. + + .. image:: _static/img/in_range.png + + The formula applied for single-channel torch.Tensor is: + + .. math:: + \text{out}(I) = \text{lower}(I) \leq \text{input}(I) \geq \text{upper}(I) + + The formula applied for multi-channel torch.Tensor is: + + .. math:: + \text{out}(I) = \bigwedge_{c=0}^{C} + \left( \text{lower}_c(I) \leq \text{input}_c(I) \geq \text{upper}_c(I) \right) + + where `C` is the number of channels. + + Args: + input: The input torch.Tensor to be filtered in the shape of :math:`(*, *, H, W)`. + lower: The lower bounds of the filter (inclusive). + upper: The upper bounds of the filter (inclusive). + return_mask: If is true, the filtered mask is returned, otherwise the filtered input image. + + Returns: + A binary mask :math:`(*, 1, H, W)` of input indicating whether elements are within the range + or filtered input image :math:`(*, *, H, W)`. + + Raises: + ValueError: If the shape of `lower`, `upper`, and `input` image channels do not match. + + .. note:: + Clarification of `lower` and `upper`: + + - If provided as a tuple, it should have the same number of elements as the channels in the input torch.Tensor. + This bound is then applied uniformly across all batches. + + - When provided as a torch.Tensor, it allows for different bounds to be applied to each batch. + The torch.Tensor shape should be (B, C, 1, 1), where B is the batch size and C is + the number of channels. + + - If the torch.Tensor has a 1-D shape, same bound will be applied across all batches. + + Examples: + >>> rng = torch.manual_seed(1) + >>> input = torch.rand(1, 3, 3, 3) + >>> lower = (0.2, 0.3, 0.4) + >>> upper = (0.8, 0.9, 1.0) + >>> mask = in_range(input, lower, upper, return_mask=True) + >>> mask + tensor([[[[1., 1., 0.], + [0., 0., 0.], + [0., 1., 1.]]]]) + >>> mask.shape + torch.Size([1, 1, 3, 3]) + + Apply different bounds (`lower` and `upper`) for each batch: + + >>> rng = torch.manual_seed(1) + >>> input_tensor = torch.rand((2, 3, 3, 3)) + >>> input_shape = input_tensor.shape + >>> lower = torch.tensor([[0.2, 0.2, 0.2], [0.2, 0.2, 0.2]]).reshape(input_shape[0], input_shape[1], 1, 1) + >>> upper = torch.tensor([[0.6, 0.6, 0.6], [0.8, 0.8, 0.8]]).reshape(input_shape[0], input_shape[1], 1, 1) + >>> mask = in_range(input_tensor, lower, upper, return_mask=True) + >>> mask + tensor([[[[0., 0., 1.], + [0., 0., 0.], + [1., 0., 0.]]], + + + [[[0., 0., 0.], + [1., 0., 0.], + [0., 0., 1.]]]]) + + """ + input_shape = input.shape + + if not isinstance(lower, (tuple, torch.Tensor)) or not isinstance(upper, (tuple, torch.Tensor)): + raise TypeError("Invalid `lower` and `upper` format. Should be tuple or torch.Tensor.") + + if not isinstance(return_mask, bool): + raise TypeError("Invalid `return_mask` format. Should be boolean.") + + if isinstance(lower, tuple) and isinstance(upper, tuple): + if len(lower) != input_shape[1] or len(upper) != input_shape[1]: + raise ValueError("Shape of `lower`, `upper` and `input` image channels must have same shape.") + + lower = ( + torch.tensor(lower, device=input.device, dtype=input.dtype) + .reshape(1, -1, 1, 1) + .repeat(input_shape[0], 1, 1, 1) + ) + upper = ( + torch.tensor(upper, device=input.device, dtype=input.dtype) + .reshape(1, -1, 1, 1) + .repeat(input_shape[0], 1, 1, 1) + ) + + elif isinstance(lower, torch.Tensor) and isinstance(upper, torch.Tensor): + valid_tensor_shape = (input_shape[0], input_shape[1], 1, 1) + if valid_tensor_shape not in (lower.shape, upper.shape): + raise ValueError( + "`lower` and `upper` bounds as Tensors must have compatible shapes with the input (B, C, 1, 1)." + ) + lower = lower.to(input) + upper = upper.to(input) + + # Apply lower and upper bounds. Combine masks with logical_and. + mask = torch.logical_and(input >= lower, input <= upper) + mask = mask.all(dim=(1), keepdim=True).to(input.dtype) + + if return_mask: + return mask + + return input * mask + + +class InRange(nn.Module): + r"""Create a module for applying lower and upper bounds to input tensors. + + Args: + input: The input torch.Tensor to be filtered. + lower: The lower bounds of the filter (inclusive). + upper: The upper bounds of the filter (inclusive). + return_mask: If is true, the filtered mask is returned, otherwise the filtered input image. + + Returns: + A binary mask :math:`(*, 1, H, W)` of input indicating whether elements are within the range + or filtered input image :math:`(*, *, H, W)`. + + .. note:: + View complete documentation in :func:`kornia.filters.in_range`. + + Examples: + >>> rng = torch.manual_seed(1) + >>> input = torch.rand(1, 3, 3, 3) + >>> lower = (0.2, 0.3, 0.4) + >>> upper = (0.8, 0.9, 1.0) + >>> mask = InRange(lower, upper, return_mask=True)(input) + >>> mask + tensor([[[[1., 1., 0.], + [0., 0., 0.], + [0., 1., 1.]]]]) + + """ + + def __init__( + self, + lower: Union[tuple[Any, ...], torch.Tensor], + upper: Union[tuple[Any, ...], torch.Tensor], + return_mask: bool = False, + ) -> None: + super().__init__() + self.lower = lower + self.upper = upper + self.return_mask = return_mask + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Select values that fall inside the configured inclusive range. + + The stored ``lower`` and ``upper`` bounds are compared against + ``input`` channel-wise. Depending on ``self.return_mask``, the module + either returns the binary in-range mask itself or uses that mask to keep + only values that satisfy the bounds. + + Args: + input: Tensor to test against the configured bounds. For images the + usual shape is :math:`(B, C, H, W)`, where :math:`B` is the + batch size, :math:`C` is the number of channels, :math:`H` is + the height, and :math:`W` is the width. + + Returns: + If ``self.return_mask`` is ``True``, a mask indicating which + entries lie within the inclusive range. Otherwise, a tensor with + values outside the range removed according to :func:`in_range`. + """ + return in_range(input, self.lower, self.upper, self.return_mask) diff --git a/kornia/filters/kernels.py b/kornia/filters/kernels.py new file mode 100644 index 0000000..453c0be --- /dev/null +++ b/kornia/filters/kernels.py @@ -0,0 +1,1049 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import math +from typing import Any, Optional, Union + +import torch + +from kornia.core._compat import deprecated +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE + + +def _check_kernel_size(kernel_size: tuple[int, ...] | int, min_value: int = 0, allow_even: bool = False) -> None: + if isinstance(kernel_size, int): + kernel_size = (kernel_size,) + + fmt = "even or odd" if allow_even else "odd" + for size in kernel_size: + KORNIA_CHECK( + isinstance(size, int) and (((size % 2 == 1) or allow_even) and size > min_value), + f"Kernel size must be an {fmt} integer bigger than {min_value}. Gotcha {size} on {kernel_size}", + ) + + +def _unpack_2d_ks(kernel_size: tuple[int, int] | int) -> tuple[int, int]: + if isinstance(kernel_size, int): + ky = kx = kernel_size + else: + KORNIA_CHECK(len(kernel_size) == 2, "2D Kernel size should have a length of 2.") + ky, kx = kernel_size + + ky = int(ky) + kx = int(kx) + + return (ky, kx) + + +def _unpack_3d_ks(kernel_size: tuple[int, int, int] | int) -> tuple[int, int, int]: + if isinstance(kernel_size, int): + kz = ky = kx = kernel_size + else: + KORNIA_CHECK(len(kernel_size) == 3, "3D Kernel size should have a length of 3.") + kz, ky, kx = kernel_size + + kz = int(kz) + ky = int(ky) + kx = int(kx) + + return (kz, ky, kx) + + +def normalize_kernel2d(input: torch.Tensor) -> torch.Tensor: + r"""Normalize both derivative and smoothing kernel.""" + KORNIA_CHECK_SHAPE(input, ["*", "H", "W"]) + + norm = input.abs().sum(dim=-1).sum(dim=-1) + + return input / (norm[..., None, None]) + + +def gaussian( + window_size: int, + sigma: torch.Tensor | float, + *, + mean: Optional[Union[torch.Tensor, float]] = None, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + r"""Compute the gaussian values based on the window and sigma values. + + Args: + window_size: the size which drives the filter amount. + sigma: gaussian standard deviation. If a tensor, should be in a shape :math:`(B, 1)` + mean: Mean of the Gaussian function (center). If not provided, it defaults to window_size // 2. + If a tensor, should be in a shape :math:`(B, 1)` + device: This value will be used if sigma is a float. Device desired to compute. + dtype: This value will be used if sigma is a float. Dtype desired for compute. + + Returns: + A tensor withshape :math:`(B, \text{kernel_size})`, with Gaussian values. + + """ + if isinstance(sigma, float): + sigma = torch.tensor([[sigma]], device=device, dtype=dtype) + + KORNIA_CHECK_IS_TENSOR(sigma) + KORNIA_CHECK_SHAPE(sigma, ["B", "1"]) + batch_size = sigma.shape[0] + + mean = float(window_size // 2) if mean is None else mean + if isinstance(mean, float): + mean = torch.tensor([[mean]], device=sigma.device, dtype=sigma.dtype) + + KORNIA_CHECK_IS_TENSOR(mean) + KORNIA_CHECK_SHAPE(mean, ["B", "1"]) + + x = (torch.arange(window_size, device=sigma.device, dtype=sigma.dtype) - mean).expand(batch_size, -1) + + if window_size % 2 == 0: + x = x + 0.5 + + gauss = torch.exp(-x.pow(2.0) / (2 * sigma.pow(2.0))) + + return gauss / gauss.sum(-1, keepdim=True) + + +def gaussian_discrete_erf( + window_size: int, + sigma: torch.Tensor | float, + *, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + r"""Discrete Gaussian by interpolating the error function. + + Adapted from: https://github.com/Project-MONAI/MONAI/blob/master/monai/networks/layers/convutils.py + Args: + window_size: the size which drives the filter amount. + sigma: gaussian standard deviation. If a tensor, should be in a shape :math:`(B, 1)` + device: This value will be used if sigma is a float. Device desired to compute. + dtype: This value will be used if sigma is a float. Dtype desired for compute. + + Returns: + A tensor withshape :math:`(B, \text{kernel_size})`, with discrete Gaussian values computed by approximation of + the error function. + + """ + if isinstance(sigma, float): + sigma = torch.tensor([[sigma]], device=device, dtype=dtype) + + KORNIA_CHECK_SHAPE(sigma, ["B", "1"]) + batch_size = sigma.shape[0] + + x = (torch.arange(window_size, device=sigma.device, dtype=sigma.dtype) - window_size // 2).expand(batch_size, -1) + + t = 0.70710678 / sigma.abs() + # t = torch.tensor(2, device=sigma.device, dtype=sigma.dtype).sqrt() / (sigma.abs() * 2) + + gauss = 0.5 * ((t * (x + 0.5)).erf() - (t * (x - 0.5)).erf()) + gauss = gauss.clamp(min=0) + + return gauss / gauss.sum(-1, keepdim=True) + + +def _modified_bessel_0(x: torch.Tensor) -> torch.Tensor: + """Adapted from:https://github.com/Project-MONAI/MONAI/blob/master/monai/networks/layers/convutils.py.""" + ax = torch.abs(x) + + out = torch.zeros_like(x) + idx_a = ax < 3.75 + + if idx_a.any(): + y = (x[idx_a] / 3.75) * (x[idx_a] / 3.75) + out[idx_a] = 1.0 + y * ( + 3.5156229 + y * (3.0899424 + y * (1.2067492 + y * (0.2659732 + y * (0.360768e-1 + y * 0.45813e-2)))) + ) + + idx_b = ~idx_a + if idx_b.any(): + y = 3.75 / ax[idx_b] + ans = 0.916281e-2 + y * (-0.2057706e-1 + y * (0.2635537e-1 + y * (-0.1647633e-1 + y * 0.392377e-2))) + coef = 0.39894228 + y * (0.1328592e-1 + y * (0.225319e-2 + y * (-0.157565e-2 + y * ans))) + out[idx_b] = (ax[idx_b].exp() / ax[idx_b].sqrt()) * coef + + return out + + +def _modified_bessel_1(x: torch.Tensor) -> torch.Tensor: + """Adapted from:https://github.com/Project-MONAI/MONAI/blob/master/monai/networks/layers/convutils.py.""" + ax = torch.abs(x) + + out = torch.zeros_like(x) + idx_a = ax < 3.75 + + if idx_a.any(): + y = (x[idx_a] / 3.75) * (x[idx_a] / 3.75) + ans = 0.51498869 + y * (0.15084934 + y * (0.2658733e-1 + y * (0.301532e-2 + y * 0.32411e-3))) + out[idx_a] = ax[idx_a] * (0.5 + y * (0.87890594 + y * ans)) + + idx_b = ~idx_a + if idx_b.any(): + y = 3.75 / ax[idx_b] + ans = 0.2282967e-1 + y * (-0.2895312e-1 + y * (0.1787654e-1 - y * 0.420059e-2)) + ans = 0.39894228 + y * (-0.3988024e-1 + y * (-0.362018e-2 + y * (0.163801e-2 + y * (-0.1031555e-1 + y * ans)))) + ans = ans * ax[idx_b].exp() / ax[idx_b].sqrt() + out[idx_b] = torch.where(x[idx_b] < 0, -ans, ans) + + return out + + +def _modified_bessel_i(n: int, x: torch.Tensor) -> torch.Tensor: + """Adapted from: https://github.com/Project-MONAI/MONAI/blob/master/monai/networks/layers/convutils.py.""" + KORNIA_CHECK(n >= 2, "n must be greater than 1.99") + + is_zero_mask = torch.isclose(x, torch.tensor(0.0, device=x.device, dtype=x.dtype)) + if is_zero_mask.all(): + return x + + x_nz = x[~is_zero_mask] + + batch_size = x_nz.shape[0] + tox = 2.0 / x_nz.abs() + + ans = torch.zeros(batch_size, device=x.device, dtype=x.dtype) + bip = torch.zeros(batch_size, device=x.device, dtype=x.dtype) + bi = torch.ones(batch_size, device=x.device, dtype=x.dtype) + + m = int(2 * (n + int(math.sqrt(40.0 * n)))) + for j in range(m, 0, -1): + bim = torch.addcmul(bip, tox, bi, value=j) + bip, bi = bi, bim + + scale_mask = bi.abs() > 1.0e10 + if scale_mask.any(): + factor = torch.where(scale_mask, 1e-10, 1.0) + ans *= factor + bi *= factor + bip *= factor + + if j == n: + ans = bip + + out_nz = ans * _modified_bessel_0(x_nz) / bi + if (n % 2) == 1: + out_nz = torch.where(x_nz < 0.0, -out_nz, out_nz) + + out = torch.zeros_like(x) + out[~is_zero_mask] = out_nz + return out + + +def gaussian_discrete( + window_size: int, + sigma: torch.Tensor | float, + *, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + r"""Discrete Gaussian kernel based on the modified Bessel functions. + + Adapted from: https://github.com/Project-MONAI/MONAI/blob/master/monai/networks/layers/convutils.py + Args: + window_size: the size which drives the filter amount. + sigma: gaussian standard deviation. If a tensor, should be in a shape :math:`(B, 1)` + device: This value will be used if sigma is a float. Device desired to compute. + dtype: This value will be used if sigma is a float. Dtype desired for compute. + + Returns: + A tensor withshape :math:`(B, \text{kernel_size})`, with discrete Gaussian values computed by modified Bessel + function. + + """ + if isinstance(sigma, float): + sigma = torch.tensor([[sigma]], device=device, dtype=dtype) + + KORNIA_CHECK_SHAPE(sigma, ["B", "1"]) + + sigma2 = sigma * sigma + tail = int(window_size // 2) + 1 + bessels = [ + _modified_bessel_0(sigma2), + _modified_bessel_1(sigma2), + *(_modified_bessel_i(k, sigma2) for k in range(2, tail)), + ] + # NOTE: on monain is exp(-sig) + # https://github.com/Project-MONAI/MONAI/blob/dev/monai/networks/layers/convutils.py#L128 + out = torch.cat(bessels[:0:-1] + bessels, -1) * sigma2.exp() + + return out / out.sum(-1, keepdim=True) + + +def laplacian_1d( + window_size: int, *, device: Optional[torch.device] = None, dtype: torch.dtype = torch.float32 +) -> torch.Tensor: + """One could also use the Laplacian of Gaussian formula to design the filter.""" + # TODO: add default dtype as None when kornia relies on torch > 1.12 + filter_1d = torch.ones(window_size, device=device, dtype=dtype) + middle = window_size // 2 + filter_1d[middle] = 1 - window_size + return filter_1d + + +def get_box_kernel1d( + kernel_size: int, *, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None +) -> torch.Tensor: + r"""Return a 1-D box filter. + + Args: + kernel_size: the size of the kernel. + device: the desired device of returned tensor. + dtype: the desired data type of returned tensor. + + Returns: + A tensor with shape :math:`(1, \text{kernel\_size})`, filled with the value + :math:`\frac{1}{\text{kernel\_size}}`. + + """ + scale = torch.tensor(1.0 / kernel_size, device=device, dtype=dtype) + return scale.expand(1, kernel_size) + + +def get_box_kernel2d( + kernel_size: tuple[int, int] | int, *, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None +) -> torch.Tensor: + r"""Return a 2-D box filter. + + Args: + kernel_size: the size of the kernel. + device: the desired device of returned tensor. + dtype: the desired data type of returned tensor. + + Returns: + A tensor with shape :math:`(1, \text{kernel\_size}[0], \text{kernel\_size}[1])`, + filled with the value :math:`\frac{1}{\text{kernel\_size}[0] \times \text{kernel\_size}[1]}`. + + """ + ky, kx = _unpack_2d_ks(kernel_size) + scale = torch.tensor(1.0 / (kx * ky), device=device, dtype=dtype) + return scale.expand(1, ky, kx) + + +def get_binary_kernel2d( + window_size: tuple[int, int] | int, *, device: Optional[torch.device] = None, dtype: torch.dtype = torch.float32 +) -> torch.Tensor: + """Create a binary kernel to extract the patches. + + If the window size is HxW will create a (H*W)x1xHxW kernel. + """ + # TODO: add default dtype as None when kornia relies on torch > 1.12 + + ky, kx = _unpack_2d_ks(window_size) + + window_range = kx * ky + + kernel = torch.zeros((window_range, window_range), device=device, dtype=dtype) + idx = torch.arange(window_range, device=device) + kernel[idx, idx] += 1.0 + return kernel.view(window_range, 1, ky, kx) + + +def get_sobel_kernel_3x3(*, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> torch.Tensor: + """Return a sobel kernel of 3x3.""" + return torch.tensor([[-1.0, 0.0, 1.0], [-2.0, 0.0, 2.0], [-1.0, 0.0, 1.0]], device=device, dtype=dtype) + + +def get_sobel_kernel_5x5_2nd_order( + *, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None +) -> torch.Tensor: + """Return a 2nd order sobel kernel of 5x5.""" + return torch.tensor( + [ + [-1.0, 0.0, 2.0, 0.0, -1.0], + [-4.0, 0.0, 8.0, 0.0, -4.0], + [-6.0, 0.0, 12.0, 0.0, -6.0], + [-4.0, 0.0, 8.0, 0.0, -4.0], + [-1.0, 0.0, 2.0, 0.0, -1.0], + ], + device=device, + dtype=dtype, + ) + + +def _get_sobel_kernel_5x5_2nd_order_xy( + *, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None +) -> torch.Tensor: + """Return a 2nd order sobel kernel of 5x5.""" + return torch.tensor( + [ + [-1.0, -2.0, 0.0, 2.0, 1.0], + [-2.0, -4.0, 0.0, 4.0, 2.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [2.0, 4.0, 0.0, -4.0, -2.0], + [1.0, 2.0, 0.0, -2.0, -1.0], + ], + device=device, + dtype=dtype, + ) + + +def get_diff_kernel_3x3(*, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> torch.Tensor: + """Return a first order derivative kernel of 3x3.""" + return torch.tensor([[-0.0, 0.0, 0.0], [-1.0, 0.0, 1.0], [-0.0, 0.0, 0.0]], device=device, dtype=dtype) + + +def get_diff_kernel3d(device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> torch.Tensor: + """Return a first order derivative kernel of 3x3x3.""" + kernel = torch.tensor( + [ + [ + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [-0.5, 0.0, 0.5], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + ], + [ + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, -0.5, 0.0], [0.0, 0.0, 0.0], [0.0, 0.5, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + ], + [ + [[0.0, 0.0, 0.0], [0.0, -0.5, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.5, 0.0], [0.0, 0.0, 0.0]], + ], + ], + device=device, + dtype=dtype, + ) + return kernel[:, None, ...] + + +def get_diff_kernel3d_2nd_order( + device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None +) -> torch.Tensor: + """Return a first order derivative kernel of 3x3x3.""" + kernel = torch.tensor( + [ + [ + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [1.0, -2.0, 1.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + ], + [ + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 1.0, 0.0], [0.0, -2.0, 0.0], [0.0, 1.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + ], + [ + [[0.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, -2.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]], + ], + [ + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + [[1.0, 0.0, -1.0], [0.0, 0.0, 0.0], [-1.0, 0.0, 1.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + ], + [ + [[0.0, 1.0, 0.0], [0.0, 0.0, 0.0], [0.0, -1.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, -1.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 0.0]], + ], + [ + [[0.0, 0.0, 0.0], [1.0, 0.0, -1.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [-1.0, 0.0, 1.0], [0.0, 0.0, 0.0]], + ], + ], + device=device, + dtype=dtype, + ) + return kernel[:, None, ...] + + +def get_sobel_kernel2d(*, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> torch.Tensor: + """Return 1st order gradient for sobel operator.""" + kernel_x = get_sobel_kernel_3x3(device=device, dtype=dtype) + kernel_y = kernel_x.transpose(0, 1) + return torch.stack([kernel_x, kernel_y]) + + +def get_diff_kernel2d(*, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> torch.Tensor: + """Return 1st order gradient for diff operator.""" + kernel_x = get_diff_kernel_3x3(device=device, dtype=dtype) + kernel_y = kernel_x.transpose(0, 1) + return torch.stack([kernel_x, kernel_y]) + + +def get_sobel_kernel2d_2nd_order( + *, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None +) -> torch.Tensor: + """Return 2nd order gradient for sobel operator.""" + gxx = get_sobel_kernel_5x5_2nd_order(device=device, dtype=dtype) + gyy = gxx.transpose(0, 1) + gxy = _get_sobel_kernel_5x5_2nd_order_xy(device=device, dtype=dtype) + return torch.stack([gxx, gxy, gyy]) + + +def get_diff_kernel2d_2nd_order( + *, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None +) -> torch.Tensor: + """Return 2nd order gradient for diff operator.""" + gxx = torch.tensor([[0.0, 0.0, 0.0], [1.0, -2.0, 1.0], [0.0, 0.0, 0.0]], device=device, dtype=dtype) + gyy = gxx.transpose(0, 1) + gxy = torch.tensor([[-1.0, 0.0, 1.0], [0.0, 0.0, 0.0], [1.0, 0.0, -1.0]], device=device, dtype=dtype) + return torch.stack([gxx, gxy, gyy]) + + +def get_spatial_gradient_kernel2d( + mode: str, + order: int, + *, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + r"""Return kernel for 1st or 2nd order image gradients. + + Uses one of the following operators: sobel, diff. + """ + KORNIA_CHECK(mode.lower() in {"sobel", "diff"}, f"Mode should be `sobel` or `diff`. Got {mode}") + KORNIA_CHECK(order in {1, 2}, f"Order should be 1 or 2. Got {order}") + + if mode == "sobel" and order == 1: + kernel: torch.Tensor = get_sobel_kernel2d(device=device, dtype=dtype) + elif mode == "sobel" and order == 2: + kernel = get_sobel_kernel2d_2nd_order(device=device, dtype=dtype) + elif mode == "diff" and order == 1: + kernel = get_diff_kernel2d(device=device, dtype=dtype) + elif mode == "diff" and order == 2: + kernel = get_diff_kernel2d_2nd_order(device=device, dtype=dtype) + else: + raise NotImplementedError(f"Not implemented for order {order} on mode {mode}") + + return kernel + + +def get_spatial_gradient_kernel3d( + mode: str, order: int, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None +) -> torch.Tensor: + r"""Return kernel for 1st or 2nd order scale pyramid gradients. + + Uses one of the following operators: sobel, diff. + """ + KORNIA_CHECK(mode.lower() in {"sobel", "diff"}, f"Mode should be `sobel` or `diff`. Got {mode}") + KORNIA_CHECK(order in {1, 2}, f"Order should be 1 or 2. Got {order}") + + if mode == "diff" and order == 1: + kernel = get_diff_kernel3d(device=device, dtype=dtype) + elif mode == "diff" and order == 2: + kernel = get_diff_kernel3d_2nd_order(device=device, dtype=dtype) + else: + raise NotImplementedError(f"Not implemented 3d gradient kernel for order {order} on mode {mode}") + + return kernel + + +def get_gaussian_kernel1d( + kernel_size: int, + sigma: float | torch.Tensor, + force_even: bool = False, + *, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + r"""Return Gaussian filter coefficients. + + Args: + kernel_size: filter size. It should be odd and positive. + sigma: gaussian standard deviation. + force_even: overrides requirement for odd kernel size. + device: This value will be used if sigma is a float. Device desired to compute. + dtype: This value will be used if sigma is a float. Dtype desired for compute. + + Returns: + gaussian filter coefficients with shape :math:`(B, \text{kernel_size})`. + + Examples: + >>> get_gaussian_kernel1d(3, 2.5) + tensor([[0.3243, 0.3513, 0.3243]]) + >>> get_gaussian_kernel1d(5, 1.5) + tensor([[0.1201, 0.2339, 0.2921, 0.2339, 0.1201]]) + >>> get_gaussian_kernel1d(5, torch.tensor([[1.5], [0.7]])) + tensor([[0.1201, 0.2339, 0.2921, 0.2339, 0.1201], + [0.0096, 0.2054, 0.5699, 0.2054, 0.0096]]) + + """ + _check_kernel_size(kernel_size, allow_even=force_even) + + return gaussian(kernel_size, sigma, device=device, dtype=dtype) + + +def get_gaussian_discrete_kernel1d( + kernel_size: int, + sigma: float | torch.Tensor, + force_even: bool = False, + *, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + r"""Return Gaussian filter coefficients based on the modified Bessel functions. + + Adapted from: https://github.com/Project-MONAI/MONAI/blob/master/monai/networks/layers/convutils.py. + + Args: + kernel_size: filter size. It should be odd and positive. + sigma: gaussian standard deviation. If a tensor, should be in a shape :math:`(B, 1)` + force_even: overrides requirement for odd kernel size. + device: This value will be used if sigma is a float. Device desired to compute. + dtype: This value will be used if sigma is a float. Dtype desired for compute. + + Returns: + 1D tensor with gaussian filter coefficients. With shape :math:`(B, \text{kernel_size})` + + Examples: + >>> get_gaussian_discrete_kernel1d(3, 2.5) + tensor([[0.3235, 0.3531, 0.3235]]) + >>> get_gaussian_discrete_kernel1d(5, 1.5) + tensor([[0.1096, 0.2323, 0.3161, 0.2323, 0.1096]]) + >>> get_gaussian_discrete_kernel1d(5, torch.tensor([[1.5],[2.4]])) + tensor([[0.1096, 0.2323, 0.3161, 0.2323, 0.1096], + [0.1635, 0.2170, 0.2389, 0.2170, 0.1635]]) + + """ + _check_kernel_size(kernel_size, allow_even=force_even) + + return gaussian_discrete(kernel_size, sigma, device=device, dtype=dtype) + + +def get_gaussian_erf_kernel1d( + kernel_size: int, + sigma: float | torch.Tensor, + force_even: bool = False, + *, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + r"""Return Gaussian filter coefficients by interpolating the error function. + + Adapted from: https://github.com/Project-MONAI/MONAI/blob/master/monai/networks/layers/convutils.py. + + Args: + kernel_size: filter size. It should be odd and positive. + sigma: gaussian standard deviation. If a tensor, should be in a shape :math:`(B, 1)` + force_even: overrides requirement for odd kernel size. + device: This value will be used if sigma is a float. Device desired to compute. + dtype: This value will be used if sigma is a float. Dtype desired for compute. + + Returns: + 1D tensor with gaussian filter coefficients. Shape :math:`(B, \text{kernel_size})` + + Examples: + >>> get_gaussian_erf_kernel1d(3, 2.5) + tensor([[0.3245, 0.3511, 0.3245]]) + >>> get_gaussian_erf_kernel1d(5, 1.5) + tensor([[0.1226, 0.2331, 0.2887, 0.2331, 0.1226]]) + >>> get_gaussian_erf_kernel1d(5, torch.tensor([[1.5], [2.1]])) + tensor([[0.1226, 0.2331, 0.2887, 0.2331, 0.1226], + [0.1574, 0.2198, 0.2456, 0.2198, 0.1574]]) + + """ + _check_kernel_size(kernel_size, allow_even=force_even) + + return gaussian_discrete_erf(kernel_size, sigma, device=device, dtype=dtype) + + +def get_gaussian_kernel2d( + kernel_size: tuple[int, int] | int, + sigma: tuple[float, float] | torch.Tensor, + force_even: bool = False, + *, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + r"""Return Gaussian filter matrix coefficients. + + Args: + kernel_size: filter sizes in the y and x direction. Sizes should be odd and positive. + sigma: gaussian standard deviation in the y and x. + force_even: overrides requirement for odd kernel size. + device: This value will be used if sigma is a float. Device desired to compute. + dtype: This value will be used if sigma is a float. Dtype desired for compute. + + Returns: + 2D tensor with gaussian filter matrix coefficients. + + Shape: + - Output: :math:`(B, \text{kernel_size}_x, \text{kernel_size}_y)` + + Examples: + >>> get_gaussian_kernel2d((5, 5), (1.5, 1.5)) + tensor([[[0.0144, 0.0281, 0.0351, 0.0281, 0.0144], + [0.0281, 0.0547, 0.0683, 0.0547, 0.0281], + [0.0351, 0.0683, 0.0853, 0.0683, 0.0351], + [0.0281, 0.0547, 0.0683, 0.0547, 0.0281], + [0.0144, 0.0281, 0.0351, 0.0281, 0.0144]]]) + >>> get_gaussian_kernel2d((3, 5), (1.5, 1.5)) + tensor([[[0.0370, 0.0720, 0.0899, 0.0720, 0.0370], + [0.0462, 0.0899, 0.1123, 0.0899, 0.0462], + [0.0370, 0.0720, 0.0899, 0.0720, 0.0370]]]) + >>> get_gaussian_kernel2d((5, 5), torch.tensor([[1.5, 1.5]])) + tensor([[[0.0144, 0.0281, 0.0351, 0.0281, 0.0144], + [0.0281, 0.0547, 0.0683, 0.0547, 0.0281], + [0.0351, 0.0683, 0.0853, 0.0683, 0.0351], + [0.0281, 0.0547, 0.0683, 0.0547, 0.0281], + [0.0144, 0.0281, 0.0351, 0.0281, 0.0144]]]) + + """ + if isinstance(sigma, tuple): + sigma = torch.tensor([sigma], device=device, dtype=dtype) + + KORNIA_CHECK_IS_TENSOR(sigma) + KORNIA_CHECK_SHAPE(sigma, ["B", "2"]) + + ksize_y, ksize_x = _unpack_2d_ks(kernel_size) + sigma_y, sigma_x = sigma[:, 0, None], sigma[:, 1, None] + + kernel_y = get_gaussian_kernel1d(ksize_y, sigma_y, force_even, device=device, dtype=dtype)[..., None] + kernel_x = get_gaussian_kernel1d(ksize_x, sigma_x, force_even, device=device, dtype=dtype)[..., None] + + return kernel_y * kernel_x.view(-1, 1, ksize_x) + + +def get_gaussian_kernel3d( + kernel_size: tuple[int, int, int] | int, + sigma: tuple[float, float, float] | torch.Tensor, + force_even: bool = False, + *, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + r"""Return Gaussian filter matrix coefficients. + + Args: + kernel_size: filter sizes in the z, y and x direction. Sizes should be odd and positive. + sigma: gaussian standard deviation in the z, y and x direction. + force_even: overrides requirement for odd kernel size. + device: This value will be used if sigma is a float. Device desired to compute. + dtype: This value will be used if sigma is a float. Dtype desired for compute. + + Returns: + 3D tensor with gaussian filter matrix coefficients. + + Shape: + - Output: :math:`(B, \text{kernel_size}_x, \text{kernel_size}_y, \text{kernel_size}_z)` + + Examples: + >>> get_gaussian_kernel3d((3, 3, 3), (1.5, 1.5, 1.5)) + tensor([[[[0.0292, 0.0364, 0.0292], + [0.0364, 0.0455, 0.0364], + [0.0292, 0.0364, 0.0292]], + + [[0.0364, 0.0455, 0.0364], + [0.0455, 0.0568, 0.0455], + [0.0364, 0.0455, 0.0364]], + + [[0.0292, 0.0364, 0.0292], + [0.0364, 0.0455, 0.0364], + [0.0292, 0.0364, 0.0292]]]]) + >>> get_gaussian_kernel3d((3, 3, 3), (1.5, 1.5, 1.5)).sum() + tensor(1.) + >>> get_gaussian_kernel3d((3, 3, 3), (1.5, 1.5, 1.5)).shape + torch.Size([1, 3, 3, 3]) + >>> get_gaussian_kernel3d((3, 7, 5), torch.tensor([[1.5, 1.5, 1.5]])).shape + torch.Size([1, 3, 7, 5]) + + """ + if isinstance(sigma, tuple): + sigma = torch.tensor([sigma], device=device, dtype=dtype) + + KORNIA_CHECK_IS_TENSOR(sigma) + KORNIA_CHECK_SHAPE(sigma, ["B", "3"]) + + ksize_z, ksize_y, ksize_x = _unpack_3d_ks(kernel_size) + sigma_z, sigma_y, sigma_x = sigma[:, 0, None], sigma[:, 1, None], sigma[:, 2, None] + + kernel_z = get_gaussian_kernel1d(ksize_z, sigma_z, force_even, device=device, dtype=dtype) + kernel_y = get_gaussian_kernel1d(ksize_y, sigma_y, force_even, device=device, dtype=dtype) + kernel_x = get_gaussian_kernel1d(ksize_x, sigma_x, force_even, device=device, dtype=dtype) + + return kernel_z.view(-1, ksize_z, 1, 1) * kernel_y.view(-1, 1, ksize_y, 1) * kernel_x.view(-1, 1, 1, ksize_x) + + +def get_laplacian_kernel1d( + kernel_size: int, *, device: Optional[torch.device] = None, dtype: torch.dtype = torch.float32 +) -> torch.Tensor: + r"""Return the coefficients of a 1D Laplacian filter. + + Args: + kernel_size: filter size. It should be odd and positive. + device: tensor device desired to create the kernel + dtype: tensor dtype desired to create the kernel + + Returns: + 1D tensor with laplacian filter coefficients. + + Shape: + - Output: math:`(\text{kernel_size})` + + Examples: + >>> get_laplacian_kernel1d(3) + tensor([ 1., -2., 1.]) + >>> get_laplacian_kernel1d(5) + tensor([ 1., 1., -4., 1., 1.]) + + """ + # TODO: add default dtype as None when kornia relies on torch > 1.12 + + _check_kernel_size(kernel_size) + + return laplacian_1d(kernel_size, device=device, dtype=dtype) + + +def get_laplacian_kernel2d( + kernel_size: tuple[int, int] | int, *, device: Optional[torch.device] = None, dtype: torch.dtype = torch.float32 +) -> torch.Tensor: + r"""Return Gaussian filter matrix coefficients. + + Args: + kernel_size: filter size should be odd. + device: tensor device desired to create the kernel + dtype: tensor dtype desired to create the kernel + + Returns: + 2D tensor with laplacian filter matrix coefficients. + + Shape: + - Output: :math:`(\text{kernel_size}_x, \text{kernel_size}_y)` + + Examples: + >>> get_laplacian_kernel2d(3) + tensor([[ 1., 1., 1.], + [ 1., -8., 1.], + [ 1., 1., 1.]]) + >>> get_laplacian_kernel2d(5) + tensor([[ 1., 1., 1., 1., 1.], + [ 1., 1., 1., 1., 1.], + [ 1., 1., -24., 1., 1.], + [ 1., 1., 1., 1., 1.], + [ 1., 1., 1., 1., 1.]]) + + """ + # TODO: add default dtype as None when kornia relies on torch > 1.12 + + ky, kx = _unpack_2d_ks(kernel_size) + _check_kernel_size((ky, kx)) + + kernel = torch.ones((ky, kx), device=device, dtype=dtype) + mid_x = kx // 2 + mid_y = ky // 2 + + kernel[mid_y, mid_x] = 1 - kernel.sum() + return kernel + + +def get_pascal_kernel_2d( + kernel_size: tuple[int, int] | int, + norm: bool = True, + *, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + """Generate pascal filter kernel by kernel size. + + Args: + kernel_size: height and width of the kernel. + norm: if to normalize the kernel or not. Default: True. + device: tensor device desired to create the kernel + dtype: tensor dtype desired to create the kernel + + Returns: + if kernel_size is an integer the kernel will be shaped as :math:`(kernel_size, kernel_size)` + otherwise the kernel will be shaped as :math: `kernel_size` + + Examples: + >>> get_pascal_kernel_2d(1) + tensor([[1.]]) + >>> get_pascal_kernel_2d(4) + tensor([[0.0156, 0.0469, 0.0469, 0.0156], + [0.0469, 0.1406, 0.1406, 0.0469], + [0.0469, 0.1406, 0.1406, 0.0469], + [0.0156, 0.0469, 0.0469, 0.0156]]) + >>> get_pascal_kernel_2d(4, norm=False) + tensor([[1., 3., 3., 1.], + [3., 9., 9., 3.], + [3., 9., 9., 3.], + [1., 3., 3., 1.]]) + + """ + ky, kx = _unpack_2d_ks(kernel_size) + ax = get_pascal_kernel_1d(kx, device=device, dtype=dtype) + ay = get_pascal_kernel_1d(ky, device=device, dtype=dtype) + + filt = ay[:, None] * ax[None, :] + if norm: + filt = filt / torch.sum(filt) + return filt + + +def get_pascal_kernel_1d( + kernel_size: int, norm: bool = False, *, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None +) -> torch.Tensor: + """Generate Yang Hui triangle (Pascal's triangle) by a given number. + + Args: + kernel_size: height and width of the kernel. + norm: if to normalize the kernel or not. Default: False. + device: tensor device desired to create the kernel + dtype: tensor dtype desired to create the kernel + + Returns: + kernel shaped as :math:`(kernel_size,)` + + Examples: + >>> get_pascal_kernel_1d(1) + tensor([1.]) + >>> get_pascal_kernel_1d(2) + tensor([1., 1.]) + >>> get_pascal_kernel_1d(3) + tensor([1., 2., 1.]) + >>> get_pascal_kernel_1d(4) + tensor([1., 3., 3., 1.]) + >>> get_pascal_kernel_1d(5) + tensor([1., 4., 6., 4., 1.]) + >>> get_pascal_kernel_1d(6) + tensor([ 1., 5., 10., 10., 5., 1.]) + + """ + pre: list[float] = [] + cur: list[float] = [] + for i in range(kernel_size): + cur = [1.0] * (i + 1) + + for j in range(1, i // 2 + 1): + value = pre[j - 1] + pre[j] + cur[j] = value + if i != 2 * j: + cur[-j - 1] = value + pre = cur + + out = torch.tensor(cur, device=device, dtype=dtype) + + if norm: + out = out / out.sum() + + return out + + +def get_canny_nms_kernel(device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> torch.Tensor: + """Return 3x3 kernels for the Canny Non-maximal suppression.""" + return torch.tensor( + [ + [[[0.0, 0.0, 0.0], [0.0, 1.0, -1.0], [0.0, 0.0, 0.0]]], + [[[0.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, -1.0]]], + [[[0.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, -1.0, 0.0]]], + [[[0.0, 0.0, 0.0], [0.0, 1.0, 0.0], [-1.0, 0.0, 0.0]]], + [[[0.0, 0.0, 0.0], [-1.0, 1.0, 0.0], [0.0, 0.0, 0.0]]], + [[[-1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]]], + [[[0.0, -1.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]]], + [[[0.0, 0.0, -1.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]]], + ], + device=device, + dtype=dtype, + ) + + +def get_hysteresis_kernel(device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> torch.Tensor: + """Return the 3x3 kernels for the Canny hysteresis.""" + return torch.tensor( + [ + [[[0.0, 0.0, 0.0], [0.0, 0.0, 1.0], [0.0, 0.0, 0.0]]], + [[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 1.0]]], + [[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 0.0]]], + [[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [1.0, 0.0, 0.0]]], + [[[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 0.0]]], + [[[1.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]], + [[[0.0, 1.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]], + [[[0.0, 0.0, 1.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]], + ], + device=device, + dtype=dtype, + ) + + +def get_hanning_kernel1d( + kernel_size: int, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None +) -> torch.Tensor: + r"""Return Hanning (also known as Hann) kernel, used in signal processing and KCF tracker. + + .. math:: w(n) = 0.5 - 0.5cos\\left(\\frac{2\\pi{n}}{M-1}\\right) + \\qquad 0 \\leq n \\leq M-1 + + See further in numpy docs https://numpy.org/doc/stable/reference/generated/numpy.hanning.html + + Args: + kernel_size: The size the of the kernel. It should be positive. + device: tensor device desired to create the kernel + dtype: tensor dtype desired to create the kernel + + Returns: + 1D tensor with Hanning filter coefficients. Shape math:`(\text{kernel_size})` + .. math:: w(n) = 0.5 - 0.5cos\\left(\\frac{2\\pi{n}}{M-1}\\right) + + Examples: + >>> get_hanning_kernel1d(4) + tensor([0.0000, 0.7500, 0.7500, 0.0000]) + + """ + _check_kernel_size(kernel_size, 2, allow_even=True) + + x = torch.arange(kernel_size, device=device, dtype=dtype) + x = 0.5 - 0.5 * torch.cos(2.0 * math.pi * x / float(kernel_size - 1)) + return x + + +def get_hanning_kernel2d( + kernel_size: tuple[int, int] | int, + device: Optional[Union[str, torch.device]] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + r"""Return 2d Hanning kernel, used in signal processing and KCF tracker. + + Args: + kernel_size: The size of the kernel for the filter. It should be positive. + device: tensor device desired to create the kernel + dtype: tensor dtype desired to create the kernel + + Returns: + 2D tensor with Hanning filter coefficients. Shape: math:`(\text{kernel_size[0], kernel_size[1]})` + .. math:: w(n) = 0.5 - 0.5cos\\left(\\frac{2\\pi{n}}{M-1}\\right) + + """ + kernel_size = _unpack_2d_ks(kernel_size) + _check_kernel_size(kernel_size, 2, allow_even=True) + + ky = get_hanning_kernel1d(kernel_size[0], device, dtype)[None].T + kx = get_hanning_kernel1d(kernel_size[1], device, dtype)[None] + kernel2d = ky @ kx + + return kernel2d + + +@deprecated(replace_with="get_gaussian_kernel1d", version="6.9.10") +def get_gaussian_kernel1d_t(*args: Any, **kwargs: Any) -> torch.Tensor: # noqa: D103 + return get_gaussian_kernel1d(*args, **kwargs) + + +@deprecated(replace_with="get_gaussian_kernel2d", version="6.9.10") +def get_gaussian_kernel2d_t(*args: Any, **kwargs: Any) -> torch.Tensor: # noqa: D103 + return get_gaussian_kernel2d(*args, **kwargs) + + +@deprecated(replace_with="get_gaussian_kernel3d", version="6.9.10") +def get_gaussian_kernel3d_t(*args: Any, **kwargs: Any) -> torch.Tensor: # noqa: D103 + return get_gaussian_kernel3d(*args, **kwargs) diff --git a/kornia/filters/kernels_geometry.py b/kornia/filters/kernels_geometry.py new file mode 100644 index 0000000..1b67c6e --- /dev/null +++ b/kornia/filters/kernels_geometry.py @@ -0,0 +1,212 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +import torch.nn.functional as F + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SHAPE +from kornia.core.utils import _extract_device_dtype +from kornia.geometry.transform import rotate, rotate3d + +from .kernels import _check_kernel_size, _unpack_2d_ks, _unpack_3d_ks + + +def get_motion_kernel2d( + kernel_size: int, angle: torch.Tensor | float, direction: torch.Tensor | float = 0.0, mode: str = "nearest" +) -> torch.Tensor: + r"""Return 2D motion blur filter. + + Args: + kernel_size: motion kernel width and height. It should be odd and positive. + angle: angle of the motion blur in degrees (anti-clockwise rotation). + direction: forward/backward direction of the motion blur. + Lower values towards -1.0 will point the motion blur towards the back (with angle provided via angle), + while higher values towards 1.0 will point the motion blur forward. A value of 0.0 leads to a + uniformly (but still angled) motion blur. + mode: interpolation mode for rotating the kernel. ``'bilinear'`` or ``'nearest'``. + + Returns: + The motion blur kernel of shape :math:`(B, k_\text{size}, k_\text{size})`. + + Examples: + >>> get_motion_kernel2d(5, 0., 0.) + tensor([[[0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.2000, 0.2000, 0.2000, 0.2000, 0.2000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000]]]) + + >>> get_motion_kernel2d(3, 215., -0.5) + tensor([[[0.0000, 0.0000, 0.1667], + [0.0000, 0.3333, 0.0000], + [0.5000, 0.0000, 0.0000]]]) + + """ + device, dtype = _extract_device_dtype( + [angle if isinstance(angle, torch.Tensor) else None, direction if isinstance(direction, torch.Tensor) else None] + ) + + # TODO: add support to kernel_size as tuple or integer + kernel_tuple = _unpack_2d_ks(kernel_size) + _check_kernel_size(kernel_size, 2) + + if not isinstance(angle, torch.Tensor): + angle = torch.tensor([angle], device=device, dtype=dtype) + + if angle.dim() == 0: + angle = angle[None] + + KORNIA_CHECK_SHAPE(angle, ["B"]) + + if not isinstance(direction, torch.Tensor): + direction = torch.tensor([direction], device=device, dtype=dtype) + + if direction.dim() == 0: + direction = direction[None] + + KORNIA_CHECK_SHAPE(direction, ["B"]) + KORNIA_CHECK( + direction.size(0) == angle.size(0), + f"direction and angle must have the same length. Got {direction} and {angle}.", + ) + + # direction from [-1, 1] to [0, 1] range + direction = (torch.clamp(direction, -1.0, 1.0) + 1.0) / 2.0 + # kernel = torch.zeros((direction.size(0), *kernel_tuple), device=device, dtype=dtype) + + # Element-wise linspace + # kernel[:, kernel_size // 2, :] = torch.stack( + # [(direction + ((1 - 2 * direction) / (kernel_size - 1)) * i) for i in range(kernel_size)], dim=-1) + # Alternatively + # m = ((1 - 2 * direction)[:, None].repeat(1, kernel_size) / (kernel_size - 1)) + # kernel[:, kernel_size // 2, :] = direction[:, None].repeat(1, kernel_size) + m * torch.arange(0, kernel_size) + k = torch.stack([(direction + ((1 - 2 * direction) / (kernel_size - 1)) * i) for i in range(kernel_size)], -1) + kernel = F.pad(k[:, None], [0, 0, kernel_size // 2, kernel_size // 2, 0, 0]) + + expected_shape = torch.Size([direction.size(0), *kernel_tuple]) + KORNIA_CHECK(kernel.shape == expected_shape, f"Kernel shape should be {expected_shape}. Gotcha {kernel.shape}") + kernel = kernel[:, None, ...] + + # rotate (counterclockwise) kernel by given angle + kernel = rotate(kernel, angle, mode=mode, align_corners=True) + kernel = kernel[:, 0] + kernel = kernel / kernel.sum(dim=(1, 2), keepdim=True) + return kernel + + +def get_motion_kernel3d( + kernel_size: int, + angle: torch.Tensor | tuple[float, float, float], + direction: torch.Tensor | float = 0.0, + mode: str = "nearest", +) -> torch.Tensor: + r"""Return 3D motion blur filter. + + Args: + kernel_size: motion kernel width, height and depth. It should be odd and positive. + angle: Range of yaw (x-axis), pitch (y-axis), roll (z-axis) to select from. + If tensor, it must be :math:`(B, 3)`. + If tuple, it must be (yaw, pitch, raw). + direction: forward/backward direction of the motion blur. + Lower values towards -1.0 will point the motion blur towards the back (with angle provided via angle), + while higher values towards 1.0 will point the motion blur forward. A value of 0.0 leads to a + uniformly (but still angled) motion blur. + mode: interpolation mode for rotating the kernel. ``'bilinear'`` or ``'nearest'``. + + Returns: + The motion blur kernel with shape :math:`(B, k_\text{size}, k_\text{size}, k_\text{size})`. + + Examples: + >>> get_motion_kernel3d(3, (0., 0., 0.), 0.) + tensor([[[[0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000]], + + [[0.0000, 0.0000, 0.0000], + [0.3333, 0.3333, 0.3333], + [0.0000, 0.0000, 0.0000]], + + [[0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000]]]]) + + >>> get_motion_kernel3d(3, (90., 90., 0.), -0.5) + tensor([[[[0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000], + [0.0000, 0.5000, 0.0000]], + + [[0.0000, 0.0000, 0.0000], + [0.0000, 0.3333, 0.0000], + [0.0000, 0.0000, 0.0000]], + + [[0.0000, 0.1667, 0.0000], + [0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000]]]]) + + """ + device, dtype = _extract_device_dtype( + [angle if isinstance(angle, torch.Tensor) else None, direction if isinstance(direction, torch.Tensor) else None] + ) + + # TODO: add support to kernel_size as tuple or integer + kernel_tuple = _unpack_3d_ks(kernel_size) + _check_kernel_size(kernel_size, 2) + + if not isinstance(angle, torch.Tensor): + angle = torch.tensor([angle], device=device, dtype=dtype) + + if angle.dim() == 1: + angle = angle[None] + + KORNIA_CHECK_SHAPE(angle, ["B", "3"]) + + if not isinstance(direction, torch.Tensor): + direction = torch.tensor([direction], device=device, dtype=dtype) + + if direction.dim() == 0: + direction = direction[None] + + KORNIA_CHECK_SHAPE(direction, ["B"]) + KORNIA_CHECK( + direction.size(0) == angle.size(0), + f"direction and angle must have the same batch size. Got {direction.shape} and {angle.shape}.", + ) + + # direction from [-1, 1] to [0, 1] range + direction = (torch.clamp(direction, -1.0, 1.0) + 1.0) / 2.0 + kernel = torch.zeros((direction.size(0), *kernel_tuple), device=device, dtype=dtype) + + # Element-wise linspace + # kernel[:, kernel_size // 2, kernel_size // 2, :] = torch.stack( + # [(direction + ((1 - 2 * direction) / (kernel_size - 1)) * i) for i in range(kernel_size)], dim=-1) + k = torch.stack([(direction + ((1 - 2 * direction) / (kernel_size - 1)) * i) for i in range(kernel_size)], -1) + kernel = F.pad( + k[:, None, None], [0, 0, kernel_size // 2, kernel_size // 2, kernel_size // 2, kernel_size // 2, 0, 0] + ) + + expected_shape = torch.Size([direction.size(0), *kernel_tuple]) + KORNIA_CHECK(kernel.shape == expected_shape, f"Kernel shape should be {expected_shape}. Gotcha {kernel.shape}") + kernel = kernel[:, None, ...] + + # rotate (counterclockwise) kernel by given angle + kernel = rotate3d(kernel, angle[:, 0], angle[:, 1], angle[:, 2], mode=mode, align_corners=True) + kernel = kernel[:, 0] + kernel = kernel / kernel.sum(dim=(1, 2, 3), keepdim=True) + + return kernel diff --git a/kornia/filters/laplacian.py b/kornia/filters/laplacian.py new file mode 100644 index 0000000..517ef0e --- /dev/null +++ b/kornia/filters/laplacian.py @@ -0,0 +1,126 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +from torch import nn + +from .filter import filter2d +from .kernels import get_laplacian_kernel2d, normalize_kernel2d + + +def laplacian( + input: torch.Tensor, kernel_size: tuple[int, int] | int, border_type: str = "reflect", normalized: bool = True +) -> torch.Tensor: + r"""Create an operator that returns a tensor using a Laplacian filter. + + .. image:: _static/img/laplacian.png + + The operator smooths the given tensor with a laplacian kernel by convolving + it to each channel. It supports batched operation. + + Args: + input: the input image tensor with shape :math:`(B, C, H, W)`. + kernel_size: the size of the kernel. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. + normalized: if True, L1 norm of the kernel is set to 1. + + Return: + the blurred image with shape :math:`(B, C, H, W)`. + + .. note:: + See a working example `here `__. + + Examples: + >>> input = torch.rand(2, 4, 5, 5) + >>> output = laplacian(input, 3) + >>> output.shape + torch.Size([2, 4, 5, 5]) + + """ + kernel = get_laplacian_kernel2d(kernel_size, device=input.device, dtype=input.dtype)[None, ...] + + if normalized: + kernel = normalize_kernel2d(kernel) + + return filter2d(input, kernel, border_type) + + +class Laplacian(nn.Module): + r"""Create an operator that returns a tensor using a Laplacian filter. + + The operator smooths the given tensor with a laplacian kernel by convolving + it to each channel. It supports batched operation. + + Args: + kernel_size: the size of the kernel. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. + normalized: if True, L1 norm of the kernel is set to 1. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + Examples: + >>> input = torch.rand(2, 4, 5, 5) + >>> laplace = Laplacian(5) + >>> output = laplace(input) + >>> output.shape + torch.Size([2, 4, 5, 5]) + + """ + + def __init__( + self, kernel_size: tuple[int, int] | int, border_type: str = "reflect", normalized: bool = True + ) -> None: + super().__init__() + self.kernel_size = kernel_size + self.border_type: str = border_type + self.normalized: bool = normalized + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}" + f"(kernel_size={self.kernel_size}, " + f"normalized={self.normalized}, " + f"border_type={self.border_type})" + ) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Compute the second-order Laplacian response of an image tensor. + + The Laplacian filter measures rapid local intensity changes by + combining second derivatives along the spatial axes. It is commonly + used for edge detection, focus measures, and highlighting fine image + detail. + + Args: + input: Image tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is the batch size, :math:`C` is the number of + channels, :math:`H` is the height, and :math:`W` is the width. + + Returns: + Tensor with shape :math:`(B, C, H, W)` containing the Laplacian + response for each batch item and channel. Positive and negative + values represent opposite directions of local curvature. + """ + return laplacian(input, self.kernel_size, self.border_type, self.normalized) diff --git a/kornia/filters/median.py b/kornia/filters/median.py new file mode 100644 index 0000000..ee26c7b --- /dev/null +++ b/kornia/filters/median.py @@ -0,0 +1,119 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE + +from .kernels import _unpack_2d_ks, get_binary_kernel2d + + +def _compute_zero_padding(kernel_size: tuple[int, int] | int) -> tuple[int, int]: + r"""Compute zero padding tuple.""" + ky, kx = _unpack_2d_ks(kernel_size) + return (ky - 1) // 2, (kx - 1) // 2 + + +def median_blur(input: torch.Tensor, kernel_size: tuple[int, int] | int) -> torch.Tensor: + r"""Blur an image using the median filter. + + .. image:: _static/img/median_blur.png + + Args: + input: the input image with shape :math:`(B,C,H,W)`. + kernel_size: the blurring kernel size. + + Returns: + the blurred input torch.Tensor with shape :math:`(B,C,H,W)`. + + .. note:: + See a working example `here `__. + + Example: + >>> input = torch.rand(2, 4, 5, 7) + >>> output = median_blur(input, (3, 3)) + >>> output.shape + torch.Size([2, 4, 5, 7]) + + """ + KORNIA_CHECK_IS_TENSOR(input) + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + + ky, kx = _unpack_2d_ks(kernel_size) + padding = _compute_zero_padding(kernel_size) + + # prepare kernel + kernel: torch.Tensor = get_binary_kernel2d(kernel_size, device=input.device, dtype=input.dtype) + b, c, h, w = input.shape + + # map the local window to single vector + features: torch.Tensor = F.conv2d(input.reshape(b * c, 1, h, w), kernel, padding=padding, stride=1) + features = features.view(b, c, ky * kx, h, w) # BxCx(K_h * K_w)xHxW + + # compute the median along the feature axis + return features.median(dim=2)[0] + + +class MedianBlur(nn.Module): + r"""Blur an image using the median filter. + + Args: + kernel_size: the blurring kernel size. + + Returns: + the blurred input torch.Tensor. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + Example: + >>> input = torch.rand(2, 4, 5, 7) + >>> blur = MedianBlur((3, 3)) + >>> output = blur(input) + >>> output.shape + torch.Size([2, 4, 5, 7]) + + """ + + def __init__(self, kernel_size: tuple[int, int] | int) -> None: + super().__init__() + self.kernel_size = kernel_size + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Replace each pixel with the median value in its local window. + + Median filtering is a non-linear smoothing operation. Instead of + averaging neighboring values, it sorts the values inside the kernel + window and chooses the middle value. This makes it effective for + reducing impulse-like noise while preserving sharper boundaries than a + simple mean filter. + + Args: + input: Image or feature tensor with shape :math:`(B, C, H, W)`, + where :math:`B` is the batch size, :math:`C` is the number of + channels, :math:`H` is the height, and :math:`W` is the width. + + Returns: + Tensor with shape :math:`(B, C, H, W)` containing the median- + filtered result for each channel independently. + """ + return median_blur(input, self.kernel_size) diff --git a/kornia/filters/motion.py b/kornia/filters/motion.py new file mode 100644 index 0000000..8f68c3b --- /dev/null +++ b/kornia/filters/motion.py @@ -0,0 +1,264 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import ClassVar + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK + +from .filter import filter2d, filter3d +from .kernels_geometry import get_motion_kernel2d, get_motion_kernel3d + +_VALID_BORDER = {"constant", "reflect", "replicate", "circular"} + + +class MotionBlur(nn.Module): + r"""Blur 2D images (4D torch.Tensor) using the motion filter. + + Args: + kernel_size: motion kernel width and height. It should be odd and positive. + angle: angle of the motion blur in degrees (anti-clockwise rotation). + direction: forward/backward direction of the motion blur. + Lower values towards -1.0 will point the motion blur towards the back (with angle provided via angle), + while higher values towards 1.0 will point the motion blur forward. A value of 0.0 leads to a + uniformly (but still angled) motion blur. + border_type: the padding mode to be applied before convolving. The expected modes are: + ``'constant'``, ``'reflect'``, ``'replicate'`` or ``'circular'``. + mode: interpolation mode for rotating the kernel. ``'bilinear'`` or ``'nearest'``. + + Returns: + the blurred input torch.Tensor. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + Examples: + >>> input = torch.rand(2, 4, 5, 7) + >>> motion_blur = MotionBlur(3, 35., 0.5) + >>> output = motion_blur(input) # 2x4x5x7 + + """ + + def __init__( + self, kernel_size: int, angle: float, direction: float, border_type: str = "constant", mode: str = "nearest" + ) -> None: + super().__init__() + self.kernel_size = kernel_size + self.angle = angle + self.direction = direction + self.border_type = border_type + self.mode = mode + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__} (kernel_size={self.kernel_size}, " + f"angle={self.angle}, direction={self.direction}, border_type={self.border_type})" + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Blur an image along a configured two-dimensional motion direction. + + The motion-blur kernel simulates linear camera or object movement in + the image plane. ``self.angle`` controls the direction of the blur, and + ``self.direction`` controls whether the kernel is centered, forward- + biased, or backward-biased along that line. + + Args: + x: Image tensor with shape :math:`(B, C, H, W)`, where :math:`B` + is the batch size, :math:`C` is the number of channels, + :math:`H` is the height, and :math:`W` is the width. + + Returns: + Tensor with shape :math:`(B, C, H, W)` containing the directional + blur response. The output uses the same batch, channel, and spatial + layout as ``x``. + """ + return motion_blur(x, self.kernel_size, self.angle, self.direction, self.border_type) + + +class MotionBlur3D(nn.Module): + r"""Blur 3D volumes (5D torch.Tensor) using the motion filter. + + Args: + kernel_size: motion kernel width and height. It should be odd and positive. + angle: Range of yaw (x-axis), pitch (y-axis), roll (z-axis) to select from. + direction: forward/backward direction of the motion blur. + Lower values towards -1.0 will point the motion blur towards the back (with angle provided via angle), + while higher values towards 1.0 will point the motion blur forward. A value of 0.0 leads to a + uniformly (but still angled) motion blur. + border_type: the padding mode to be applied before convolving. The expected modes are: + ``'constant'``, ``'reflect'``, ``'replicate'`` or ``'circular'``. + mode: interpolation mode for rotating the kernel. ``'bilinear'`` or ``'nearest'``. + + Returns: + the blurred input torch.Tensor. + + Shape: + - Input: :math:`(B, C, D, H, W)` + - Output: :math:`(B, C, D, H, W)` + + Examples: + >>> input = torch.rand(2, 4, 5, 7, 9) + >>> motion_blur = MotionBlur3D(3, 35., 0.5) + >>> output = motion_blur(input) # 2x4x5x7x9 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, -1, -1, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, -1, -1, -1, -1] + ONNX_EXPORT_PSEUDO_SHAPE: ClassVar[list[int]] = [1, 3, 80, 80, 80] + + def __init__( + self, + kernel_size: int, + angle: float | tuple[float, float, float] | torch.Tensor, + direction: float | torch.Tensor, + border_type: str = "constant", + mode: str = "nearest", + ) -> None: + super().__init__() + self.kernel_size = kernel_size + KORNIA_CHECK( + isinstance(angle, (torch.Tensor, float, list, tuple)), + f"Angle should be a torch.Tensor, float or a sequence of floats. Got {angle}", + ) + if isinstance(angle, float): + self.angle = (angle, angle, angle) + elif isinstance(angle, (tuple, list)) and len(angle) == 3: + self.angle = (angle[0], angle[1], angle[2]) + + self.direction = direction + self.border_type = border_type + self.mode = mode + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__} (kernel_size={self.kernel_size}, " + f"angle={self.angle}, direction={self.direction}, border_type={self.border_type})" + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Blur a volume along a configured three-dimensional motion direction. + + This module extends motion blur from images to volumetric tensors. The + configured kernel is applied through depth, height, and width so that + movement can be modeled in three spatial dimensions. + + Args: + x: Volume tensor with shape :math:`(B, C, D, H, W)`, where + :math:`B` is the batch size, :math:`C` is the channel count, + :math:`D` is the depth, :math:`H` is the height, and + :math:`W` is the width. + + Returns: + Tensor with shape :math:`(B, C, D, H, W)` containing the blurred + volume. The output keeps the same dimensional order as ``x``. + """ + return motion_blur3d(x, self.kernel_size, self.angle, self.direction, self.border_type) + + +def motion_blur( + input: torch.Tensor, + kernel_size: int, + angle: float | torch.Tensor, + direction: float | torch.Tensor, + border_type: str = "constant", + mode: str = "nearest", +) -> torch.Tensor: + r"""Perform motion blur on torch.Tensor images. + + .. image:: _static/img/motion_blur.png + + Args: + input: the input torch.Tensor with shape :math:`(B, C, H, W)`. + kernel_size: motion kernel width and height. It should be odd and positive. + angle (Union[torch.Tensor, float]): angle of the motion blur in degrees (anti-clockwise rotation). + If torch.Tensor, it must be :math:`(B,)`. + direction : forward/backward direction of the motion blur. + Lower values towards -1.0 will point the motion blur towards the back (with angle provided via angle), + while higher values towards 1.0 will point the motion blur forward. A value of 0.0 leads to a + uniformly (but still angled) motion blur. + If torch.Tensor, it must be :math:`(B,)`. + border_type: the padding mode to be applied before convolving. The expected modes are: + ``'constant'``, ``'reflect'``, ``'replicate'`` or ``'circular'``. Default: ``'constant'``. + mode: interpolation mode for rotating the kernel. ``'bilinear'`` or ``'nearest'``. + + Return: + the blurred image with shape :math:`(B, C, H, W)`. + + Example: + >>> input = torch.randn(1, 3, 80, 90).repeat(2, 1, 1, 1) + >>> # perform exact motion blur across the batch + >>> out_1 = motion_blur(input, 5, 90., 1) + >>> torch.allclose(out_1[0], out_1[1]) + True + >>> # perform element-wise motion blur across the batch + >>> out_1 = motion_blur(input, 5, torch.tensor([90., 180,]), torch.tensor([1., -1.])) + >>> torch.allclose(out_1[0], out_1[1]) + False + + """ + kernel = get_motion_kernel2d(kernel_size, angle, direction, mode) + return filter2d(input, kernel, border_type) + + +def motion_blur3d( + input: torch.Tensor, + kernel_size: int, + angle: tuple[float, float, float] | torch.Tensor, + direction: float | torch.Tensor, + border_type: str = "constant", + mode: str = "nearest", +) -> torch.Tensor: + r"""Perform motion blur on 3D volumes (5D torch.Tensor). + + Args: + input: the input torch.Tensor with shape :math:`(B, C, D, H, W)`. + kernel_size: motion kernel width, height and depth. It should be odd and positive. + angle: Range of yaw (x-axis), pitch (y-axis), roll (z-axis) to select from. + If torch.Tensor, it must be :math:`(B, 3)`. + direction: forward/backward direction of the motion blur. + Lower values towards -1.0 will point the motion blur towards the back (with angle provided via angle), + while higher values towards 1.0 will point the motion blur forward. A value of 0.0 leads to a + uniformly (but still angled) motion blur. + If torch.Tensor, it must be :math:`(B,)`. + border_type: the padding mode to be applied before convolving. The expected modes are: + ``'constant'``, ``'reflect'``, ``'replicate'`` or ``'circular'``. Default: ``'constant'``. + mode: interpolation mode for rotating the kernel. ``'bilinear'`` or ``'nearest'``. + + Return: + the blurred image with shape :math:`(B, C, D, H, W)`. + + Example: + >>> input = torch.randn(1, 3, 120, 80, 90).repeat(2, 1, 1, 1, 1) + >>> # perform exact motion blur across the batch + >>> out_1 = motion_blur3d(input, 5, (0., 90., 90.), 1) + >>> torch.allclose(out_1[0], out_1[1]) + True + >>> # perform element-wise motion blur across the batch + >>> out_1 = motion_blur3d(input, 5, torch.tensor([[0., 90., 90.], [90., 180., 0.]]), torch.tensor([1., -1.])) + >>> torch.allclose(out_1[0], out_1[1]) + False + + """ + kernel = get_motion_kernel3d(kernel_size, angle, direction, mode) + return filter3d(input, kernel, border_type) diff --git a/kornia/filters/otsu_thresholding.py b/kornia/filters/otsu_thresholding.py new file mode 100644 index 0000000..e8c3008 --- /dev/null +++ b/kornia/filters/otsu_thresholding.py @@ -0,0 +1,230 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional, Tuple + +import torch + +from kornia.core.check import KORNIA_CHECK +from kornia.core.utils import _torch_histc_cast +from kornia.enhance.histogram import histogram as diff_histogram + + +class OtsuThreshold(torch.nn.Module): + """Otsu thresholding module for PyTorch tensors.""" + + def __init__(self) -> None: + """Initialize the OtsuThreshold module.""" + super().__init__() + + @staticmethod + def __histogram(xs: torch.Tensor, bins: int, diff: bool = False) -> Tuple[torch.Tensor, torch.Tensor]: + """Compute a histogram for each row of xs, CUDA compatible. + + Args: + xs (torch.Tensor): 2D tensor (n, N) with values to histogram. + bins (int): Number of bins. + diff: denote if the differentiable histagram will be used. Default: False + + Returns: + Tuple[torch.Tensor, torch.Tensor]: Normalized histograms and bin edges. + """ + # Ensure input is float for histogram computation if it's integer type + # For torch.histc, input should be floating point or quantized. + if xs.dtype in [torch.uint8, torch.int8, torch.int16, torch.int32, torch.int64]: + xs = xs.to(torch.float32) + + min_val = xs.min() + max_val = xs.max() + + histograms = [] + bin_edges = torch.linspace(min_val.item(), max_val.item(), bins, device=xs.device) + + for i in range(xs.shape[0]): + if diff: + hist = diff_histogram(xs[i].view(1, -1), bin_edges, torch.tensor(0.001)).squeeze() + else: + # Use torch.histc for non-differentiable histogram + # Note: torch.histogram is in PyTorch 1.10+, and should replace histc in future versions when + # no longer supporting older pytorch versions. + hist = _torch_histc_cast(xs[i], bins=bins, min=min_val.item(), max=max_val.item()) + + # Normalize and append the histogram + histograms.append(hist / hist.sum()) + + return torch.stack(histograms), bin_edges + + def transform_input( + self, x: torch.Tensor, original_shape: Optional[torch.Size] = None + ) -> Tuple[torch.Tensor, torch.Size]: + """Flatten the input to make it compatible with threshold computation. + + Args: + x (torch.Tensor): Image or batch of images. + original_shape (Optional[torch.Size]): Shape to preserve. + + Returns: + Tuple[torch.Tensor, torch.Size]: Flattened tensor, original shape. + """ + if original_shape is None: + original_shape = x.shape + dimensionality: int = x.dim() + + if dimensionality <= 2: + return x.flatten().unsqueeze(0), original_shape + elif dimensionality == 3: + return x.flatten(start_dim=1), original_shape + elif dimensionality == 4: + b, c, h, w = x.shape + return self.transform_input(x.reshape(b * c, h, w), original_shape=original_shape) + elif dimensionality == 5: + f, b, c, h, w = x.shape + return self.transform_input(x.reshape(f * b * c, h, w), original_shape=original_shape) + else: + raise ValueError(f"Unsupported tensor dimensionality: {dimensionality}") + + def forward( + self, x: torch.Tensor, nbins: int = 256, slow_and_differentiable: bool = False + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Apply Otsu thresholding to the input x. + + Args: + x (torch.Tensor): Image or batch of images to threshold. + nbins (int, optional): Number of bins for histogram computation. Default is 256. + slow_and_differentiable (bool, optional): If True, use a differentiable histogram computation. + Default is False. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: Thresholded tensor, threshold values. + """ + # Flatten input and store original shape + x_flattened, orig_shape = self.transform_input(x) + nchannel = x_flattened.shape[0] + + # Check tensor type compatibility + KORNIA_CHECK( + x.dtype + in [ + torch.uint8, + torch.int8, + torch.int16, + torch.int32, + torch.int64, + torch.float32, + torch.float64, + torch.float16, + torch.bfloat16, + ], + "Tensor dtype not supported for Otsu thresholding.", + ) + + # Compute histogram and bin edges + histograms, bin_edges = self.__histogram(x_flattened, bins=nbins, diff=slow_and_differentiable) + + # Initialize thresholds + best_thresholds = torch.zeros(nchannel, device=x.device, dtype=x.dtype) + + # Vectorized computation of optimal thresholds + bin_values = torch.arange(nbins, device=histograms.device, dtype=torch.float32) + total_weight = torch.sum(histograms, dim=1) # Shape: (nchannel,) + total_sum = torch.sum(histograms * bin_values, dim=1) # Shape: (nchannel,) + cumsum_weight = torch.cumsum(histograms, dim=1) # Shape: (nchannel, nbins) + cumsum_sum = torch.cumsum(histograms * bin_values, dim=1) # Shape: (nchannel, nbins) + + # Compute weights and sums for background and foreground + weight_bg = cumsum_weight[:, :-1] # Shape: (nchannel, nbins-1) + sum_bg = cumsum_sum[:, :-1] # Shape: (nchannel, nbins-1) + weight_fg = total_weight[:, None] - weight_bg # Shape: (nchannel, nbins-1) + sum_fg = total_sum[:, None] - sum_bg # Shape: (nchannel, nbins-1) + + # Compute means, avoiding division by zero + mean_bg = torch.where(weight_bg > 0, sum_bg / weight_bg, torch.tensor(0.0, device=histograms.device)) + mean_fg = torch.where(weight_fg > 0, sum_fg / weight_fg, torch.tensor(0.0, device=histograms.device)) + + # Compute inter-class variance, setting invalid cases to -1 + valid = (weight_bg > 0) & (weight_fg > 0) + inter_class_var = torch.where( + valid, weight_bg * weight_fg * (mean_bg - mean_fg) ** 2, torch.tensor(-1.0, device=histograms.device) + ) + + # Find the maximum inter-class variance and corresponding threshold + t_max = torch.argmax(inter_class_var, dim=1) # Shape: (nchannel,) + max_var = inter_class_var.gather(1, t_max[:, None]).squeeze(1) # Shape: (nchannel,) + best_thresholds = torch.where( + max_var > 0, bin_edges[t_max + 1], torch.tensor(0.0, device=histograms.device) + ).to(x.dtype) + + # Apply thresholding: keep values strictly greater than the threshold + thresholded = (x_flattened > best_thresholds[:, None]).to(x.dtype) * x_flattened + thresholded = thresholded.reshape(orig_shape) + + return thresholded, best_thresholds + + +def otsu_threshold( + x: torch.Tensor, + nbins: int = 256, + slow_and_differentiable: bool = False, + return_mask: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Apply automatic image thresholding using Otsu algorithm to the input tensor. + + Args: + x (Tensor): Input tensor (image or batch of images). + nbins (int): Number of bins for histogram computation, default is 256. + slow_and_differentiable (bool): If True, use a differentiable histogram computation. Default is False. + return_mask (bool): If True, return a binary mask indicating the thresholded pixels. If False, + return the thresholded image. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: Thresholded tensor and the computed threshold values. + + Raises: + ValueError: If the input tensor has unsupported dimensionality or dtype. + + .. note:: + - The input tensor can be of various types, but float types are preferred for accuracy + in histogram computation, especially on CPU. Integer types will be cast to float. + - If `use_thresh` is True, the threshold must have been computed previously and set in the module. + - If `threshold` is provided, it overrides the computed threshold. + + .. note:: + You may found more information about the Otsu algorithm here: https://en.wikipedia.org/wiki/Otsu's_method + + Example: + >>> import torch + >>> from kornia.filters.otsu_thresholding import otsu_threshold + >>> x = torch.tensor([[10, 20, 30], [40, 50, 60], [70, 80, 90]]) + >>> x + tensor([[10, 20, 30], + [40, 50, 60], + [70, 80, 90]]) + >>> otsu_threshold(x) + (tensor([[ 0, 0, 0], + [ 0, 50, 60], + [70, 80, 90]]), tensor([40])) + """ + module = OtsuThreshold() + + result, threshold = module(x, nbins=nbins, slow_and_differentiable=slow_and_differentiable) + + if return_mask: + return result > 0, threshold + + return result, threshold diff --git a/kornia/filters/sobel.py b/kornia/filters/sobel.py new file mode 100644 index 0000000..9627c86 --- /dev/null +++ b/kornia/filters/sobel.py @@ -0,0 +1,335 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import ClassVar + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE + +from .kernels import get_spatial_gradient_kernel2d, get_spatial_gradient_kernel3d, normalize_kernel2d + + +def spatial_gradient(input: torch.Tensor, mode: str = "sobel", order: int = 1, normalized: bool = True) -> torch.Tensor: + r"""Compute the first order image derivative in both x and y using a Sobel operator. + + .. image:: _static/img/spatial_gradient.png + + Args: + input: input image torch.Tensor with shape :math:`(B, C, H, W)`. + mode: derivatives modality, can be: `sobel` or `diff`. + order: the order of the derivatives. + normalized: whether the output is normalized. + + Return: + the derivatives of the input feature map. with shape :math:`(B, C, 2, H, W)`. + + .. note:: + See a working example `here `__. + + Examples: + >>> input = torch.rand(1, 3, 4, 4) + >>> output = spatial_gradient(input) # 1x3x2x4x4 + >>> output.shape + torch.Size([1, 3, 2, 4, 4]) + + """ + KORNIA_CHECK_IS_TENSOR(input) + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + + # allocate kernel + kernel = get_spatial_gradient_kernel2d(mode, order, device=input.device, dtype=input.dtype) + if normalized: + kernel = normalize_kernel2d(kernel) + + # prepare kernel + b, c, h, w = input.shape + tmp_kernel = kernel[:, None, ...] + + # Pad with "replicate for spatial dims, but with torch.zeros for channel + spatial_pad = [kernel.size(1) // 2, kernel.size(1) // 2, kernel.size(2) // 2, kernel.size(2) // 2] + out_channels: int = 3 if order == 2 else 2 + padded_inp: torch.Tensor = F.pad(input.reshape(b * c, 1, h, w), spatial_pad, "replicate") + out = F.conv2d(padded_inp, tmp_kernel, groups=1, padding=0, stride=1) + return out.reshape(b, c, out_channels, h, w) + + +def spatial_gradient3d(input: torch.Tensor, mode: str = "diff", order: int = 1) -> torch.Tensor: + r"""Compute the first and second order volume derivative in x, y and d using a diff operator. + + Args: + input: input features torch.Tensor with shape :math:`(B, C, D, H, W)`. + mode: derivatives modality, can be: `sobel` or `diff`. + order: the order of the derivatives. + + Return: + the spatial gradients of the input feature map with shape math:`(B, C, 3, D, H, W)` + or :math:`(B, C, 6, D, H, W)`. + + Examples: + >>> input = torch.rand(1, 4, 2, 4, 4) + >>> output = spatial_gradient3d(input) + >>> output.shape + torch.Size([1, 4, 3, 2, 4, 4]) + + """ + KORNIA_CHECK_IS_TENSOR(input) + KORNIA_CHECK_SHAPE(input, ["B", "C", "D", "H", "W"]) + + b, c, d, h, w = input.shape + dev = input.device + dtype = input.dtype + if (mode == "diff") and (order == 1): + # we go for the special case implementation due to conv3d bad speed + x: torch.Tensor = F.pad(input, 6 * [1], "replicate") + center = slice(1, -1) + left = slice(0, -2) + right = slice(2, None) + out = torch.empty(b, c, 3, d, h, w, device=dev, dtype=dtype) + out[..., 0, :, :, :] = x[..., center, center, right] - x[..., center, center, left] + out[..., 1, :, :, :] = x[..., center, right, center] - x[..., center, left, center] + out[..., 2, :, :, :] = x[..., right, center, center] - x[..., left, center, center] + out = 0.5 * out + else: + # prepare kernel + # allocate kernel + kernel = get_spatial_gradient_kernel3d(mode, order, device=dev, dtype=dtype) + + tmp_kernel = kernel.repeat(c, 1, 1, 1, 1) + + # convolve input torch.Tensor with grad kernel + kernel_flip = tmp_kernel.flip(-3) + + # Pad with "replicate for spatial dims, but with torch.zeros for channel + spatial_pad = [ + kernel.size(2) // 2, + kernel.size(2) // 2, + kernel.size(3) // 2, + kernel.size(3) // 2, + kernel.size(4) // 2, + kernel.size(4) // 2, + ] + out_ch: int = 6 if order == 2 else 3 + out = F.conv3d(F.pad(input, spatial_pad, "replicate"), kernel_flip, padding=0, groups=c).view( + b, c, out_ch, d, h, w + ) + return out + + +def sobel(input: torch.Tensor, normalized: bool = True, eps: float = 1e-6) -> torch.Tensor: + r"""Compute the Sobel operator and returns the magnitude per channel. + + .. image:: _static/img/sobel.png + + Args: + input: the input image with shape :math:`(B,C,H,W)`. + normalized: if True, L1 norm of the kernel is set to 1. + eps: regularization number to avoid NaN during backprop. + + Return: + the sobel edge gradient magnitudes map with shape :math:`(B,C,H,W)`. + + .. note:: + See a working example `here `__. + + Example: + >>> input = torch.rand(1, 3, 4, 4) + >>> output = sobel(input) # 1x3x4x4 + >>> output.shape + torch.Size([1, 3, 4, 4]) + + """ + KORNIA_CHECK_IS_TENSOR(input) + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + + # comput the x/y gradients + edges: torch.Tensor = spatial_gradient(input, normalized=normalized) + + # unpack the edges + gx: torch.Tensor = edges[:, :, 0] + gy: torch.Tensor = edges[:, :, 1] + + # compute gradient maginitude + magnitude: torch.Tensor = torch.sqrt(gx * gx + gy * gy + eps) + + return magnitude + + +class SpatialGradient(nn.Module): + r"""Compute the first order image derivative in both x and y using a Sobel operator. + + Args: + mode: derivatives modality, can be: `sobel` or `diff`. + order: the order of the derivatives. + normalized: whether the output is normalized. + + Return: + the sobel edges of the input feature map. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output: :math:`(B, C, 2, H, W)` + + Examples: + >>> input = torch.rand(1, 3, 4, 4) + >>> output = SpatialGradient()(input) # 1x3x2x4x4 + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, -1, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, -1, 2, -1, -1] + + def __init__(self, mode: str = "sobel", order: int = 1, normalized: bool = True) -> None: + super().__init__() + self.normalized: bool = normalized + self.order: int = order + self.mode: str = mode + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(order={self.order}, normalized={self.normalized}, mode={self.mode})" + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Compute horizontal and vertical spatial derivatives. + + The module applies Sobel or difference kernels, depending on + ``self.mode``, to estimate image gradients along the width and height + axes. First-order derivatives describe local slope; second-order + derivatives describe how that slope changes. + + Args: + input: Image tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is the batch size, :math:`C` is the number of + channels, :math:`H` is the height, and :math:`W` is the width. + + Returns: + Gradient tensor from :func:`spatial_gradient`. For first-order + derivatives the shape is typically :math:`(B, C, 2, H, W)`, where + the derivative axis of size ``2`` stores the response along + :math:`x` (width) and :math:`y` (height). Higher orders may expose + additional derivative components according to the functional API. + """ + return spatial_gradient(input, self.mode, self.order, self.normalized) + + +class SpatialGradient3d(nn.Module): + r"""Compute the first and second order volume derivative in x, y and d using a diff operator. + + Args: + mode: derivatives modality, can be: `sobel` or `diff`. + order: the order of the derivatives. + + Return: + the spatial gradients of the input feature map. + + Shape: + - Input: :math:`(B, C, D, H, W)`. D, H, W are spatial dimensions, gradient is calculated w.r.t to them. + - Output: :math:`(B, C, 3, D, H, W)` or :math:`(B, C, 6, D, H, W)` + + Examples: + >>> input = torch.rand(1, 4, 2, 4, 4) + >>> output = SpatialGradient3d()(input) + >>> output.shape + torch.Size([1, 4, 3, 2, 4, 4]) + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, -1, -1, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, -1, -1, -1, -1, -1] + + def __init__(self, mode: str = "diff", order: int = 1) -> None: + super().__init__() + self.order: int = order + self.mode: str = mode + self.kernel = get_spatial_gradient_kernel3d(mode, order) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(order={self.order}, mode={self.mode})" + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Compute spatial derivatives through depth, height, and width. + + This is the volumetric counterpart of :class:`SpatialGradient`. It + estimates changes along the three spatial axes of a five-dimensional + tensor, which is useful for 3D images, videos represented as volumes, + or feature maps with an explicit depth dimension. + + Args: + input: Volume tensor with shape :math:`(B, C, D, H, W)`, where + :math:`B` is the batch size, :math:`C` is the channel count, + :math:`D` is the depth, :math:`H` is the height, and + :math:`W` is the width. + + Returns: + Derivative tensor from :func:`spatial_gradient3d`. The output keeps + the input batch and channel dimensions and includes a derivative + axis for the spatial directions requested by ``self.order``. + """ + return spatial_gradient3d(input, self.mode, self.order) + + +class Sobel(nn.Module): + r"""Compute the Sobel operator and returns the magnitude per channel. + + Args: + normalized: if True, L1 norm of the kernel is set to 1. + eps: regularization number to avoid NaN during backprop. + + Return: + the sobel edge gradient magnitudes map. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + Examples: + >>> input = torch.rand(1, 3, 4, 4) + >>> output = Sobel()(input) # 1x3x4x4 + + """ + + def __init__(self, normalized: bool = True, eps: float = 1e-6) -> None: + super().__init__() + self.normalized: bool = normalized + self.eps: float = eps + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(normalized={self.normalized})" + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Compute the Sobel gradient magnitude for every image channel. + + Sobel filtering estimates horizontal and vertical derivatives and then + combines them into a magnitude response. Large values indicate strong + local intensity changes, which often correspond to edges or texture + boundaries. + + Args: + input: Image tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is the batch size, :math:`C` is the number of + channels, :math:`H` is the height, and :math:`W` is the width. + + Returns: + Tensor with shape :math:`(B, C, H, W)` containing the gradient + magnitude for each channel. The ``self.eps`` value is used by the + underlying function to keep magnitude computation numerically + stable near zero. + """ + return sobel(input, self.normalized, self.eps) diff --git a/kornia/filters/unsharp.py b/kornia/filters/unsharp.py new file mode 100644 index 0000000..be15190 --- /dev/null +++ b/kornia/filters/unsharp.py @@ -0,0 +1,117 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +from torch import nn + +from .gaussian import gaussian_blur2d + + +def unsharp_mask( + input: torch.Tensor, + kernel_size: tuple[int, int] | int, + sigma: tuple[float, float] | torch.Tensor, + border_type: str = "reflect", +) -> torch.Tensor: + r"""Create an operator that sharpens a torch.Tensor by applying operation out = 2 * image - gaussian_blur2d(image). + + .. image:: _static/img/unsharp_mask.png + + Args: + input: the input torch.Tensor with shape :math:`(B,C,H,W)`. + kernel_size: the size of the kernel. + sigma: the standard deviation of the kernel. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. + + Returns: + the blurred torch.Tensor with shape :math:`(B,C,H,W)`. + + Examples: + >>> input = torch.rand(2, 4, 5, 5) + >>> output = unsharp_mask(input, (3, 3), (1.5, 1.5)) + >>> output.shape + torch.Size([2, 4, 5, 5]) + + """ + data_blur: torch.Tensor = gaussian_blur2d(input, kernel_size, sigma, border_type) + return torch.lerp(data_blur, input, weight=2.0) + + +class UnsharpMask(nn.Module): + r"""Create an operator that sharpens image with: out = 2 * image - gaussian_blur2d(image). + + Args: + kernel_size: the size of the kernel. + sigma: the standard deviation of the kernel. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. + + Returns: + the sharpened torch.Tensor with shape :math:`(B,C,H,W)`. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output: :math:`(B, C, H, W)` + + .. note:: + See a working example `here `__. + + Examples: + >>> input = torch.rand(2, 4, 5, 5) + >>> sharpen = UnsharpMask((3, 3), (1.5, 1.5)) + >>> output = sharpen(input) + >>> output.shape + torch.Size([2, 4, 5, 5]) + + """ + + def __init__( + self, + kernel_size: tuple[int, int] | int, + sigma: tuple[float, float] | torch.Tensor, + border_type: str = "reflect", + ) -> None: + super().__init__() + self.kernel_size = kernel_size + self.sigma = sigma + self.border_type = border_type + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Sharpen an image by adding back high-frequency detail. + + Unsharp masking first builds a blurred version of the image and then + uses the difference between the original and blurred images as a detail + signal. Adding that detail back emphasizes edges and fine texture while + keeping the tensor layout unchanged. + + Args: + input: Image tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is the batch size, :math:`C` is the number of + channels, :math:`H` is the image height, and :math:`W` is the + image width. + + Returns: + Tensor with shape :math:`(B, C, H, W)` containing the sharpened + image. The amount and spatial scale of sharpening are controlled by + the configured Gaussian kernel size, sigma, and border handling. + """ + return unsharp_mask(input, self.kernel_size, self.sigma, self.border_type) diff --git a/kornia/geometry/__init__.py b/kornia/geometry/__init__.py new file mode 100644 index 0000000..923ed1a --- /dev/null +++ b/kornia/geometry/__init__.py @@ -0,0 +1,40 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Geometry — Geometric vision operations for Kornia. + +This subpackage provides modules for transformations, camera models, and geometry processing. +""" + +from .bbox import * +from .calibration import * +from .camera import * +from .conversions import * +from .depth import * +from .epipolar import * +from .grid import * +from .homography import * +from .liegroup import * +from .linalg import * +from .line import * +from .plane import * +from .pointcloud import * +from .pose import * +from .ransac import * +from .solvers import * +from .subpix import * +from .transform import * diff --git a/kornia/geometry/bbox.py b/kornia/geometry/bbox.py new file mode 100644 index 0000000..f0f0e15 --- /dev/null +++ b/kornia/geometry/bbox.py @@ -0,0 +1,589 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import warnings +from typing import Optional + +import torch + +from .linalg import transform_points + +__all__ = [ + "bbox_generator", + "bbox_generator3d", + "bbox_to_mask", + "bbox_to_mask3d", + "infer_bbox_shape", + "infer_bbox_shape3d", + "nms", + "transform_bbox", + "validate_bbox", + "validate_bbox3d", +] + + +def validate_bbox(boxes: torch.Tensor) -> bool: + """Validate if a 2D bounding box usable or not. This function checks if the boxes are rectangular or not. + + Args: + boxes: a tensor containing the coordinates of the bounding boxes to be extracted. The tensor must have the shape + of Bx4x2, where each box is defined in the following ``clockwise`` order: top-left, top-right, bottom-right, + bottom-left. The coordinates must be in the x, y order. + + """ + if not (len(boxes.shape) in [3, 4] and boxes.shape[-2:] == torch.Size([4, 2])): + return False + + if len(boxes.shape) == 4: + boxes = boxes.view(-1, 4, 2) + + x_tl, y_tl = boxes[..., 0, 0], boxes[..., 0, 1] + x_tr, y_tr = boxes[..., 1, 0], boxes[..., 1, 1] + x_br, y_br = boxes[..., 2, 0], boxes[..., 2, 1] + x_bl, y_bl = boxes[..., 3, 0], boxes[..., 3, 1] + + width_t, width_b = x_tr - x_tl + 1, x_br - x_bl + 1 + height_t, height_b = y_tr - y_tl + 1, y_br - y_bl + 1 + + # Replace torch.allclose with exportable operations + width_diff = torch.abs(width_t - width_b) + height_diff = torch.abs(height_t - height_b) + + # Check if differences are within tolerance (1e-4) + if torch.any(width_diff > 1e-4): + return False + + if torch.any(height_diff > 1e-4): + return False + + return True + + +def validate_bbox3d(boxes: torch.Tensor) -> bool: + """Validate if a 3D bounding box usable or not. This function checks if the boxes are cube or not. + + Args: + boxes: a tensor containing the coordinates of the bounding boxes to be extracted. The tensor must have the shape + of Bx8x3, where each box is defined in the following ``clockwise`` order: front-top-left, front-top-right, + front-bottom-right, front-bottom-left, back-top-left, back-top-right, back-bottom-right, back-bottom-left. + The coordinates must be in the x, y, z order. + + """ + if not (len(boxes.shape) in [3, 4] and boxes.shape[-2:] == torch.Size([8, 3])): + raise AssertionError(f"Box shape must be (B, 8, 3) or (B, N, 8, 3). Got {boxes.shape}.") + + if len(boxes.shape) == 4: + boxes = boxes.view(-1, 8, 3) + + left = torch.index_select(boxes, 1, torch.tensor([1, 2, 5, 6], device=boxes.device, dtype=torch.long))[:, :, 0] + right = torch.index_select(boxes, 1, torch.tensor([0, 3, 4, 7], device=boxes.device, dtype=torch.long))[:, :, 0] + widths = left - right + 1 + if not torch.allclose(widths.permute(1, 0), widths[:, 0]): + raise AssertionError(f"Boxes must have be cube, while get different widths {widths}.") + + bot = torch.index_select(boxes, 1, torch.tensor([2, 3, 6, 7], device=boxes.device, dtype=torch.long))[:, :, 1] + upper = torch.index_select(boxes, 1, torch.tensor([0, 1, 4, 5], device=boxes.device, dtype=torch.long))[:, :, 1] + heights = bot - upper + 1 + if not torch.allclose(heights.permute(1, 0), heights[:, 0]): + raise AssertionError(f"Boxes must have be cube, while get different heights {heights}.") + + depths = boxes[:, 4:, 2] - boxes[:, :4, 2] + 1 + if not torch.allclose(depths.permute(1, 0), depths[:, 0]): + raise AssertionError(f"Boxes must have be cube, while get different depths {depths}.") + + return True + + +def infer_bbox_shape(boxes: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + r"""Auto-infer the output sizes for the given 2D bounding boxes. + + Args: + boxes: a tensor containing the coordinates of the bounding boxes to be extracted. The tensor must have the shape + of Bx4x2, where each box is defined in the following ``clockwise`` order: top-left, top-right, bottom-right, + bottom-left. The coordinates must be in the x, y order. + + Returns: + - Bounding box heights, shape of :math:`(B,)`. + - Boundingbox widths, shape of :math:`(B,)`. + + Example: + >>> boxes = torch.tensor([[ + ... [1., 1.], + ... [2., 1.], + ... [2., 2.], + ... [1., 2.], + ... ], [ + ... [1., 1.], + ... [3., 1.], + ... [3., 2.], + ... [1., 2.], + ... ]]) # 2x4x2 + >>> infer_bbox_shape(boxes) + (tensor([2., 2.]), tensor([2., 3.])) + + """ + validate_bbox(boxes) + width: torch.Tensor = boxes[:, 1, 0] - boxes[:, 0, 0] + 1 + height: torch.Tensor = boxes[:, 2, 1] - boxes[:, 0, 1] + 1 + return height, width + + +def infer_bbox_shape3d(boxes: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + r"""Auto-infer the output sizes for the given 3D bounding boxes. + + Args: + boxes: a tensor containing the coordinates of the bounding boxes to be extracted. The tensor must have the shape + of Bx8x3, where each box is defined in the following ``clockwise`` order: front-top-left, front-top-right, + front-bottom-right, front-bottom-left, back-top-left, back-top-right, back-bottom-right, back-bottom-left. + The coordinates must be in the x, y, z order. + + Returns: + - Bounding box depths, shape of :math:`(B,)`. + - Bounding box heights, shape of :math:`(B,)`. + - Bounding box widths, shape of :math:`(B,)`. + + Example: + >>> boxes = torch.tensor([[[ 0, 1, 2], + ... [10, 1, 2], + ... [10, 21, 2], + ... [ 0, 21, 2], + ... [ 0, 1, 32], + ... [10, 1, 32], + ... [10, 21, 32], + ... [ 0, 21, 32]], + ... [[ 3, 4, 5], + ... [43, 4, 5], + ... [43, 54, 5], + ... [ 3, 54, 5], + ... [ 3, 4, 65], + ... [43, 4, 65], + ... [43, 54, 65], + ... [ 3, 54, 65]]]) # 2x8x3 + >>> infer_bbox_shape3d(boxes) + (tensor([31, 61]), tensor([21, 51]), tensor([11, 41])) + + """ + validate_bbox3d(boxes) + + left = torch.index_select(boxes, 1, torch.tensor([1, 2, 5, 6], device=boxes.device, dtype=torch.long))[:, :, 0] + right = torch.index_select(boxes, 1, torch.tensor([0, 3, 4, 7], device=boxes.device, dtype=torch.long))[:, :, 0] + widths = (left - right + 1)[:, 0] + + bot = torch.index_select(boxes, 1, torch.tensor([2, 3, 6, 7], device=boxes.device, dtype=torch.long))[:, :, 1] + upper = torch.index_select(boxes, 1, torch.tensor([0, 1, 4, 5], device=boxes.device, dtype=torch.long))[:, :, 1] + heights = (bot - upper + 1)[:, 0] + + depths = (boxes[:, 4:, 2] - boxes[:, :4, 2] + 1)[:, 0] + return depths, heights, widths + + +def bbox_to_mask(boxes: torch.Tensor, width: int, height: int) -> torch.Tensor: + """Convert 2D bounding boxes to masks. Covered area is 1. and the remaining is 0. + + Args: + boxes: a tensor containing the coordinates of the bounding boxes to be extracted. The tensor must have the shape + of Bx4x2, where each box is defined in the following ``clockwise`` order: top-left, top-right, bottom-right + and bottom-left. The coordinates must be in the x, y order. + width: width of the masked image. + height: height of the masked image. + + Returns: + the output mask tensor. + + Note: + It is currently non-differentiable. + + Examples: + >>> boxes = torch.tensor([[ + ... [1., 1.], + ... [3., 1.], + ... [3., 2.], + ... [1., 2.], + ... ]]) # 1x4x2 + >>> bbox_to_mask(boxes, 5, 5) + tensor([[[0., 0., 0., 0., 0.], + [0., 1., 1., 1., 0.], + [0., 1., 1., 1., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.]]]) + + """ + validate_bbox(boxes) + # zero padding the surroundings + yy = torch.arange(height, device=boxes.device, dtype=boxes.dtype).view(height, 1) + xx = torch.arange(width, device=boxes.device, dtype=boxes.dtype).view(1, width) + x_min = boxes[:, 0, 0].view(-1, 1, 1) + y_min = boxes[:, 0, 1].view(-1, 1, 1) + x_max = boxes[:, 2, 0].view(-1, 1, 1) + y_max = boxes[:, 2, 1].view(-1, 1, 1) + mask = (xx >= x_min) & (xx <= x_max) & (yy >= y_min) & (yy <= y_max) + return mask.to(boxes.dtype) + + +def bbox_to_mask3d(boxes: torch.Tensor, size: tuple[int, int, int]) -> torch.Tensor: + """Convert 3D bounding boxes to masks. Covered area is 1. and the remaining is 0. + + Args: + boxes: a tensor containing the coordinates of the bounding boxes to be extracted. The tensor must have the shape + of Bx8x3, where each box is defined in the following ``clockwise`` order: front-top-left, front-top-right, + front-bottom-right, front-bottom-left, back-top-left, back-top-right, back-bottom-right, back-bottom-left. + The coordinates must be in the x, y, z order. + size: depth, height and width of the masked image. + + Returns: + the output mask tensor. + + Examples: + >>> boxes = torch.tensor([[ + ... [1., 1., 1.], + ... [2., 1., 1.], + ... [2., 2., 1.], + ... [1., 2., 1.], + ... [1., 1., 2.], + ... [2., 1., 2.], + ... [2., 2., 2.], + ... [1., 2., 2.], + ... ]]) # 1x8x3 + >>> bbox_to_mask3d(boxes, (4, 5, 5)) + tensor([[[[[0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.]], + + [[0., 0., 0., 0., 0.], + [0., 1., 1., 0., 0.], + [0., 1., 1., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.]], + + [[0., 0., 0., 0., 0.], + [0., 1., 1., 0., 0.], + [0., 1., 1., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.]], + + [[0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.]]]]]) + + """ + validate_bbox3d(boxes) + D0, D1, D2 = size # get depth, height, width + + z_min = boxes[:, 0, 2].long() + z_max = boxes[:, 4, 2].long() + y_min = boxes[:, 1, 1].long() + y_max = boxes[:, 2, 1].long() + x_min = boxes[:, 0, 0].long() + x_max = boxes[:, 1, 0].long() + + z = torch.arange(D0, device=boxes.device, dtype=torch.long) + y = torch.arange(D1, device=boxes.device, dtype=torch.long) + x = torch.arange(D2, device=boxes.device, dtype=torch.long) + + # Compute mask as union of planes in one step + m = ( + ((z[None, :] >= z_min[:, None]) & (z[None, :] <= z_max[:, None]))[:, None, :, None, None] + | ((y[None, :] >= y_min[:, None]) & (y[None, :] <= y_max[:, None]))[:, None, None, :, None] + | ((x[None, :] >= x_min[:, None]) & (x[None, :] <= x_max[:, None]))[:, None, None, None, :] + ).float() # Shape: (N, 1, D0, D1, D2) + + # Compute conditions + cond1 = m.all(dim=3, keepdim=True).all(dim=2, keepdim=True) + cond2 = m.all(dim=4, keepdim=True).all(dim=2, keepdim=True) + cond3 = m.all(dim=3, keepdim=True).all(dim=4, keepdim=True) + + m_out = cond1 * cond2 * cond3 # Broadcasting to (N, 1, D0, D1, D2) + return m_out.float() + + +def bbox_generator( + x_start: torch.Tensor, y_start: torch.Tensor, width: torch.Tensor, height: torch.Tensor +) -> torch.Tensor: + """Generate 2D bounding boxes according to the provided start coords, width and height. + + Args: + x_start: a tensor containing the x coordinates of the bounding boxes to be extracted. Shape must be a scalar + tensor or :math:`(B,)`. + y_start: a tensor containing the y coordinates of the bounding boxes to be extracted. Shape must be a scalar + tensor or :math:`(B,)`. + width: widths of the masked image. Shape must be a scalar tensor or :math:`(B,)`. + height: heights of the masked image. Shape must be a scalar tensor or :math:`(B,)`. + + Returns: + the bounding box tensor. + + Examples: + >>> x_start = torch.tensor([0, 1]) + >>> y_start = torch.tensor([1, 0]) + >>> width = torch.tensor([5, 3]) + >>> height = torch.tensor([7, 4]) + >>> bbox_generator(x_start, y_start, width, height) + tensor([[[0, 1], + [4, 1], + [4, 7], + [0, 7]], + + [[1, 0], + [3, 0], + [3, 3], + [1, 3]]]) + + """ + if not (x_start.shape == y_start.shape and x_start.dim() in [0, 1]): + raise AssertionError(f"`x_start` and `y_start` must be a scalar or (B,). Got {x_start}, {y_start}.") + if not (width.shape == height.shape and width.dim() in [0, 1]): + raise AssertionError(f"`width` and `height` must be a scalar or (B,). Got {width}, {height}.") + if not x_start.dtype == y_start.dtype == width.dtype == height.dtype: + raise AssertionError( + "All tensors must be in the same dtype. Got " + f"`x_start`({x_start.dtype}), `y_start`({x_start.dtype}), `width`({width.dtype}), `height`({height.dtype})." + ) + if not x_start.device == y_start.device == width.device == height.device: + raise AssertionError( + "All tensors must be in the same device. Got " + f"`x_start`({x_start.device}), `y_start`({x_start.device}), " + f"`width`({width.device}), `height`({height.device})." + ) + + bbox = torch.tensor([[[0, 0], [0, 0], [0, 0], [0, 0]]], device=x_start.device, dtype=x_start.dtype).repeat( + 1 if x_start.dim() == 0 else len(x_start), 1, 1 + ) + + bbox[:, :, 0] += x_start.view(-1, 1) + bbox[:, :, 1] += y_start.view(-1, 1) + bbox[:, 1, 0] += width - 1 + bbox[:, 2, 0] += width - 1 + bbox[:, 2, 1] += height - 1 + bbox[:, 3, 1] += height - 1 + + return bbox + + +def bbox_generator3d( + x_start: torch.Tensor, + y_start: torch.Tensor, + z_start: torch.Tensor, + width: torch.Tensor, + height: torch.Tensor, + depth: torch.Tensor, +) -> torch.Tensor: + """Generate 3D bounding boxes according to the provided start coords, width, height and depth. + + Args: + x_start: a tensor containing the x coordinates of the bounding boxes to be extracted. Shape must be a scalar + tensor or :math:`(B,)`. + y_start: a tensor containing the y coordinates of the bounding boxes to be extracted. Shape must be a scalar + tensor or :math:`(B,)`. + z_start: a tensor containing the z coordinates of the bounding boxes to be extracted. Shape must be a scalar + tensor or :math:`(B,)`. + width: widths of the masked image. Shape must be a scalar tensor or :math:`(B,)`. + height: heights of the masked image. Shape must be a scalar tensor or :math:`(B,)`. + depth: depths of the masked image. Shape must be a scalar tensor or :math:`(B,)`. + + Returns: + the 3d bounding box tensor :math:`(B, 8, 3)`. + + Examples: + >>> x_start = torch.tensor([0, 3]) + >>> y_start = torch.tensor([1, 4]) + >>> z_start = torch.tensor([2, 5]) + >>> width = torch.tensor([10, 40]) + >>> height = torch.tensor([20, 50]) + >>> depth = torch.tensor([30, 60]) + >>> bbox_generator3d(x_start, y_start, z_start, width, height, depth) + tensor([[[ 0, 1, 2], + [10, 1, 2], + [10, 21, 2], + [ 0, 21, 2], + [ 0, 1, 32], + [10, 1, 32], + [10, 21, 32], + [ 0, 21, 32]], + + [[ 3, 4, 5], + [43, 4, 5], + [43, 54, 5], + [ 3, 54, 5], + [ 3, 4, 65], + [43, 4, 65], + [43, 54, 65], + [ 3, 54, 65]]]) + + """ + if not (x_start.shape == y_start.shape == z_start.shape and x_start.dim() in [0, 1]): + raise AssertionError( + f"`x_start`, `y_start` and `z_start` must be a scalar or (B,). Got {x_start}, {y_start}, {z_start}." + ) + if not (width.shape == height.shape == depth.shape and width.dim() in [0, 1]): + raise AssertionError(f"`width`, `height` and `depth` must be a scalar or (B,). Got {width}, {height}, {depth}.") + if not x_start.dtype == y_start.dtype == z_start.dtype == width.dtype == height.dtype == depth.dtype: + raise AssertionError( + "All tensors must be in the same dtype. " + f"Got `x_start`({x_start.dtype}), `y_start`({x_start.dtype}), `z_start`({x_start.dtype}), " + f"`width`({width.dtype}), `height`({height.dtype}) and `depth`({depth.dtype})." + ) + if not x_start.device == y_start.device == z_start.device == width.device == height.device == depth.device: + raise AssertionError( + "All tensors must be in the same device. " + f"Got `x_start`({x_start.device}), `y_start`({x_start.device}), `z_start`({x_start.device}), " + f"`width`({width.device}), `height`({height.device}) and `depth`({depth.device})." + ) + + # front + bbox = torch.tensor( + [[[0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0]]], device=x_start.device, dtype=x_start.dtype + ).repeat(len(x_start), 1, 1) + + bbox[:, :, 0] += x_start.view(-1, 1) + bbox[:, :, 1] += y_start.view(-1, 1) + bbox[:, :, 2] += z_start.view(-1, 1) + bbox[:, 1, 0] += width + bbox[:, 2, 0] += width + bbox[:, 2, 1] += height + bbox[:, 3, 1] += height + + # back + bbox_back = bbox.clone() + bbox_back[:, :, -1] += depth.unsqueeze(dim=1).repeat(1, 4) + bbox = torch.cat([bbox, bbox_back], dim=1) + + return bbox + + +def transform_bbox( + trans_mat: torch.Tensor, boxes: torch.Tensor, mode: str = "xyxy", restore_coordinates: Optional[bool] = None +) -> torch.Tensor: + r"""Apply a transformation matrix to a box or batch of boxes. + + Args: + trans_mat: The transformation matrix to be applied with a shape of :math:`(3, 3)` + or batched as :math:`(B, 3, 3)`. + boxes: The boxes to be transformed with a common shape of :math:`(N, 4)` or batched as :math:`(B, N, 4)`, the + polygon shape of :math:`(B, N, 4, 2)` is also supported. + mode: The format in which the boxes are provided. If set to 'xyxy' the boxes are assumed to be in the format + ``xmin, ymin, xmax, ymax``. If set to 'xywh' the boxes are assumed to be in the format + ``xmin, ymin, width, height`` + restore_coordinates: In case the boxes are flipped, adding a post processing step to restore the + coordinates to a valid bounding box. + + Returns: + The set of transformed points in the specified mode + + """ + if not isinstance(mode, str): + raise TypeError(f"Mode must be a string. Got {type(mode)}") + + if mode not in ("xyxy", "xywh"): + raise ValueError(f"Mode must be one of 'xyxy', 'xywh'. Got {mode}") + + # (B, N, 4, 2) shaped polygon boxes do not need to be restored. + if restore_coordinates is None and not (boxes.shape[-2:] == torch.Size([4, 2])): + warnings.warn( + "Previous behaviour produces incorrect box coordinates if a flip transformation performed on boxes." + "The previous wrong behaviour has been corrected and will be removed in the future versions." + "If you wish to keep the previous behaviour, please set `restore_coordinates=False`." + "Otherwise, set `restore_coordinates=True` as an acknowledgement.", + stacklevel=1, + ) + + # convert boxes to format xyxy + if mode == "xywh": + boxes[..., 2] = boxes[..., 0] + boxes[..., 2] # x + w + boxes[..., 3] = boxes[..., 1] + boxes[..., 3] # y + h + + transformed_boxes: torch.Tensor = transform_points(trans_mat, boxes.view(boxes.shape[0], -1, 2)) + transformed_boxes = transformed_boxes.view_as(boxes) + + if (restore_coordinates is None or restore_coordinates) and not (boxes.shape[-2:] == torch.Size([4, 2])): + restored_boxes = transformed_boxes.clone() + # In case the boxes are flipped, we ensure it is ordered like left-top -> right-bot points + restored_boxes[..., 0] = torch.min(transformed_boxes[..., [0, 2]], dim=-1)[0] + restored_boxes[..., 1] = torch.min(transformed_boxes[..., [1, 3]], dim=-1)[0] + restored_boxes[..., 2] = torch.max(transformed_boxes[..., [0, 2]], dim=-1)[0] + restored_boxes[..., 3] = torch.max(transformed_boxes[..., [1, 3]], dim=-1)[0] + transformed_boxes = restored_boxes + + if mode == "xywh": + transformed_boxes[..., 2] = transformed_boxes[..., 2] - transformed_boxes[..., 0] + transformed_boxes[..., 3] = transformed_boxes[..., 3] - transformed_boxes[..., 1] + + return transformed_boxes + + +def nms(boxes: torch.Tensor, scores: torch.Tensor, iou_threshold: float) -> torch.Tensor: + """Perform non-maxima suppression (NMS) on tensor of bounding boxes according to the intersection-over-union (IoU). + + Args: + boxes: tensor containing the encoded bounding boxes with the shape :math:`(N, (x_1, y_1, x_2, y_2))`. + scores: tensor containing the scores associated to each bounding box with shape :math:`(N,)`. + iou_threshold: the throshold to discard the overlapping boxes. + + Return: + A tensor mask with the indices to keep from the input set of boxes and scores. + + Example: + >>> boxes = torch.tensor([ + ... [10., 10., 20., 20.], + ... [15., 5., 15., 25.], + ... [100., 100., 200., 200.], + ... [100., 100., 200., 200.]]) + >>> scores = torch.tensor([0.9, 0.8, 0.7, 0.9]) + >>> nms(boxes, scores, iou_threshold=0.8) + tensor([0, 3, 1]) + + """ + if len(boxes.shape) != 2 and boxes.shape[-1] != 4: + raise ValueError(f"boxes expected as Nx4. Got: {boxes.shape}.") + + if len(scores.shape) != 1: + raise ValueError(f"scores expected as N. Got: {scores.shape}.") + + if boxes.shape[0] != scores.shape[0]: + raise ValueError(f"boxes and scores mus have same shape. Got: {boxes.shape, scores.shape}.") + + x1, y1, x2, y2 = boxes.unbind(-1) + areas = (x2 - x1) * (y2 - y1) + + _, order = scores.sort(descending=True) + + keep = [] + while order.shape[0] > 0: + i = order[0] + keep.append(i) + xx1 = torch.max(x1[i], x1[order[1:]]) + yy1 = torch.max(y1[i], y1[order[1:]]) + xx2 = torch.min(x2[i], x2[order[1:]]) + yy2 = torch.min(y2[i], y2[order[1:]]) + + w = torch.clamp(xx2 - xx1, min=0.0) + h = torch.clamp(yy2 - yy1, min=0.0) + inter = w * h + ovr = inter / (areas[i] + areas[order[1:]] - inter) + + inds = torch.where(ovr <= iou_threshold)[0] + order = order[inds + 1] + + if len(keep) > 0: + return torch.stack(keep) + + return torch.tensor(keep) diff --git a/kornia/geometry/boxes.py b/kornia/geometry/boxes.py new file mode 100644 index 0000000..b9bd40f --- /dev/null +++ b/kornia/geometry/boxes.py @@ -0,0 +1,1260 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional, Tuple, cast + +import torch +from torch import Size + +from kornia.core.ops import eye_like +from kornia.geometry.bbox import validate_bbox +from kornia.geometry.linalg import transform_points + +__all__ = ["Boxes", "Boxes3D"] + + +def _is_floating_point_dtype(dtype: torch.dtype) -> bool: + return dtype in (torch.float16, torch.float32, torch.float64, torch.bfloat16, torch.half) + + +def _merge_box_list(boxes: list[torch.Tensor], method: str = "pad") -> tuple[torch.Tensor, list[int]]: + r"""Merge a list of boxes into one tensor.""" + if not all(box.shape[-2:] == torch.Size([4, 2]) and box.dim() == 3 for box in boxes): + raise TypeError(f"Input boxes must be a list of (N, 4, 2) shaped. Got: {[box.shape for box in boxes]}.") + + if method == "pad": + max_N = max(box.shape[0] for box in boxes) + stats = [max_N - box.shape[0] for box in boxes] + output = torch.nn.utils.rnn.pad_sequence(boxes, batch_first=True) + else: + raise NotImplementedError(f"`{method}` is not implemented.") + + return output, stats + + +def _transform_boxes(boxes: torch.Tensor, M: torch.Tensor) -> torch.Tensor: + """Transform 3D and 2D in kornia format by applying the transformation matrix M. + + Boxes and the transformation matrix could be batched or not. + + Args: + boxes: 2D quadrilaterals or 3D hexahedrons in kornia format. + M: the transformation matrix of shape :math:`(3, 3)` or :math:`(B, 3, 3)` for 2D and :math:`(4, 4)` or + :math:`(B, 4, 4)` for 3D hexahedron. + + """ + M = M if M.is_floating_point() else M.float() + + # Work with batch as kornia.transform_points only supports a batch of points. + boxes_per_batch, n_points_per_box, coordinates_dimension = boxes.shape[-3:] + if boxes_per_batch == 0: + return boxes + points = boxes.view(-1, n_points_per_box * boxes_per_batch, coordinates_dimension) + M = M if M.ndim == 3 else M.unsqueeze(0) + + if points.shape[0] != M.shape[0]: + raise ValueError( + f"Batch size mismatch. Got {points.shape[0]} for boxes and {M.shape[0]} for the transformation matrix." + ) + + transformed_boxes: torch.Tensor = transform_points(M, points) + transformed_boxes = transformed_boxes.view_as(boxes) + return transformed_boxes + + +def _boxes_to_polygons( + xmin: torch.Tensor, ymin: torch.Tensor, width: torch.Tensor, height: torch.Tensor +) -> torch.Tensor: + if not xmin.ndim == ymin.ndim == width.ndim == height.ndim == 2: + raise ValueError("We expect to create a batch of 2D boxes (quadrilaterals) in vertices format (B, N, 4, 2)") + + # Create (B,N,4,2) with all points in top left position of boxes + polygons = torch.zeros((xmin.shape[0], xmin.shape[1], 4, 2), device=xmin.device, dtype=xmin.dtype) + polygons[..., 0] = xmin.unsqueeze(-1) + polygons[..., 1] = ymin.unsqueeze(-1) + # Shift top-right, bottom-right, bottom-left points to the right coordinates + polygons[..., 1, 0] += width - 1 # Top right + polygons[..., 2, 0] += width - 1 # Bottom right + polygons[..., 2, 1] += height - 1 # Bottom right + polygons[..., 3, 1] += height - 1 # Bottom left + return polygons + + +def _boxes_to_quadrilaterals(boxes: torch.Tensor, mode: str = "xyxy", validate_boxes: bool = True) -> torch.Tensor: + """Convert from boxes to quadrilaterals.""" + mode = mode.lower() + + if mode.startswith("vertices"): + batched = boxes.ndim == 4 + if not (3 <= boxes.ndim <= 4 and boxes.shape[-2:] == torch.Size([4, 2])): + raise ValueError(f"Boxes shape must be (N, 4, 2) or (B, N, 4, 2) when {mode} mode. Got {boxes.shape}.") + elif mode.startswith("xy"): + batched = boxes.ndim == 3 + if not (2 <= boxes.ndim <= 3 and boxes.shape[-1] == 4): + raise ValueError(f"Boxes shape must be (N, 4) or (B, N, 4) when {mode} mode. Got {boxes.shape}.") + else: + raise ValueError(f"Unknown mode {mode}") + + boxes = boxes if boxes.is_floating_point() else boxes.float() + boxes = boxes if batched else boxes.unsqueeze(0) + + if mode.startswith("vertices"): + if mode == "vertices": + quadrilaterals = boxes.clone() + # Here, vertices are quadrilaterals with width and height defined as `width = xmax - xmin` and + # `height = ymax - ymin`. We need to convert to `width = xmax - xmin + 1` and `height = ymax - ymin + 1` to + # match with internal Boxes Kornia representation. + quadrilaterals[..., 1:3, 0] = quadrilaterals[..., 1:3, 0] - 1 + quadrilaterals[..., 2:, 1] = quadrilaterals[..., 2:, 1] - 1 + elif mode == "vertices_plus": + # Avoid passing reference + quadrilaterals = boxes.clone() + else: + raise ValueError(f"Unknown mode {mode}") + not validate_boxes or validate_bbox(quadrilaterals) + elif mode.startswith("xy"): + if mode == "xyxy": + height, width = boxes[..., 3] - boxes[..., 1], boxes[..., 2] - boxes[..., 0] + elif mode == "xyxy_plus": + height, width = boxes[..., 3] - boxes[..., 1] + 1, boxes[..., 2] - boxes[..., 0] + 1 + elif mode == "xywh": + height, width = boxes[..., 3], boxes[..., 2] + else: + raise ValueError(f"Unknown mode {mode}") + + if validate_boxes: + if (width <= 0).any(): + raise ValueError("Some boxes have negative widths or 0.") + if (height <= 0).any(): + raise ValueError("Some boxes have negative heights or 0.") + + xmin, ymin = boxes[..., 0], boxes[..., 1] + quadrilaterals = _boxes_to_polygons(xmin, ymin, width, height) + else: + raise ValueError(f"Unknown mode {mode}") + + quadrilaterals = quadrilaterals if batched else quadrilaterals.squeeze(0) + + return quadrilaterals + + +def _boxes3d_to_polygons3d( + xmin: torch.Tensor, + ymin: torch.Tensor, + zmin: torch.Tensor, + width: torch.Tensor, + height: torch.Tensor, + depth: torch.Tensor, +) -> torch.Tensor: + if not xmin.ndim == ymin.ndim == zmin.ndim == width.ndim == height.ndim == depth.ndim == 2: + raise ValueError("We expect to create a batch of 3D boxes (hexahedrons) in vertices format (B, N, 8, 3)") + + # Front + # Create (B,N,4,3) with all points in front top left position of boxes + front_vertices = torch.zeros((xmin.shape[0], xmin.shape[1], 4, 3), device=xmin.device, dtype=xmin.dtype) + front_vertices[..., 0] = xmin.unsqueeze(-1) + front_vertices[..., 1] = ymin.unsqueeze(-1) + front_vertices[..., 2] = zmin.unsqueeze(-1) + # Shift front-top-right, front-bottom-right, front-bottom-left points to the right coordinates + front_vertices[..., 1, 0] += width - 1 # Top right + front_vertices[..., 2, 0] += width - 1 # Bottom right + front_vertices[..., 2, 1] += height - 1 # Bottom right + front_vertices[..., 3, 1] += height - 1 # Bottom left + + # Back + back_vertices = front_vertices.clone() + back_vertices[..., 2] += depth.unsqueeze(-1) - 1 + + polygons3d = torch.cat([front_vertices, back_vertices], dim=-2) + return polygons3d + + +class Boxes: + r"""2D boxes containing N or BxN boxes. + + Args: + boxes: 2D boxes, shape of :math:`(N, 4, 2)`, :math:`(B, N, 4, 2)` or a list of :math:`(N, 4, 2)`. + See below for more details. + raise_if_not_floating_point: flag to control floating point casting behaviour when `boxes` is not a + floating point tensor. True to raise an error when `boxes` isn't a floating point tensor, False + to cast to float. + mode: the box format of the input boxes. + + Note: + **2D boxes format** is defined as a floating data type tensor of shape ``Nx4x2`` or ``BxNx4x2`` + where each box is a `quadrilateral `_ defined by it's + 4 vertices coordinates (A, B, C, D). Coordinates must be in ``x, y`` order. The height and width of + a box is defined as ``width = xmax - xmin + 1`` and ``height = ymax - ymin + 1``. Examples of + `quadrilaterals `_ are rectangles, rhombus and trapezoids. + + """ + + def __init__( + self, + boxes: torch.Tensor | list[torch.Tensor], + raise_if_not_floating_point: bool = True, + mode: str = "vertices_plus", + ) -> None: + self._N: Optional[list[int]] = None + + if isinstance(boxes, list): + boxes, self._N = _merge_box_list(boxes) + + if not isinstance(boxes, torch.Tensor): + raise TypeError(f"Input boxes is not a Tensor. Got: {type(boxes)}.") + + if not boxes.is_floating_point(): + if raise_if_not_floating_point: + raise ValueError(f"Coordinates must be in floating point. Got {boxes.dtype}") + + boxes = boxes.float() + + if len(boxes.shape) == 0: + boxes = boxes.reshape((-1, 4)) + + if not (3 <= boxes.ndim <= 4 and boxes.shape[-2:] == (4, 2)): + raise ValueError(f"Boxes shape must be (N, 4, 2) or (B, N, 4, 2). Got {boxes.shape}.") + + self._is_batched = False if boxes.ndim == 3 else True + + self._data = boxes + self._mode = mode + + def __getitem__(self, key: slice | int | torch.Tensor) -> Boxes: + new_box = type(self)(self._data[key], False) + new_box._mode = self._mode + return new_box + + def __setitem__(self, key: slice | int | torch.Tensor, value: Boxes) -> Boxes: + self._data[key] = value._data + return self + + @property + def shape(self) -> tuple[int, ...] | Size: + """Return the tensor shape used to store the boxes. + + Returns: + Shape of :attr:`data`. For unbatched boxes this is usually + :math:`(N, 4, 2)`, where :math:`N` is the number of boxes, ``4`` + is the number of corner vertices, and ``2`` stores ``(x, y)``. + For batched boxes the shape is usually :math:`(B, N, 4, 2)`, where + :math:`B` is the batch size. + """ + return self.data.shape + + def get_boxes_shape(self) -> tuple[torch.Tensor, torch.Tensor]: + r"""Compute boxes heights and widths. + + Returns: + - Boxes heights, shape of :math:`(N,)` or :math:`(B,N)`. + - Boxes widths, shape of :math:`(N,)` or :math:`(B,N)`. + + Example: + >>> boxes_xyxy = torch.tensor([[[1,1,2,2],[1,1,3,2]]]) + >>> boxes = Boxes.from_tensor(boxes_xyxy) + >>> boxes.get_boxes_shape() + (tensor([[1., 1.]]), tensor([[1., 2.]])) + + """ + boxes_xywh = cast(torch.Tensor, self.to_tensor("xywh", as_padded_sequence=True)) + widths, heights = boxes_xywh[..., 2], boxes_xywh[..., 3] + return heights, widths + + def merge(self, boxes: Boxes, inplace: bool = False) -> Boxes: + """Merge boxes. + + Say, current instance holds :math:`(B, N, 4, 2)` and the incoming boxes holds :math:`(B, M, 4, 2)`, + the merge results in :math:`(B, N + M, 4, 2)`. + + Args: + boxes: 2D boxes. + inplace: do transform in-place and return self. + + """ + data = torch.cat([self._data, boxes.data], dim=1) + if inplace: + self._data = data + return self + + obj = self.clone() + obj._data = data + return obj + + def index_put( + self, + indices: tuple[torch.Tensor, ...] | list[torch.Tensor], + values: torch.Tensor | Boxes, + inplace: bool = False, + ) -> Boxes: + """Write box coordinates at selected tensor indices. + + This mirrors :meth:`torch.Tensor.index_put_` for the internal + quadrilateral tensor. It is useful when a subset of boxes in a batch + must be replaced while keeping the :class:`Boxes` wrapper and metadata. + + Args: + indices: Index tuple or list accepted by ``Tensor.index_put_``. + The indices address entries in the stored tensor, commonly + shaped :math:`(B, N, 4, 2)` or :math:`(N, 4, 2)`. + values: Replacement coordinates. If a :class:`Boxes` object is + passed, its :attr:`data` tensor is used. + inplace: If ``True``, update this object and return ``self``. If + ``False``, clone the current data first and return a new + :class:`Boxes` instance. + + Returns: + :class:`Boxes` containing the updated coordinates. + """ + if inplace: + _data = self._data + else: + _data = self._data.clone() + + if isinstance(values, Boxes): + _data.index_put_(indices, values.data) + else: + _data.index_put_(indices, values) + + if inplace: + return self + + obj = self.clone() + obj._data = _data + return obj + + def pad(self, padding_size: torch.Tensor) -> Boxes: + """Pad a bounding box. + + Args: + padding_size: (B, 4) + + """ + if not (len(padding_size.shape) == 2 and padding_size.size(1) == 4): + raise RuntimeError(f"Expected padding_size as (B, 4). Got {padding_size.shape}.") + self._data[..., 0] += padding_size[..., None, :1].to(device=self._data.device) # left padding + self._data[..., 1] += padding_size[..., None, 2:3].to(device=self._data.device) # top padding + return self + + def unpad(self, padding_size: torch.Tensor) -> Boxes: + """Pad a bounding box. + + Args: + padding_size: (B, 4) + + """ + if not (len(padding_size.shape) == 2 and padding_size.size(1) == 4): + raise RuntimeError(f"Expected padding_size as (B, 4). Got {padding_size.shape}.") + self._data[..., 0] -= padding_size[..., None, :1].to(device=self._data.device) # left padding + self._data[..., 1] -= padding_size[..., None, 2:3].to(device=self._data.device) # top padding + return self + + def clamp( + self, + topleft: Optional[torch.Tensor | tuple[int, int]] = None, + botright: Optional[torch.Tensor | tuple[int, int]] = None, + inplace: bool = False, + ) -> Boxes: + """Clamp every box vertex inside per-image coordinate limits. + + Coordinates below ``topleft`` are raised to the lower bound and + coordinates above ``botright`` are lowered to the upper bound. The + implementation expects tensor bounds with one ``(x, y)`` pair per batch + element. + + Args: + topleft: Tensor of shape :math:`(B, 2)` containing the minimum + ``x`` and ``y`` coordinate allowed for each batch item. + botright: Tensor of shape :math:`(B, 2)` containing the maximum + ``x`` and ``y`` coordinate allowed for each batch item. + inplace: If ``True``, clamp this object in place. Otherwise, return + a new :class:`Boxes` object with clamped data. + + Returns: + :class:`Boxes` whose vertex coordinates are restricted to the + provided bounds. + """ + if not (isinstance(topleft, torch.Tensor) and isinstance(botright, torch.Tensor)): + raise NotImplementedError + if inplace: + _data = self._data + else: + _data = self._data.clone() + topleft_x = topleft[:, None, :1].repeat(1, _data.size(1), 4) + _data[..., 0][_data[..., 0] < topleft_x] = topleft_x[_data[..., 0] < topleft_x] + + topleft_y = topleft[:, None, 1:].repeat(1, _data.size(1), 4) + _data[..., 1][_data[..., 1] < topleft_y] = topleft_y[_data[..., 1] < topleft_y] + + botright_x = botright[:, None, :1].repeat(1, _data.size(1), 4) + _data[..., 0][_data[..., 0] > botright_x] = botright_x[_data[..., 0] > botright_x] + + botright_y = botright[:, None, 1:].repeat(1, _data.size(1), 4) + _data[..., 1][_data[..., 1] > botright_y] = botright_y[_data[..., 1] > botright_y] + if inplace: + return self + + obj = self.clone() + obj._data = _data + return obj + + def trim(self, correspondence_preserve: bool = False, inplace: bool = False) -> Boxes: + """Trim out zero padded boxes. + + Given box arrangements of shape :math:`(4, 4, Box)`: + + == === == === == === == === == + -- Box -- Box -- Box -- Box -- + -- 0 -- 0 -- Box -- Box -- + -- 0 -- Box -- 0 -- 0 -- + -- 0 -- 0 -- 0 -- 0 -- + == === == === == === == === == + + Nothing will change if correspondence_preserve is True. Only pure zero layers will be removed, resulting in + shape :math:`(4, 3, Box)`: + + == === == === == === == === == + -- Box -- Box -- Box -- Box -- + -- 0 -- 0 -- Box -- Box -- + -- 0 -- Box -- 0 -- 0 -- + == === == === == === == === == + + Otherwise, you will get :math:`(4, 2, Box)`: + + == === == === == === == === == + -- Box -- Box -- Box -- Box -- + -- 0 -- Box -- Box -- Box -- + == === == === == === == === == + """ + raise NotImplementedError + + def filter_boxes_by_area( + self, min_area: Optional[float] = None, max_area: Optional[float] = None, inplace: bool = False + ) -> Boxes: + """Remove boxes whose polygon area is outside the requested range. + + The box area is computed from its four vertices. Boxes smaller than + ``min_area`` or larger than ``max_area`` are not dropped from the + tensor; their coordinates are replaced with zeros so the original batch + and box dimensions stay unchanged. + + Args: + min_area: Optional lower inclusive area threshold. Boxes with area + below this value are zeroed. + max_area: Optional upper inclusive area threshold. Boxes with area + above this value are zeroed. + inplace: If ``True``, update this object in place. Otherwise, + return a filtered clone. + + Returns: + :class:`Boxes` with the same shape as the input container and + out-of-range boxes replaced by zero coordinates. + """ + area = self.compute_area() + if inplace: + _data = self._data + else: + _data = self._data.clone() + if min_area is not None: + _data[area < min_area] = 0.0 + if max_area is not None: + _data[area > max_area] = 0.0 + if inplace: + return self + + obj = self.clone() + obj._data = _data + return obj + + def compute_area(self) -> torch.Tensor: + """Return :math:`(B, N)`.""" + coords = self._data.view((-1, 4, 2)) if self._data.ndim == 4 else self._data + # calculate centroid of the box + centroid = coords.mean(dim=1, keepdim=True) + # calculate the angle from centroid to each corner + angles = torch.atan2(coords[..., 1] - centroid[..., 1], coords[..., 0] - centroid[..., 0]) + # sort the corners by angle to get an order for shoelace formula + _, clockwise_indices = torch.sort(angles, dim=1, descending=True) + # gather the corners in the new order + ordered_corners = torch.gather(coords, 1, clockwise_indices.unsqueeze(-1).expand(-1, -1, 2)) + x, y = ordered_corners[..., 0], ordered_corners[..., 1] + # Gaussian/Shoelace formula https://en.wikipedia.org/wiki/Shoelace_formula + area = 0.5 * torch.abs(torch.sum((x * torch.roll(y, 1, 1)) - (y * torch.roll(x, 1, 1)), dim=1)) + return area.view(self._data.shape[:2]) if self._data.ndim == 4 else area + + @classmethod + def from_tensor( + cls, boxes: torch.Tensor | list[torch.Tensor], mode: str = "xyxy", validate_boxes: bool = True + ) -> Boxes: + r"""Create :class:`Boxes` from boxes stored in another format. + + Args: + boxes: 2D boxes, shape of :math:`(N, 4)`, :math:`(B, N, 4)`, :math:`(N, 4, 2)` or :math:`(B, N, 4, 2)`. + mode: The format in which the boxes are provided. + validate_boxes: Check if boxes are valid. Default is True. + + * 'xyxy': boxes are assumed to be in the format ``xmin, ymin, xmax, ymax`` where ``width = xmax - xmin`` + and ``height = ymax - ymin``. With shape :math:`(N, 4)`, :math:`(B, N, 4)`. + * 'xyxy_plus': similar to 'xyxy' mode but where box width and length are defined as + ``width = xmax - xmin + 1`` and ``height = ymax - ymin + 1``. + With shape :math:`(N, 4)`, :math:`(B, N, 4)`. + * 'xywh': boxes are assumed to be in the format ``xmin, ymin, width, height`` where + ``width = xmax - xmin`` and ``height = ymax - ymin``. With shape :math:`(N, 4)`, :math:`(B, N, 4)`. + * 'vertices': boxes are defined by their vertices points in the following ``clockwise`` order: + *top-left, top-right, bottom-right, bottom-left*. Vertices coordinates are in (x,y) order. Finally, + box width and height are defined as ``width = xmax - xmin`` and ``height = ymax - ymin``. + With shape :math:`(N, 4, 2)` or :math:`(B, N, 4, 2)`. + * 'vertices_plus': similar to 'vertices' mode but where box width and length are defined as + ``width = xmax - xmin + 1`` and ``height = ymax - ymin + 1``. ymin + 1``. + With shape :math:`(N, 4, 2)` or :math:`(B, N, 4, 2)`. + + validate_boxes: check if boxes are valid rectangles or not. Valid rectangles are those with width + and height >= 1 (>= 2 when mode ends with '_plus' suffix). + + Returns: + :class:`Boxes` class containing the original `boxes` in the format specified by ``mode``. + + Examples: + >>> boxes_xyxy = torch.as_tensor([[0, 3, 1, 4], [5, 1, 8, 4]]) + >>> boxes = Boxes.from_tensor(boxes_xyxy, mode='xyxy') + >>> boxes.data # (2, 4, 2) + tensor([[[0., 3.], + [0., 3.], + [0., 3.], + [0., 3.]], + + [[5., 1.], + [7., 1.], + [7., 3.], + [5., 3.]]]) + + """ + quadrilaterals: torch.Tensor | list[torch.Tensor] + if isinstance(boxes, torch.Tensor): + quadrilaterals = _boxes_to_quadrilaterals(boxes, mode=mode, validate_boxes=validate_boxes) + else: + quadrilaterals = [_boxes_to_quadrilaterals(box, mode, validate_boxes) for box in boxes] + + return cls(quadrilaterals, False, mode) + + def to_tensor( + self, mode: Optional[str] = None, as_padded_sequence: bool = False + ) -> torch.Tensor | list[torch.Tensor]: + r"""Cast :class:`Boxes` to a tensor. + + ``mode`` controls which 2D boxes format should be use to represent boxes in the tensor. + + Args: + mode: the output box format. It could be: + + * 'xyxy': boxes are defined as ``xmin, ymin, xmax, ymax`` where ``width = xmax - xmin`` and + ``height = ymax - ymin``. + * 'xyxy_plus': similar to 'xyxy' mode but where box width and length are defined as + ``width = xmax - xmin + 1`` and ``height = ymax - ymin + 1``. + * 'xywh': boxes are defined as ``xmin, ymin, width, height`` where ``width = xmax - xmin`` + and ``height = ymax - ymin``. + * 'vertices': boxes are defined by their vertices points in the following ``clockwise`` order: + *top-left, top-right, bottom-right, bottom-left*. Vertices coordinates are in (x,y) order. Finally, + box width and height are defined as ``width = xmax - xmin`` and ``height = ymax - ymin``. + * 'vertices_plus': similar to 'vertices' mode but where box width and length are defined as + ``width = xmax - xmin + 1`` and ``height = ymax - ymin + 1``. ymin + 1``. + as_padded_sequence: whether to keep the pads for a list of boxes. This parameter is only valid + if the boxes are from a box list whilst `from_tensor`. + + Returns: + Boxes tensor in the ``mode`` format. The shape depends with the ``mode`` value: + + * 'vertices' or 'verticies_plus': :math:`(N, 4, 2)` or :math:`(B, N, 4, 2)`. + * Any other value: :math:`(N, 4)` or :math:`(B, N, 4)`. + + Examples: + >>> boxes_xyxy = torch.as_tensor([[0, 3, 1, 4], [5, 1, 8, 4]]) + >>> boxes = Boxes.from_tensor(boxes_xyxy) + >>> assert (boxes_xyxy == boxes.to_tensor(mode='xyxy')).all() + + """ + batched_boxes = self._data if self._is_batched else self._data.unsqueeze(0) + + boxes: torch.Tensor | list[torch.Tensor] + + # Create boxes in xyxy_plus format. + boxes = torch.stack([batched_boxes.amin(dim=-2), batched_boxes.amax(dim=-2)], dim=-2).view( + batched_boxes.shape[0], batched_boxes.shape[1], 4 + ) + + if mode is None: + mode = self.mode + + mode = mode.lower() + + if mode in ("xyxy", "xyxy_plus"): + pass + elif mode in ("xywh", "vertices", "vertices_plus"): + height, width = boxes[..., 3] - boxes[..., 1] + 1, boxes[..., 2] - boxes[..., 0] + 1 + boxes[..., 2] = width + boxes[..., 3] = height + else: + raise ValueError(f"Unknown mode {mode}") + + if mode in ("xyxy", "vertices"): + offset = torch.as_tensor([0, 0, 1, 1], device=boxes.device, dtype=boxes.dtype) + boxes = boxes + offset + + if mode.startswith("vertices"): + boxes = _boxes_to_polygons(boxes[..., 0], boxes[..., 1], boxes[..., 2], boxes[..., 3]) + + if self._N is not None and not as_padded_sequence: + boxes = [torch.nn.functional.pad(o, (len(o.shape) - 1) * [0, 0] + [0, -n]) for o, n in zip(boxes, self._N)] + else: + boxes = boxes if self._is_batched else boxes.squeeze(0) + return boxes + + def to_mask(self, height: int, width: int) -> torch.Tensor: + """Convert 2D boxes to masks. Covered area is 1 and the remaining is 0. + + Args: + height: height of the masked image/images. + width: width of the masked image/images. + + Returns: + the output mask tensor, shape of :math:`(N, width, height)` or :math:`(B,N, width, height)` and dtype of + :func:`Boxes.dtype` (it can be any floating point dtype). + + Note: + It is currently non-differentiable. + + Examples: + >>> boxes = Boxes(torch.tensor([[ # Equivalent to boxes = Boxes.from_tensor([[1,1,4,3]]) + ... [1., 1.], + ... [4., 1.], + ... [4., 3.], + ... [1., 3.], + ... ]])) # 1x4x2 + >>> boxes.to_mask(5, 5) + tensor([[[0., 0., 0., 0., 0.], + [0., 1., 1., 1., 1.], + [0., 1., 1., 1., 1.], + [0., 1., 1., 1., 1.], + [0., 0., 0., 0., 0.]]]) + + """ + if self._data.requires_grad: + raise RuntimeError( + "Boxes.to_tensor isn't differentiable. Please, create boxes from tensors with `requires_grad=False`." + ) + + is_batched = self._is_batched + dtype = self.dtype + device = self.device + + # ----------------- + # CPU Hotpath (loop) + # ----------------- + if device.type != "cuda": + if self._is_batched: # (B, N, 4, 2) + mask = torch.zeros( + (self._data.shape[0], self._data.shape[1], height, width), dtype=self.dtype, device=self.device + ) + else: # (N, 4, 2) + mask = torch.zeros((self._data.shape[0], height, width), dtype=self.dtype, device=self.device) + + # Boxes coordinates can be outside the image size after transforms. Clamp values to the image size + clipped_boxes_xyxy = cast(torch.Tensor, self.to_tensor("xyxy", as_padded_sequence=True)) + clipped_boxes_xyxy[..., ::2].clamp_(0, width) + clipped_boxes_xyxy[..., 1::2].clamp_(0, height) + + # Reshape mask to (BxN, H, W) and boxes to (BxN, 4) to iterate over all of them. + # Cast boxes coordinates to be integer to use them as indexes. Use round to handle decimal values. + for mask_channel, box_xyxy in zip( + mask.view(-1, height, width), clipped_boxes_xyxy.view(-1, 4).round().int() + ): + # Mask channel dimensions: (height, width) + mask_channel[box_xyxy[1] : box_xyxy[3], box_xyxy[0] : box_xyxy[2]] = 1 + + return mask + + # ----------------- + # GPU Hotpath (vectorized) + # ----------------- + out_shape: Tuple[int, ...] + if is_batched: + out_shape = (self.shape[0], self.shape[1], height, width) + else: + out_shape = (self.shape[0], height, width) + + clipped_boxes_xyxy = cast(torch.Tensor, self.to_tensor("xyxy", as_padded_sequence=True)) + clipped_boxes_xyxy[..., ::2].clamp_(0, width) + clipped_boxes_xyxy[..., 1::2].clamp_(0, height) + + xyxy = clipped_boxes_xyxy.view(-1, 4).round().long() + + x1, y1, x2, y2 = xyxy[:, 0], xyxy[:, 1], xyxy[:, 2], xyxy[:, 3] + x1 = x1.clamp(0, width) + x2 = x2.clamp(0, width) + y1 = y1.clamp(0, height) + y2 = y2.clamp(0, height) + + ys = torch.arange(height, device=device) + xs = torch.arange(width, device=device) + + y_mask = (ys[None, :] >= y1[:, None]) & (ys[None, :] < y2[:, None]) + x_mask = (xs[None, :] >= x1[:, None]) & (xs[None, :] < x2[:, None]) + + masks = (y_mask.unsqueeze(2) & x_mask.unsqueeze(1)).to(dtype) + return masks.view(*out_shape) + + def transform_boxes(self, M: torch.Tensor, inplace: bool = False) -> Boxes: + r"""Apply a transformation matrix to the 2D boxes. + + Args: + M: The transformation matrix to be applied, shape of :math:`(3, 3)` or :math:`(B, 3, 3)`. + inplace: do transform in-place and return self. + + Returns: + The transformed boxes. + + """ + if not 2 <= M.ndim <= 3 or M.shape[-2:] != (3, 3): + raise ValueError(f"The transformation matrix shape must be (3, 3) or (B, 3, 3). Got {M.shape}.") + + transformed_boxes = _transform_boxes(self._data, M) + if inplace: + self._data = transformed_boxes + return self + + obj = self.clone() + obj._data = transformed_boxes + return obj + + def transform_boxes_(self, M: torch.Tensor) -> Boxes: + """Inplace version of :func:`Boxes.transform_boxes`.""" + return self.transform_boxes(M, inplace=True) + + def translate(self, size: torch.Tensor, method: str = "warp", inplace: bool = False) -> Boxes: + """Translate boxes by the provided size. + + Args: + size: translate size for x, y direction, shape of :math:`(B, 2)`. + method: "warp" or "fast". + inplace: do transform in-place and return self. + + Returns: + The transformed boxes. + + """ + if method == "fast": + raise NotImplementedError + elif method == "warp": + pass + else: + raise NotImplementedError + + M: torch.Tensor = eye_like(3, size) + M[:, :2, 2] = size + return self.transform_boxes(M, inplace=inplace) + + @property + def data(self) -> torch.Tensor: + """Return the raw quadrilateral coordinate tensor. + + Returns: + Tensor storing four vertices per box in ``(x, y)`` order. The + common shapes are :math:`(N, 4, 2)` for unbatched boxes and + :math:`(B, N, 4, 2)` for batched boxes, where :math:`B` is batch + size and :math:`N` is the number of boxes. + """ + return self._data + + @property + def mode(self) -> str: + """Return the box format remembered by this container. + + Returns: + Mode string used as the default by :meth:`to_tensor`, such as + ``"xyxy"``, ``"xywh"``, ``"vertices"``, or their ``"_plus"`` + variants. + """ + return self._mode + + @property + def device(self) -> torch.device: + """Returns boxes device.""" + return self._data.device + + @property + def dtype(self) -> torch.dtype: + """Returns boxes dtype.""" + return self._data.dtype + + def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> Boxes: + """Like :func:`torch.nn.Module.to()` method.""" + # In torchscript, dtype is a int and not a class. https://github.com/pytorch/pytorch/issues/51941 + if dtype is not None and not _is_floating_point_dtype(dtype): + raise ValueError("Boxes must be in floating point") + self._data = self._data.to(device=device, dtype=dtype) + return self + + def clone(self) -> Boxes: + """Create an independent copy of the box container. + + Returns: + New :class:`Boxes` object with cloned tensor storage. Metadata such + as the current mode, original list lengths, and batched flag is + preserved. + """ + obj = type(self)(self._data.clone(), False) + obj._mode = self._mode + obj._N = self._N + obj._is_batched = self._is_batched + return obj + + def type(self, dtype: torch.dtype) -> Boxes: + """Cast the stored box coordinates to a new dtype. + + Args: + dtype: Target floating-point dtype for the coordinate tensor. + + Returns: + ``self`` after converting :attr:`data` in place. + """ + self._data = self._data.type(dtype) + return self + + +class VideoBoxes(Boxes): + temporal_channel_size: int + + @classmethod + def from_tensor( # type: ignore[override] + cls, boxes: torch.Tensor | list[torch.Tensor], validate_boxes: bool = True + ) -> VideoBoxes: + if isinstance(boxes, (list,)) or (boxes.dim() != 5 or boxes.shape[-2:] != torch.Size([4, 2])): + raise ValueError("Input box type is not yet supported. Please input an `BxTxNx4x2` tensor directly.") + + temporal_channel_size = boxes.size(1) + + quadrilaterals = _boxes_to_quadrilaterals( + boxes.view(boxes.size(0) * boxes.size(1), -1, boxes.size(3), boxes.size(4)), + mode="vertices_plus", + validate_boxes=validate_boxes, + ) + out = cls(quadrilaterals, False, "vertices_plus") + out.temporal_channel_size = temporal_channel_size + return out + + def to_tensor(self, mode: Optional[str] = None) -> torch.Tensor | list[torch.Tensor]: # type: ignore[override] + out = super().to_tensor(mode, as_padded_sequence=False) + if isinstance(out, torch.Tensor): + return out.view(-1, self.temporal_channel_size, *out.shape[1:]) + # If returns a list of boxes. + return [_out.view(-1, self.temporal_channel_size, *_out.shape[1:]) for _out in out] + + def clone(self) -> VideoBoxes: + obj = type(self)(self._data.clone(), False) + obj._mode = self._mode + obj._N = self._N + obj._is_batched = self._is_batched + obj.temporal_channel_size = self.temporal_channel_size + return obj + + +class Boxes3D: + r"""3D boxes containing N or BxN boxes. + + Args: + boxes: 3D boxes, shape of :math:`(N,8,3)` or :math:`(B,N,8,3)`. See below for more details. + raise_if_not_floating_point: flag to control floating point casting behaviour when `boxes` is not a floating + point tensor. True to raise an error when `boxes` isn't a floating point tensor, False to cast to float. + + Note: + **3D boxes format** is defined as a floating data type tensor of shape ``Nx8x3`` or ``BxNx8x3`` where each box + is a `hexahedron `_ defined by it's 8 vertices coordinates. + Coordinates must be in ``x, y, z`` order. The height, width and depth of a box is defined as + ``width = xmax - xmin + 1``, ``height = ymax - ymin + 1`` and ``depth = zmax - zmin + 1``. Examples of + `hexahedrons `_ are cubes and rhombohedrons. + + """ + + def __init__( + self, boxes: torch.Tensor, raise_if_not_floating_point: bool = True, mode: str = "xyzxyz_plus" + ) -> None: + if not isinstance(boxes, torch.Tensor): + raise TypeError(f"Input boxes is not a Tensor. Got: {type(boxes)}.") + + if not boxes.is_floating_point(): + if raise_if_not_floating_point: + raise ValueError(f"Coordinates must be in floating point. Got {boxes.dtype}.") + + boxes = boxes.float() + + if len(boxes.shape) == 0: + boxes = boxes.reshape((-1, 6)) + + if not (3 <= boxes.ndim <= 4 and boxes.shape[-2:] == (8, 3)): + raise ValueError(f"3D bbox shape must be (N, 8, 3) or (B, N, 8, 3). Got {boxes.shape}.") + + self._is_batched = False if boxes.ndim == 3 else True + + self._data = boxes + self._mode = mode + + def __getitem__(self, key: slice | int | torch.Tensor) -> Boxes3D: + new_box = Boxes3D(self._data[key], False, mode="xyzxyz_plus") + new_box._mode = self._mode + return new_box + + def __setitem__(self, key: slice | int | torch.Tensor, value: Boxes3D) -> Boxes3D: + self._data[key] = value._data + return self + + @property + def shape(self) -> tuple[int, ...] | Size: + """Return the tensor shape used to store 3D boxes. + + Returns: + Shape of :attr:`data`. For unbatched boxes this is usually + :math:`(N, 8, 3)`, where :math:`N` is the number of boxes, ``8`` is + the number of cuboid corners, and ``3`` stores ``(x, y, z)``. For + batched boxes the shape is usually :math:`(B, N, 8, 3)`. + """ + return self.data.shape + + def get_boxes_shape(self) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + r"""Compute boxes heights and widths. + + Returns: + - Boxes depths, shape of :math:`(N,)` or :math:`(B,N)`. + - Boxes heights, shape of :math:`(N,)` or :math:`(B,N)`. + - Boxes widths, shape of :math:`(N,)` or :math:`(B,N)`. + + Example: + >>> boxes_xyzxyz = torch.tensor([[ 0, 1, 2, 10, 21, 32], [3, 4, 5, 43, 54, 65]]) + >>> boxes3d = Boxes3D.from_tensor(boxes_xyzxyz) + >>> boxes3d.get_boxes_shape() + (tensor([30., 60.]), tensor([20., 50.]), tensor([10., 40.])) + + """ + boxes_xyzwhd = self.to_tensor(mode="xyzwhd") + widths, heights, depths = boxes_xyzwhd[..., 3], boxes_xyzwhd[..., 4], boxes_xyzwhd[..., 5] + return depths, heights, widths + + @classmethod + def from_tensor(cls, boxes: torch.Tensor, mode: str = "xyzxyz", validate_boxes: bool = True) -> Boxes3D: + r"""Create :class:`Boxes3D` from 3D boxes stored in another format. + + Args: + boxes: 3D boxes, shape of :math:`(N,6)` or :math:`(B,N,6)`. + mode: The format in which the 3D boxes are provided. + + * 'xyzxyz': boxes are assumed to be in the format ``xmin, ymin, zmin, xmax, ymax, zmax`` where + ``width = xmax - xmin``, ``height = ymax - ymin`` and ``depth = zmax - zmin``. + * 'xyzxyz_plus': similar to 'xyzxyz' mode but where box width, length and depth are defined as + ``width = xmax - xmin + 1``, ``height = ymax - ymin + 1`` and ``depth = zmax - zmin + 1``. + * 'xyzwhd': boxes are assumed to be in the format ``xmin, ymin, zmin, width, height, depth`` where + ``width = xmax - xmin``, ``height = ymax - ymin`` and ``depth = zmax - zmin``. + + validate_boxes: check if boxes are valid rectangles or not. Valid rectangles are those with width, height + and depth >= 1 (>= 2 when mode ends with '_plus' suffix). + + Returns: + :class:`Boxes3D` class containing the original `boxes` in the format specified by ``mode``. + + Examples: + >>> boxes_xyzxyz = torch.as_tensor([[0, 3, 6, 1, 4, 8], [5, 1, 3, 8, 4, 9]]) + >>> boxes = Boxes3D.from_tensor(boxes_xyzxyz, mode='xyzxyz') + >>> boxes.data # (2, 8, 3) + tensor([[[0., 3., 6.], + [0., 3., 6.], + [0., 3., 6.], + [0., 3., 6.], + [0., 3., 7.], + [0., 3., 7.], + [0., 3., 7.], + [0., 3., 7.]], + + [[5., 1., 3.], + [7., 1., 3.], + [7., 3., 3.], + [5., 3., 3.], + [5., 1., 8.], + [7., 1., 8.], + [7., 3., 8.], + [5., 3., 8.]]]) + + """ + if not (2 <= boxes.ndim <= 3 and boxes.shape[-1] == 6): + raise ValueError(f"BBox shape must be (N, 6) or (B, N, 6). Got {boxes.shape}.") + + batched = boxes.ndim == 3 + boxes = boxes if batched else boxes.unsqueeze(0) + boxes = boxes if boxes.is_floating_point() else boxes.float() + + xmin, ymin, zmin = boxes[..., 0], boxes[..., 1], boxes[..., 2] + mode = mode.lower() + if mode == "xyzxyz": + width = boxes[..., 3] - boxes[..., 0] + height = boxes[..., 4] - boxes[..., 1] + depth = boxes[..., 5] - boxes[..., 2] + elif mode == "xyzxyz_plus": + width = boxes[..., 3] - boxes[..., 0] + 1 + height = boxes[..., 4] - boxes[..., 1] + 1 + depth = boxes[..., 5] - boxes[..., 2] + 1 + elif mode == "xyzwhd": + depth, height, width = boxes[..., 4], boxes[..., 3], boxes[..., 5] + else: + raise ValueError(f"Unknown mode {mode}") + + if validate_boxes: + if (width <= 0).any(): + raise ValueError("Some boxes have negative widths or 0.") + if (height <= 0).any(): + raise ValueError("Some boxes have negative heights or 0.") + if (depth <= 0).any(): + raise ValueError("Some boxes have negative depths or 0.") + + hexahedrons = _boxes3d_to_polygons3d(xmin, ymin, zmin, width, height, depth) + hexahedrons = hexahedrons if batched else hexahedrons.squeeze(0) + return cls(hexahedrons, raise_if_not_floating_point=False, mode=mode) + + def to_tensor(self, mode: str = "xyzxyz") -> torch.Tensor: + r"""Cast :class:`Boxes3D` to a tensor. + + ``mode`` controls which 3D boxes format should be use to represent boxes in the tensor. + + Args: + mode: The format in which the boxes are provided. + + * 'xyzxyz': boxes are assumed to be in the format ``xmin, ymin, zmin, xmax, ymax, zmax`` where + ``width = xmax - xmin``, ``height = ymax - ymin`` and ``depth = zmax - zmin``. + * 'xyzxyz_plus': similar to 'xyzxyz' mode but where box width, length and depth are defined as + ``width = xmax - xmin + 1``, ``height = ymax - ymin + 1`` and ``depth = zmax - zmin + 1``. + * 'xyzwhd': boxes are assumed to be in the format ``xmin, ymin, zmin, width, height, depth`` where + ``width = xmax - xmin``, ``height = ymax - ymin`` and ``depth = zmax - zmin``. + * 'vertices': boxes are defined by their vertices points in the following ``clockwise`` order: + *front-top-left, front-top-right, front-bottom-right, front-bottom-left, back-top-left, + back-top-right, back-bottom-right, back-bottom-left*. Vertices coordinates are in (x,y, z) order. + Finally, box width, height and depth are defined as ``width = xmax - xmin``, ``height = ymax - ymin`` + and ``depth = zmax - zmin``. + * 'vertices_plus': similar to 'vertices' mode but where box width, length and depth are defined as + ``width = xmax - xmin + 1`` and ``height = ymax - ymin + 1``. + + Returns: + 3D Boxes tensor in the ``mode`` format. The shape depends with the ``mode`` value: + + * 'vertices' or 'verticies_plus': :math:`(N, 8, 3)` or :math:`(B, N, 8, 3)`. + * Any other value: :math:`(N, 6)` or :math:`(B, N, 6)`. + + Note: + It is currently non-differentiable due to a bug. See github issue + `#1304 `_. + + Examples: + >>> boxes_xyzxyz = torch.as_tensor([[0, 3, 6, 1, 4, 8], [5, 1, 3, 8, 4, 9]]) + >>> boxes = Boxes3D.from_tensor(boxes_xyzxyz, mode='xyzxyz') + >>> assert (boxes.to_tensor(mode='xyzxyz') == boxes_xyzxyz).all() + + """ + if self._data.requires_grad: + raise RuntimeError( + "Boxes3D.to_tensor doesn't support computing gradients since they aren't accurate. " + "Please, create boxes from tensors with `requires_grad=False`. " + "This is a known bug. Help is needed to fix it. For more information, " + "see https://github.com/kornia/kornia/issues/1396." + ) + + batched_boxes = self._data if self._is_batched else self._data.unsqueeze(0) + + # Create boxes in xyzxyz_plus format. + boxes = torch.stack([batched_boxes.amin(dim=-2), batched_boxes.amax(dim=-2)], dim=-2).view( + batched_boxes.shape[0], batched_boxes.shape[1], 6 + ) + + mode = mode.lower() + if mode in ("xyzxyz", "xyzxyz_plus"): + pass + elif mode in ("xyzwhd", "vertices", "vertices_plus"): + width = boxes[..., 3] - boxes[..., 0] + 1 + height = boxes[..., 4] - boxes[..., 1] + 1 + depth = boxes[..., 5] - boxes[..., 2] + 1 + boxes[..., 3] = width + boxes[..., 4] = height + boxes[..., 5] = depth + else: + raise ValueError(f"Unknown mode {mode}") + + if mode in ("xyzxyz", "vertices"): + offset = torch.as_tensor([0, 0, 0, 1, 1, 1], device=boxes.device, dtype=boxes.dtype) + boxes = boxes + offset + + if mode.startswith("vertices"): + xmin, ymin, zmin = boxes[..., 0], boxes[..., 1], boxes[..., 2] + width, height, depth = boxes[..., 3], boxes[..., 4], boxes[..., 5] + + boxes = _boxes3d_to_polygons3d(xmin, ymin, zmin, width, height, depth) + + boxes = boxes if self._is_batched else boxes.squeeze(0) + return boxes + + def to_mask(self, depth: int, height: int, width: int) -> torch.Tensor: + """Convert ·D boxes to masks. Covered area is 1 and the remaining is 0. + + Args: + depth: depth of the masked image/images. + height: height of the masked image/images. + width: width of the masked image/images. + + Returns: + the output mask tensor, shape of :math:`(N, depth, width, height)` or :math:`(B,N, depth, width, height)` + and dtype of :func:`Boxes3D.dtype` (it can be any floating point dtype). + + Note: + It is currently non-differentiable. + + Examples: + >>> boxes = Boxes3D(torch.tensor([[ # Equivalent to boxes = Boxes.3Dfrom_tensor([[1,1,1,3,3,2]]) + ... [1., 1., 1.], + ... [3., 1., 1.], + ... [3., 3., 1.], + ... [1., 3., 1.], + ... [1., 1., 2.], + ... [3., 1., 2.], + ... [3., 3., 2.], + ... [1., 3., 2.], + ... ]])) # 1x8x3 + >>> boxes.to_mask(4, 5, 5) + tensor([[[[0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.]], + + [[0., 0., 0., 0., 0.], + [0., 1., 1., 1., 0.], + [0., 1., 1., 1., 0.], + [0., 1., 1., 1., 0.], + [0., 0., 0., 0., 0.]], + + [[0., 0., 0., 0., 0.], + [0., 1., 1., 1., 0.], + [0., 1., 1., 1., 0.], + [0., 1., 1., 1., 0.], + [0., 0., 0., 0., 0.]], + + [[0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.]]]]) + + """ + if self._data.requires_grad: + raise RuntimeError( + "Boxes.to_tensor isn't differentiable. Please, create boxes from tensors with `requires_grad=False`." + ) + + if self._is_batched: # (B, N, 8, 3) + mask = torch.zeros( + (self._data.shape[0], self._data.shape[1], depth, height, width), + dtype=self._data.dtype, + device=self._data.device, + ) + else: # (N, 8, 3) + mask = torch.zeros( + (self._data.shape[0], depth, height, width), dtype=self._data.dtype, device=self._data.device + ) + + # Boxes coordinates can be outside the image size after transforms. Clamp values to the image size + clipped_boxes_xyzxyz = self.to_tensor("xyzxyz") + clipped_boxes_xyzxyz[..., ::3].clamp_(0, width) + clipped_boxes_xyzxyz[..., 1::3].clamp_(0, height) + clipped_boxes_xyzxyz[..., 2::3].clamp_(0, depth) + + # Reshape mask to (BxN, D, H, W) and boxes to (BxN, 6) to iterate over all of them. + # Cast boxes coordinates to be integer to use them as indexes. Use round to handle decimal values. + for mask_channel, box_xyzxyz in zip( + mask.view(-1, depth, height, width), clipped_boxes_xyzxyz.view(-1, 6).round().int() + ): + # Mask channel dimensions: (depth, height, width) + mask_channel[ + box_xyzxyz[2] : box_xyzxyz[5], box_xyzxyz[1] : box_xyzxyz[4], box_xyzxyz[0] : box_xyzxyz[3] + ] = 1 + + return mask + + def transform_boxes(self, M: torch.Tensor, inplace: bool = False) -> Boxes3D: + r"""Apply a transformation matrix to the 3D boxes. + + Args: + M: The transformation matrix to be applied, shape of :math:`(4, 4)` or :math:`(B, 4, 4)`. + inplace: do transform in-place and return self. + + Returns: + The transformed boxes. + + """ + if not 2 <= M.ndim <= 3 or M.shape[-2:] != (4, 4): + raise ValueError(f"The transformation matrix shape must be (4, 4) or (B, 4, 4). Got {M.shape}.") + + transformed_boxes = _transform_boxes(self._data, M) + if inplace: + self._data = transformed_boxes + return self + + return Boxes3D(transformed_boxes, False, "xyzxyz_plus") + + def transform_boxes_(self, M: torch.Tensor) -> Boxes3D: + """Inplace version of :func:`Boxes3D.transform_boxes`.""" + return self.transform_boxes(M, inplace=True) + + @property + def data(self) -> torch.Tensor: + """Return the raw 3D corner-coordinate tensor. + + Returns: + Tensor containing eight 3D corner coordinates per box, usually + shaped :math:`(N, 8, 3)` or :math:`(B, N, 8, 3)`. + """ + return self._data + + @property + def mode(self) -> str: + """Return the 3D box format remembered by this container. + + Returns: + Mode string describing how this container should be interpreted + during tensor conversion. + """ + return self._mode + + @property + def device(self) -> torch.device: + """Returns boxes device.""" + return self._data.device + + @property + def dtype(self) -> torch.dtype: + """Returns boxes dtype.""" + return self._data.dtype + + def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> Boxes3D: + """Like :func:`torch.nn.Module.to()` method.""" + # In torchscript, dtype is a int and not a class. https://github.com/pytorch/pytorch/issues/51941 + if dtype is not None and not _is_floating_point_dtype(dtype): + raise ValueError("Boxes must be in floating point") + self._data = self._data.to(device=device, dtype=dtype) + return self diff --git a/kornia/geometry/calibration/__init__.py b/kornia/geometry/calibration/__init__.py new file mode 100644 index 0000000..a660560 --- /dev/null +++ b/kornia/geometry/calibration/__init__.py @@ -0,0 +1,27 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Geometry Calibration — Camera calibration utilities for Kornia. + +This subpackage provides functions for camera calibration, distortion, and undistortion. +""" + +from .distort import distort_points, tilt_projection +from .pnp import solve_pnp_dlt +from .undistort import undistort_image, undistort_points + +__all__ = ["distort_points", "solve_pnp_dlt", "tilt_projection", "undistort_image", "undistort_points"] diff --git a/kornia/geometry/calibration/distort.py b/kornia/geometry/calibration/distort.py new file mode 100644 index 0000000..e0383f0 --- /dev/null +++ b/kornia/geometry/calibration/distort.py @@ -0,0 +1,175 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Optional + +import torch +import torch.nn.functional as F + + +# Based on https://github.com/opencv/opencv/blob/master/modules/calib3d/src/distortion_model.hpp#L75 +def tilt_projection(taux: torch.Tensor, tauy: torch.Tensor, return_inverse: bool = False) -> torch.Tensor: + r"""Estimate the tilt projection matrix or the inverse tilt projection matrix. + + Args: + taux: Rotation angle in radians around the :math:`x`-axis with shape :math:`(*, 1)`. + tauy: Rotation angle in radians around the :math:`y`-axis with shape :math:`(*, 1)`. + return_inverse: False to obtain the tilt projection matrix. True for the inverse matrix. + + Returns: + torch.Tensor: Inverse tilt projection matrix with shape :math:`(*, 3, 3)`. + + """ + if taux.shape != tauy.shape: + raise ValueError(f"Shape of taux {taux.shape} and tauy {tauy.shape} do not match.") + + ndim: int = taux.dim() + taux = taux.reshape(-1) + tauy = tauy.reshape(-1) + + cTx = torch.cos(taux) + sTx = torch.sin(taux) + cTy = torch.cos(tauy) + sTy = torch.sin(tauy) + zero = torch.zeros_like(cTx) + one = torch.ones_like(cTx) + + Rx = torch.stack([one, zero, zero, zero, cTx, sTx, zero, -sTx, cTx], -1).reshape(-1, 3, 3) + Ry = torch.stack([cTy, zero, -sTy, zero, one, zero, sTy, zero, cTy], -1).reshape(-1, 3, 3) + R = Ry @ Rx + + if return_inverse: + invR22 = 1 / R[..., 2, 2] + invPz = torch.stack( + [invR22, zero, R[..., 0, 2] * invR22, zero, invR22, R[..., 1, 2] * invR22, zero, zero, one], -1 + ).reshape(-1, 3, 3) + + inv_tilt = R.transpose(-1, -2) @ invPz + if ndim == 0: + inv_tilt = torch.squeeze(inv_tilt) + + return inv_tilt + + Pz = torch.stack( + [R[..., 2, 2], zero, -R[..., 0, 2], zero, R[..., 2, 2], -R[..., 1, 2], zero, zero, one], -1 + ).reshape(-1, 3, 3) + + tilt = Pz @ R.transpose(-1, -2) + if ndim == 0: + tilt = torch.squeeze(tilt) + + return tilt + + +def distort_points( + points: torch.Tensor, K: torch.Tensor, dist: torch.Tensor, new_K: Optional[torch.Tensor] = None +) -> torch.Tensor: + r"""Distortion of a set of 2D points based on the lens distortion model. + + Radial :math:`(k_1, k_2, k_3, k_4, k_4, k_6)`, + tangential :math:`(p_1, p_2)`, thin prism :math:`(s_1, s_2, s_3, s_4)`, and tilt :math:`(\tau_x, \tau_y)` + distortion models are considered in this function. + + Args: + points: Input image points with shape :math:`(*, N, 2)`. + K: Intrinsic camera matrix with shape :math:`(*, 3, 3)`. + dist: Distortion coefficients + :math:`(k_1,k_2,p_1,p_2[,k_3[,k_4,k_5,k_6[,s_1,s_2,s_3,s_4[,\tau_x,\tau_y]]]])`. This is + a vector with 4, 5, 8, 12 or 14 elements with shape :math:`(*, n)`. + new_K: Intrinsic camera matrix of the distorted image. By default, it is the same as K but you may additionally + scale and shift the result by using a different matrix. Shape: :math:`(*, 3, 3)`. Default: None. + + Returns: + Undistorted 2D points with shape :math:`(*, N, 2)`. + + Example: + >>> points = torch.rand(1, 1, 2) + >>> K = torch.eye(3)[None] + >>> dist_coeff = torch.rand(1, 4) + >>> points_dist = distort_points(points, K, dist_coeff) + + """ + if points.dim() < 2 and points.shape[-1] != 2: + raise ValueError(f"points shape is invalid. Got {points.shape}.") + + if K.shape[-2:] != (3, 3): + raise ValueError(f"K matrix shape is invalid. Got {K.shape}.") + + if new_K is None: + new_K = K + elif new_K.shape[-2:] != (3, 3): + raise ValueError(f"new_K matrix shape is invalid. Got {new_K.shape}.") + + if dist.shape[-1] not in [4, 5, 8, 12, 14]: + raise ValueError(f"Invalid number of distortion coefficients. Got {dist.shape[-1]}") + + # Adding torch.zeros to obtain vector with 14 coeffs. + if dist.shape[-1] < 14: + dist = F.pad(dist, [0, 14 - dist.shape[-1]]) + + # Convert 2D points from pixels to normalized camera coordinates + new_cx: torch.Tensor = new_K[..., 0:1, 2] # princial point in x (Bx1) + new_cy: torch.Tensor = new_K[..., 1:2, 2] # princial point in y (Bx1) + new_fx: torch.Tensor = new_K[..., 0:1, 0] # focal in x (Bx1) + new_fy: torch.Tensor = new_K[..., 1:2, 1] # focal in y (Bx1) + + # This is equivalent to K^-1 [u,v,1]^T + x: torch.Tensor = (points[..., 0] - new_cx) / new_fx # (BxN - Bx1)/Bx1 -> BxN or (N,) + y: torch.Tensor = (points[..., 1] - new_cy) / new_fy # (BxN - Bx1)/Bx1 -> BxN or (N,) + + # Distort points + r2 = x * x + y * y + r4 = r2 * r2 + r6 = r4 * r2 + + rad_poly = (1 + dist[..., 0:1] * r2 + dist[..., 1:2] * r4 + dist[..., 4:5] * r6) / ( + 1 + dist[..., 5:6] * r2 + dist[..., 6:7] * r4 + dist[..., 7:8] * r6 + ) + xd = ( + x * rad_poly + + 2 * dist[..., 2:3] * x * y + + dist[..., 3:4] * (r2 + 2 * x * x) + + dist[..., 8:9] * r2 + + dist[..., 9:10] * r4 + ) + yd = ( + y * rad_poly + + dist[..., 2:3] * (r2 + 2 * y * y) + + 2 * dist[..., 3:4] * x * y + + dist[..., 10:11] * r2 + + dist[..., 11:12] * r4 + ) + + # Compensate for tilt distortion + if torch.any(dist[..., 12] != 0) or torch.any(dist[..., 13] != 0): + tilt = tilt_projection(dist[..., 12], dist[..., 13]) + + # Transposed untilt points (instead of [x,y,1]^T, we obtain [x,y,1]) + points_untilt = torch.stack([xd, yd, torch.ones_like(xd)], -1) @ tilt.transpose(-2, -1) + xd = points_untilt[..., 0] / points_untilt[..., 2] + yd = points_untilt[..., 1] / points_untilt[..., 2] + + # Convert points from normalized camera coordinates to pixel coordinates + cx: torch.Tensor = K[..., 0:1, 2] # princial point in x (Bx1) + cy: torch.Tensor = K[..., 1:2, 2] # princial point in y (Bx1) + fx: torch.Tensor = K[..., 0:1, 0] # focal in x (Bx1) + fy: torch.Tensor = K[..., 1:2, 1] # focal in y (Bx1) + + x = fx * xd + cx + y = fy * yd + cy + + return torch.stack([x, y], -1) diff --git a/kornia/geometry/calibration/pnp.py b/kornia/geometry/calibration/pnp.py new file mode 100644 index 0000000..509fdd4 --- /dev/null +++ b/kornia/geometry/calibration/pnp.py @@ -0,0 +1,251 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Optional, Tuple + +import torch +from torch.linalg import qr as linalg_qr + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SAME_SHAPE, KORNIA_CHECK_SHAPE +from kornia.core.ops import eye_like +from kornia.core.utils import _torch_linalg_svdvals, _torch_svd_cast +from kornia.geometry.conversions import convert_points_to_homogeneous, normalize_points_with_intrinsics +from kornia.geometry.linalg import transform_points + + +def _mean_isotropic_scale_normalize(points: torch.Tensor, eps: float = 1e-8) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Normalize points. + + Args: + points : torch.Tensor containing the points to be normalized with shape :math:`(B, N, D)`. + eps : Small value to avoid division by zero error. + + Returns: + Tuple containing the normalized points in the shape :math:`(B, N, D)` and the transformation matrix + in the shape :math:`(B, D+1, D+1)`. + + """ + KORNIA_CHECK_SHAPE(points, ["B", "N", "D"]) + x_mean = torch.mean(points, dim=1, keepdim=True) # Bx1xD + scale = (points - x_mean).norm(dim=-1, p=2).mean(dim=-1) # B + + D_int = points.shape[-1] + scale = (D_int**0.5) / (scale + eps) # B + transform = eye_like(D_int + 1, points) # (B, D+1, D+1) + + idxs = torch.arange(D_int, dtype=torch.int64, device=points.device) + transform[:, idxs, idxs] = transform[:, idxs, idxs] * scale[:, None] + transform[:, idxs, D_int] = transform[:, idxs, D_int] + (-scale[:, None] * x_mean[:, 0, idxs]) + + points_norm = transform_points(transform, points) # BxNxD + + return (points_norm, transform) + + +def solve_pnp_dlt( + world_points: torch.Tensor, + img_points: torch.Tensor, + intrinsics: torch.Tensor, + weights: Optional[torch.Tensor] = None, + svd_eps: float = 1e-4, +) -> torch.Tensor: + r"""Attempt to solve the Perspective-n-Point (PnP) problem using Direct Linear Transform (DLT). + + Given a batch (torch.where batch size is :math:`B`) of :math:`N` 3D points + (torch.where :math:`N \geq 6`) in the world space, a batch of :math:`N` + corresponding 2D points in the image space and a batch of + intrinsic matrices, this function tries to estimate a batch of + world to camera transformation matrices. + + This implementation needs at least 6 points (i.e. :math:`N \geq 6`) to + provide solutions. + + This function cannot be used if all the 3D world points (of any element + of the batch) lie on a line or if all the 3D world points (of any element + of the batch) lie on a plane. This function attempts to check for these + conditions and throws an AssertionError if found. Do note that this check + is sensitive to the value of the svd_eps parameter. + + Another bad condition occurs when the camera and the points lie on a + twisted cubic. However, this function does not check for this condition. + + Args: + world_points : A torch.Tensor with shape :math:`(B, N, 3)` representing + the points in the world space. + img_points : A torch.Tensor with shape :math:`(B, N, 2)` representing + the points in the image space. + intrinsics : A torch.Tensor with shape :math:`(B, 3, 3)` representing + the intrinsic matrices. + weights : A torch.Tensor with shape :math:`(B, N)` representing the + weights for each point. If None, all points are considered to be equally important. + svd_eps : A small float value to avoid numerical precision issues. + + Returns: + A torch.Tensor with shape :math:`(B, 3, 4)` representing the estimated world to + camera transformation matrices (also known as the extrinsic matrices). + + Example: + >>> world_points = torch.tensor([[ + ... [ 5. , -5. , 0. ], [ 0. , 0. , 1.5], + ... [ 2.5, 3. , 6. ], [ 9. , -2. , 3. ], + ... [-4. , 5. , 2. ], [-5. , 5. , 1. ], + ... ]], dtype=torch.float64) + >>> + >>> img_points = torch.tensor([[ + ... [1409.1504, -800.936 ], [ 407.0207, -182.1229], + ... [ 392.7021, 177.9428], [1016.838 , -2.9416], + ... [ -63.1116, 142.9204], [-219.3874, 99.666 ], + ... ]], dtype=torch.float64) + >>> + >>> intrinsics = torch.tensor([[ + ... [ 500., 0., 250.], + ... [ 0., 500., 250.], + ... [ 0., 0., 1.], + ... ]], dtype=torch.float64) + >>> + >>> print(world_points.shape, img_points.shape, intrinsics.shape) + torch.Size([1, 6, 3]) torch.Size([1, 6, 2]) torch.Size([1, 3, 3]) + >>> + >>> pred_world_to_cam = kornia.geometry.solve_pnp_dlt(world_points, img_points, intrinsics) + >>> + >>> print(pred_world_to_cam.shape) + torch.Size([1, 3, 4]) + >>> + >>> pred_world_to_cam + tensor([[[ 0.9392, -0.3432, -0.0130, 1.6734], + [ 0.3390, 0.9324, -0.1254, -4.3634], + [ 0.0552, 0.1134, 0.9920, 3.7785]]], dtype=torch.float64) + + """ + # This function was implemented based on ideas inspired from multiple references. + # ============ + # References: + # ============ + # 1. https://team.inria.fr/lagadic/camera_localization/tutorial-pose-dlt-opencv.html + # 2. https://github.com/opencv/opencv/blob/68d15fc62edad980f1ffa15ee478438335f39cc3/modules/calib3d/src/calibration.cpp # noqa: E501 + # 3. http://rpg.ifi.uzh.ch/docs/teaching/2020/03_camera_calibration.pdf + # 4. http://www.cs.cmu.edu/~16385/s17/Slides/11.3_Pose_Estimation.pdf + # 5. https://www.ece.mcmaster.ca/~shirani/vision/hartley_ch7.pdf + accepted_dtypes = (torch.float32, torch.float64) + KORNIA_CHECK_IS_TENSOR(world_points) + KORNIA_CHECK_IS_TENSOR(img_points) + KORNIA_CHECK_IS_TENSOR(intrinsics) + KORNIA_CHECK(isinstance(svd_eps, float)) + KORNIA_CHECK(world_points.dtype in accepted_dtypes) + KORNIA_CHECK(img_points.dtype in accepted_dtypes) + KORNIA_CHECK(intrinsics.dtype in accepted_dtypes) + KORNIA_CHECK_SHAPE(world_points, ["B", "N", "3"]) + KORNIA_CHECK_SHAPE(img_points, ["B", "N", "2"]) + KORNIA_CHECK_SHAPE(intrinsics, ["B", "3", "3"]) + KORNIA_CHECK_SAME_SHAPE(world_points[:, :, 0], img_points[:, :, 0]) + KORNIA_CHECK(world_points.shape[1] >= 6) + if weights is not None: + KORNIA_CHECK_IS_TENSOR(weights) + + B, N = world_points.shape[:2] + + # Getting normalized world points. + world_points_norm, world_transform_norm = _mean_isotropic_scale_normalize(world_points) + + # Checking if world_points_norm (of any element of the batch) has rank = 3. This + # function cannot be used if all world points (of any element of the batch) lie + # on a line or if all world points (of any element of the batch) lie on a plane. + s = _torch_linalg_svdvals(world_points_norm) + if torch.any(s[:, -1] < svd_eps): + raise AssertionError( + "The last singular value of one/more of the elements of the batch is smaller " + f"than {svd_eps}. This function cannot be used if all world_points (of any " + "element of the batch) lie on a line or if all world_points (of any " + "element of the batch) lie on a plane." + ) + world_points_norm_h = convert_points_to_homogeneous(world_points_norm) + + # Normalizing img_points_inv + img_points_inv = normalize_points_with_intrinsics(img_points, intrinsics) + img_points_norm, img_transform_norm = _mean_isotropic_scale_normalize(img_points_inv) + inv_img_transform_norm = torch.inverse(img_transform_norm) + + # Setting up the system (the matrix A in Ax=0) + system = torch.zeros((B, 2 * N, 12), dtype=world_points.dtype, device=world_points.device) + system[:, 0::2, 0:4] = world_points_norm_h + system[:, 1::2, 4:8] = world_points_norm_h + system[:, 0::2, 8:12] = world_points_norm_h * (-1) * img_points_norm[..., 0:1] + system[:, 1::2, 8:12] = world_points_norm_h * (-1) * img_points_norm[..., 1:2] + + # Apply weights to the system if provided + if weights is not None: + if weights.shape != (B, N): + raise AssertionError(f"Weights should have shape (B, N). Got {weights.shape}.") + weights_expanded = weights.unsqueeze(-1).repeat(1, 1, 2).view(B, 2 * N, 1) + # Multiply the system matrix by the expanded weights + system = weights_expanded * system + + # Getting the solution vectors. + _, _, v = _torch_svd_cast(system) + solution = v[..., -1] + + # Reshaping the solution vectors to the correct shape. + solution = solution.reshape(B, 3, 4) + + # Creating solution_4x4 + solution_4x4 = eye_like(4, solution) + solution_4x4[:, :3, :] = solution + + # De-normalizing the solution + intermediate = torch.bmm(solution_4x4, world_transform_norm) + solution = torch.bmm(inv_img_transform_norm, intermediate[:, :3, :]) + + # We obtained one solution for each element of the batch. We may + # need to multiply each solution with a scalar. This is because + # if x is a solution to Ax=0, then cx is also a solution. We can + # find the required scalars by using the properties of + # rotation matrices. We do this in two parts: + + # First, we fix the sign by making sure that the determinant of + # the all the rotation matrices are non-negative (since determinant + # of a rotation matrix should be 1). + det = torch.det(solution[:, :3, :3]) + ones_tensor = torch.ones_like(det) + sign_fix = torch.where(det < 0, ones_tensor * -1, ones_tensor) + solution = solution * sign_fix[:, None, None] + + # Then, we make sure that norm of the 0th columns of the rotation + # matrices are 1. Do note that the norm of any column of a rotation + # matrix should be 1. Here we use the 0th column to calculate norm_col. + # We then multiply solution with mul_factor. + norm_col = torch.norm(input=solution[:, :3, 0], p=2, dim=1) + mul_factor = (1 / norm_col)[:, None, None] + temp = solution * mul_factor + + # To make sure that the rotation matrix would be orthogonal, we apply + # QR decomposition. + ortho, right = linalg_qr(temp[:, :3, :3]) + + # We may need to fix the signs of the columns of the ortho matrix. + # If right[i, j, j] is negative, then we need to flip the signs of + # the column ortho[i, :, j]. The below code performs the necessary + # operations in a better way. + mask = eye_like(3, ortho) + col_sign_fix = torch.sign(mask * right) + rot_mat = torch.bmm(ortho, col_sign_fix) + + # Preparing the final output. + pred_world_to_cam = torch.cat([rot_mat, temp[:, :3, 3:4]], dim=-1) + + # TODO: Implement algorithm to refine the solution. + + return pred_world_to_cam diff --git a/kornia/geometry/calibration/undistort.py b/kornia/geometry/calibration/undistort.py new file mode 100644 index 0000000..52bbf4e --- /dev/null +++ b/kornia/geometry/calibration/undistort.py @@ -0,0 +1,200 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional + +import torch +import torch.nn.functional as F + +from kornia.core.check import KORNIA_CHECK_SHAPE +from kornia.geometry.grid import create_meshgrid +from kornia.geometry.linalg import transform_points +from kornia.geometry.transform import remap + +from .distort import distort_points, tilt_projection + + +# Based on https://github.com/opencv/opencv/blob/master/modules/calib3d/src/undistort.dispatch.cpp#L384 +def undistort_points( + points: torch.Tensor, K: torch.Tensor, dist: torch.Tensor, new_K: Optional[torch.Tensor] = None, num_iters: int = 5 +) -> torch.Tensor: + r"""Compensate for lens distortion a set of 2D image points. + + Radial :math:`(k_1, k_2, k_3, k_4, k_5, k_6)`, + tangential :math:`(p_1, p_2)`, thin prism :math:`(s_1, s_2, s_3, s_4)`, and tilt :math:`(\tau_x, \tau_y)` + distortion models are considered in this function. + + Args: + points: Input image points with shape :math:`(*, N, 2)`. + K: Intrinsic camera matrix with shape :math:`(*, 3, 3)`. + dist: Distortion coefficients + :math:`(k_1,k_2,p_1,p_2[,k_3[,k_4,k_5,k_6[,s_1,s_2,s_3,s_4[,\tau_x,\tau_y]]]])`. This is + a vector with 4, 5, 8, 12 or 14 elements with shape :math:`(*, n)`. + new_K: Intrinsic camera matrix of the distorted image. By default, it is the same as K but you may additionally + scale and shift the result by using a different matrix. Shape: :math:`(*, 3, 3)`. Default: None. + num_iters: Number of undistortion iterations. Default: 5. + + Returns: + Undistorted 2D points with shape :math:`(*, N, 2)`. + + Example: + >>> _ = torch.manual_seed(0) + >>> x = torch.rand(1, 4, 2) + >>> K = torch.eye(3)[None] + >>> dist = torch.rand(1, 4) + >>> undistort_points(x, K, dist) + tensor([[[-0.1513, -0.1165], + [ 0.0711, 0.1100], + [-0.0697, 0.0228], + [-0.1843, -0.1606]]]) + + """ + KORNIA_CHECK_SHAPE(points, ["*", "N", "2"]) + KORNIA_CHECK_SHAPE(K, ["*", "3", "3"]) + + if points.dim() < 2 and points.shape[-1] != 2: + raise ValueError(f"points shape is invalid. Got {points.shape}.") + + if new_K is None: + new_K = K + else: + KORNIA_CHECK_SHAPE(new_K, ["*", "3", "3"]) + + if dist.shape[-1] not in [4, 5, 8, 12, 14]: + raise ValueError(f"Invalid number of distortion coefficients. Got {dist.shape[-1]}") + + # Adding torch.zeros to obtain vector with 14 coeffs. + if dist.shape[-1] < 14: + dist = F.pad(dist, [0, 14 - dist.shape[-1]]) + + # Convert 2D points from pixels to normalized camera coordinates + cx: torch.Tensor = K[..., 0:1, 2] # princial point in x (Bx1) + cy: torch.Tensor = K[..., 1:2, 2] # princial point in y (Bx1) + fx: torch.Tensor = K[..., 0:1, 0] # focal in x (Bx1) + fy: torch.Tensor = K[..., 1:2, 1] # focal in y (Bx1) + + # This is equivalent to K^-1 [u,v,1]^T + x: torch.Tensor = (points[..., 0] - cx) / fx # (BxN - Bx1)/Bx1 -> BxN + y: torch.Tensor = (points[..., 1] - cy) / fy # (BxN - Bx1)/Bx1 -> BxN + + # Compensate for tilt distortion + if torch.any(dist[..., 12] != 0) or torch.any(dist[..., 13] != 0): + inv_tilt = tilt_projection(dist[..., 12], dist[..., 13], True) + + # Transposed untilt points (instead of [x,y,1]^T, we obtain [x,y,1]) + x, y = transform_points(inv_tilt, torch.stack([x, y], dim=-1)).unbind(-1) + + # Iteratively undistort points + x0, y0 = x, y + for _ in range(num_iters): + r2 = x * x + y * y + + inv_rad_poly = (1 + dist[..., 5:6] * r2 + dist[..., 6:7] * r2 * r2 + dist[..., 7:8] * r2**3) / ( + 1 + dist[..., 0:1] * r2 + dist[..., 1:2] * r2 * r2 + dist[..., 4:5] * r2**3 + ) + deltaX = ( + 2 * dist[..., 2:3] * x * y + + dist[..., 3:4] * (r2 + 2 * x * x) + + dist[..., 8:9] * r2 + + dist[..., 9:10] * r2 * r2 + ) + deltaY = ( + dist[..., 2:3] * (r2 + 2 * y * y) + + 2 * dist[..., 3:4] * x * y + + dist[..., 10:11] * r2 + + dist[..., 11:12] * r2 * r2 + ) + + x = (x0 - deltaX) * inv_rad_poly + y = (y0 - deltaY) * inv_rad_poly + + # Convert points from normalized camera coordinates to pixel coordinates + new_cx: torch.Tensor = new_K[..., 0:1, 2] # princial point in x (Bx1) + new_cy: torch.Tensor = new_K[..., 1:2, 2] # princial point in y (Bx1) + new_fx: torch.Tensor = new_K[..., 0:1, 0] # focal in x (Bx1) + new_fy: torch.Tensor = new_K[..., 1:2, 1] # focal in y (Bx1) + x = new_fx * x + new_cx + y = new_fy * y + new_cy + return torch.stack([x, y], -1) + + +# Based on https://github.com/opencv/opencv/blob/master/modules/calib3d/src/undistort.dispatch.cpp#L287 +def undistort_image(image: torch.Tensor, K: torch.Tensor, dist: torch.Tensor) -> torch.Tensor: + r"""Compensate an image for lens distortion. + + Radial :math:`(k_1, k_2, k_3, k_4, k_4, k_6)`, + tangential :math:`(p_1, p_2)`, thin prism :math:`(s_1, s_2, s_3, s_4)`, and tilt :math:`(\tau_x, \tau_y)` + distortion models are considered in this function. + + Args: + image: Input image with shape :math:`(*, C, H, W)`. + K: Intrinsic camera matrix with shape :math:`(*, 3, 3)`. + dist: Distortion coefficients + :math:`(k_1,k_2,p_1,p_2[,k_3[,k_4,k_5,k_6[,s_1,s_2,s_3,s_4[,\tau_x,\tau_y]]]])`. This is + a vector with 4, 5, 8, 12 or 14 elements with shape :math:`(*, n)`. + + Returns: + Undistorted image with shape :math:`(*, C, H, W)`. + + Example: + >>> img = torch.rand(1, 3, 5, 5) + >>> K = torch.eye(3)[None] + >>> dist_coeff = torch.rand(1, 4) + >>> out = undistort_image(img, K, dist_coeff) + >>> out.shape + torch.Size([1, 3, 5, 5]) + + """ + if len(image.shape) < 3: + raise ValueError(f"Image shape is invalid. Got: {image.shape}.") + + if K.shape[-2:] != (3, 3): + raise ValueError(f"K matrix shape is invalid. Got {K.shape}.") + + if dist.shape[-1] not in [4, 5, 8, 12, 14]: + raise ValueError(f"Invalid number of distortion coefficients. Got {dist.shape[-1]}.") + + if not image.is_floating_point(): + raise ValueError(f"Invalid input image data type. Input should be float. Got {image.dtype}.") + + if image.shape[:-3] != K.shape[:-2] or image.shape[:-3] != dist.shape[:-1]: + # Input with image shape (1, C, H, W), K shape (3, 3), dist shape (4) + # allowed to avoid a breaking change. + if not all((image.shape[:-3] == (1,), K.shape[:-2] == (), dist.shape[:-1] == ())): + raise ValueError( + "Input shape is invalid. Input batch dimensions should match. " + f"Got {image.shape[:-3]}, {K.shape[:-2]}, {dist.shape[:-1]}." + ) + + channels, rows, cols = image.shape[-3:] + B = image.numel() // (channels * rows * cols) + + # Create point coordinates for each pixel of the image + xy_grid: torch.Tensor = create_meshgrid(rows, cols, False, image.device, image.dtype) + pts = xy_grid.reshape(-1, 2) # (rows*cols)x2 matrix of pixel coordinates + + # Distort points and define maps + ptsd: torch.Tensor = distort_points(pts, K, dist) # Bx(rows*cols)x2 + mapx: torch.Tensor = ptsd[..., 0].reshape(B, rows, cols) # B x rows x cols, float + mapy: torch.Tensor = ptsd[..., 1].reshape(B, rows, cols) # B x rows x cols, float + + # Remap image to undistort + out = remap(image.reshape(B, channels, rows, cols), mapx, mapy, align_corners=True) + + return out.view_as(image) diff --git a/kornia/geometry/camera/__init__.py b/kornia/geometry/camera/__init__.py new file mode 100644 index 0000000..763c13e --- /dev/null +++ b/kornia/geometry/camera/__init__.py @@ -0,0 +1,58 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Geometry Camera — Camera models and camera-related utilities for Kornia. + +This subpackage provides camera models, distortion, and projection functions. +""" + +from .distortion_affine import distort_points_affine, dx_distort_points_affine, undistort_points_affine +from .distortion_kannala_brandt import ( + distort_points_kannala_brandt, + dx_distort_points_kannala_brandt, + undistort_points_kannala_brandt, +) +from .perspective import project_points, unproject_points +from .pinhole import PinholeCamera, cam2pixel, pixel2cam +from .projection_orthographic import ( + dx_project_points_orthographic, + project_points_orthographic, + unproject_points_orthographic, +) +from .projection_z1 import dx_project_points_z1, project_points_z1, unproject_points_z1 +from .stereo import StereoCamera + +__all__ = [ + "PinholeCamera", + "StereoCamera", + "cam2pixel", + "distort_points_affine", + "distort_points_kannala_brandt", + "dx_distort_points_affine", + "dx_distort_points_kannala_brandt", + "dx_project_points_orthographic", + "dx_project_points_z1", + "pixel2cam", + "project_points", + "project_points_orthographic", + "project_points_z1", + "undistort_points_affine", + "undistort_points_kannala_brandt", + "unproject_points", + "unproject_points_orthographic", + "unproject_points_z1", +] diff --git a/kornia/geometry/camera/distortion_affine.py b/kornia/geometry/camera/distortion_affine.py new file mode 100644 index 0000000..7487680 --- /dev/null +++ b/kornia/geometry/camera/distortion_affine.py @@ -0,0 +1,132 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""nn.Module containing the affine distortion model.""" + +# inspired by: https://github.com/farm-ng/sophus-rs/blob/main/src/sensor/affine.rs +import torch + +from kornia.core.check import KORNIA_CHECK_SHAPE + + +def distort_points_affine(projected_points_in_camera_z1_plane: torch.Tensor, params: torch.Tensor) -> torch.Tensor: + r"""Distort one or more points from the canonical z=1 plane into the camera frame. + + .. math:: + \begin{bmatrix} u \\ v \end{bmatrix} = + \begin{bmatrix} f_x & 0 \\ 0 & f_y \end{bmatrix} + \begin{bmatrix} x \\ y \end{bmatrix} + + \begin{bmatrix} c_x \\ c_y \end{bmatrix} + + Args: + projected_points_in_camera_z1_plane: torch.Tensor representing the points to distort with shape (..., 2). + params: torch.Tensor representing the parameters of the affine distortion model with shape (..., 4). + + Returns: + torch.Tensor representing the distorted points with shape (..., 2). + + Example: + >>> points = torch.tensor([319.5, 239.5]) # center of a 640x480 image + >>> params = torch.tensor([600., 600., 319.5, 239.5]) + >>> distort_points_affine(points, params) + tensor([192019.5000, 143939.5000]) + + """ + KORNIA_CHECK_SHAPE(projected_points_in_camera_z1_plane, ["*", "2"]) + KORNIA_CHECK_SHAPE(params, ["*", "4"]) + + x = projected_points_in_camera_z1_plane[..., 0] + y = projected_points_in_camera_z1_plane[..., 1] + + fx, fy = params[..., 0], params[..., 1] + cx, cy = params[..., 2], params[..., 3] + + u = fx * x + cx + v = fy * y + cy + + return torch.stack([u, v], dim=-1) + + +def undistort_points_affine(distorted_points_in_camera: torch.Tensor, params: torch.Tensor) -> torch.Tensor: + r"""Undistort one or more points from the camera frame into the canonical z=1 plane. + + .. math:: + \begin{bmatrix} x \\ y \end{bmatrix} = + \begin{bmatrix} u \\ v \end{bmatrix} - + \begin{bmatrix} c_x \\ c_y \end{bmatrix} + \begin{bmatrix} f_x & 0 \\ 0 & f_y \end{bmatrix}^{-1} + + Args: + distorted_points_in_camera: torch.Tensor representing the points to undistort with shape (..., 2). + params: torch.Tensor representing the parameters of the affine distortion model with shape (..., 4). + + Returns: + torch.Tensor representing the undistorted points with shape (..., 2). + + Example: + >>> points = torch.tensor([319.5, 239.5]) # center of a 640x480 image + >>> params = torch.tensor([600., 600., 319.5, 239.5]) + >>> undistort_points_affine(points, params) + tensor([0., 0.]) + + """ + KORNIA_CHECK_SHAPE(distorted_points_in_camera, ["*", "2"]) + KORNIA_CHECK_SHAPE(params, ["*", "4"]) + + u = distorted_points_in_camera[..., 0] + v = distorted_points_in_camera[..., 1] + + fx, fy = params[..., 0], params[..., 1] + cx, cy = params[..., 2], params[..., 3] + + x = (u - cx) / fx + y = (v - cy) / fy + + return torch.stack([x, y], dim=-1) + + +def dx_distort_points_affine(projected_points_in_camera_z1_plane: torch.Tensor, params: torch.Tensor) -> torch.Tensor: + r"""Compute the derivative of the x distortion with respect to the x coordinate. + + .. math:: + \frac{\partial u}{\partial x} = + \begin{bmatrix} f_x & 0 \\ 0 & f_y \end{bmatrix} + + Args: + projected_points_in_camera_z1_plane: torch.Tensor representing the points to distort with shape (..., 2). + params: torch.Tensor representing the parameters of the affine distortion model with shape (..., 4). + + Returns: + torch.Tensor representing the derivative of the x distortion with respect to the x coordinate + with shape (..., 2). + + Example: + >>> points = torch.tensor([319.5, 239.5]) # center of a 640x480 image + >>> params = torch.tensor([600., 600., 319.5, 239.5]) + >>> dx_distort_points_affine(points, params) + tensor([[600., 0.], + [ 0., 600.]]) + + """ + KORNIA_CHECK_SHAPE(projected_points_in_camera_z1_plane, ["*", "2"]) + KORNIA_CHECK_SHAPE(params, ["*", "4"]) + + fx, fy = params[..., 0], params[..., 1] + + zeros = torch.zeros_like(fx) + + return torch.stack([torch.stack([fx, zeros], dim=-1), torch.stack([zeros, fy], dim=-1)], dim=-2) diff --git a/kornia/geometry/camera/distortion_kannala_brandt.py b/kornia/geometry/camera/distortion_kannala_brandt.py new file mode 100644 index 0000000..e155cbe --- /dev/null +++ b/kornia/geometry/camera/distortion_kannala_brandt.py @@ -0,0 +1,237 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# inspired by: shttps://github.com/farm-ng/sophus-rs/blob/main/src/sensor/kannala_brandt.rs + +import torch + +from kornia.core.check import KORNIA_CHECK_SHAPE +from kornia.geometry.camera.distortion_affine import distort_points_affine + + +def _distort_points_kannala_brandt_impl( + projected_points_in_camera_z1_plane: torch.Tensor, + params: torch.Tensor, + radius_sq: torch.Tensor, +) -> torch.Tensor: + # https://github.com/farm-ng/sophus-rs/blob/20f6cac68f17fe1ac41d0aa8a27489e2b886806f/ + # src/sensor/kannala_brandt.rs#L51-L67 + x = projected_points_in_camera_z1_plane[..., 0] + y = projected_points_in_camera_z1_plane[..., 1] + + fx, fy = params[..., 0], params[..., 1] + cx, cy = params[..., 2], params[..., 3] + + k0 = params[..., 4] + k1 = params[..., 5] + k2 = params[..., 6] + k3 = params[..., 7] + + radius = radius_sq.sqrt() + radius_inverse = 1.0 / radius + theta = radius.atan2(torch.ones_like(radius)) + theta2 = theta**2 + theta4 = theta2**2 + theta6 = theta2 * theta4 + theta8 = theta4**2 + + r_distorted = theta * (1.0 + k0 * theta2 + k1 * theta4 + k2 * theta6 + k3 * theta8) + + scaling = r_distorted * radius_inverse + + u = fx * scaling * x + cx + v = fy * scaling * y + cy + + return torch.stack([u, v], dim=-1) + + +def distort_points_kannala_brandt( + projected_points_in_camera_z1_plane: torch.Tensor, params: torch.Tensor +) -> torch.Tensor: + r"""Distort points from the canonical z=1 plane into the camera frame using the Kannala-Brandt model. + + Args: + projected_points_in_camera_z1_plane: torch.Tensor representing the points to distort with shape (..., 2). + params: torch.Tensor representing the parameters of the Kannala-Brandt distortion model with shape (..., 8). + + Returns: + torch.Tensor representing the distorted points with shape (..., 2). + + Example: + >>> points = torch.tensor([319.5, 239.5]) # center of a 640x480 image + >>> params = torch.tensor([1000.0, 1000.0, 320.0, 280.0, 0.1, 0.01, 0.001, 0.0001]) + >>> distort_points_kannala_brandt(points, params) + tensor([1982.6832, 1526.3619]) + + """ + KORNIA_CHECK_SHAPE(projected_points_in_camera_z1_plane, ["*", "2"]) + KORNIA_CHECK_SHAPE(params, ["*", "8"]) + + x = projected_points_in_camera_z1_plane[..., 0] + y = projected_points_in_camera_z1_plane[..., 1] + + radius_sq = x**2 + y**2 + + distorted_points = torch.where( + radius_sq[..., None] > 1e-8, + _distort_points_kannala_brandt_impl( + projected_points_in_camera_z1_plane, + params, + radius_sq, + ), + distort_points_affine(projected_points_in_camera_z1_plane, params[..., :4]), + ) + + return distorted_points + + +def undistort_points_kannala_brandt(distorted_points_in_camera: torch.Tensor, params: torch.Tensor) -> torch.Tensor: + r"""Undistort points from the camera frame into the canonical z=1 plane using the Kannala-Brandt model. + + Args: + distorted_points_in_camera: torch.Tensor representing the points to undistort with shape (..., 2). + params: torch.Tensor representing the parameters of the Kannala-Brandt distortion model with shape (..., 8). + + Returns: + torch.Tensor representing the undistorted points with shape (..., 2). + + Example: + >>> points = torch.tensor([319.5, 239.5]) # center of a 640x480 image + >>> params = torch.tensor([1000.0, 1000.0, 320.0, 280.0, 0.1, 0.01, 0.001, 0.0001]) + >>> undistort_points_kannala_brandt(points, params).shape + torch.Size([2]) + + """ + KORNIA_CHECK_SHAPE(distorted_points_in_camera, ["*", "2"]) + KORNIA_CHECK_SHAPE(params, ["*", "8"]) + + iters = 10 + eps = 1e-8 + device = distorted_points_in_camera.device + out_dtype = distorted_points_in_camera.dtype + + pts = distorted_points_in_camera.to(device=device, dtype=params.dtype) + p = params.to(device=device, dtype=params.dtype) + + x = pts[..., 0] + y = pts[..., 1] + + fx = p[..., 0] + fy = p[..., 1] + cx = p[..., 2] + cy = p[..., 3] + k0 = p[..., 4] + k1 = p[..., 5] + k2 = p[..., 6] + k3 = p[..., 7] + + un = (x - cx) / fx + vn = (y - cy) / fy + + rth2 = un * un + vn * vn + rth = rth2.sqrt() + + th = rth.clamp(min=1e-16).sqrt() + + # gauss-newton + for _ in range(iters): + th2 = th * th + inner = k0 + th2 * (k1 + th2 * (k2 + th2 * k3)) + thd = th * (1.0 + th2 * inner) + d_thd = 1.0 + th2 * (3.0 * k0 + th2 * (5.0 * k1 + th2 * (7.0 * k2 + 9.0 * k3 * th2))) + step = (thd - rth) / (d_thd + 1e-12) + th = th - step + + radius_undistorted = th.tan() + denom = rth + eps + mag = radius_undistorted.abs() / denom + undistorted = torch.stack([mag * un, mag * vn], dim=-1) + + return undistorted.to(device=device, dtype=out_dtype) + + +def dx_distort_points_kannala_brandt( + projected_points_in_camera_z1_plane: torch.Tensor, params: torch.Tensor +) -> torch.Tensor: + r"""Compute the derivative of the x distortion with respect to the x coordinate. + + .. math:: + \frac{\partial u}{\partial x} = + \begin{bmatrix} f_x & 0 \\ 0 & f_y \end{bmatrix} + + Args: + projected_points_in_camera_z1_plane: torch.Tensor representing the points to distort with shape (..., 2). + params: torch.Tensor representing the parameters of the Kannala-Brandt distortion model with shape (..., 8). + + Returns: + torch.Tensor representing the derivative of the x distortion with respect to the x coordinate + with shape (..., 2). + + Example: + >>> points = torch.tensor([1., 2.]) + >>> params = torch.tensor([1000.0, 1000.0, 320.0, 280.0, 0.1, 0.01, 0.001, 0.0001]) + >>> dx_distort_points_kannala_brandt(points, params) + tensor([[ 486.0507, -213.5573], + [-213.5573, 165.7147]]) + + """ + KORNIA_CHECK_SHAPE(projected_points_in_camera_z1_plane, ["*", "2"]) + KORNIA_CHECK_SHAPE(params, ["*", "8"]) + + a = projected_points_in_camera_z1_plane[..., 0] + b = projected_points_in_camera_z1_plane[..., 1] + + fx, fy = params[..., 0], params[..., 1] + + k0 = params[..., 4] + k1 = params[..., 5] + k2 = params[..., 6] + k3 = params[..., 7] + + # TODO: return identity matrix if a and b are zero + # radius_sq = a ** 2 + b ** 2 + + c0 = a.pow(2.0) + c1 = b.pow(2.0) + c2 = c0 + c1 + c3 = c2.pow(5.0 / 2.0) + c4 = c2 + 1.0 + c5 = c2.sqrt().atan() + c6 = c5.pow(2.0) + c7 = c6 * k0 + c8 = c5.pow(4.0) + c9 = c8 * k1 + c10 = c5.pow(6.0) + c11 = c10 * k2 + c12 = c5.pow(8.0) * k3 + c13 = 1.0 * c4 * c5 * (c11 + c12 + c7 + c9 + 1.0) + c14 = c13 * c3 + c15 = c2.pow(3.0 / 2.0) + c16 = c13 * c15 + c17 = 1.0 * c11 + 1.0 * c12 + 2.0 * c6 * (4.0 * c10 * k3 + 2.0 * c6 * k1 + 3.0 * c8 * k2 + k0) + c18 = c17 * c2.pow(2.0) + c19 = 1.0 / c4 + c20 = c19 / c2.pow(3.0) + c21 = a * b * c19 * (-c13 * c2 + c15 * c17) / c3 + + return torch.stack( + [ + torch.stack([c20 * fx * (-c0 * c16 + c0 * c18 + c14), c21 * fx], dim=-1), + torch.stack([c21 * fy, c20 * fy * (-c1 * c16 + c1 * c18 + c14)], dim=-1), + ], + dim=-2, + ) diff --git a/kornia/geometry/camera/perspective.py b/kornia/geometry/camera/perspective.py new file mode 100644 index 0000000..a111e8f --- /dev/null +++ b/kornia/geometry/camera/perspective.py @@ -0,0 +1,98 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch +import torch.nn.functional as F + +from kornia.geometry.conversions import ( + convert_points_from_homogeneous, + convert_points_to_homogeneous, + denormalize_points_with_intrinsics, + normalize_points_with_intrinsics, +) + + +def project_points(point_3d: torch.Tensor, camera_matrix: torch.Tensor) -> torch.Tensor: + r"""Project a 3d point onto the 2d camera plane. + + Args: + point_3d: tensor containing the 3d points to be projected + to the camera plane. The shape of the tensor can be :math:`(*, 3)`. + camera_matrix: tensor containing the intrinsics camera + matrix. The tensor shape must be :math:`(*, 3, 3)`. + + Returns: + tensor of (u, v) cam coordinates with shape :math:`(*, 2)`. + + Example: + >>> _ = torch.manual_seed(0) + >>> X = torch.rand(1, 3) + >>> K = torch.eye(3)[None] + >>> project_points(X, K) + tensor([[5.6088, 8.6827]]) + + """ + # projection eq. [u, v, w]' = K * [x y z 1]' + # u = fx * X / Z + cx + # v = fy * Y / Z + cy + # project back using depth dividing in a safe way + xy_coords: torch.Tensor = convert_points_from_homogeneous(point_3d) + return denormalize_points_with_intrinsics(xy_coords, camera_matrix) + + +def unproject_points( + point_2d: torch.Tensor, depth: torch.Tensor, camera_matrix: torch.Tensor, normalize: bool = False +) -> torch.Tensor: + r"""Unproject a 2d point in 3d. + + Transform coordinates in the pixel frame to the camera frame. + + Args: + point_2d: tensor containing the 2d to be projected to + world coordinates. The shape of the tensor can be :math:`(*, 2)`. + depth: tensor containing the depth value of each 2d + points. The tensor shape must be equal to point2d :math:`(*, 1)`. + camera_matrix: tensor containing the intrinsics camera + matrix. The tensor shape must be :math:`(*, 3, 3)`. + normalize: whether to normalize the pointcloud. This + must be set to `True` when the depth is represented as the Euclidean + ray length from the camera position. + + Returns: + tensor of (x, y, z) world coordinates with shape :math:`(*, 3)`. + + Example: + >>> _ = torch.manual_seed(0) + >>> x = torch.rand(1, 2) + >>> depth = torch.ones(1, 1) + >>> K = torch.eye(3)[None] + >>> unproject_points(x, depth, K) + tensor([[0.4963, 0.7682, 1.0000]]) + + """ + if not isinstance(depth, torch.Tensor): + raise TypeError(f"Input depth type is not a torch.Tensor. Got {type(depth)}") + + if not depth.shape[-1] == 1: + raise ValueError(f"Input depth must be in the shape of (*, 1). Got {depth.shape}") + + xy: torch.Tensor = normalize_points_with_intrinsics(point_2d, camera_matrix) + xyz: torch.Tensor = convert_points_to_homogeneous(xy) + if normalize: + xyz = F.normalize(xyz, dim=-1, p=2.0) + + return xyz * depth diff --git a/kornia/geometry/camera/pinhole.py b/kornia/geometry/camera/pinhole.py new file mode 100644 index 0000000..09230b8 --- /dev/null +++ b/kornia/geometry/camera/pinhole.py @@ -0,0 +1,799 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Iterable, List, Union + +import torch + +from kornia.core.check import KORNIA_CHECK_SAME_DEVICE +from kornia.core.utils import _torch_inverse_cast +from kornia.geometry.conversions import convert_points_from_homogeneous, convert_points_to_homogeneous +from kornia.geometry.linalg import inverse_transformation, transform_points + + +class PinholeCamera: + r"""Class that represents a Pinhole Camera model. + + Args: + intrinsics: torch.Tensor with shape :math:`(B, 4, 4)` + containing the full 4x4 camera calibration matrix. + extrinsics: torch.Tensor with shape :math:`(B, 4, 4)` + containing the full 4x4 rotation-translation matrix. + height: torch.Tensor with shape :math:`(B)` containing the image height. + width: torch.Tensor with shape :math:`(B)` containing the image width. + + .. note:: + We assume that the class attributes are in batch form in order to take + advantage of PyTorch parallelism to boost computing performance. + + """ + + def __init__( + self, intrinsics: torch.Tensor, extrinsics: torch.Tensor, height: torch.Tensor, width: torch.Tensor + ) -> None: + # verify batch size and shapes + self._check_valid([intrinsics, extrinsics, height, width]) + self._check_valid_params(intrinsics, "intrinsics") + self._check_valid_params(extrinsics, "extrinsics") + self._check_valid_shape(height, "height") + self._check_valid_shape(width, "width") + self._check_consistent_device([intrinsics, extrinsics, height, width]) + # set class attributes + self.height: torch.Tensor = height + self.width: torch.Tensor = width + self._intrinsics: torch.Tensor = intrinsics + self._extrinsics: torch.Tensor = extrinsics + + @staticmethod + def _check_valid(data_iter: Iterable[torch.Tensor]) -> bool: + if not all(data.shape[0] for data in data_iter): + raise ValueError("Arguments shapes must match") + return True + + @staticmethod + def _check_valid_params(data: torch.Tensor, data_name: str) -> bool: + if len(data.shape) not in (3, 4) and data.shape[-2:] != (4, 4): # Shouldn't this be an OR logic than AND? + raise ValueError( + f"Argument {data_name} shape must be in the following shape Bx4x4 or BxNx4x4. Got {data.shape}" + ) + return True + + @staticmethod + def _check_valid_shape(data: torch.Tensor, data_name: str) -> bool: + if not len(data.shape) == 1: + raise ValueError(f"Argument {data_name} shape must be in the following shape B. Got {data.shape}") + return True + + @staticmethod + def _check_consistent_device(data_iter: List[torch.Tensor]) -> None: + first = data_iter[0] + for data in data_iter: + KORNIA_CHECK_SAME_DEVICE(data, first) + + def device(self) -> torch.device: + r"""Return the device for camera buffers. + + Returns: + Union[str, torch.device, None] type + + """ + return self._intrinsics.device + + @property + def intrinsics(self) -> torch.Tensor: + r"""The full 4x4 intrinsics matrix. + + Returns: + torch.Tensor of shape :math:`(B, 4, 4)`. + + """ + if not self._check_valid_params(self._intrinsics, "intrinsics"): + raise AssertionError + return self._intrinsics + + @property + def extrinsics(self) -> torch.Tensor: + r"""The full 4x4 extrinsics matrix. + + Returns: + torch.Tensor of shape :math:`(B, 4, 4)`. + + """ + if not self._check_valid_params(self._extrinsics, "extrinsics"): + raise AssertionError + return self._extrinsics + + @property + def batch_size(self) -> int: + r"""Return the batch size of the storage. + + Returns: + scalar with the batch size. + + """ + return self.intrinsics.shape[0] + + @property + def fx(self) -> torch.Tensor: + r"""Return the focal length in the x-direction. + + Returns: + torch.Tensor of shape :math:`(B)`. + + """ + return self.intrinsics[..., 0, 0] + + @property + def fy(self) -> torch.Tensor: + r"""Return the focal length in the y-direction. + + Returns: + torch.Tensor of shape :math:`(B)`. + + """ + return self.intrinsics[..., 1, 1] + + @property + def cx(self) -> torch.Tensor: + r"""Return the x-coordinate of the principal point. + + Returns: + torch.Tensor of shape :math:`(B)`. + + """ + return self.intrinsics[..., 0, 2] + + @property + def cy(self) -> torch.Tensor: + r"""Return the y-coordinate of the principal point. + + Returns: + torch.Tensor of shape :math:`(B)`. + + """ + return self.intrinsics[..., 1, 2] + + @property + def tx(self) -> torch.Tensor: + r"""Return the x-coordinate of the translation vector. + + Returns: + torch.Tensor of shape :math:`(B)`. + + """ + return self.extrinsics[..., 0, -1] + + @tx.setter + def tx(self, value: Union[torch.Tensor, float]) -> None: + r"""Set the x-coordinate of the translation vector with the given value.""" + self.extrinsics[..., 0, -1] = value + + @property + def ty(self) -> torch.Tensor: + r"""Return the y-coordinate of the translation vector. + + Returns: + torch.Tensor of shape :math:`(B)`. + + """ + return self.extrinsics[..., 1, -1] + + @ty.setter + def ty(self, value: Union[torch.Tensor, float]) -> None: + r"""Set the y-coordinate of the translation vector with the given value.""" + self.extrinsics[..., 1, -1] = value + + @property + def tz(self) -> torch.Tensor: + r"""Returns the z-coordinate of the translation vector. + + Returns: + torch.Tensor of shape :math:`(B)`. + + """ + return self.extrinsics[..., 2, -1] + + @tz.setter + def tz(self, value: Union[torch.Tensor, float]) -> None: + r"""Set the y-coordinate of the translation vector with the given value.""" + self.extrinsics[..., 2, -1] = value + + @property + def rt_matrix(self) -> torch.Tensor: + r"""Return the 3x4 rotation-translation matrix. + + Returns: + torch.Tensor of shape :math:`(B, 3, 4)`. + + """ + return self.extrinsics[..., :3, :4] + + @property + def camera_matrix(self) -> torch.Tensor: + r"""Return the 3x3 camera matrix containing the intrinsics. + + Returns: + torch.Tensor of shape :math:`(B, 3, 3)`. + + """ + return self.intrinsics[..., :3, :3] + + @property + def rotation_matrix(self) -> torch.Tensor: + r"""Return the 3x3 rotation matrix from the extrinsics. + + Returns: + torch.Tensor of shape :math:`(B, 3, 3)`. + + """ + return self.extrinsics[..., :3, :3] + + @property + def translation_vector(self) -> torch.Tensor: + r"""Return the translation vector from the extrinsics. + + Returns: + torch.Tensor of shape :math:`(B, 3, 1)`. + + """ + return self.extrinsics[..., :3, -1:] + + def clone(self) -> "PinholeCamera": + r"""Return a deep copy of the current object instance.""" + height: torch.Tensor = self.height.clone() + width: torch.Tensor = self.width.clone() + intrinsics: torch.Tensor = self.intrinsics.clone() + extrinsics: torch.Tensor = self.extrinsics.clone() + return PinholeCamera(intrinsics, extrinsics, height, width) + + def intrinsics_inverse(self) -> torch.Tensor: + r"""Return the inverse of the 4x4 instrisics matrix. + + Returns: + torch.Tensor of shape :math:`(B, 4, 4)`. + + """ + return self.intrinsics.inverse() + + def scale(self, scale_factor: torch.Tensor) -> "PinholeCamera": + r"""Scale the pinhole model. + + Args: + scale_factor: a torch.Tensor with the scale factor. It has + to be broadcastable with class members. The expected shape is + :math:`(B)` or :math:`(1)`. + + Returns: + the camera model with scaled parameters. + + """ + # scale the intrinsic parameters + intrinsics: torch.Tensor = self.intrinsics.clone() + intrinsics[..., 0, 0] *= scale_factor + intrinsics[..., 1, 1] *= scale_factor + intrinsics[..., 0, 2] *= scale_factor + intrinsics[..., 1, 2] *= scale_factor + # scale the image height/width + height: torch.Tensor = scale_factor * self.height.clone() + width: torch.Tensor = scale_factor * self.width.clone() + return PinholeCamera(intrinsics, self.extrinsics, height, width) + + def scale_(self, scale_factor: Union[float, torch.Tensor]) -> "PinholeCamera": + r"""Scale the pinhole model in-place. + + Args: + scale_factor: a torch.Tensor with the scale factor. It has + to be broadcastable with class members. The expected shape is + :math:`(B)` or :math:`(1)`. + + Returns: + the camera model with scaled parameters. + + """ + # scale the intrinsic parameters + self.intrinsics[..., 0, 0] *= scale_factor + self.intrinsics[..., 1, 1] *= scale_factor + self.intrinsics[..., 0, 2] *= scale_factor + self.intrinsics[..., 1, 2] *= scale_factor + # scale the image height/width + self.height *= scale_factor + self.width *= scale_factor + return self + + def project(self, point_3d: torch.Tensor) -> torch.Tensor: + r"""Project a 3d point in world coordinates onto the 2d camera plane. + + Args: + point_3d: torch.Tensor containing the 3d points to be projected + to the camera plane. The shape of the torch.Tensor can be :math:`(*, 3)`. + + Returns: + torch.Tensor of (u, v) cam coordinates with shape :math:`(*, 2)`. + + Example: + >>> _ = torch.manual_seed(0) + >>> X = torch.rand(1, 3) + >>> K = torch.eye(4)[None] + >>> E = torch.eye(4)[None] + >>> h = torch.ones(1) + >>> w = torch.ones(1) + >>> pinhole = kornia.geometry.camera.PinholeCamera(K, E, h, w) + >>> pinhole.project(X) + tensor([[5.6088, 8.6827]]) + + """ + P = self.intrinsics @ self.extrinsics + return convert_points_from_homogeneous(transform_points(P, point_3d)) + + def unproject(self, point_2d: torch.Tensor, depth: torch.Tensor) -> torch.Tensor: + r"""Unproject a 2d point in 3d. + + Transform coordinates in the pixel frame to the world frame. + + Args: + point_2d: torch.Tensor containing the 2d to be projected to + world coordinates. The shape of the torch.Tensor can be :math:`(*, 2)`. + depth: torch.Tensor containing the depth value of each 2d + points. The torch.Tensor shape must be equal to point2d :math:`(*, 1)`. + normalize: whether to F.normalize the pointcloud. This + must be set to `True` when the depth is represented as the Euclidean + ray length from the camera position. + + Returns: + torch.Tensor of (x, y, z) world coordinates with shape :math:`(*, 3)`. + + Example: + >>> _ = torch.manual_seed(0) + >>> x = torch.rand(1, 2) + >>> depth = torch.ones(1, 1) + >>> K = torch.eye(4)[None] + >>> E = torch.eye(4)[None] + >>> h = torch.ones(1) + >>> w = torch.ones(1) + >>> pinhole = kornia.geometry.camera.PinholeCamera(K, E, h, w) + >>> pinhole.unproject(x, depth) + tensor([[0.4963, 0.7682, 1.0000]]) + + """ + P = self.intrinsics @ self.extrinsics + P_inv = _torch_inverse_cast(P) + return transform_points(P_inv, convert_points_to_homogeneous(point_2d) * depth) + + # NOTE: just for test. Decide if we keep it. + @classmethod + def from_parameters( + self, + fx: torch.Tensor, + fy: torch.Tensor, + cx: torch.Tensor, + cy: torch.Tensor, + height: int, + width: int, + tx: torch.Tensor, + ty: torch.Tensor, + tz: torch.Tensor, + batch_size: int, + device: Union[str, torch.device, None], + dtype: torch.dtype, + ) -> "PinholeCamera": + r"""Construct a batched pinhole camera from scalar parameter tensors. + + This helper allocates batched :math:`4 \times 4` intrinsic and + extrinsic matrices, fills focal lengths/principal point/translation, + and wraps them into a :class:`PinholeCamera` instance. + + Args: + fx: Horizontal focal length per batch element. + fy: Vertical focal length per batch element. + cx: Principal point x-coordinate per batch element. + cy: Principal point y-coordinate per batch element. + height: Image height in pixels. + width: Image width in pixels. + tx: Camera translation along x per batch element. + ty: Camera translation along y per batch element. + tz: Camera translation along z per batch element. + batch_size: Number of cameras in the batch. + device: Target device for the created tensors. + dtype: Target floating-point dtype for the created tensors. + + Returns: + Batched :class:`PinholeCamera` configured from the provided + intrinsic and translation parameters. + """ + # create the camera matrix + intrinsics = torch.zeros(batch_size, 4, 4, device=device, dtype=dtype) + intrinsics[..., 0, 0] += fx + intrinsics[..., 1, 1] += fy + intrinsics[..., 0, 2] += cx + intrinsics[..., 1, 2] += cy + intrinsics[..., 2, 2] += 1.0 + intrinsics[..., 3, 3] += 1.0 + # create the pose matrix + extrinsics = torch.eye(4, device=device, dtype=dtype).repeat(batch_size, 1, 1) + extrinsics[..., 0, -1] += tx + extrinsics[..., 1, -1] += ty + extrinsics[..., 2, -1] += tz + # create image hegith and width + height_tmp = torch.zeros(batch_size, device=device, dtype=dtype) + height_tmp[..., 0] += height + width_tmp = torch.zeros(batch_size, device=device, dtype=dtype) + width_tmp[..., 0] += width + return self(intrinsics, extrinsics, height_tmp, width_tmp) + + +class PinholeCamerasList(PinholeCamera): + r"""Class that represents a list of pinhole cameras. + + The class inherits from :class:`~kornia.PinholeCamera` meaning that + it will keep the same class properties but with an extra dimension. + + .. note:: + The underlying data torch.Tensor will be stacked in the first dimension. + That's it, given a list of two camera instances, the intrinsics torch.Tensor + will have a shape :math:`(B, N, 4, 4)` where :math:`B` is the batch + size and :math:`N` is the numbers of cameras (in this case two). + + Args: + pinholes_list: a python tuple or list containing a set of `PinholeCamera` instances. + + """ + + def __init__(self, pinholes_list: Iterable[PinholeCamera]) -> None: + self._initialize_parameters(pinholes_list) + + def _initialize_parameters(self, pinholes: Iterable[PinholeCamera]) -> "PinholeCamerasList": + r"""Initialise the class attributes given a cameras list.""" + if not isinstance(pinholes, (list, tuple)): + raise TypeError(f"pinhole must of type list or tuple. Got {type(pinholes)}") + height, width = [], [] + intrinsics, extrinsics = [], [] + for pinhole in pinholes: + if not isinstance(pinhole, PinholeCamera): + raise TypeError(f"Argument pinhole must be from type PinholeCamera. Got {type(pinhole)}") + height.append(pinhole.height) + width.append(pinhole.width) + intrinsics.append(pinhole.intrinsics) + extrinsics.append(pinhole.extrinsics) + # torch.cat and set members. We will assume BxNx4x4 + self.height: torch.Tensor = torch.stack(height, dim=1) + self.width: torch.Tensor = torch.stack(width, dim=1) + self._intrinsics: torch.Tensor = torch.stack(intrinsics, dim=1) + self._extrinsics: torch.Tensor = torch.stack(extrinsics, dim=1) + return self + + @property + def num_cameras(self) -> int: + r"""Return the number of pinholes cameras per batch.""" + num_cameras: int = -1 + if self.intrinsics is not None: + num_cameras = int(self.intrinsics.shape[1]) + return num_cameras + + def get_pinhole(self, idx: int) -> PinholeCamera: + r"""Return a PinholeCamera object with parameters such as Bx4x4.""" + height: torch.Tensor = self.height[..., idx] + width: torch.Tensor = self.width[..., idx] + intrinsics: torch.Tensor = self.intrinsics[:, idx] + extrinsics: torch.Tensor = self.extrinsics[:, idx] + return PinholeCamera(intrinsics, extrinsics, height, width) + + +def pinhole_matrix(pinholes: torch.Tensor, eps: float = 1e-6) -> torch.Tensor: + r"""Return the pinhole matrix from a pinhole model. + + .. note:: + This method is going to be deprecated in version 0.2 in favour of + :attr:`kornia.PinholeCamera.camera_matrix`. + + Args: + pinholes: torch.Tensor of pinhole models. + eps: epsilon for numerical stability. + + Returns: + torch.Tensor of pinhole matrices. + + Shape: + - Input: :math:`(N, 12)` + - Output: :math:`(N, 4, 4)` + + Example: + >>> rng = torch.manual_seed(0) + >>> pinhole = torch.rand(1, 12) # Nx12 + >>> pinhole_matrix(pinhole) # Nx4x4 + tensor([[[4.9626e-01, 1.0000e-06, 8.8477e-02, 1.0000e-06], + [1.0000e-06, 7.6822e-01, 1.3203e-01, 1.0000e-06], + [1.0000e-06, 1.0000e-06, 1.0000e+00, 1.0000e-06], + [1.0000e-06, 1.0000e-06, 1.0000e-06, 1.0000e+00]]]) + + """ + if not (len(pinholes.shape) == 2 and pinholes.shape[1] == 12): + raise AssertionError(pinholes.shape) + # unpack pinhole values + fx, fy, cx, cy = torch.chunk(pinholes[..., :4], 4, dim=1) # Nx1 + # create output container + k = torch.eye(4, device=pinholes.device, dtype=pinholes.dtype) + eps + k = k.view(1, 4, 4).repeat(pinholes.shape[0], 1, 1) # Nx4x4 + # fill output with pinhole values + k[..., 0, 0:1] = fx + k[..., 0, 2:3] = cx + k[..., 1, 1:2] = fy + k[..., 1, 2:3] = cy + return k + + +def inverse_pinhole_matrix(pinhole: torch.Tensor, eps: float = 1e-6) -> torch.Tensor: + r"""Return the inverted pinhole matrix from a pinhole model. + + .. note:: + This method is going to be deprecated in version 0.2 in favour of + :attr:`kornia.PinholeCamera.intrinsics_inverse()`. + + Args: + pinhole: torch.Tensor with pinhole models. + eps: epsilon for numerical stability. + + Returns: + torch.Tensor of inverted pinhole matrices. + + Shape: + - Input: :math:`(N, 12)` + - Output: :math:`(N, 4, 4)` + + Example: + >>> rng = torch.manual_seed(0) + >>> pinhole = torch.rand(1, 12) # Nx12 + >>> inverse_pinhole_matrix(pinhole) # Nx4x4 + tensor([[[ 2.0151, 0.0000, -0.1783, 0.0000], + [ 0.0000, 1.3017, -0.1719, 0.0000], + [ 0.0000, 0.0000, 1.0000, 0.0000], + [ 0.0000, 0.0000, 0.0000, 1.0000]]]) + + """ + if not (len(pinhole.shape) == 2 and pinhole.shape[1] == 12): + raise AssertionError(pinhole.shape) + # unpack pinhole values + fx, fy, cx, cy = torch.chunk(pinhole[..., :4], 4, dim=1) # Nx1 + # create output container + k = torch.eye(4, device=pinhole.device, dtype=pinhole.dtype) + k = k.view(1, 4, 4).repeat(pinhole.shape[0], 1, 1) # Nx4x4 + # fill output with inverse values + k[..., 0, 0:1] = 1.0 / (fx + eps) + k[..., 1, 1:2] = 1.0 / (fy + eps) + k[..., 0, 2:3] = -1.0 * cx / (fx + eps) + k[..., 1, 2:3] = -1.0 * cy / (fy + eps) + return k + + +def scale_pinhole(pinholes: torch.Tensor, scale: torch.Tensor) -> torch.Tensor: + r"""Scale the pinhole matrix for each pinhole model. + + .. note:: + This method is going to be deprecated in version 0.2 in favour of + :attr:`kornia.PinholeCamera.scale()`. + + Args: + pinholes: torch.Tensor with the pinhole model. + scale: torch.Tensor of scales. + + Returns: + torch.Tensor of scaled pinholes. + + Shape: + - Input: :math:`(N, 12)` and :math:`(N, 1)` + - Output: :math:`(N, 12)` + + Example: + >>> rng = torch.manual_seed(0) + >>> pinhole_i = torch.rand(1, 12) # Nx12 + >>> scales = 2.0 * torch.ones(1) # N + >>> scale_pinhole(pinhole_i, scales) # Nx12 + tensor([[0.9925, 1.5364, 0.1770, 0.2641, 0.6148, 1.2682, 0.4901, 0.8964, 0.4556, + 0.6323, 0.3489, 0.4017]]) + + """ + if not (len(pinholes.shape) == 2 and pinholes.shape[1] == 12): + raise AssertionError(pinholes.shape) + if len(scale.shape) != 1: + raise AssertionError(scale.shape) + pinholes_scaled = pinholes.clone() + pinholes_scaled[..., :6] = pinholes[..., :6] * scale.unsqueeze(-1) + return pinholes_scaled + + +def get_optical_pose_base(pinholes: torch.Tensor) -> torch.Tensor: + """Compute extrinsic transformation matrices for pinholes. + + Args: + pinholes: torch.Tensor of form [fx fy cx cy h w rx ry rz tx ty tz] + of size (N, 12). + + Returns: + torch.Tensor of extrinsic transformation matrices of size (N, 4, 4). + + """ + if not (len(pinholes.shape) == 2 and pinholes.shape[1] == 12): + raise AssertionError(pinholes.shape) + # TODO: where is rtvec_to_pose? + raise NotImplementedError + # TODO: We have rtvec_to_pose in torchgeometry + # https://github.com/whh14/torchgeometry/blob/master/torchgeometry/conversions.py#L240 + # But it relies on axis_angle_to_rotation_matrix + # And since then, it was changed from returning Nx4x4 matrix to Nx3x3 + # return rtvec_to_pose(optical_pose_parent) type: ignore + + +def homography_i_H_ref(pinhole_i: torch.Tensor, pinhole_ref: torch.Tensor) -> torch.Tensor: + r"""Homography from reference to ith pinhole. + + .. note:: + The pinhole model is represented in a single vector as follows: + + .. math:: + pinhole = (f_x, f_y, c_x, c_y, height, width, + r_x, r_y, r_z, t_x, t_y, t_z) + + torch.where: + :math:`(r_x, r_y, r_z)` is the rotation vector in angle-axis + convention. + + :math:`(t_x, t_y, t_z)` is the translation vector. + + .. math:: + + H_{ref}^{i} = K_{i} * T_{ref}^{i} * K_{ref}^{-1} + + Args: + pinhole_i: torch.Tensor with pinhole model for ith frame. + pinhole_ref: torch.Tensor with pinhole model for reference frame. + + Returns: + tensors that convert depth points (u, v, d) from pinhole_ref to pinhole_i. + + Shape: + - Input: :math:`(N, 12)` and :math:`(N, 12)` + - Output: :math:`(N, 4, 4)` + + Example: + pinhole_i = torch.rand(1, 12) # Nx12 + pinhole_ref = torch.rand(1, 12) # Nx12 + homography_i_H_ref(pinhole_i, pinhole_ref) # Nx4x4 + + """ + # TODO: Add doctest once having `rtvec_to_pose`. + if not (len(pinhole_i.shape) == 2 and pinhole_i.shape[1] == 12): + raise AssertionError(pinhole_i.shape) + if pinhole_i.shape != pinhole_ref.shape: + raise AssertionError(pinhole_ref.shape) + i_pose_base = get_optical_pose_base(pinhole_i) + ref_pose_base = get_optical_pose_base(pinhole_ref) + i_pose_ref = torch.matmul(i_pose_base, inverse_transformation(ref_pose_base)) + return torch.matmul(pinhole_matrix(pinhole_i), torch.matmul(i_pose_ref, inverse_pinhole_matrix(pinhole_ref))) + + +# based on: +# https://github.com/ClementPinard/SfmLearner-Pytorch/blob/master/inverse_warp.py#L26 + + +def pixel2cam(depth: torch.Tensor, intrinsics_inv: torch.Tensor, pixel_coords: torch.Tensor) -> torch.Tensor: + r"""Transform coordinates in the pixel frame to the camera frame. + + Args: + depth: the source depth maps. Shape must be Bx1xHxW. + intrinsics_inv: the inverse intrinsics camera matrix. Shape must be Bx4x4. + pixel_coords: the grid with (u, v, 1) pixel coordinates. Shape must be BxHxWx3. + + Returns: + torch.Tensor of shape BxHxWx3 with (x, y, z) cam coordinates. + + """ + if not len(depth.shape) == 4 and depth.shape[1] == 1: + raise ValueError(f"Input depth has to be in the shape of Bx1xHxW. Got {depth.shape}") + if not len(intrinsics_inv.shape) == 3: + raise ValueError(f"Input intrinsics_inv has to be in the shape of Bx4x4. Got {intrinsics_inv.shape}") + if not len(pixel_coords.shape) == 4 and pixel_coords.shape[3] == 3: + raise ValueError(f"Input pixel_coords has to be in the shape of BxHxWx3. Got {intrinsics_inv.shape}") + cam_coords: torch.Tensor = transform_points(intrinsics_inv[:, None], pixel_coords) + return cam_coords * depth.permute(0, 2, 3, 1) + + +# based on +# https://github.com/ClementPinard/SfmLearner-Pytorch/blob/master/inverse_warp.py#L43 + + +def cam2pixel(cam_coords_src: torch.Tensor, dst_proj_src: torch.Tensor, eps: float = 1e-12) -> torch.Tensor: + r"""Transform coordinates in the camera frame to the pixel frame. + + Args: + cam_coords_src: (x, y, z) coordinates defined in the first camera coordinates system. Shape must be BxHxWx3. + dst_proj_src: the projection matrix between the + reference and the non reference camera frame. Shape must be Bx4x4. + eps: small value to avoid division by zero error. + + Returns: + torch.Tensor of shape BxHxWx2 with (u, v) pixel coordinates. + + """ + if not len(cam_coords_src.shape) == 4 and cam_coords_src.shape[3] == 3: + raise ValueError(f"Input cam_coords_src has to be in the shape of BxHxWx3. Got {cam_coords_src.shape}") + if not len(dst_proj_src.shape) == 3 and dst_proj_src.shape[-2:] == (4, 4): + raise ValueError(f"Input dst_proj_src has to be in the shape of Bx4x4. Got {dst_proj_src.shape}") + # apply projection matrix to points + point_coords: torch.Tensor = transform_points(dst_proj_src[:, None], cam_coords_src) + x_coord: torch.Tensor = point_coords[..., 0] + y_coord: torch.Tensor = point_coords[..., 1] + z_coord: torch.Tensor = point_coords[..., 2] + + # compute pixel coordinates + u_coord: torch.Tensor = x_coord / (z_coord + eps) + v_coord: torch.Tensor = y_coord / (z_coord + eps) + + # torch.stack and return the coordinates, that's the actual flow + pixel_coords_dst: torch.Tensor = torch.stack([u_coord, v_coord], dim=-1) + return pixel_coords_dst # BxHxWx2 + + +# layer api +'''class PinholeMatrix(nn.Module): + r"""Create an object that returns the pinhole matrix from a pinhole model + + Args: + pinholes (torch.Tensor): torch.tensor of pinhole models. + + Returns: + torch.Tensor: torch.tensor of pinhole matrices. + + Shape: + - Input: :math:`(N, 12)` + - Output: :math:`(N, 4, 4)` + + Example: + >>> pinhole = torch.rand(1, 12) # Nx12 + >>> transform = PinholeMatrix() + >>> pinhole_matrix = transform(pinhole) # Nx4x4 + """ + + def __init__(self): + super(PinholeMatrix, self).__init__() + + def forward(self, input): + return pinhole_matrix(input) + + +class InversePinholeMatrix(nn.Module): + r"""Return and object that inverts a pinhole matrix from a pinhole model + + Args: + pinholes (torch.Tensor): torch.tensor with pinhole models. + + Returns: + torch.Tensor: torch.tensor of inverted pinhole matrices. + + Shape: + - Input: :math:`(N, 12)` + - Output: :math:`(N, 4, 4)` + + Example: + >>> pinhole = torch.rand(1, 12) # Nx12 + >>> transform = kornia.InversePinholeMatrix() + >>> pinhole_matrix_inv = transform(pinhole) # Nx4x4 + """ + + def __init__(self): + super(InversePinholeMatrix, self).__init__() + + def forward(self, input): + return inverse_pinhole_matrix(input)''' diff --git a/kornia/geometry/camera/projection_orthographic.py b/kornia/geometry/camera/projection_orthographic.py new file mode 100644 index 0000000..1551d35 --- /dev/null +++ b/kornia/geometry/camera/projection_orthographic.py @@ -0,0 +1,98 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""nn.Module containing functions for orthographic projection.""" + +# inspired by: https://github.com/farm-ng/sophus-rs/blob/main/src/sensor/ortho_camera.rs +import torch + +from kornia.core.check import KORNIA_CHECK_SHAPE + + +def project_points_orthographic(points_in_camera: torch.Tensor) -> torch.Tensor: + r"""Project points from the camera frame into the canonical z=1 plane through orthographic projection. + + .. math:: + \begin{bmatrix} u \\ v \end{bmatrix} = + \begin{bmatrix} x \\ y \\ z \end{bmatrix} + + + Args: + points_in_camera: torch.Tensor representing the points to project. + + Returns: + torch.Tensor representing the projected points. + + Example: + >>> points = torch.tensor([1., 2., 3.]) + >>> project_points_orthographic(points) + tensor([1., 2.]) + + """ + KORNIA_CHECK_SHAPE(points_in_camera, ["*", "3"]) + return points_in_camera[..., :2] + + +def unproject_points_orthographic(points_in_camera: torch.Tensor, extension: torch.Tensor) -> torch.Tensor: + r"""Unproject one or more points from the canonical z=1 plane into the camera frame. + + .. math:: + \begin{bmatrix} x \\ y \\ z \end{bmatrix} = + \begin{bmatrix} u \\ v \\ w \end{bmatrix} + + Args: + points_in_camera: torch.Tensor representing the points to unproject with shape (..., 2). + extension: torch.Tensor representing the extension of the points to unproject with shape (..., 1). + + Returns: + torch.Tensor representing the unprojected points with shape (..., 3). + + Example: + >>> points = torch.tensor([1., 2.]) + >>> extension = torch.tensor([3.]) + >>> unproject_points_orthographic(points, extension) + tensor([1., 2., 3.]) + + """ + KORNIA_CHECK_SHAPE(points_in_camera, ["*", "2"]) + + if len(points_in_camera.shape) != len(extension.shape): + extension = extension[..., None] + + return torch.cat([points_in_camera, extension], dim=-1) + + +def dx_project_points_orthographic(points_in_camera: torch.Tensor) -> torch.Tensor: + r"""Compute the derivative of the x projection with respect to the x coordinate. + + .. math:: + \frac{\partial u}{\partial x} = 1 + + Args: + points_in_camera: torch.Tensor representing the points to project. + + Returns: + torch.Tensor representing the derivative of the x projection with respect to the x coordinate. + + Example: + >>> points = torch.tensor([1., 2., 3.]) + >>> dx_project_points_orthographic(points) + tensor([1.]) + + """ + KORNIA_CHECK_SHAPE(points_in_camera, ["*", "3"]) + return torch.ones_like(points_in_camera[..., 0:1]) diff --git a/kornia/geometry/camera/projection_z1.py b/kornia/geometry/camera/projection_z1.py new file mode 100644 index 0000000..06c0ad3 --- /dev/null +++ b/kornia/geometry/camera/projection_z1.py @@ -0,0 +1,143 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""nn.Module for the projection of points in the canonical z=1 plane.""" + +# inspired by: https://github.com/farm-ng/sophus-rs/blob/main/src/sensor/perspective_camera.rs +from __future__ import annotations + +from typing import Optional + +import torch + +from kornia.core.check import KORNIA_CHECK_SHAPE + + +def project_points_z1(points_in_camera: torch.Tensor) -> torch.Tensor: + r"""Project one or more points from the camera frame into the canonical z=1 plane through perspective division. + + .. math:: + + \begin{bmatrix} u \\ v \\ w \end{bmatrix} = + \begin{bmatrix} x \\ y \\ z \end{bmatrix} / z + + .. note:: + + This function has a precondition that the points are in front of the camera, i.e. z > 0. + If this is not the case, the points will be projected to the canonical plane, but the resulting + points will be behind the camera and causing numerical issues for z == 0. + + Args: + points_in_camera: torch.Tensor representing the points to project with shape (..., 3). + + Returns: + torch.Tensor representing the projected points with shape (..., 2). + + Example: + >>> points = torch.tensor([1., 2., 3.]) + >>> project_points_z1(points) + tensor([0.3333, 0.6667]) + + """ + KORNIA_CHECK_SHAPE(points_in_camera, ["*", "3"]) + return points_in_camera[..., :2] / points_in_camera[..., 2:3] + + +def unproject_points_z1( + points_in_cam_canonical: torch.Tensor, extension: Optional[torch.Tensor] = None +) -> torch.Tensor: + r"""Unproject one or more points from the canonical z=1 plane into the camera frame. + + .. math:: + \begin{bmatrix} x \\ y \\ z \end{bmatrix} = + \begin{bmatrix} u \\ v \end{bmatrix} \cdot w + + Args: + points_in_cam_canonical: torch.Tensor representing the points to unproject with shape (..., 2). + extension: torch.Tensor representing the extension (depth) of the points to unproject with shape (..., 1). + + Returns: + torch.Tensor representing the unprojected points with shape (..., 3). + + Example: + >>> points = torch.tensor([1., 2.]) + >>> extension = torch.tensor([3.]) + >>> unproject_points_z1(points, extension) + tensor([3., 6., 3.]) + + """ + KORNIA_CHECK_SHAPE(points_in_cam_canonical, ["*", "2"]) + + if extension is None: + extension = torch.ones( + points_in_cam_canonical.shape[:-1] + (1,), + device=points_in_cam_canonical.device, + dtype=points_in_cam_canonical.dtype, + ) # (..., 1) + elif extension.shape[0] > 1: + extension = extension[..., None] # (..., 1) + + return torch.cat([points_in_cam_canonical * extension, extension], dim=-1) + + +def dx_project_points_z1(points_in_camera: torch.Tensor) -> torch.Tensor: + r"""Compute the derivative of the x projection with respect to the x coordinate. + + Returns point derivative of inverse depth point projection with respect to the x coordinate. + + .. math:: + \frac{\partial \pi}{\partial x} = + \begin{bmatrix} + \frac{1}{z} & 0 & -\frac{x}{z^2} \\ + 0 & \frac{1}{z} & -\frac{y}{z^2} + \end{bmatrix} + + .. note:: + This function has a precondition that the points are in front of the camera, i.e. z > 0. + If this is not the case, the points will be projected to the canonical plane, but the resulting + points will be behind the camera and causing numerical issues for z == 0. + + Args: + points_in_camera: torch.Tensor representing the points to project with shape (..., 3). + + Returns: + torch.Tensor representing the derivative of the x projection with respect to the x coordinate + with shape (..., 2, 3). + + Example: + >>> points = torch.tensor([1., 2., 3.]) + >>> dx_project_points_z1(points) + tensor([[ 0.3333, 0.0000, -0.1111], + [ 0.0000, 0.3333, -0.2222]]) + + """ + KORNIA_CHECK_SHAPE(points_in_camera, ["*", "3"]) + + x = points_in_camera[..., 0] + y = points_in_camera[..., 1] + z = points_in_camera[..., 2] + + z_inv = 1.0 / z + z_sq = z_inv * z_inv + zeros = torch.zeros_like(z_inv) + return torch.stack( + [ + torch.stack([z_inv, zeros, -x * z_sq], dim=-1), + torch.stack([zeros, z_inv, -y * z_sq], dim=-1), + ], + dim=-2, + ) diff --git a/kornia/geometry/camera/stereo.py b/kornia/geometry/camera/stereo.py new file mode 100644 index 0000000..f40df30 --- /dev/null +++ b/kornia/geometry/camera/stereo.py @@ -0,0 +1,346 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any + +import torch + +from kornia.geometry.grid import create_meshgrid +from kornia.geometry.linalg import transform_points + + +class StereoException(Exception): + """Handle errors related to stereo camera calibration and rectification.""" + + def __init__(self, msg: str, *args: Any, **kwargs: Any) -> None: + r"""Construct custom exception for the :module:`~kornia.geometry.camera.stereo` module. + + Adds a general helper module redirecting the user to the proper documentation site. + + Args: + msg: Custom message to add to the general message. + *args: Additional argument passthrough + **kwargs: Additional argument passthrough + + """ + doc_help = ( + "\n Please check documents here: " + "https://kornia.readthedocs.io/en/latest/geometry.camera.stereo.html for further information and examples." + ) + final_msg = msg + doc_help + # type ignore because of mypy error: + # Too many arguments for "__init__" of "BaseException" + super().__init__(final_msg, *args, **kwargs) + + +class StereoCamera: + """Represent a horizontal stereo camera setup for depth estimation. + + Args: + rectified_left_camera: The intrinsic matrix for the left camera. + rectified_right_camera: The intrinsic matrix for the right camera. + """ + + def __init__(self, rectified_left_camera: torch.Tensor, rectified_right_camera: torch.Tensor) -> None: + r"""Class representing a horizontal stereo camera setup. + + Args: + rectified_left_camera: The rectified left camera projection matrix + of shape :math:`(B, 3, 4)` + rectified_right_camera: The rectified right camera projection matrix + of shape :math:`(B, 3, 4)` + + """ + self._check_stereo_camera(rectified_left_camera, rectified_right_camera) + self.rectified_left_camera: torch.Tensor = rectified_left_camera + self.rectified_right_camera: torch.Tensor = rectified_right_camera + + self.device = self.rectified_left_camera.device + self.dtype = self.rectified_left_camera.dtype + + self._Q_matrix = self._init_Q_matrix() + + @staticmethod + def _check_stereo_camera(rectified_left_camera: torch.Tensor, rectified_right_camera: torch.Tensor) -> None: + r"""Ensure user specified correct camera matrices. + + Args: + rectified_left_camera: The rectified left camera projection matrix + of shape :math:`(B, 3, 4)` + rectified_right_camera: The rectified right camera projection matrix + of shape :math:`(B, 3, 4)` + + """ + # Ensure correct shapes + if len(rectified_left_camera.shape) != 3: + raise StereoException( + f"Expected 'rectified_left_camera' to have 3 dimensions. Got {rectified_left_camera.shape}." + ) + + if len(rectified_right_camera.shape) != 3: + raise StereoException( + f"Expected 'rectified_right_camera' to have 3 dimension. Got {rectified_right_camera.shape}." + ) + + if rectified_left_camera.shape[:1] == (3, 4): + raise StereoException( + f"Expected each 'rectified_left_camera' to be of shape (3, 4).Got {rectified_left_camera.shape[:1]}." + ) + + if rectified_right_camera.shape[:1] == (3, 4): + raise StereoException( + f"Expected each 'rectified_right_camera' to be of shape (3, 4).Got {rectified_right_camera.shape[:1]}." + ) + + # Ensure same devices for cameras. + if rectified_left_camera.device != rectified_right_camera.device: + raise StereoException( + "Expected 'rectified_left_camera' and 'rectified_right_camera' " + "to be on the same devices." + f"Got {rectified_left_camera.device} and {rectified_right_camera.device}." + ) + + # Ensure same dtypes for cameras. + if rectified_left_camera.dtype != rectified_right_camera.dtype: + raise StereoException( + "Expected 'rectified_left_camera' and 'rectified_right_camera' to" + "have same dtype." + f"Got {rectified_left_camera.dtype} and {rectified_right_camera.dtype}." + ) + + # Ensure all intrinsics parameters (fx, fy, cx, cy) are the same in both cameras. + if not torch.all(torch.eq(rectified_left_camera[..., :, :3], rectified_right_camera[..., :, :3])): + raise StereoException( + "Expected 'left_rectified_camera' and 'rectified_right_camera' to have" + "same parameters except for the last column." + f"Got {rectified_left_camera[..., :, :3]} and {rectified_right_camera[..., :, :3]}." + ) + + # Ensure that tx * fx is negative and exists. + tx_fx = rectified_right_camera[..., 0, 3] + if torch.all(torch.gt(tx_fx, 0)): + raise StereoException(f"Expected :math:`T_x * f_x` to be negative. Got {tx_fx}.") + + @property + def batch_size(self) -> int: + r"""Return the batch size of the storage. + + Returns: + scalar with the batch size + + """ + return self.rectified_left_camera.shape[0] + + @property + def fx(self) -> torch.Tensor: + r"""Return the focal length in the x-direction. + + Note that the focal lengths of the rectified left and right + camera are assumed to be equal. + + Returns: + torch.Tensor of shape :math:`(B)` + + """ + return self.rectified_left_camera[..., 0, 0] + + @property + def fy(self) -> torch.Tensor: + r"""Returns the focal length in the y-direction. + + Note that the focal lengths of the rectified left and right + camera are assumed to be equal. + + Returns: + torch.Tensor of shape :math:`(B)` + + """ + return self.rectified_left_camera[..., 1, 1] + + @property + def cx_left(self) -> torch.Tensor: + r"""Return the x-coordinate of the principal point for the left camera. + + Returns: + torch.Tensor of shape :math:`(B)` + + """ + return self.rectified_left_camera[..., 0, 2] + + @property + def cx_right(self) -> torch.Tensor: + r"""Return the x-coordinate of the principal point for the right camera. + + Returns: + torch.Tensor of shape :math:`(B)` + + """ + return self.rectified_right_camera[..., 0, 2] + + @property + def cy(self) -> torch.Tensor: + r"""Return the y-coordinate of the principal point. + + Note that the y-coordinate of the principal points + is assumed to be equal for the left and right camera. + + Returns: + torch.Tensor of shape :math:`(B)` + + """ + return self.rectified_left_camera[..., 1, 2] + + @property + def tx(self) -> torch.Tensor: + r"""The horizontal baseline between the two cameras. + + Returns: + torch.Tensor of shape :math:`(B)` + + """ + return -self.rectified_right_camera[..., 0, 3] / self.fx + + @property + def Q(self) -> torch.Tensor: + r"""The Q matrix of the horizontal stereo setup. + + This matrix is used for reprojecting a disparity torch.Tensor to + the corresponding point cloud. Note that this is in a general form that allows different focal + lengths in the x and y direction. + + Return: + The Q matrix of shape :math:`(B, 4, 4)`. + + """ + return self._Q_matrix + + def _init_Q_matrix(self) -> torch.Tensor: + r"""Initialize the Q matrix of the horizontal stereo setup. See the Q property. + + Returns: + The Q matrix of shape :math:`(B, 4, 4)`. + + """ + Q = torch.zeros((self.batch_size, 4, 4), device=self.device, dtype=self.dtype) + baseline: torch.Tensor = -self.tx + Q[:, 0, 0] = self.fy * baseline + Q[:, 0, 3] = -self.fy * self.cx_left * baseline + Q[:, 1, 1] = self.fx * baseline + Q[:, 1, 3] = -self.fx * self.cy * baseline + Q[:, 2, 3] = self.fx * self.fy * baseline + Q[:, 3, 2] = -self.fy + Q[:, 3, 3] = self.fy * (self.cx_left - self.cx_right) # NOTE: This is usually zero. + return Q + + def reproject_disparity_to_3D(self, disparity_tensor: torch.Tensor) -> torch.Tensor: + r"""Reproject the disparity torch.Tensor to a 3D point cloud. + + Args: + disparity_tensor: Disparity torch.Tensor of shape :math:`(B, 1, H, W)`. + + Returns: + The 3D point cloud of shape :math:`(B, H, W, 3)` + + """ + return reproject_disparity_to_3D(disparity_tensor, self.Q) + + +def _check_disparity_tensor(disparity_tensor: torch.Tensor) -> None: + r"""Ensure correct user provided correct disparity torch.Tensor. + + Args: + disparity_tensor: The disparity torch.Tensor of shape :math:`(B, 1, H, W)`. + + """ + if not isinstance(disparity_tensor, torch.Tensor): + raise StereoException( + f"Expected 'disparity_tensor' to be an instance of torch.Tensor but got {type(disparity_tensor)}." + ) + + if len(disparity_tensor.shape) != 4: + raise StereoException(f"Expected 'disparity_tensor' to have 4 dimensions. Got {disparity_tensor.shape}.") + + if disparity_tensor.shape[-1] != 1: + raise StereoException( + "Expected dimension 1 of 'disparity_tensor' to be 1 for as single channeled disparity map." + f"Got {disparity_tensor.shape}." + ) + + if disparity_tensor.dtype not in (torch.bfloat16, torch.float16, torch.float32, torch.float64): + raise StereoException( + "Expected 'disparity_tensor' to have dtype torch.bfloat16, torch.float16, torch.float32 or torch.float64." + f"Got {disparity_tensor.dtype}" + ) + + +def _check_Q_matrix(Q_matrix: torch.Tensor) -> None: + r"""Ensure Q matrix is of correct form. + + Args: + Q_matrix: The Q matrix for reprojecting disparity to a point cloud of shape :math:`(B, 4, 4)` + + """ + if not isinstance(Q_matrix, torch.Tensor): + raise StereoException(f"Expected 'Q_matrix' to be an instance of torch.Tensor but got {type(Q_matrix)}.") + + if not len(Q_matrix.shape) == 3: + raise StereoException(f"Expected 'Q_matrix' to have 3 dimensions. Got {Q_matrix.shape}") + + if not Q_matrix.shape[1:] == (4, 4): + raise StereoException(f"Expected last two dimensions of 'Q_matrix' to be of shape (4, 4). Got {Q_matrix.shape}") + + if Q_matrix.dtype not in (torch.bfloat16, torch.float16, torch.float32, torch.float64): + raise StereoException( + "Expected 'Q_matrix' to be of type torch.bfloat16, torch.float16, torch.float32 or torch.float64. " + f"Got {Q_matrix.dtype}" + ) + + +def reproject_disparity_to_3D(disparity_tensor: torch.Tensor, Q_matrix: torch.Tensor) -> torch.Tensor: + r"""Reproject the disparity torch.Tensor to a 3D point cloud. + + Args: + disparity_tensor: Disparity torch.Tensor of shape :math:`(B, H, W, 1)`. + Q_matrix: torch.Tensor of Q matrices of shapes :math:`(B, 4, 4)`. + + Returns: + The 3D point cloud of shape :math:`(B, H, W, 3)` + + """ + _check_Q_matrix(Q_matrix) + _check_disparity_tensor(disparity_tensor) + + batch_size, rows, cols, _ = disparity_tensor.shape + dtype = disparity_tensor.dtype + device = disparity_tensor.device + + uv = create_meshgrid(rows, cols, normalized_coordinates=False, device=device, dtype=dtype) + uv = uv.expand(batch_size, -1, -1, -1) + v, u = torch.unbind(uv, dim=-1) + v, u = torch.unsqueeze(v, -1), torch.unsqueeze(u, -1) + uvd = torch.stack((u, v, disparity_tensor), 1).reshape(batch_size, 3, -1).permute(0, 2, 1) + points = transform_points(Q_matrix, uvd).reshape(batch_size, rows, cols, 3) + + # Final check that everything went well. + if not points.shape == (batch_size, rows, cols, 3): + raise StereoException( + "Something went wrong in `reproject_disparity_to_3D`. Expected the final output" + f"to be of shape {(batch_size, rows, cols, 3)}." + f"But the computed point cloud had shape {points.shape}. " + "Please ensure input are correct. If this is an error, please submit an issue." + ) + return points diff --git a/kornia/geometry/camera/utils.py b/kornia/geometry/camera/utils.py new file mode 100644 index 0000000..5fbccc5 --- /dev/null +++ b/kornia/geometry/camera/utils.py @@ -0,0 +1,247 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import math +from typing import Union + +import torch + +# from torch import Tensor (use torch.Tensor instead) +from kornia.geometry.camera import PinholeCamera + + +def create_camera_dimensions( + device: Union[str, torch.device, None], dtype: torch.dtype, n_cams1: int = 3, n_cams2: int = 2 +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Create camera dimensions for ray sampling. + + Args: + device: Union[str, torch.device, None] for tensors + dtype: Data type for tensors + n_cams1: Number of cameras in first group (default: 3) + n_cams2: Number of cameras in second group (default: 2) + + Returns: + Tuple of (heights, widths, num_img_rays) tensors + """ + heights: torch.Tensor = torch.cat( + ( + torch.tensor([200] * n_cams1, device=device, dtype=dtype), + torch.tensor([100] * n_cams2, device=device, dtype=dtype), + ) + ) + widths: torch.Tensor = torch.cat( + ( + torch.tensor([300] * n_cams1, device=device, dtype=dtype), + torch.tensor([400] * n_cams2, device=device, dtype=dtype), + ) + ) + num_img_rays: torch.Tensor = torch.cat( + ( + torch.tensor([10] * n_cams1, device=device, dtype=dtype), + torch.tensor([15] * n_cams2, device=device, dtype=dtype), + ) + ) + return heights, widths, num_img_rays + + +def create_intrinsics( + fxs: list[float | int], + fys: list[float | int], + cxs: torch.Tensor | list[float | int], + cys: torch.Tensor | list[float | int], + device: Union[str, torch.device, None], + dtype: torch.dtype, +) -> torch.Tensor: + """Create intrinsic camera matrices from focal lengths and principal points. + + Args: + fxs: Focal length in x direction + fys: Focal length in y direction + cxs: Principal point x coordinate + cys: Principal point y coordinate + device: Union[str, torch.device, None] for tensors + dtype: Data type for tensors + + Returns: + Stacked intrinsic matrices of shape (N, 4, 4) + """ + intrinsics_batch: list[torch.Tensor] = [] + # Convert cxs and cys to lists if they are tensors + cxs_list = cxs.tolist() if isinstance(cxs, torch.Tensor) else cxs + cys_list = cys.tolist() if isinstance(cys, torch.Tensor) else cys + for fx, fy, cx, cy in zip(fxs, fys, cxs_list, cys_list): + intrinsics = torch.eye(4, device=device, dtype=dtype) + intrinsics[0, 0] = fx + intrinsics[1, 1] = fy + intrinsics[0, 2] = cx + intrinsics[1, 2] = cy + intrinsics_batch.append(intrinsics) + return torch.stack(intrinsics_batch) + + +def create_extrinsics_with_rotation( + alphas: list[float], + betas: list[float], + gammas: list[float], + txs: list[float], + tys: list[float], + tzs: list[float], + device: Union[str, torch.device, None], + dtype: torch.dtype, +) -> torch.Tensor: + """Create extrinsic camera matrices with rotation and translation. + + Args: + alphas: Rotation angles around x-axis + betas: Rotation angles around y-axis + gammas: Rotation angles around z-axis + txs: Translation in x direction + tys: Translation in y direction + tzs: Translation in z direction + device: Union[str, torch.device, None] for tensors + dtype: Data type for tensors + + Returns: + Stacked extrinsic matrices of shape (N, 4, 4) + """ + extrinsics_batch: list[torch.Tensor] = [] + for alpha, beta, gamma, tx, ty, tz in zip(alphas, betas, gammas, txs, tys, tzs): + Rx = torch.eye(3, device=device, dtype=dtype) + Rx[1, 1] = torch.cos(alpha) + Rx[1, 2] = torch.sin(alpha) + Rx[2, 1] = -Rx[1, 2] + Rx[2, 2] = Rx[1, 1] + + Ry = torch.eye(3, device=device, dtype=dtype) + Ry[0, 0] = torch.cos(beta) + Ry[0, 2] = -torch.sin(beta) + Ry[2, 0] = -Ry[0, 2] + Ry[2, 2] = Ry[0, 0] + + Rz = torch.eye(3, device=device, dtype=dtype) + Rz[0, 0] = torch.cos(gamma) + Rz[0, 1] = torch.sin(gamma) + Rz[1, 0] = -Rz[0, 1] + Rz[1, 1] = Rz[0, 0] + + Ryz = torch.matmul(Ry, Rz) + R = torch.matmul(Rx, Ryz) + + extrinsics = torch.eye(4, device=device, dtype=dtype) + extrinsics[..., 0, -1] = tx + extrinsics[..., 1, -1] = ty + extrinsics[..., 2, -1] = tz + extrinsics[:3, :3] = R + + extrinsics_batch.append(extrinsics) + return torch.stack(extrinsics_batch) + + +def create_pinhole_camera( + height: float, width: float, device: Union[str, torch.device, None], dtype: torch.dtype +) -> PinholeCamera: + """Create a single PinholeCamera with default parameters. + + Args: + height: Camera image height + width: Camera image width + device: Union[str, torch.device, None] for tensors + dtype: Data type for tensors + + Returns: + PinholeCamera: A PinholeCamera instance + """ + fx = width + fy = height + cx = (width - 1.0) / 2.0 + cy = (height - 1.0) / 2.0 + + tx = 0.0 + ty = 0.0 + tz = 1.0 + + alpha = math.pi / 2.0 + beta = 0.0 + gamma = -math.pi / 2.0 + + intrinsics = create_intrinsics([fx], [fy], [cx], [cy], device=device, dtype=dtype) + extrinsics = create_extrinsics_with_rotation([alpha], [beta], [gamma], [tx], [ty], [tz], device=device, dtype=dtype) + + return PinholeCamera( + intrinsics, + extrinsics, + torch.tensor([height], device=device, dtype=dtype), + torch.tensor([width], device=device, dtype=dtype), + ) + + +def create_four_cameras(device: Union[str, torch.device, None], dtype: torch.dtype) -> PinholeCamera: + """Create four PinholeCameras with predefined parameters. + + Args: + device: Union[str, torch.device, None] for tensors + dtype: Data type for tensors + + Returns: + PinholeCamera: A PinholeCamera instance with 4 cameras in batch + """ + height = torch.tensor([5, 4, 4, 4], device=device, dtype=dtype) + width = torch.tensor([9, 7, 7, 7], device=device, dtype=dtype) + + fx = width.tolist() + fy = height.tolist() + + cx = (width - 1.0) / 2.0 + cy = (height - 1.0) / 2.0 + + tx = [0.0, 0.0, 0.0, 0.0] + ty = [0.0, 0.0, 0.0, 0.0] + tz = [11.0, 11.0, 11.0, 5.0] + + pi = math.pi + alpha = [pi / 2.0, pi / 2.0, pi / 2.0, 0.0] + beta = [0.0, 0.0, 0.0, pi] + gamma = [-pi / 2.0, 0.0, pi / 2.0, 0.0] + + intrinsics = create_intrinsics(fx, fy, cx, cy, device=device, dtype=dtype) + extrinsics = create_extrinsics_with_rotation(alpha, beta, gamma, tx, ty, tz, device=device, dtype=dtype) + + cameras = PinholeCamera(intrinsics, extrinsics, height, width) + return cameras + + +def create_random_images_for_cameras(cameras: PinholeCamera) -> list[torch.Tensor]: + """Create random images for a given set of cameras.""" + torch.manual_seed(112) + imgs: list[torch.Tensor] = [] + for height, width in zip(cameras.height.tolist(), cameras.width.tolist()): + image_data = torch.randint(0, 255, (3, int(height), int(width)), dtype=torch.uint8) # (C, H, W) + imgs.append(image_data) # (C, H, W) + return imgs + + +def create_red_images_for_cameras(cameras: PinholeCamera, device: Union[str, torch.device, None]) -> list[torch.Tensor]: + """Create red images for a given set of cameras.""" + imgs: list[torch.Tensor] = [] + for height, width in zip(cameras.height.tolist(), cameras.width.tolist()): + image_data = torch.zeros(3, int(height), int(width), dtype=torch.uint8) # (C, H, W) + image_data[0, ...] = 255 # Red channel + imgs.append(image_data.to(device=device)) + return imgs diff --git a/kornia/geometry/conversions.py b/kornia/geometry/conversions.py new file mode 100644 index 0000000..b8f0ebf --- /dev/null +++ b/kornia/geometry/conversions.py @@ -0,0 +1,1615 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional + +import torch +import torch.nn.functional as F + +from kornia.constants import pi +from kornia.core._compat import deprecated +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SHAPE +from kornia.core.utils import _torch_inverse_cast + +__all__ = [ + "ARKitQTVecs_to_ColmapQTVecs", + "Rt_to_matrix4x4", + "angle_axis_to_quaternion", + "angle_axis_to_rotation_matrix", + "angle_to_rotation_matrix", + "axis_angle_to_quaternion", + "axis_angle_to_rotation_matrix", + "camtoworld_graphics_to_vision_4x4", + "camtoworld_graphics_to_vision_Rt", + "camtoworld_to_worldtocam_Rt", + "camtoworld_vision_to_graphics_4x4", + "camtoworld_vision_to_graphics_Rt", + "cart2pol", + "convert_affinematrix_to_homography", + "convert_affinematrix_to_homography3d", + "convert_points_from_homogeneous", + "convert_points_to_homogeneous", + "deg2rad", + "denormalize_homography", + "denormalize_pixel_coordinates", + "denormalize_pixel_coordinates3d", + "denormalize_points_with_intrinsics", + "euler_from_quaternion", + "matrix4x4_to_Rt", + "normal_transform_pixel", + "normal_transform_pixel3d", + "normalize_homography", + "normalize_homography3d", + "normalize_pixel_coordinates", + "normalize_pixel_coordinates3d", + "normalize_points_with_intrinsics", + "normalize_quaternion", + "pol2cart", + "quaternion_exp_to_log", + "quaternion_from_euler", + "quaternion_log_to_exp", + "quaternion_to_angle_axis", + "quaternion_to_axis_angle", + "quaternion_to_rotation_matrix", + "rad2deg", + "rotation_matrix_to_angle_axis", + "rotation_matrix_to_axis_angle", + "rotation_matrix_to_quaternion", + "vector_to_skew_symmetric_matrix", + "worldtocam_to_camtoworld_Rt", +] + + +def rad2deg(tensor: torch.Tensor) -> torch.Tensor: + r"""Convert angles from radians to degrees. + + Args: + tensor: torch.Tensor of arbitrary shape. + + Returns: + torch.Tensor with same shape as input. + + Example: + >>> input = torch.tensor(3.1415926535) + >>> rad2deg(input) + tensor(180.) + + """ + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(tensor)}") + + return 180.0 * tensor / pi.to(tensor.device).type(tensor.dtype) + + +def deg2rad(tensor: torch.Tensor) -> torch.Tensor: + r"""Convert angles from degrees to radians. + + Args: + tensor: torch.Tensor of arbitrary shape. + + Returns: + tensor with same shape as input. + + Examples: + >>> input = torch.tensor(180.) + >>> deg2rad(input) + tensor(3.1416) + + """ + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(tensor)}") + + return tensor * pi.to(tensor.device).type(tensor.dtype) / 180.0 + + +def pol2cart(rho: torch.Tensor, phi: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + r"""Convert polar coordinates to cartesian coordinates. + + Args: + rho: torch.Tensor of arbitrary shape. + phi: torch.Tensor of same arbitrary shape. + + Returns: + - x: torch.Tensor with same shape as input. + - y: torch.Tensor with same shape as input. + + Example: + >>> rho = torch.rand(1, 3, 3) + >>> phi = torch.rand(1, 3, 3) + >>> x, y = pol2cart(rho, phi) + + """ + if not (isinstance(rho, torch.Tensor) & isinstance(phi, torch.Tensor)): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(rho)}, {type(phi)}") + + x = rho * torch.cos(phi) + y = rho * torch.sin(phi) + return x, y + + +def cart2pol(x: torch.Tensor, y: torch.Tensor, eps: float = 1.0e-8) -> tuple[torch.Tensor, torch.Tensor]: + """Convert cartesian coordinates to polar coordinates. + + Args: + x: torch.Tensor of arbitrary shape. + y: torch.Tensor of same arbitrary shape. + eps: To avoid division by zero. + + Returns: + - rho: torch.Tensor with same shape as input. + - phi: torch.Tensor with same shape as input. + + Example: + >>> x = torch.rand(1, 3, 3) + >>> y = torch.rand(1, 3, 3) + >>> rho, phi = cart2pol(x, y) + + """ + if not (isinstance(x, torch.Tensor) & isinstance(y, torch.Tensor)): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(x)}, {type(y)}") + + rho = torch.sqrt(x**2 + y**2 + eps) + phi = torch.atan2(y, x) + return rho, phi + + +def convert_points_from_homogeneous(points: torch.Tensor, eps: float = 1e-8) -> torch.Tensor: + r"""Convert points from homogeneous to Euclidean space. + + Args: + points: the points to be transformed of shape :math:`(B, N, D)`. + eps: to avoid division by zero. + + Returns: + the points in Euclidean space :math:`(B, N, D-1)`. + + Examples: + >>> input = torch.tensor([[0., 0., 1.]]) + >>> convert_points_from_homogeneous(input) + tensor([[0., 0.]]) + + """ + if not isinstance(points, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(points)}") + + if len(points.shape) < 2: + raise ValueError(f"Input must be at least a 2D tensor. Got {points.shape}") + + # we check for points at max_val + z_vec: torch.Tensor = points[..., -1:] + + # set the results of division by zeror/near-zero to 1.0 + # follow the convention of opencv: + # https://github.com/opencv/opencv/pull/14411/files + mask: torch.Tensor = torch.abs(z_vec) > eps + scale = torch.where(mask, 1.0 / (z_vec + eps), torch.ones_like(z_vec)) + + return scale * points[..., :-1] + + +def convert_points_to_homogeneous(points: torch.Tensor) -> torch.Tensor: + r"""Convert points from Euclidean to homogeneous space. + + Args: + points: the points to be transformed with shape :math:`(*, N, D)`. + + Returns: + the points in homogeneous coordinates :math:`(*, N, D+1)`. + + Examples: + >>> input = torch.tensor([[0., 0.]]) + >>> convert_points_to_homogeneous(input) + tensor([[0., 0., 1.]]) + + """ + if not isinstance(points, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(points)}") + if len(points.shape) < 2: + raise ValueError(f"Input must be at least a 2D tensor. Got {points.shape}") + + return F.pad(points, [0, 1], "constant", 1.0) + + +def _convert_affinematrix_to_homography_impl(A: torch.Tensor) -> torch.Tensor: + H: torch.Tensor = F.pad(A, [0, 0, 0, 1], "constant", value=0.0) + H[..., -1, -1] += 1.0 + return H + + +def convert_affinematrix_to_homography(A: torch.Tensor) -> torch.Tensor: + r"""Convert batch of affine matrices. + + Args: + A: the affine matrix with shape :math:`(B,2,3)`. + + Returns: + the homography matrix with shape of :math:`(B,3,3)`. + + Examples: + >>> A = torch.tensor([[[1., 0., 0.], + ... [0., 1., 0.]]]) + >>> convert_affinematrix_to_homography(A) + tensor([[[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]]) + + """ + if not isinstance(A, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(A)}") + + if not (len(A.shape) == 3 and A.shape[-2:] == (2, 3)): + raise ValueError(f"Input matrix must be a Bx2x3 tensor. Got {A.shape}") + + return _convert_affinematrix_to_homography_impl(A) + + +def convert_affinematrix_to_homography3d(A: torch.Tensor) -> torch.Tensor: + r"""Convert batch of 3d affine matrices. + + Args: + A: the affine matrix with shape :math:`(B,3,4)`. + + Returns: + the homography matrix with shape of :math:`(B,4,4)`. + + Examples: + >>> A = torch.tensor([[[1., 0., 0., 0.], + ... [0., 1., 0., 0.], + ... [0., 0., 1., 0.]]]) + >>> convert_affinematrix_to_homography3d(A) + tensor([[[1., 0., 0., 0.], + [0., 1., 0., 0.], + [0., 0., 1., 0.], + [0., 0., 0., 1.]]]) + + """ + if not isinstance(A, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(A)}") + + if not (len(A.shape) == 3 and A.shape[-2:] == (3, 4)): + raise ValueError(f"Input matrix must be a Bx3x4 tensor. Got {A.shape}") + + return _convert_affinematrix_to_homography_impl(A) + + +def axis_angle_to_rotation_matrix(axis_angle: torch.Tensor) -> torch.Tensor: + r"""Convert 3d vector of axis-angle rotation to 3x3 rotation matrix. + + Args: + axis_angle: tensor of 3d vector of axis-angle rotations in radians with shape :math:`(N, 3)`. + + Returns: + tensor of rotation matrices of shape :math:`(N, 3, 3)`. + + Example: + >>> input = torch.tensor([[0., 0., 0.]]) + >>> axis_angle_to_rotation_matrix(input) # doctest: +ELLIPSIS + tensor([[[1., ...0., 0.], + [0., 1., ...0.], + [...0., 0., 1.]]]) + + >>> input = torch.tensor([[1.5708, 0., 0.]]) + >>> axis_angle_to_rotation_matrix(input) + tensor([[[ 1.0000e+00, 0.0000e+00, 0.0000e+00], + [ 0.0000e+00, -3.6200e-06, -1.0000e+00], + [ 0.0000e+00, 1.0000e+00, -3.6200e-06]]]) + + """ + if not isinstance(axis_angle, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(axis_angle)}") + + if not axis_angle.shape[-1] == 3: + raise ValueError(f"Input size must be a (*, 3) tensor. Got {axis_angle.shape}") + + def _compute_rotation_matrix(axis_angle: torch.Tensor, theta2: torch.Tensor, eps: float = 1e-6) -> torch.Tensor: + theta = torch.sqrt(theta2.clamp(min=1e-12)) # clamping to ensure no nan gradients + wxyz = axis_angle / (theta.unsqueeze(-1) + eps) # (B, 3) + wx, wy, wz = wxyz.unbind(dim=1) # (B,) + + cos_theta = torch.cos(theta) + sin_theta = torch.sin(theta) + one_minus_cos = 1.0 - cos_theta + + wxwy = wx * wy + wxwz = wx * wz + wywz = wy * wz + + r00 = cos_theta + wx * wx * one_minus_cos + r01 = wxwy * one_minus_cos - wz * sin_theta + r02 = wy * sin_theta + wxwz * one_minus_cos + + r10 = wz * sin_theta + wxwy * one_minus_cos + r11 = cos_theta + wy * wy * one_minus_cos + r12 = -wx * sin_theta + wywz * one_minus_cos + + r20 = -wy * sin_theta + wxwz * one_minus_cos + r21 = wx * sin_theta + wywz * one_minus_cos + r22 = cos_theta + wz * wz * one_minus_cos + + rot = torch.stack( + [ + torch.stack([r00, r01, r02], dim=-1), + torch.stack([r10, r11, r12], dim=-1), + torch.stack([r20, r21, r22], dim=-1), + ], + dim=1, + ) + + return rot + + def _compute_rotation_matrix_taylor(axis_angle: torch.Tensor) -> torch.Tensor: + rx, ry, rz = axis_angle.unbind(-1) + k_one = torch.ones_like(rx) + + rot = torch.stack( + [ + k_one, + -rz, + ry, + rz, + k_one, + -rx, + -ry, + rx, + k_one, + ], + dim=-1, + ).view(-1, 3, 3) + + return rot + + theta2 = (axis_angle * axis_angle).sum(dim=-1) + + rot_normal = _compute_rotation_matrix(axis_angle, theta2) # (N,3,3) + rot_taylor = _compute_rotation_matrix_taylor(axis_angle) # (N,3,3) + + mask = (theta2 > 1e-6).view(-1, 1, 1) # shape (N,1,1) + + rotation_matrix = torch.where(mask, rot_normal, rot_taylor) + + return rotation_matrix + + +@deprecated(replace_with="axis_angle_to_rotation_matrix", version="0.7.0") +def angle_axis_to_rotation_matrix(axis_angle: torch.Tensor) -> torch.Tensor: # noqa: D103 + return axis_angle_to_rotation_matrix(axis_angle) + + +def rotation_matrix_to_axis_angle(rotation_matrix: torch.Tensor) -> torch.Tensor: + r"""Convert 3x3 rotation matrix to Rodrigues vector in radians. + + Args: + rotation_matrix: rotation matrix of shape :math:`(N, 3, 3)`. + + Returns: + Rodrigues vector transformation of shape :math:`(N, 3)`. + + Example: + >>> input = torch.tensor([[1., 0., 0.], + ... [0., 1., 0.], + ... [0., 0., 1.]]) + >>> rotation_matrix_to_axis_angle(input) + tensor([0., 0., 0.]) + + >>> input = torch.tensor([[1., 0., 0.], + ... [0., 0., -1.], + ... [0., 1., 0.]]) + >>> rotation_matrix_to_axis_angle(input) + tensor([1.5708, 0.0000, 0.0000]) + + """ + if not isinstance(rotation_matrix, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(rotation_matrix)}") + + if not rotation_matrix.shape[-2:] == (3, 3): + raise ValueError(f"Input size must be a (*, 3, 3) tensor. Got {rotation_matrix.shape}") + quaternion: torch.Tensor = rotation_matrix_to_quaternion(rotation_matrix) + return quaternion_to_axis_angle(quaternion) + + +@deprecated(replace_with="rotation_matrix_to_axis_angle", version="0.7.0") +def rotation_matrix_to_angle_axis(rotation_matrix: torch.Tensor) -> torch.Tensor: # noqa: D103 + return rotation_matrix_to_axis_angle(rotation_matrix) + + +def rotation_matrix_to_quaternion(rotation_matrix: torch.Tensor, eps: float = 1.0e-8) -> torch.Tensor: + r"""Convert 3x3 rotation matrix to 4d quaternion vector. + + The quaternion vector has components in (w, x, y, z) format. + + Args: + rotation_matrix: the rotation matrix to convert with shape :math:`(*, 3, 3)`. + eps: small value to avoid zero division. + + Return: + the rotation in quaternion with shape :math:`(*, 4)`. + + Example: + >>> input = torch.tensor([[1., 0., 0.], + ... [0., 1., 0.], + ... [0., 0., 1.]]) + >>> rotation_matrix_to_quaternion(input, eps=torch.finfo(input.dtype).eps) + tensor([1., 0., 0., 0.]) + + """ + if not isinstance(rotation_matrix, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(rotation_matrix)}") + + if not rotation_matrix.shape[-2:] == (3, 3): + raise ValueError(f"Input size must be a (*, 3, 3) tensor. Got {rotation_matrix.shape}") + + def safe_zero_division(numerator: torch.Tensor, denominator: torch.Tensor) -> torch.Tensor: + eps: float = torch.finfo(numerator.dtype).tiny + return numerator / torch.clamp(denominator, min=eps) + + rotation_matrix_vec: torch.Tensor = rotation_matrix.reshape(*rotation_matrix.shape[:-2], 9) + + m00, m01, m02, m10, m11, m12, m20, m21, m22 = torch.chunk(rotation_matrix_vec, chunks=9, dim=-1) + + trace: torch.Tensor = m00 + m11 + m22 + + def trace_positive_cond() -> torch.Tensor: + sq = torch.sqrt(trace + 1.0 + eps) * 2.0 # sq = 4 * qw. + qw = 0.25 * sq + qx = safe_zero_division(m21 - m12, sq) + qy = safe_zero_division(m02 - m20, sq) + qz = safe_zero_division(m10 - m01, sq) + return torch.cat((qw, qx, qy, qz), dim=-1) + + def cond_1() -> torch.Tensor: + sq = torch.sqrt(1.0 + m00 - m11 - m22 + eps) * 2.0 # sq = 4 * qx. + qw = safe_zero_division(m21 - m12, sq) + qx = 0.25 * sq + qy = safe_zero_division(m01 + m10, sq) + qz = safe_zero_division(m02 + m20, sq) + return torch.cat((qw, qx, qy, qz), dim=-1) + + def cond_2() -> torch.Tensor: + sq = torch.sqrt(1.0 + m11 - m00 - m22 + eps) * 2.0 # sq = 4 * qy. + qw = safe_zero_division(m02 - m20, sq) + qx = safe_zero_division(m01 + m10, sq) + qy = 0.25 * sq + qz = safe_zero_division(m12 + m21, sq) + return torch.cat((qw, qx, qy, qz), dim=-1) + + def cond_3() -> torch.Tensor: + sq = torch.sqrt(1.0 + m22 - m00 - m11 + eps) * 2.0 # sq = 4 * qz. + qw = safe_zero_division(m10 - m01, sq) + qx = safe_zero_division(m02 + m20, sq) + qy = safe_zero_division(m12 + m21, sq) + qz = 0.25 * sq + return torch.cat((qw, qx, qy, qz), dim=-1) + + where_2 = torch.where(m11 > m22, cond_2(), cond_3()) + where_1 = torch.where((m00 > m11) & (m00 > m22), cond_1(), where_2) + + quaternion: torch.Tensor = torch.where(trace > 0.0, trace_positive_cond(), where_1) + return quaternion + + +def normalize_quaternion(quaternion: torch.Tensor, eps: float = 1.0e-12) -> torch.Tensor: + r"""Normalize a quaternion. + + The quaternion should be in (x, y, z, w) or (w, x, y, z) format. + + Args: + quaternion: a tensor containing a quaternion to be normalized. + The tensor can be of shape :math:`(*, 4)`. + eps: small value to avoid division by zero. + + Return: + the normalized quaternion of shape :math:`(*, 4)`. + + Example: + >>> quaternion = torch.tensor((1., 0., 1., 0.)) + >>> normalize_quaternion(quaternion) + tensor([0.7071, 0.0000, 0.7071, 0.0000]) + + """ + if not isinstance(quaternion, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(quaternion)}") + + if not quaternion.shape[-1] == 4: + raise ValueError(f"Input must be a tensor of shape (*, 4). Got {quaternion.shape}") + return F.normalize(quaternion, p=2.0, dim=-1, eps=eps) + + +# based on: +# https://github.com/matthew-brett/transforms3d/blob/8965c48401d9e8e66b6a8c37c65f2fc200a076fa/transforms3d/quaternions.py#L101 +# https://github.com/tensorflow/graphics/blob/master/tensorflow_graphics/geometry/transformation/rotation_matrix_3d.py#L247 + + +def quaternion_to_rotation_matrix(quaternion: torch.Tensor) -> torch.Tensor: + r"""Convert a quaternion to a rotation matrix. + + The quaternion should be in (w, x, y, z) format. + + Args: + quaternion: a tensor containing a quaternion to be converted. + The tensor can be of shape :math:`(*, 4)`. + + Return: + the rotation matrix of shape :math:`(*, 3, 3)`. + + Example: + >>> quaternion = torch.tensor((0., 0., 0., 1.)) + >>> quaternion_to_rotation_matrix(quaternion) + tensor([[-1., 0., 0.], + [ 0., -1., 0.], + [ 0., 0., 1.]]) + + """ + if not isinstance(quaternion, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(quaternion)}") + + if not quaternion.shape[-1] == 4: + raise ValueError(f"Input must be a tensor of shape (*, 4). Got {quaternion.shape}") + + # normalize the input quaternion + quaternion_norm: torch.Tensor = normalize_quaternion(quaternion) + + # unpack the normalized quaternion components + w = quaternion_norm[..., 0] + x = quaternion_norm[..., 1] + y = quaternion_norm[..., 2] + z = quaternion_norm[..., 3] + + # compute the actual conversion + tx: torch.Tensor = 2.0 * x + ty: torch.Tensor = 2.0 * y + tz: torch.Tensor = 2.0 * z + twx: torch.Tensor = tx * w + twy: torch.Tensor = ty * w + twz: torch.Tensor = tz * w + txx: torch.Tensor = tx * x + txy: torch.Tensor = ty * x + txz: torch.Tensor = tz * x + tyy: torch.Tensor = ty * y + tyz: torch.Tensor = tz * y + tzz: torch.Tensor = tz * z + one: torch.Tensor = torch.tensor(1.0) + + matrix_flat: torch.Tensor = torch.stack( + ( + one - (tyy + tzz), + txy - twz, + txz + twy, + txy + twz, + one - (txx + tzz), + tyz - twx, + txz - twy, + tyz + twx, + one - (txx + tyy), + ), + dim=-1, + ) + + # this slightly awkward construction of the output shape is to satisfy torchscript + output_shape = [*list(quaternion.shape[:-1]), 3, 3] + matrix = matrix_flat.reshape(output_shape) + + return matrix + + +def quaternion_to_axis_angle(quaternion: torch.Tensor) -> torch.Tensor: + """Convert quaternion vector to axis angle of rotation in radians. + + The quaternion should be in (w, x, y, z) format. + + Adapted from ceres C++ library: ceres-solver/include/ceres/rotation.h + + Args: + quaternion: tensor with quaternions. + + Return: + tensor with axis angle of rotation. + + Shape: + - Input: :math:`(*, 4)` where `*` means, any number of dimensions + - Output: :math:`(*, 3)` + + Example: + >>> quaternion = torch.tensor((1., 0., 0., 0.)) + >>> quaternion_to_axis_angle(quaternion) + tensor([0., 0., 0.]) + + """ + if not isinstance(quaternion, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(quaternion)}") + + if not quaternion.shape[-1] == 4: + raise ValueError(f"Input must be a tensor of shape Nx4 or 4. Got {quaternion.shape}") + + # unpack input and compute conversion + q1: torch.Tensor = torch.tensor([]) + q2: torch.Tensor = torch.tensor([]) + q3: torch.Tensor = torch.tensor([]) + cos_theta: torch.Tensor = torch.tensor([]) + + cos_theta = quaternion[..., 0] + q1 = quaternion[..., 1] + q2 = quaternion[..., 2] + q3 = quaternion[..., 3] + + sin_squared_theta: torch.Tensor = q1 * q1 + q2 * q2 + q3 * q3 + + sin_theta: torch.Tensor = torch.sqrt(sin_squared_theta) + two_theta: torch.Tensor = 2.0 * torch.where( + cos_theta < 0.0, torch.atan2(-sin_theta, -cos_theta), torch.atan2(sin_theta, cos_theta) + ) + + k_pos: torch.Tensor = two_theta / sin_theta + k_neg: torch.Tensor = 2.0 * torch.ones_like(sin_theta) + k: torch.Tensor = torch.where(sin_squared_theta > 0.0, k_pos, k_neg) + + axis_angle: torch.Tensor = torch.zeros_like(quaternion)[..., :3] + axis_angle[..., 0] += q1 * k + axis_angle[..., 1] += q2 * k + axis_angle[..., 2] += q3 * k + return axis_angle + + +@deprecated(replace_with="quaternion_to_axis_angle", version="0.7.0") +def quaternion_to_angle_axis(quaternion: torch.Tensor) -> torch.Tensor: # noqa: D103 + return quaternion_to_axis_angle(quaternion) + + +def quaternion_log_to_exp(quaternion: torch.Tensor, eps: float = 1.0e-8) -> torch.Tensor: + r"""Apply exponential map to log quaternion. + + The quaternion should be in (w, x, y, z) format. + + Args: + quaternion: a tensor containing a quaternion to be converted. + The tensor can be of shape :math:`(*, 3)`. + eps: a small number for clamping. + + Return: + the quaternion exponential map of shape :math:`(*, 4)`. + + Example: + >>> quaternion = torch.tensor((0., 0., 0.)) + >>> quaternion_log_to_exp(quaternion, eps=torch.finfo(quaternion.dtype).eps) + tensor([1., 0., 0., 0.]) + + """ + if not isinstance(quaternion, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(quaternion)}") + + if not quaternion.shape[-1] == 3: + raise ValueError(f"Input must be a tensor of shape (*, 3). Got {quaternion.shape}") + + # compute quaternion norm + norm_q: torch.Tensor = torch.norm(quaternion, p=2, dim=-1, keepdim=True).clamp(min=eps) + + # compute scalar and vector + quaternion_vector: torch.Tensor = quaternion * torch.sin(norm_q) / norm_q + quaternion_scalar: torch.Tensor = torch.cos(norm_q) + + # compose quaternion and return + quaternion_exp: torch.Tensor = torch.tensor([]) + quaternion_exp = torch.cat((quaternion_scalar, quaternion_vector), dim=-1) + + return quaternion_exp + + +def quaternion_exp_to_log(quaternion: torch.Tensor, eps: float = 1.0e-8) -> torch.Tensor: + r"""Apply the log map to a quaternion. + + The quaternion should be in (w, x, y, z) format. + + Args: + quaternion: a tensor containing a quaternion to be converted. + The tensor can be of shape :math:`(*, 4)`. + eps: a small number for clamping. + + Return: + the quaternion log map of shape :math:`(*, 3)`. + + Example: + >>> quaternion = torch.tensor((1., 0., 0., 0.)) + >>> quaternion_exp_to_log(quaternion, eps=torch.finfo(quaternion.dtype).eps) + tensor([0., 0., 0.]) + + """ + if not isinstance(quaternion, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(quaternion)}") + + if not quaternion.shape[-1] == 4: + raise ValueError(f"Input must be a tensor of shape (*, 4). Got {quaternion.shape}") + + # unpack quaternion vector and scalar + quaternion_vector: torch.Tensor = torch.tensor([]) + quaternion_scalar: torch.Tensor = torch.tensor([]) + + quaternion_scalar = quaternion[..., 0:1] + quaternion_vector = quaternion[..., 1:4] + + # compute quaternion norm + norm_q: torch.Tensor = torch.norm(quaternion_vector, p=2, dim=-1, keepdim=True).clamp(min=eps) + + # apply log map + quaternion_log: torch.Tensor = ( + quaternion_vector * torch.acos(torch.clamp(quaternion_scalar, min=-1.0, max=1.0)) / norm_q + ) + + return quaternion_log + + +# based on: +# https://github.com/facebookresearch/QuaterNet/blob/master/common/quaternion.py#L138 + + +def axis_angle_to_quaternion(axis_angle: torch.Tensor) -> torch.Tensor: + r"""Convert an axis angle to a quaternion. + + The quaternion vector has components in (w, x, y, z) format. + + Adapted from ceres C++ library: ceres-solver/include/ceres/rotation.h + + Args: + axis_angle: tensor with axis angle in radians. + + Return: + tensor with quaternion. + + Shape: + - Input: :math:`(*, 3)` where `*` means, any number of dimensions + - Output: :math:`(*, 4)` + + Example: + >>> axis_angle = torch.tensor((0., 1., 0.)) + >>> axis_angle_to_quaternion(axis_angle) + tensor([0.8776, 0.0000, 0.4794, 0.0000]) + + """ + if not isinstance(axis_angle, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(axis_angle)}") + + if not axis_angle.shape[-1] == 3: + raise ValueError(f"Input must be a tensor of shape Nx3 or 3. Got {axis_angle.shape}") + + # unpack input and compute conversion + a0: torch.Tensor = axis_angle[..., 0:1] + a1: torch.Tensor = axis_angle[..., 1:2] + a2: torch.Tensor = axis_angle[..., 2:3] + theta_squared: torch.Tensor = a0 * a0 + a1 * a1 + a2 * a2 + + theta: torch.Tensor = torch.sqrt(theta_squared) + half_theta: torch.Tensor = theta * 0.5 + + mask: torch.Tensor = theta_squared > 0.0 + ones: torch.Tensor = torch.ones_like(half_theta) + + k_neg: torch.Tensor = 0.5 * ones + k_pos: torch.Tensor = torch.sin(half_theta) / theta + k: torch.Tensor = torch.where(mask, k_pos, k_neg) + w: torch.Tensor = torch.where(mask, torch.cos(half_theta), ones) + + quaternion: torch.Tensor = torch.zeros( + size=(*axis_angle.shape[:-1], 4), dtype=axis_angle.dtype, device=axis_angle.device + ) + quaternion[..., 1:2] = a0 * k + quaternion[..., 2:3] = a1 * k + quaternion[..., 3:4] = a2 * k + quaternion[..., 0:1] = w + return quaternion + + +@deprecated(replace_with="axis_angle_to_quaternion", version="0.7.0") +def angle_axis_to_quaternion(axis_angle: torch.Tensor) -> torch.Tensor: # noqa: D103 + return axis_angle_to_quaternion(axis_angle) + + +# inspired by: https://stackoverflow.com/questions/56207448/efficient-quaternions-to-euler-transformation + + +def euler_from_quaternion( + w: torch.Tensor, x: torch.Tensor, y: torch.Tensor, z: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Convert a quaternion coefficients to Euler angles. + + Returned angles are in radians in XYZ convention. + + Args: + w: quaternion :math:`q_w` coefficient. + x: quaternion :math:`q_x` coefficient. + y: quaternion :math:`q_y` coefficient. + z: quaternion :math:`q_z` coefficient. + + Return: + A tuple with euler angles`roll`, `pitch`, `yaw`. + + """ + KORNIA_CHECK(w.shape == x.shape) + KORNIA_CHECK(x.shape == y.shape) + KORNIA_CHECK(y.shape == z.shape) + + yy = y * y + + sinr_cosp = 2.0 * (w * x + y * z) + cosr_cosp = 1.0 - 2.0 * (x * x + yy) + roll = sinr_cosp.atan2(cosr_cosp) + + sinp = 2.0 * (w * y - z * x) + sinp = sinp.clamp(min=-1.0, max=1.0) + pitch = sinp.asin() + + siny_cosp = 2.0 * (w * z + x * y) + cosy_cosp = 1.0 - 2.0 * (yy + z * z) + yaw = siny_cosp.atan2(cosy_cosp) + + return roll, pitch, yaw + + +def quaternion_from_euler( + roll: torch.Tensor, pitch: torch.Tensor, yaw: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Convert Euler angles to quaternion coefficients. + + Euler angles are assumed to be in radians in XYZ convention. + + Args: + roll: the roll euler angle. + pitch: the pitch euler angle. + yaw: the yaw euler angle. + + Return: + A tuple with quaternion coefficients in order of `wxyz`. + + """ + KORNIA_CHECK(roll.shape == pitch.shape) + KORNIA_CHECK(pitch.shape == yaw.shape) + + roll_half = roll * 0.5 + pitch_half = pitch * 0.5 + yaw_half = yaw * 0.5 + + cy = yaw_half.cos() + sy = yaw_half.sin() + cp = pitch_half.cos() + sp = pitch_half.sin() + cr = roll_half.cos() + sr = roll_half.sin() + + qw = cy * cp * cr + sy * sp * sr + qx = cy * cp * sr - sy * sp * cr + qy = sy * cp * sr + cy * sp * cr + qz = sy * cp * cr - cy * sp * sr + + return qw, qx, qy, qz + + +# based on: +# https://github.com/ClementPinard/SfmLearner-Pytorch/blob/master/inverse_warp.py#L65-L71 + + +def normalize_pixel_coordinates( + pixel_coordinates: torch.Tensor, height: int, width: int, eps: float = 1e-8 +) -> torch.Tensor: + r"""Normalize pixel coordinates between -1 and 1. + + Normalized, -1 if on extreme left, 1 if on extreme right (x = w-1). + + Args: + pixel_coordinates: the grid with pixel coordinates. Shape can be :math:`(*, 2)`. + width: the maximum width in the x-axis. + height: the maximum height in the y-axis. + eps: safe division by zero. + + Return: + the normalized pixel coordinates with shape :math:`(*, 2)`. + + Examples: + >>> coords = torch.tensor([[50., 100.]]) + >>> normalize_pixel_coordinates(coords, 100, 50) + tensor([[1.0408, 1.0202]]) + + """ + if pixel_coordinates.shape[-1] != 2: + raise ValueError(f"Input pixel_coordinates must be of shape (*, 2). Got {pixel_coordinates.shape}") + + # compute normalization factor + hw: torch.Tensor = torch.stack( + [ + torch.tensor(width, device=pixel_coordinates.device, dtype=pixel_coordinates.dtype), + torch.tensor(height, device=pixel_coordinates.device, dtype=pixel_coordinates.dtype), + ] + ) + + factor: torch.Tensor = torch.tensor(2.0, device=pixel_coordinates.device, dtype=pixel_coordinates.dtype) / ( + hw - 1 + ).clamp(eps) + + return factor * pixel_coordinates - 1 + + +def denormalize_pixel_coordinates( + pixel_coordinates: torch.Tensor, height: int, width: int, eps: float = 1e-8 +) -> torch.Tensor: + r"""Denormalize pixel coordinates. + + The input is assumed to be -1 if on extreme left, 1 if on extreme right (x = w-1). + + Args: + pixel_coordinates: the normalized grid coordinates. Shape can be :math:`(*, 2)`. + width: the maximum width in the x-axis. + height: the maximum height in the y-axis. + eps: safe division by zero. + + Return: + the denormalized pixel coordinates with shape :math:`(*, 2)`. + + Examples: + >>> coords = torch.tensor([[-1., -1.]]) + >>> denormalize_pixel_coordinates(coords, 100, 50) + tensor([[0., 0.]]) + + """ + if pixel_coordinates.shape[-1] != 2: + raise ValueError(f"Input pixel_coordinates must be of shape (*, 2). Got {pixel_coordinates.shape}") + # compute normalization factor + hw: torch.Tensor = ( + torch.stack([torch.tensor(width), torch.tensor(height)]) + .to(pixel_coordinates.device) + .to(pixel_coordinates.dtype) + ) + + factor: torch.Tensor = torch.tensor(2.0) / (hw - 1).clamp(eps) + + return torch.tensor(1.0) / factor * (pixel_coordinates + 1) + + +def normalize_pixel_coordinates3d( + pixel_coordinates: torch.Tensor, depth: int, height: int, width: int, eps: float = 1e-8 +) -> torch.Tensor: + r"""Normalize pixel coordinates between -1 and 1. + + Normalized, -1 if on extreme left, 1 if on extreme right (x = w-1). + + Args: + pixel_coordinates: the grid with pixel coordinates. Shape can be :math:`(*, 3)`. + depth: the maximum depth in the z-axis. + height: the maximum height in the y-axis. + width: the maximum width in the x-axis. + eps: safe division by zero. + + Return: + the normalized pixel coordinates. + + """ + if pixel_coordinates.shape[-1] != 3: + raise ValueError(f"Input pixel_coordinates must be of shape (*, 3). Got {pixel_coordinates.shape}") + # compute normalization factor + dhw: torch.Tensor = ( + torch.stack([torch.tensor(depth), torch.tensor(width), torch.tensor(height)]) + .to(pixel_coordinates.device) + .to(pixel_coordinates.dtype) + ) + + factor: torch.Tensor = torch.tensor(2.0) / (dhw - 1).clamp(eps) + + return factor * pixel_coordinates - 1 + + +def denormalize_pixel_coordinates3d( + pixel_coordinates: torch.Tensor, depth: int, height: int, width: int, eps: float = 1e-8 +) -> torch.Tensor: + r"""Denormalize pixel coordinates. + + The input is assumed to be -1 if on extreme left, 1 if on extreme right (x = w-1). + + Args: + pixel_coordinates: the normalized grid coordinates. Shape can be :math:`(*, 3)`. + depth: the maximum depth in the x-axis. + height: the maximum height in the y-axis. + width: the maximum width in the x-axis. + eps: safe division by zero. + + Return: + the denormalized pixel coordinates. + + """ + if pixel_coordinates.shape[-1] != 3: + raise ValueError(f"Input pixel_coordinates must be of shape (*, 3). Got {pixel_coordinates.shape}") + # compute normalization factor + dhw: torch.Tensor = ( + torch.stack([torch.tensor(depth), torch.tensor(width), torch.tensor(height)]) + .to(pixel_coordinates.device) + .to(pixel_coordinates.dtype) + ) + + factor: torch.Tensor = torch.tensor(2.0) / (dhw - 1).clamp(eps) + + return torch.tensor(1.0) / factor * (pixel_coordinates + 1) + + +def angle_to_rotation_matrix(angle: torch.Tensor) -> torch.Tensor: + r"""Create a rotation matrix out of angles in degrees. + + Args: + angle: tensor of angles in degrees, any shape :math:`(*)`. + + Returns: + tensor of rotation matrices with shape :math:`(*, 2, 2)`. + + Example: + >>> input = torch.rand(1, 3) # Nx3 + >>> output = angle_to_rotation_matrix(input) # Nx3x2x2 + + """ + ang_rad = deg2rad(angle) + cos_a: torch.Tensor = torch.cos(ang_rad) + sin_a: torch.Tensor = torch.sin(ang_rad) + return torch.stack([cos_a, sin_a, -sin_a, cos_a], dim=-1).view(*angle.shape, 2, 2) + + +def normalize_homography( + dst_pix_trans_src_pix: torch.Tensor, dsize_src: tuple[int, int], dsize_dst: tuple[int, int] +) -> torch.Tensor: + r"""Normalize a given homography in pixels to [-1, 1]. + + Args: + dst_pix_trans_src_pix: homography/ies from source to destination to be + normalized. :math:`(B, 3, 3)` + dsize_src: size of the source image (height, width). + dsize_dst: size of the destination image (height, width). + + Returns: + the normalized homography of shape :math:`(B, 3, 3)`. + + """ + if not isinstance(dst_pix_trans_src_pix, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(dst_pix_trans_src_pix)}") + + if not (len(dst_pix_trans_src_pix.shape) == 3 or dst_pix_trans_src_pix.shape[-2:] == (3, 3)): + raise ValueError(f"Input dst_pix_trans_src_pix must be a Bx3x3 tensor. Got {dst_pix_trans_src_pix.shape}") + + # source and destination sizes + src_h, src_w = dsize_src + dst_h, dst_w = dsize_dst + + # compute the transformation pixel/norm for src/dst + src_norm_trans_src_pix: torch.Tensor = normal_transform_pixel(src_h, src_w).to(dst_pix_trans_src_pix) + + src_pix_trans_src_norm = _torch_inverse_cast(src_norm_trans_src_pix) + dst_norm_trans_dst_pix: torch.Tensor = normal_transform_pixel(dst_h, dst_w).to(dst_pix_trans_src_pix) + + # compute chain transformations + dst_norm_trans_src_norm: torch.Tensor = dst_norm_trans_dst_pix @ (dst_pix_trans_src_pix @ src_pix_trans_src_norm) + return dst_norm_trans_src_norm + + +def normal_transform_pixel( + height: int, + width: int, + eps: float = 1e-14, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + r"""Compute the normalization matrix from image size in pixels to [-1, 1]. + + Args: + height: image height. + width: image width. + eps: epsilon to prevent divide-by-zero errors + device: device to place the result on. + dtype: dtype of the result. + + Returns: + normalized transform with shape :math:`(1, 3, 3)`. + + """ + # prevent divide by zero bugs + width_denom: float = eps if width == 1 else width - 1.0 + height_denom: float = eps if height == 1 else height - 1.0 + + sx: float = 2.0 / width_denom + sy: float = 2.0 / height_denom + + # Construct the matrix in one shot (no in-place mutation). + tr_mat = torch.tensor( + [[sx, 0.0, -1.0], [0.0, sy, -1.0], [0.0, 0.0, 1.0]], + device=device, + dtype=dtype, + ) # 3x3 + + return tr_mat.unsqueeze(0) # 1x3x3 + + +def normal_transform_pixel3d( + depth: int, + height: int, + width: int, + eps: float = 1e-14, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + r"""Compute the normalization matrix from image size in pixels to [-1, 1]. + + Args: + depth: image depth. + height: image height. + width: image width. + eps: epsilon to prevent divide-by-zero errors + device: device to place the result on. + dtype: dtype of the result. + + Returns: + normalized transform with shape :math:`(1, 4, 4)`. + + """ + tr_mat = torch.tensor( + [[1.0, 0.0, 0.0, -1.0], [0.0, 1.0, 0.0, -1.0], [0.0, 0.0, 1.0, -1.0], [0.0, 0.0, 0.0, 1.0]], + device=device, + dtype=dtype, + ) # 4x4 + + # prevent divide by zero bugs + width_denom: float = eps if width == 1 else width - 1.0 + height_denom: float = eps if height == 1 else height - 1.0 + depth_denom: float = eps if depth == 1 else depth - 1.0 + + tr_mat[0, 0] = tr_mat[0, 0] * 2.0 / width_denom + tr_mat[1, 1] = tr_mat[1, 1] * 2.0 / height_denom + tr_mat[2, 2] = tr_mat[2, 2] * 2.0 / depth_denom + + return tr_mat.unsqueeze(0) # 1x4x4 + + +def denormalize_homography( + dst_pix_trans_src_pix: torch.Tensor, dsize_src: tuple[int, int], dsize_dst: tuple[int, int] +) -> torch.Tensor: + r"""De-normalize a given homography in pixels from [-1, 1] to actual height and width. + + Args: + dst_pix_trans_src_pix: homography/ies from source to destination to be + denormalized. :math:`(B, 3, 3)` + dsize_src: size of the source image (height, width). + dsize_dst: size of the destination image (height, width). + + Returns: + the denormalized homography of shape :math:`(B, 3, 3)`. + + """ + if not isinstance(dst_pix_trans_src_pix, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(dst_pix_trans_src_pix)}") + + if not (len(dst_pix_trans_src_pix.shape) == 3 or dst_pix_trans_src_pix.shape[-2:] == (3, 3)): + raise ValueError(f"Input dst_pix_trans_src_pix must be a Bx3x3 tensor. Got {dst_pix_trans_src_pix.shape}") + + # source and destination sizes + src_h, src_w = dsize_src + dst_h, dst_w = dsize_dst + + # compute the transformation pixel/norm for src/dst + src_norm_trans_src_pix: torch.Tensor = normal_transform_pixel(src_h, src_w).to(dst_pix_trans_src_pix) + + dst_norm_trans_dst_pix: torch.Tensor = normal_transform_pixel(dst_h, dst_w).to(dst_pix_trans_src_pix) + dst_denorm_trans_dst_pix = _torch_inverse_cast(dst_norm_trans_dst_pix) + # compute chain transformations + dst_norm_trans_src_norm: torch.Tensor = dst_denorm_trans_dst_pix @ (dst_pix_trans_src_pix @ src_norm_trans_src_pix) + return dst_norm_trans_src_norm + + +def normalize_homography3d( + dst_pix_trans_src_pix: torch.Tensor, dsize_src: tuple[int, int, int], dsize_dst: tuple[int, int, int] +) -> torch.Tensor: + r"""Normalize a given homography in pixels to [-1, 1]. + + Args: + dst_pix_trans_src_pix: homography/ies from source to destination to be + normalized. :math:`(B, 4, 4)` + dsize_src: size of the source image (depth, height, width). + dsize_dst: size of the destination image (depth, height, width). + + Returns: + the normalized homography. + + Shape: + Output: :math:`(B, 4, 4)` + + """ + if not isinstance(dst_pix_trans_src_pix, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(dst_pix_trans_src_pix)}") + + if not (len(dst_pix_trans_src_pix.shape) == 3 or dst_pix_trans_src_pix.shape[-2:] == (4, 4)): + raise ValueError(f"Input dst_pix_trans_src_pix must be a Bx3x3 tensor. Got {dst_pix_trans_src_pix.shape}") + + # source and destination sizes + src_d, src_h, src_w = dsize_src + dst_d, dst_h, dst_w = dsize_dst + # compute the transformation pixel/norm for src/dst + src_norm_trans_src_pix: torch.Tensor = normal_transform_pixel3d(src_d, src_h, src_w).to(dst_pix_trans_src_pix) + + src_pix_trans_src_norm = _torch_inverse_cast(src_norm_trans_src_pix) + dst_norm_trans_dst_pix: torch.Tensor = normal_transform_pixel3d(dst_d, dst_h, dst_w).to(dst_pix_trans_src_pix) + # compute chain transformations + dst_norm_trans_src_norm: torch.Tensor = dst_norm_trans_dst_pix @ (dst_pix_trans_src_pix @ src_pix_trans_src_norm) + return dst_norm_trans_src_norm + + +def normalize_points_with_intrinsics(point_2d: torch.Tensor, camera_matrix: torch.Tensor) -> torch.Tensor: + """Normalize points with intrinsics. Useful for conversion of keypoints to be used with essential matrix. + + Args: + point_2d: tensor containing the 2d points in the image pixel coordinates. The shape of the tensor can be + :math:`(*, 2)`. + camera_matrix: tensor containing the intrinsics camera matrix. The tensor shape must be :math:`(*, 3, 3)`. + + Returns: + tensor of (u, v) cam coordinates with shape :math:`(*, 2)`. + + Example: + >>> _ = torch.manual_seed(0) + >>> X = torch.rand(1, 2) + >>> K = torch.eye(3)[None] + >>> normalize_points_with_intrinsics(X, K) + tensor([[0.4963, 0.7682]]) + + """ + KORNIA_CHECK_SHAPE(point_2d, ["*", "2"]) + KORNIA_CHECK_SHAPE(camera_matrix, ["*", "3", "3"]) + # projection eq. K_inv * [u v 1]' + # x = (u - cx) * Z / fx + # y = (v - cy) * Z / fy + + # unpack coordinates + cxcy = camera_matrix[..., :2, 2] + fxfy = camera_matrix[..., :2, :2].diagonal(dim1=-2, dim2=-1) + if len(cxcy.shape) < len(point_2d.shape): # broadcast intrinsics: + cxcy, fxfy = cxcy.unsqueeze(-2), fxfy.unsqueeze(-2) + xy = (point_2d - cxcy) / fxfy + return xy + + +def denormalize_points_with_intrinsics(point_2d_norm: torch.Tensor, camera_matrix: torch.Tensor) -> torch.Tensor: + """Normalize points with intrinsics. Useful for conversion of keypoints to be used with essential matrix. + + Args: + point_2d_norm: tensor containing the 2d points in the image pixel coordinates. The shape of the tensor can be + :math:`(*, 2)`. + camera_matrix: tensor containing the intrinsics camera matrix. The tensor shape must be :math:`(*, 3, 3)`. + + Returns: + tensor of (u, v) cam coordinates with shape :math:`(*, 2)`. + + Example: + >>> _ = torch.manual_seed(0) + >>> X = torch.rand(1, 2) + >>> K = torch.eye(3)[None] + >>> denormalize_points_with_intrinsics(X, K) + tensor([[0.4963, 0.7682]]) + + """ + KORNIA_CHECK_SHAPE(point_2d_norm, ["*", "2"]) + KORNIA_CHECK_SHAPE(camera_matrix, ["*", "3", "3"]) + # projection eq. [u, v, w]' = K * [x y z 1]' + # u = fx * X + cx + # v = fy * Y + cy + + fxfy = camera_matrix[..., :2, :2].diagonal(dim1=-2, dim2=-1) # (*, 2) + cxcy = camera_matrix[..., :2, 2] # (*, 2) + if len(cxcy.shape) < len(point_2d_norm.shape): + fxfy, cxcy = fxfy.unsqueeze(-2), cxcy.unsqueeze(-2) + return point_2d_norm * fxfy + cxcy + + +def Rt_to_matrix4x4(R: torch.Tensor, t: torch.Tensor) -> torch.Tensor: + r"""Combine 3x3 rotation matrix R and 1x3 translation vector t into 4x4 extrinsics. + + Args: + R: Rotation matrix, :math:`(B, 3, 3).` + t: Translation matrix :math:`(B, 3, 1)`. + + Returns: + the extrinsics :math:`(B, 4, 4)`. + + Example: + >>> R, t = torch.eye(3)[None], torch.ones(3).reshape(1, 3, 1) + >>> Rt_to_matrix4x4(R, t) + tensor([[[1., 0., 0., 1.], + [0., 1., 0., 1.], + [0., 0., 1., 1.], + [0., 0., 0., 1.]]]) + + """ + KORNIA_CHECK_SHAPE(R, ["B", "3", "3"]) + KORNIA_CHECK_SHAPE(t, ["B", "3", "1"]) + Rt = torch.cat([R, t], dim=2) + return convert_affinematrix_to_homography3d(Rt) + + +def matrix4x4_to_Rt(extrinsics: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + r"""Convert 4x4 extrinsics into 3x3 rotation matrix R and 1x3 translation vector ts. + + Args: + extrinsics: pose matrix :math:`(B, 4, 4)`. + + Returns: + R: Rotation matrix, :math:`(B, 3, 3).` + t: Translation matrix :math:`(B, 3, 1)`. + + Example: + >>> ext = torch.eye(4)[None] + >>> matrix4x4_to_Rt(ext) + (tensor([[[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]]), tensor([[[0.], + [0.], + [0.]]])) + + """ + KORNIA_CHECK_SHAPE(extrinsics, ["B", "4", "4"]) + R, t = extrinsics[:, :3, :3], extrinsics[:, :3, 3:] + return R, t + + +def camtoworld_graphics_to_vision_4x4(extrinsics_graphics: torch.Tensor) -> torch.Tensor: + r"""Convert graphics coordinate frame (e.g. OpenGL) to vision coordinate frame (e.g. OpenCV.). + + I.e. flips y and z axis. Graphics convention: [+x, +y, +z] == [right, up, backwards]. + Vision convention: [+x, +y, +z] == [right, down, forwards]. + + Args: + extrinsics_graphics: pose matrix :math:`(B, 4, 4)`. + + Returns: + extrinsics: pose matrix :math:`(B, 4, 4)`. + + Example: + >>> ext = torch.eye(4)[None] + >>> camtoworld_graphics_to_vision_4x4(ext) + tensor([[[ 1., 0., 0., 0.], + [ 0., -1., 0., 0.], + [ 0., 0., -1., 0.], + [ 0., 0., 0., 1.]]]) + + """ + KORNIA_CHECK_SHAPE(extrinsics_graphics, ["B", "4", "4"]) + invert_yz = torch.tensor( + [[[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1.0]]], + dtype=extrinsics_graphics.dtype, + device=extrinsics_graphics.device, + ) + return extrinsics_graphics @ invert_yz + + +def camtoworld_graphics_to_vision_Rt(R: torch.Tensor, t: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + r"""Convert graphics coordinate frame (e.g. OpenGL) to vision coordinate frame (e.g. OpenCV.). + + I.e. flips y and z axis. Graphics convention: [+x, +y, +z] == [right, up, backwards]. + Vision convention: [+x, +y, +z] == [right, down, forwards]. + + Args: + R: Rotation matrix, :math:`(B, 3, 3).` + t: Translation matrix :math:`(B, 3, 1)`. + + Returns: + R: Rotation matrix, :math:`(B, 3, 3).` + t: Translation matrix :math:`(B, 3, 1)`. + + Example: + >>> R, t = torch.eye(3)[None], torch.ones(3).reshape(1, 3, 1) + >>> camtoworld_graphics_to_vision_Rt(R, t) + (tensor([[[ 1., 0., 0.], + [ 0., -1., 0.], + [ 0., 0., -1.]]]), tensor([[[1.], + [1.], + [1.]]])) + + """ + KORNIA_CHECK_SHAPE(R, ["B", "3", "3"]) + KORNIA_CHECK_SHAPE(t, ["B", "3", "1"]) + mat4x4 = camtoworld_graphics_to_vision_4x4(Rt_to_matrix4x4(R, t)) + return matrix4x4_to_Rt(mat4x4) + + +def camtoworld_vision_to_graphics_4x4(extrinsics_vision: torch.Tensor) -> torch.Tensor: + r"""Convert vision coordinate frame (e.g. OpenCV) to graphics coordinate frame (e.g. OpenGK.). + + I.e. flips y and z axis Graphics convention: [+x, +y, +z] == [right, up, backwards]. + Vision convention: [+x, +y, +z] == [right, down, forwards]. + + Args: + extrinsics_vision: pose matrix :math:`(B, 4, 4)`. + + Returns: + extrinsics: pose matrix :math:`(B, 4, 4)`. + + Example: + >>> ext = torch.eye(4)[None] + >>> camtoworld_vision_to_graphics_4x4(ext) + tensor([[[ 1., 0., 0., 0.], + [ 0., -1., 0., 0.], + [ 0., 0., -1., 0.], + [ 0., 0., 0., 1.]]]) + + """ + KORNIA_CHECK_SHAPE(extrinsics_vision, ["B", "4", "4"]) + invert_yz = torch.tensor( + [[[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1.0]]], + dtype=extrinsics_vision.dtype, + device=extrinsics_vision.device, + ) + return extrinsics_vision @ invert_yz + + +def camtoworld_vision_to_graphics_Rt(R: torch.Tensor, t: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + r"""Convert graphics coordinate frame (e.g. OpenGL) to vision coordinate frame (e.g. OpenCV.). + + I.e. flips y and z axis. Graphics convention: [+x, +y, +z] == [right, up, backwards]. + Vision convention: [+x, +y, +z] == [right, down, forwards] + + Args: + R: Rotation matrix, :math:`(B, 3, 3).` + t: Translation matrix :math:`(B, 3, 1)`. + + Returns: + R: Rotation matrix, :math:`(B, 3, 3).` + t: Translation matrix :math:`(B, 3, 1)`. + + Example: + >>> R, t = torch.eye(3)[None], torch.ones(3).reshape(1, 3, 1) + >>> camtoworld_vision_to_graphics_Rt(R, t) + (tensor([[[ 1., 0., 0.], + [ 0., -1., 0.], + [ 0., 0., -1.]]]), tensor([[[1.], + [1.], + [1.]]])) + + """ + KORNIA_CHECK_SHAPE(R, ["B", "3", "3"]) + KORNIA_CHECK_SHAPE(t, ["B", "3", "1"]) + mat4x4 = camtoworld_vision_to_graphics_4x4(Rt_to_matrix4x4(R, t)) + return matrix4x4_to_Rt(mat4x4) + + +def camtoworld_to_worldtocam_Rt(R: torch.Tensor, t: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + r"""Convert camtoworld to worldtocam frame used in Colmap. + + See + long-url: https://colmap.github.io/format.html#output-format + + Args: + R: Rotation matrix, :math:`(B, 3, 3).` + t: Translation matrix :math:`(B, 3, 1)`. + + Returns: + Rinv: Rotation matrix, :math:`(B, 3, 3).` + tinv: Translation matrix :math:`(B, 3, 1)`. + + Example: + >>> R, t = torch.eye(3)[None], torch.ones(3).reshape(1, 3, 1) + >>> camtoworld_to_worldtocam_Rt(R, t) + (tensor([[[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]]), tensor([[[-1.], + [-1.], + [-1.]]])) + + """ + KORNIA_CHECK_SHAPE(R, ["B", "3", "3"]) + KORNIA_CHECK_SHAPE(t, ["B", "3", "1"]) + + R_inv = R.transpose(1, 2) + new_t: torch.Tensor = -R_inv @ t + + return (R_inv, new_t) + + +def worldtocam_to_camtoworld_Rt(R: torch.Tensor, t: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + r"""Convert worldtocam frame used in Colmap to camtoworld. + + Args: + R: Rotation matrix, :math:`(B, 3, 3).` + t: Translation matrix :math:`(B, 3, 1)`. + + Returns: + Rinv: Rotation matrix, :math:`(B, 3, 3).` + tinv: Translation matrix :math:`(B, 3, 1)`. + + Example: + >>> R, t = torch.eye(3)[None], torch.ones(3).reshape(1, 3, 1) + >>> worldtocam_to_camtoworld_Rt(R, t) + (tensor([[[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]]), tensor([[[-1.], + [-1.], + [-1.]]])) + + """ + KORNIA_CHECK_SHAPE(R, ["B", "3", "3"]) + KORNIA_CHECK_SHAPE(t, ["B", "3", "1"]) + + R_inv = R.transpose(1, 2) + new_t: torch.Tensor = -R_inv @ t + + return (R_inv, new_t) + + +def ARKitQTVecs_to_ColmapQTVecs(qvec: torch.Tensor, tvec: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + r"""Convert output of Apple ARKit screen pose to the camera-to-world transformation, expected by Colmap. + + Both poses in quaternion representation. + + Args: + qvec: ARKit rotation quaternion :math:`(B, 4)`, [w, x, y, z] format. + tvec: translation vector :math:`(B, 3, 1)`, [x, y, z] + + Returns: + qvec: Colmap rotation quaternion :math:`(B, 4)`, [w, x, y, z] format. + tvec: translation vector :math:`(B, 3, 1)`, [x, y, z] + + Example: + >>> q, t = torch.tensor([0, 1, 0, 1.])[None], torch.ones(3).reshape(1, 3, 1) + >>> ARKitQTVecs_to_ColmapQTVecs(q, t) + (tensor([[0.7071, 0.0000, 0.7071, 0.0000]]), tensor([[[-1.0000], + [-1.0000], + [ 1.0000]]])) + + """ + # ToDo: integrate QuaterniaonAPI + + Rcg = quaternion_to_rotation_matrix(qvec) + Rcv, Tcv = camtoworld_graphics_to_vision_Rt(Rcg, tvec) + R_colmap, t_colmap = camtoworld_to_worldtocam_Rt(Rcv, Tcv) + t_colmap = t_colmap.reshape(-1, 3, 1) + q_colmap = rotation_matrix_to_quaternion(R_colmap.contiguous()) + return q_colmap, t_colmap + + +def vector_to_skew_symmetric_matrix(vec: torch.Tensor) -> torch.Tensor: + r"""Convert a vector to a skew symmetric matrix. + + A vector :math:`(v1, v2, v3)` has a corresponding skew-symmetric matrix, which is of the form: + + .. math:: + \begin{bmatrix} 0 & -v3 & v2 \\ + v3 & 0 & -v1 \\ + -v2 & v1 & 0\end{bmatrix} + + Args: + vec: tensor of shape :math:`(B, 3)`. + + Returns: + tensor of shape :math:`(B, 3, 3)`. + + Example: + >>> vec = torch.tensor([1.0, 2.0, 3.0]) + >>> vector_to_skew_symmetric_matrix(vec) + tensor([[ 0., -3., 2.], + [ 3., 0., -1.], + [-2., 1., 0.]]) + + """ + # KORNIA_CHECK_SHAPE(vec, ["B", "3"]) + if vec.shape[-1] != 3 or len(vec.shape) > 2: + raise ValueError(f"Input vector must be of shape (B, 3) or (3,). Got {vec.shape}") + v1, v2, v3 = vec[..., 0], vec[..., 1], vec[..., 2] + zeros = torch.zeros_like(v1) + skew_symmetric_matrix = torch.stack( + [ + torch.stack([zeros, -v3, v2], dim=-1), + torch.stack([v3, zeros, -v1], dim=-1), + torch.stack([-v2, v1, zeros], dim=-1), + ], + dim=-2, + ) + return skew_symmetric_matrix diff --git a/kornia/geometry/depth.py b/kornia/geometry/depth.py new file mode 100644 index 0000000..4d368ce --- /dev/null +++ b/kornia/geometry/depth.py @@ -0,0 +1,617 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""nn.Module containing operators to work on RGB-Depth images.""" + +from __future__ import annotations + +from typing import Optional + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE +from kornia.filters.sobel import spatial_gradient +from kornia.geometry.grid import create_meshgrid + +from .camera import PinholeCamera, cam2pixel, pixel2cam, project_points, unproject_points +from .conversions import normalize_pixel_coordinates, normalize_points_with_intrinsics +from .linalg import convert_points_to_homogeneous, transform_points + +"""nn.Module containing operators to work on RGB-Depth images.""" + +__all__ = [ + "DepthWarper", + "depth_from_disparity", + "depth_from_plane_equation", + "depth_to_3d", + "depth_to_3d_v2", + "depth_to_normals", + "depth_warp", + "unproject_meshgrid", + "warp_frame_depth", +] + + +def unproject_meshgrid( + height: int, + width: int, + camera_matrix: torch.Tensor, + normalize_points: bool = False, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + """Compute a 3d point per pixel given its depth value and the camera intrinsics. + + .. tip:: + + This function should be used in conjunction with :py:func:`kornia.geometry.depth.depth_to_3d_v2` to cache + the meshgrid computation when warping multiple frames with the same camera intrinsics. + + Args: + height: height of image. + width: width of image. + camera_matrix: tensor containing the camera intrinsics with shape :math:`(3, 3)`. + normalize_points: whether to normalize the pointcloud. This must be set to `True` when the depth is + represented as the Euclidean ray length from the camera position. + device: device to place the result on. + dtype: dtype of the result. + + Return: + tensor with a 3d point per pixel of the same resolution as the input :math:`(*, H, W, 3)`. + + """ + KORNIA_CHECK_SHAPE(camera_matrix, ["*", "3", "3"]) + + # create base coordinates grid + points_uv: torch.Tensor = create_meshgrid( + height, width, normalized_coordinates=False, device=device, dtype=dtype + ).squeeze() # HxWx2 + + # project pixels to camera frame + camera_matrix_tmp: torch.Tensor = camera_matrix[:, None, None] # Bx1x1x3x3 + + points_xy = normalize_points_with_intrinsics(points_uv, camera_matrix_tmp) # HxWx2 + + # unproject pixels to camera frame + points_xyz = convert_points_to_homogeneous(points_xy) # HxWx3 + + if normalize_points: + points_xyz = F.normalize(points_xyz, dim=-1, p=2) + + return points_xyz + + +def depth_to_3d_v2( + depth: torch.Tensor, + camera_matrix: torch.Tensor, + normalize_points: bool = False, + xyz_grid: Optional[torch.Tensor] = None, +) -> torch.Tensor: + # NOTE: when this replaces the `depth_to_3d` behaviour, a deprecated function should be added here, instead + # of just replace the other function. + """Compute a 3d point per pixel given its depth value and the camera intrinsics. + + .. note:: + + This is an alternative implementation of :py:func:`kornia.geometry.depth.depth_to_3d` + that does not require the creation of a meshgrid. + + Args: + depth: image tensor containing a depth value per pixel with shape :math:`(*, H, W)`. + camera_matrix: tensor containing the camera intrinsics with shape :math:`(*, 3, 3)`. + normalize_points: whether to normalise the pointcloud. This must be set to `True` when the depth is + represented as the Euclidean ray length from the camera position. + xyz_grid: explicit xyz point values. + + Return: + tensor with a 3d point per pixel of the same resolution as the input :math:`(*, H, W, 3)`. + + Example: + >>> depth = torch.rand(4, 4) + >>> K = torch.eye(3).repeat(2,1,1) + >>> depth_to_3d_v2(depth, K).shape + torch.Size([2, 4, 4, 3]) + + """ + KORNIA_CHECK_SHAPE(depth, ["*", "H", "W"]) + KORNIA_CHECK_SHAPE(camera_matrix, ["*", "3", "3"]) + + # create base grid if not provided + height, width = depth.shape[-2:] + points_xyz: torch.Tensor = ( + xyz_grid + if xyz_grid is not None + else unproject_meshgrid(height, width, camera_matrix, normalize_points, depth.device, depth.dtype) + ) + + KORNIA_CHECK_SHAPE(points_xyz, ["*", "H", "W", "3"]) + + return points_xyz * depth[..., None] # HxWx3 + + +def depth_to_3d(depth: torch.Tensor, camera_matrix: torch.Tensor, normalize_points: bool = False) -> torch.Tensor: + """Compute a 3d point per pixel given its depth value and the camera intrinsics. + + .. note:: + + This is an alternative implementation of `depth_to_3d` that does not require the creation of a meshgrid. + In future, we will support only this implementation. + + Args: + depth: image tensor containing a depth value per pixel with shape :math:`(B, 1, H, W)`. + camera_matrix: tensor containing the camera intrinsics with shape :math:`(B, 3, 3)`. + normalize_points: whether to normalise the pointcloud. This must be set to `True` when the depth is + represented as the Euclidean ray length from the camera position. + + Return: + tensor with a 3d point per pixel of the same resolution as the input :math:`(B, 3, H, W)`. + + Example: + >>> depth = torch.rand(1, 1, 4, 4) + >>> K = torch.eye(3)[None] + >>> depth_to_3d(depth, K).shape + torch.Size([1, 3, 4, 4]) + + """ + KORNIA_CHECK_IS_TENSOR(depth) + KORNIA_CHECK_IS_TENSOR(camera_matrix) + KORNIA_CHECK_SHAPE(depth, ["B", "1", "H", "W"]) + KORNIA_CHECK_SHAPE(camera_matrix, ["B", "3", "3"]) + + # create base coordinates grid + _, _, height, width = depth.shape + points_2d: torch.Tensor = create_meshgrid( + height, width, normalized_coordinates=False, device=depth.device, dtype=depth.dtype + ) # 1xHxWx2 + + # depth should come in Bx1xHxW + points_depth: torch.Tensor = depth.permute(0, 2, 3, 1) # 1xHxWx1 + + # project pixels to camera frame + camera_matrix_tmp: torch.Tensor = camera_matrix[:, None, None] # Bx1x1x3x3 + points_3d: torch.Tensor = unproject_points( + points_2d, points_depth, camera_matrix_tmp, normalize=normalize_points + ) # BxHxWx3 + + return points_3d.permute(0, 3, 1, 2) # Bx3xHxW + + +def depth_to_normals(depth: torch.Tensor, camera_matrix: torch.Tensor, normalize_points: bool = False) -> torch.Tensor: + """Compute the normal surface per pixel. + + Args: + depth: image tensor containing a depth value per pixel with shape :math:`(B, 1, H, W)`. + camera_matrix: tensor containing the camera intrinsics with shape :math:`(B, 3, 3)`. + normalize_points: whether to normalize the pointcloud. This must be set to `True` when the depth is + represented as the Euclidean ray length from the camera position. + + Return: + tensor with a normal surface vector per pixel of the same resolution as the input :math:`(B, 3, H, W)`. + + Example: + >>> depth = torch.rand(1, 1, 4, 4) + >>> K = torch.eye(3)[None] + >>> depth_to_normals(depth, K).shape + torch.Size([1, 3, 4, 4]) + + """ + KORNIA_CHECK_IS_TENSOR(depth) + KORNIA_CHECK_IS_TENSOR(camera_matrix) + KORNIA_CHECK_SHAPE(depth, ["B", "1", "H", "W"]) + KORNIA_CHECK_SHAPE(camera_matrix, ["B", "3", "3"]) + + # compute the 3d points from depth; permute to channel-first for spatial_gradient + xyz: torch.Tensor = depth_to_3d_v2(depth.squeeze(1), camera_matrix, normalize_points).permute(0, 3, 1, 2) # Bx3xHxW + + # compute the pointcloud spatial gradients + gradients: torch.Tensor = spatial_gradient(xyz) # Bx3x2xHxW + + # Rearrange to (B, H, W, 3) before cross product so the 3 XYZ components are + # contiguous in memory. Cross product along dim=1 on a (B,3,H,W) tensor strides + # H*W elements between components, causing severe cache thrashing on CPU. + a: torch.Tensor = gradients[:, :, 0].permute(0, 2, 3, 1).contiguous() # BxHxWx3 + b: torch.Tensor = gradients[:, :, 1].permute(0, 2, 3, 1).contiguous() # BxHxWx3 + + normals: torch.Tensor = torch.linalg.cross(a, b, dim=-1) # BxHxWx3 + return F.normalize(normals, dim=-1, p=2).permute(0, 3, 1, 2) # Bx3xHxW + + +def depth_from_plane_equation( + plane_normals: torch.Tensor, + plane_offsets: torch.Tensor, + points_uv: torch.Tensor, + camera_matrix: torch.Tensor, + eps: float = 1e-8, +) -> torch.Tensor: + """Compute depth values from plane equations and pixel coordinates. + + Args: + plane_normals (torch.Tensor): Plane normal vectors of shape (B, 3). + plane_offsets (torch.Tensor): Plane offsets of shape (B, 1). + points_uv (torch.Tensor): Pixel coordinates of shape (B, N, 2). + camera_matrix (torch.Tensor): Camera intrinsic matrix of shape (B, 3, 3). + eps: epsilon for numerical stability. + + Returns: + torch.Tensor: Computed depth values at the given pixels, shape (B, N). + + """ + KORNIA_CHECK_SHAPE(plane_normals, ["B", "3"]) + KORNIA_CHECK_SHAPE(plane_offsets, ["B", "1"]) + KORNIA_CHECK_SHAPE(points_uv, ["B", "N", "2"]) + KORNIA_CHECK_SHAPE(camera_matrix, ["B", "3", "3"]) + + # Normalize pixel coordinates + points_xy = normalize_points_with_intrinsics(points_uv, camera_matrix) # (B, N, 2) + rays = convert_points_to_homogeneous(points_xy) # (B, N, 3) + + # Reshape plane normals to match rays + plane_normals_exp = plane_normals.unsqueeze(1) # (B, 1, 3) + # No need to unsqueeze plane_offsets; it is already (B, 1) + + # Compute the denominator of the depth equation + denom = torch.sum(rays * plane_normals_exp, dim=-1) # (B, N) + denom_abs = torch.abs(denom) + zero_mask = denom_abs < eps + denom = torch.where(zero_mask, eps * torch.sign(denom), denom) + + # Compute depth from plane equation + depth = plane_offsets / denom # plane_offsets: (B, 1), denom: (B, N) -> depth: (B, N) + return depth + + +def warp_frame_depth( + image_src: torch.Tensor, + depth_dst: torch.Tensor, + src_trans_dst: torch.Tensor, + camera_matrix: torch.Tensor, + normalize_points: bool = False, +) -> torch.Tensor: + """Warp a tensor from a source to destination frame by the depth in the destination. + + Compute 3d points from the depth, transform them using given transformation, then project the point cloud to an + image plane. + + Args: + image_src: image tensor in the source frame with shape :math:`(B,D,H,W)`. + depth_dst: depth tensor in the destination frame with shape :math:`(B,1,H,W)`. + src_trans_dst: transformation matrix from destination to source with shape :math:`(B,4,4)`. + camera_matrix: tensor containing the camera intrinsics with shape :math:`(B,3,3)`. + normalize_points: whether to normalize the pointcloud. This must be set to ``True`` when the depth + is represented as the Euclidean ray length from the camera position. + + Return: + the warped tensor in the source frame with shape :math:`(B,3,H,W)`. + + """ + KORNIA_CHECK_SHAPE(image_src, ["B", "D", "H", "W"]) + KORNIA_CHECK_SHAPE(depth_dst, ["B", "1", "H", "W"]) + KORNIA_CHECK_SHAPE(src_trans_dst, ["B", "4", "4"]) + KORNIA_CHECK_SHAPE(camera_matrix, ["B", "3", "3"]) + + # unproject source points to camera frame as (B, H, W, 3) directly — avoids two permutes + points_3d_dst: torch.Tensor = depth_to_3d_v2(depth_dst.squeeze(1), camera_matrix, normalize_points) # BxHxWx3 + + # apply transformation to the 3d points + points_3d_src = transform_points(src_trans_dst[:, None], points_3d_dst) # BxHxWx3 + + # project back to pixels + camera_matrix_tmp: torch.Tensor = camera_matrix[:, None, None] # Bx1x1xHxW + points_2d_src: torch.Tensor = project_points(points_3d_src, camera_matrix_tmp) # BxHxWx2 + + # normalize points between [-1 / 1] + height, width = depth_dst.shape[-2:] + points_2d_src_norm: torch.Tensor = normalize_pixel_coordinates(points_2d_src, height, width) # BxHxWx2 + + return F.grid_sample(image_src, points_2d_src_norm, align_corners=True) + + +class DepthWarper(nn.Module): + r"""Warp a patch by depth. + + .. math:: + P_{src}^{\{dst\}} = K_{dst} * T_{src}^{\{dst\}} + + I_{src} = \\omega(I_{dst}, P_{src}^{\{dst\}}, D_{src}) + + Args: + pinholes_dst: the pinhole models for the destination frame. + height: the height of the image to warp. + width: the width of the image to warp. + mode: interpolation mode to calculate output values ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values ``'zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: interpolation flag. + + """ + + # All per-instance, not global (thread safe, multiple warps) + def __init__( + self, + pinhole_dst: PinholeCamera, + height: int, + width: int, + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = True, + ) -> None: + super().__init__() + self.width: int = width + self.height: int = height + self.mode: str = mode + self.padding_mode: str = padding_mode + self.eps = 1e-6 + self.align_corners: bool = align_corners + + # state members + # _pinhole_dst is Type[PinholeCamera], enforce in constructor + if not isinstance(pinhole_dst, PinholeCamera): + raise TypeError(f"Expected pinhole_dst as PinholeCamera, got {type(pinhole_dst)}") + self._pinhole_dst: PinholeCamera = pinhole_dst + self._pinhole_src: None | PinholeCamera = None + self._dst_proj_src: None | torch.Tensor = None + + # Meshgrid only depends on (height, width), can be staticmethod cached + self.grid: torch.Tensor = self._create_meshgrid(height, width) + + @staticmethod + def _create_meshgrid(height: int, width: int) -> torch.Tensor: + grid: torch.Tensor = create_meshgrid(height, width, normalized_coordinates=False) # 1xHxWx2 + return convert_points_to_homogeneous(grid) # append ones to last dim + + def compute_projection_matrix(self, pinhole_src: PinholeCamera) -> DepthWarper: + """Compute the projection matrix from the source to destination frame.""" + # Inline type checks for faster fail-fast + if type(self._pinhole_dst) is not PinholeCamera: + raise TypeError( + f"Member self._pinhole_dst expected to be of class PinholeCamera. Got {type(self._pinhole_dst)}" + ) + if type(pinhole_src) is not PinholeCamera: + raise TypeError(f"Argument pinhole_src expected to be of class PinholeCamera. Got {type(pinhole_src)}") + # Compute transformation matrix: dst_extrinsics @ inv(src_extrinsics) + batch_shape = pinhole_src.extrinsics.shape[:-2] + device = pinhole_src.extrinsics.device + dtype = pinhole_src.extrinsics.dtype + + # Create 4x4 identity matrices efficiently + inv_extr = torch.eye(4, device=device, dtype=dtype).expand(*batch_shape, 4, 4).contiguous() + dst_trans_src = torch.eye(4, device=device, dtype=dtype).expand(*batch_shape, 4, 4).contiguous() + + # Inline inverse transformation + src_rmat = pinhole_src.extrinsics[..., :3, :3] + src_tvec = pinhole_src.extrinsics[..., :3, 3:] + inv_rmat = torch.transpose(src_rmat, -1, -2) + inv_tvec = torch.matmul(-inv_rmat, src_tvec) + + # Set rotation and translation parts + inv_extr[..., :3, :3] = inv_rmat + inv_extr[..., :3, 3:] = inv_tvec + + # Compose with dst extrinsics + dst_rmat = self._pinhole_dst.extrinsics[..., :3, :3] + dst_tvec = self._pinhole_dst.extrinsics[..., :3, 3:] + composed_rmat = torch.matmul(dst_rmat, inv_rmat) + composed_tvec = torch.matmul(dst_rmat, inv_tvec) + dst_tvec + + dst_trans_src[..., :3, :3] = composed_rmat + dst_trans_src[..., :3, 3:] = composed_tvec + + # intrinsics (Nx3x3) @ extrinsics (Nx4x4) + dst_proj_src = torch.matmul(self._pinhole_dst.intrinsics, dst_trans_src) + + self._pinhole_src = pinhole_src + self._dst_proj_src = dst_proj_src + return self + + def _compute_projection(self, x: float, y: float, invd: float) -> torch.Tensor: + if self._dst_proj_src is None or self._pinhole_src is None: + raise ValueError("Please, call compute_projection_matrix.") + + point = torch.tensor( + [[[x], [y], [invd], [1.0]]], device=self._dst_proj_src.device, dtype=self._dst_proj_src.dtype + ) + flow = torch.matmul(self._dst_proj_src, point) + z = 1.0 / flow[:, 2] + _x = flow[:, 0] * z + _y = flow[:, 1] * z + return torch.cat([_x, _y], 1) + + def compute_subpixel_step(self) -> torch.Tensor: + """Compute the inverse depth step for sub pixel accurate sampling of the depth cost volume, per camera. + + Szeliski, Richard, and Daniel Scharstein. "Symmetric sub-pixel stereo matching." European Conference on Computer + Vision. Springer Berlin Heidelberg, 2002. + """ + if self._dst_proj_src is None: + raise RuntimeError("Expected torch.Tensor, but got None Type from the projection matrix") + + delta_d = 0.01 + center_x = self.width / 2 + center_y = self.height / 2 + + # Batch both invds in one call (for potential fused kernels in future) for efficiency + invds = (1.0 - delta_d, 1.0 + delta_d) + # Instead of two calls, process both at once with minimal tensor construction + points = ( + torch.tensor( + [[center_x, center_y, invds[0], 1.0], [center_x, center_y, invds[1], 1.0]], + dtype=self._dst_proj_src.dtype, + device=self._dst_proj_src.device, + ) + .transpose(0, 1) + .unsqueeze(0) + ) # (1, 4, 2) + # Repeat projection matrix for batch + proj = self._dst_proj_src + flow = torch.matmul(proj, points) # (N, 3/4, 2) + zs = 1.0 / flow[:, 2] # (N, 2) + xs = flow[:, 0] * zs + ys = flow[:, 1] * zs + xys = torch.stack((xs, ys), dim=-1) # (N, 2, 2) + dxy = torch.norm(xys[:, 1] - xys[:, 0], p=2, dim=1) / 2.0 + dxdd = dxy / delta_d + # half pixel sampling, min for all cameras + return torch.min(0.5 / dxdd) + + def warp_grid(self, depth_src: torch.Tensor) -> torch.Tensor: + """Compute a grid for warping a given the depth from the reference pinhole camera. + + The function `compute_projection_matrix` has to be called beforehand in order to have precomputed the relative + projection matrices encoding the relative pose and the intrinsics between the reference and a non reference + camera. + """ + # TODO: add type and value checkings + if self._dst_proj_src is None or self._pinhole_src is None: + raise ValueError("Please, call compute_projection_matrix.") + + if len(depth_src.shape) != 4: + raise ValueError(f"Input depth_src has to be in the shape of Bx1xHxW. Got {depth_src.shape}") + + # unpack depth attributes + batch_size, _, _, _ = depth_src.shape + device: torch.device = depth_src.device + dtype: torch.dtype = depth_src.dtype + + # expand the base coordinate grid according to the input batch size + pixel_coords: torch.Tensor = self.grid.to(device=device, dtype=dtype).expand(batch_size, -1, -1, -1) # BxHxWx3 + + # reproject the pixel coordinates to the camera frame + cam_coords_src: torch.Tensor = pixel2cam( + depth_src, self._pinhole_src.intrinsics_inverse().to(device=device, dtype=dtype), pixel_coords + ) # BxHxWx3 + + # reproject the camera coordinates to the pixel + pixel_coords_src: torch.Tensor = cam2pixel( + cam_coords_src, self._dst_proj_src.to(device=device, dtype=dtype) + ) # (B*N)xHxWx2 + + # normalize between -1 and 1 the coordinates + pixel_coords_src_norm: torch.Tensor = normalize_pixel_coordinates(pixel_coords_src, self.height, self.width) + return pixel_coords_src_norm + + def forward(self, depth_src: torch.Tensor, patch_dst: torch.Tensor) -> torch.Tensor: + """Warp a tensor from destination frame to reference given the depth in the reference frame. + + Args: + depth_src: the depth in the reference frame. The tensor must have a shape :math:`(B, 1, H, W)`. + patch_dst: the patch in the destination frame. The tensor must have a shape :math:`(B, C, H, W)`. + + Return: + the warped patch from destination frame to reference. + + Shape: + - Output: :math:`(N, C, H, W)` where C = number of channels. + + Example: + >>> # pinholes camera models + >>> pinhole_dst = PinholeCamera(torch.randn(1, 4, 4), torch.randn(1, 4, 4), + ... torch.tensor([32]), torch.tensor([32])) + >>> pinhole_src = PinholeCamera(torch.randn(1, 4, 4), torch.randn(1, 4, 4), + ... torch.tensor([32]), torch.tensor([32])) + >>> # create the depth warper, compute the projection matrix + >>> warper = DepthWarper(pinhole_dst, 32, 32) + >>> _ = warper.compute_projection_matrix(pinhole_src) + >>> # warp the destination frame to reference by depth + >>> depth_src = torch.ones(1, 1, 32, 32) # Nx1xHxW + >>> image_dst = torch.rand(1, 3, 32, 32) # NxCxHxW + >>> image_src = warper(depth_src, image_dst) # NxCxHxW + + """ + return F.grid_sample( + patch_dst, + self.warp_grid(depth_src), + mode=self.mode, + padding_mode=self.padding_mode, + align_corners=self.align_corners, + ) + + +def depth_warp( + pinhole_dst: PinholeCamera, + pinhole_src: PinholeCamera, + depth_src: torch.Tensor, + patch_dst: torch.Tensor, + height: int, + width: int, + align_corners: bool = True, +) -> torch.Tensor: + """Warp a tensor from destination frame to reference given the depth in the reference frame. + + See :class:`~kornia.geometry.warp.DepthWarper` for details. + + Example: + >>> # pinholes camera models + >>> pinhole_dst = PinholeCamera(torch.randn(1, 4, 4), torch.randn(1, 4, 4), + ... torch.tensor([32]), torch.tensor([32])) + >>> pinhole_src = PinholeCamera(torch.randn(1, 4, 4), torch.randn(1, 4, 4), + ... torch.tensor([32]), torch.tensor([32])) + >>> # warp the destination frame to reference by depth + >>> depth_src = torch.ones(1, 1, 32, 32) # Nx1xHxW + >>> image_dst = torch.rand(1, 3, 32, 32) # NxCxHxW + >>> image_src = depth_warp(pinhole_dst, pinhole_src, depth_src, image_dst, 32, 32) # NxCxHxW + + """ + # Cache and reuse warper and projection matrix (single use/call) + # Inlined for performance, use local variables and freed objects + # instead of class members where possible. + warper = DepthWarper(pinhole_dst, height, width, align_corners=align_corners) + # projection matrix is required for each call, avoid double checking in class + warper.compute_projection_matrix(pinhole_src) + # __call__ implemented by nn.Module (likely calls forward, not shown). + return warper(depth_src, patch_dst) + + +def depth_from_disparity( + disparity: torch.Tensor, baseline: float | torch.Tensor, focal: float | torch.Tensor +) -> torch.Tensor: + """Compute depth from disparity. + + Args: + disparity: Disparity tensor of shape :math:`(*, H, W)`. + baseline: float/tensor containing the distance between the two lenses. + focal: float/tensor containing the focal length. + + Return: + Depth map of the shape :math:`(*, H, W)`. + + Example: + >>> disparity = torch.rand(4, 1, 4, 4) + >>> baseline = torch.rand(1) + >>> focal = torch.rand(1) + >>> depth_from_disparity(disparity, baseline, focal).shape + torch.Size([4, 1, 4, 4]) + + """ + KORNIA_CHECK_IS_TENSOR(disparity, f"Input disparity type is not a torch.Tensor. Got {type(disparity)}.") + KORNIA_CHECK_SHAPE(disparity, ["*", "H", "W"]) + KORNIA_CHECK( + isinstance(baseline, (float, torch.Tensor)), + f"Input baseline should be either a float or torch.Tensor. Got {type(baseline)}", + ) + KORNIA_CHECK( + isinstance(focal, (float, torch.Tensor)), + f"Input focal should be either a float or torch.Tensor. Got {type(focal)}", + ) + + if isinstance(baseline, torch.Tensor): + KORNIA_CHECK_SHAPE(baseline, ["1"]) + + if isinstance(focal, torch.Tensor): + KORNIA_CHECK_SHAPE(focal, ["1"]) + + return baseline * focal / (disparity + 1e-8) diff --git a/kornia/geometry/epipolar/__init__.py b/kornia/geometry/epipolar/__init__.py new file mode 100644 index 0000000..f065d0d --- /dev/null +++ b/kornia/geometry/epipolar/__init__.py @@ -0,0 +1,88 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from ._metrics import ( + left_to_right_epipolar_distance, + right_to_left_epipolar_distance, + sampson_epipolar_distance, + symmetrical_epipolar_distance, +) +from .essential import ( + decompose_essential_matrix, + decompose_essential_matrix_no_svd, + essential_from_fundamental, + essential_from_Rt, + find_essential, + motion_from_essential, + motion_from_essential_choose_solution, + relative_camera_motion, +) +from .fundamental import ( + compute_correspond_epilines, + find_fundamental, + fundamental_from_essential, + fundamental_from_projections, + get_closest_point_on_epipolar_line, + get_perpendicular, + normalize_points, + normalize_transformation, +) +from .numeric import cross_product_matrix +from .projection import ( + KRt_from_projection, + depth_from_point, + intrinsics_like, + projection_from_KRt, + projections_from_fundamental, + random_intrinsics, + scale_intrinsics, +) +from .scene import generate_scene +from .triangulation import triangulate_points + +__all__ = [ + "KRt_from_projection", + "compute_correspond_epilines", + "cross_product_matrix", + "decompose_essential_matrix", + "decompose_essential_matrix_no_svd", + "depth_from_point", + "essential_from_Rt", + "essential_from_fundamental", + "find_essential", + "find_fundamental", + "fundamental_from_essential", + "fundamental_from_projections", + "generate_scene", + "get_closest_point_on_epipolar_line", + "get_perpendicular", + "intrinsics_like", + "left_to_right_epipolar_distance", + "motion_from_essential", + "motion_from_essential_choose_solution", + "normalize_points", + "normalize_transformation", + "projection_from_KRt", + "projections_from_fundamental", + "random_intrinsics", + "relative_camera_motion", + "right_to_left_epipolar_distance", + "sampson_epipolar_distance", + "scale_intrinsics", + "symmetrical_epipolar_distance", + "triangulate_points", +] diff --git a/kornia/geometry/epipolar/_metrics.py b/kornia/geometry/epipolar/_metrics.py new file mode 100644 index 0000000..235bd20 --- /dev/null +++ b/kornia/geometry/epipolar/_metrics.py @@ -0,0 +1,334 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Module including useful metrics for Structure from Motion.""" + +from torch import Tensor, ones_like + +from kornia.core.check import KORNIA_CHECK_IS_TENSOR +from kornia.geometry.conversions import convert_points_to_homogeneous +from kornia.geometry.linalg import point_line_distance + + +def _sampson_epipolar_distance_manual_impl_( + pts1: Tensor, pts2: Tensor, Fm: Tensor, squared: bool = True, eps: float = 1e-8 +) -> Tensor: + """Return Sampson distance for correspondences given the fundamental matrix. + + Args: + pts1: correspondences from the left images with shape :math:`(*, N, (2|3))`. If they are not homogeneous, + converted automatically. + pts2: correspondences from the right images with shape :math:`(*, N, (2|3))`. If they are not homogeneous, + converted automatically. + Fm: Fundamental matrices with shape :math:`(*, 3, 3)`. Called Fm to avoid ambiguity with torch.nn.functional. + squared: if True (default), the squared distance is returned. + eps: Small constant for safe sqrt. + + Returns: + the computed Sampson distance with shape :math:`(*, N)`. + + """ + if not isinstance(Fm, Tensor): + raise TypeError(f"Fm type is not a torch.Tensor. Got {type(Fm)}") + + if (len(Fm.shape) < 3) or not Fm.shape[-2:] == (3, 3): + raise ValueError(f"Fm must be a (*, 3, 3) tensor. Got {Fm.shape}") + + # Extract coords; support 2D (w=1) and 3D homogeneous + x = pts1[..., :, 0] + y = pts1[..., :, 1] + u = pts2[..., :, 0] + v = pts2[..., :, 1] + # homogeneous weights with correct dtype/shape + w1 = pts1[..., :, 2] if pts1.shape[-1] == 3 else ones_like(x) + w2 = pts2[..., :, 2] if pts2.shape[-1] == 3 else ones_like(u) + + # Grab F entries and add a length-1 axis to broadcast across N + f00 = Fm[..., 0, 0][..., None] + f01 = Fm[..., 0, 1][..., None] + f02 = Fm[..., 0, 2][..., None] + f10 = Fm[..., 1, 0][..., None] + f11 = Fm[..., 1, 1][..., None] + f12 = Fm[..., 1, 2][..., None] + f20 = Fm[..., 2, 0][..., None] + f21 = Fm[..., 2, 1][..., None] + f22 = Fm[..., 2, 2][..., None] + + # Fx = F @ [x,y,w1] + Fx0 = f00 * x + f01 * y + f02 * w1 + Fx1 = f10 * x + f11 * y + f12 * w1 + Fx2 = f20 * x + f21 * y + f22 * w1 + + # (F^T x')_{1:2} for x' = [u,v,w2] + # (first two coordinates only) + Ft0 = f00 * u + f10 * v + f20 * w2 # (F^T x')_0 + Ft1 = f01 * u + f11 * v + f21 * w2 # (F^T x')_1 + + # Numerator: (x'^T F x)^2 = (u*Fx0 + v*Fx1 + w2*Fx2)^2 + num = (u * Fx0 + v * Fx1 + w2 * Fx2) ** 2 + # Denominator: ||(F x)_{1:2}||^2 + ||(F^T x')_{1:2}||^2 + den = Fx0 * Fx0 + Fx1 * Fx1 + Ft0 * Ft0 + Ft1 * Ft1 + eps + out: Tensor = num / den + if squared: + return out + return (out + eps).sqrt() + + +def _sampson_epipolar_distance_matmul_impl_( + pts1: Tensor, pts2: Tensor, Fm: Tensor, squared: bool = True, eps: float = 1e-8 +) -> Tensor: + """Return Sampson distance for correspondences given the fundamental matrix. + + Args: + pts1: correspondences from the left images with shape :math:`(*, N, (2|3))`. If they are not homogeneous, + converted automatically. + pts2: correspondences from the right images with shape :math:`(*, N, (2|3))`. If they are not homogeneous, + converted automatically. + Fm: Fundamental matrices with shape :math:`(*, 3, 3)`. Called Fm to avoid ambiguity with torch.nn.functional. + squared: if True (default), the squared distance is returned. + eps: Small constant for safe sqrt. + + Returns: + the computed Sampson distance with shape :math:`(*, N)`. + + """ + if pts1.shape[-1] == 2: + pts1 = convert_points_to_homogeneous(pts1) + + if pts2.shape[-1] == 2: + pts2 = convert_points_to_homogeneous(pts2) + + # From Hartley and Zisserman, Sampson error (11.9) + # sam = (x'^T F x) ** 2 / ( (((Fx)_1**2) + (Fx)_2**2)) + (((F^Tx')_1**2) + (F^Tx')_2**2)) ) + + # line1_in_2 = (F @ pts1.transpose(dim0=-2, dim1=-1)).transpose(dim0=-2, dim1=-1) + # line2_in_1 = (F.transpose(dim0=-2, dim1=-1) @ pts2.transpose(dim0=-2, dim1=-1)).transpose(dim0=-2, dim1=-1) + + # Instead we can just transpose F once and switch the order of multiplication + F_t: Tensor = Fm.transpose(dim0=-2, dim1=-1) + line1_in_2: Tensor = pts1 @ F_t + line2_in_1: Tensor = pts2 @ Fm + + # numerator = (x'^T F x) ** 2 + numerator: Tensor = (pts2 * line1_in_2).sum(dim=-1).pow(2) + + # denominator = (((Fx)_1**2) + (Fx)_2**2)) + (((F^Tx')_1**2) + (F^Tx')_2**2)) + denominator: Tensor = line1_in_2[..., :2].norm(2, dim=-1).pow(2) + line2_in_1[..., :2].norm(2, dim=-1).pow(2) + out: Tensor = numerator / denominator + if squared: + return out + return (out + eps).sqrt() + + +def sampson_epipolar_distance( + pts1: Tensor, + pts2: Tensor, + Fm: Tensor, + squared: bool = True, + eps: float = 1e-8, + use_matmul_at_less_than_points: int = 10000, +) -> Tensor: + """Return Sampson distance for correspondences given the fundamental matrix. + + Args: + pts1: correspondences from the left images with shape :math:`(*, N, (2|3))`. If they are not homogeneous, + converted automatically. + pts2: correspondences from the right images with shape :math:`(*, N, (2|3))`. If they are not homogeneous, + converted automatically. + Fm: Fundamental matrices with shape :math:`(*, 3, 3)`. Called Fm to avoid ambiguity with torch.nn.functional. + squared: if True (default), the squared distance is returned. + eps: Small constant for safe sqrt. + use_matmul_at_less_than_points: If the number of points is less than this value, use the matmul implementation. + + Returns: + the computed Sampson distance with shape :math:`(*, N)`. + + """ + if Fm.device.type == "cuda": + num_points = pts1.shape[-2] + if num_points < use_matmul_at_less_than_points: + return _sampson_epipolar_distance_matmul_impl_(pts1, pts2, Fm, squared, eps) + return _sampson_epipolar_distance_manual_impl_(pts1, pts2, Fm, squared, eps) + + +def _symmetrical_epipolar_distance_manual_impl_( + pts1: Tensor, pts2: Tensor, Fm: Tensor, squared: bool = True, eps: float = 1e-8 +) -> Tensor: + """Return symmetric epipolar distance for correspondences given the fundamental matrix (CPU-optimized).""" + if not isinstance(Fm, Tensor): + raise TypeError(f"Fm type is not a torch.Tensor. Got {type(Fm)}") + + if (len(Fm.shape) < 3) or Fm.shape[-2:] != (3, 3): + raise ValueError(f"Fm must be a (*, 3, 3) tensor. Got {Fm.shape}") + + # Extract coords; support 2D (w=1) and 3D homogeneous + x = pts1[..., :, 0] + y = pts1[..., :, 1] + u = pts2[..., :, 0] + v = pts2[..., :, 1] + + # homogeneous weights with correct dtype/shape + w1 = pts1[..., :, 2] if pts1.shape[-1] == 3 else ones_like(x) + w2 = pts2[..., :, 2] if pts2.shape[-1] == 3 else ones_like(u) + + # Grab F entries and add a length-1 axis to broadcast across N + f00 = Fm[..., 0, 0][..., None] + f01 = Fm[..., 0, 1][..., None] + f02 = Fm[..., 0, 2][..., None] + f10 = Fm[..., 1, 0][..., None] + f11 = Fm[..., 1, 1][..., None] + f12 = Fm[..., 1, 2][..., None] + f20 = Fm[..., 2, 0][..., None] + f21 = Fm[..., 2, 1][..., None] + f22 = Fm[..., 2, 2][..., None] + + # Fx = F @ [x, y, w1]^T (compute components explicitly) + Fx0 = f00 * x + f01 * y + f02 * w1 + Fx1 = f10 * x + f11 * y + f12 * w1 + Fx2 = f20 * x + f21 * y + f22 * w1 + + # (F^T x')_{0:1} for x' = [u, v, w2] + Ft0 = f00 * u + f10 * v + f20 * w2 # (F^T x')_0 + Ft1 = f01 * u + f11 * v + f21 * w2 # (F^T x')_1 + + # Numerator: (x'^T F x)^2 = (u*Fx0 + v*Fx1 + w2*Fx2)^2 + num = (u * Fx0 + v * Fx1 + w2 * Fx2).pow(2) + + # denominator_inv = 1/|| (F x)_{1:2} ||^2 + 1/|| (F^T x')_{1:2} ||^2 + inv1 = 1.0 / (Fx0.pow(2) + Fx1.pow(2) + eps) + inv2 = 1.0 / (Ft0.pow(2) + Ft1.pow(2) + eps) + den_inv = inv1 + inv2 + + out: Tensor = num * den_inv + if squared: + return out + return (out + eps).sqrt() + + +def _symmetrical_epipolar_distance_matmul_impl_( + pts1: Tensor, pts2: Tensor, Fm: Tensor, squared: bool = True, eps: float = 1e-8 +) -> Tensor: + if pts1.shape[-1] == 2: + pts1 = convert_points_to_homogeneous(pts1) + if pts2.shape[-1] == 2: + pts2 = convert_points_to_homogeneous(pts2) + + F_t: Tensor = Fm.transpose(dim0=-2, dim1=-1) + line1_in_2: Tensor = pts1 @ F_t + line2_in_1: Tensor = pts2 @ Fm + + numerator: Tensor = (pts2 * line1_in_2).sum(dim=-1).pow(2) + + denominator_inv: Tensor = 1.0 / (line1_in_2[..., :2].norm(2, dim=-1).pow(2) + eps) + 1.0 / ( + line2_in_1[..., :2].norm(2, dim=-1).pow(2) + eps + ) + out: Tensor = numerator * denominator_inv + if squared: + return out + return (out + eps).sqrt() + + +def symmetrical_epipolar_distance( + pts1: Tensor, pts2: Tensor, Fm: Tensor, squared: bool = True, eps: float = 1e-8 +) -> Tensor: + """Return symmetrical epipolar distance for correspondences given the fundamental matrix. + + Args: + pts1: correspondences from the left images with shape :math:`(*, N, (2|3))`. If they are not homogeneous, + converted automatically. + pts2: correspondences from the right images with shape :math:`(*, N, (2|3))`. If they are not homogeneous, + converted automatically. + Fm: Fundamental matrices with shape :math:`(*, 3, 3)`. Called Fm to avoid ambiguity with torch.nn.functional. + squared: if True (default), the squared distance is returned. + eps: Small constant for safe sqrt. + + Returns: + the computed Symmetrical distance with shape :math:`(*, N)`. + + """ + if Fm.device.type == "cuda": + num_points = pts1.shape[-2] + if num_points < 10000: + return _symmetrical_epipolar_distance_matmul_impl_(pts1, pts2, Fm, squared, eps) + return _symmetrical_epipolar_distance_manual_impl_(pts1, pts2, Fm, squared, eps) + + +def left_to_right_epipolar_distance(pts1: Tensor, pts2: Tensor, Fm: Tensor) -> Tensor: + r"""Return one-sided epipolar distance for correspondences given the fundamental matrix. + + This method measures the distance from points in the right images to the epilines + of the corresponding points in the left images as they reflect in the right images. + + Args: + pts1: correspondences from the left images with shape + :math:`(*, N, 2 or 3)`. If they are not homogeneous, converted automatically. + pts2: correspondences from the right images with shape + :math:`(*, N, 2 or 3)`. If they are not homogeneous, converted automatically. + Fm: Fundamental matrices with shape :math:`(*, 3, 3)`. Called Fm to + avoid ambiguity with torch.nn.functional. + + Returns: + the computed Symmetrical distance with shape :math:`(*, N)`. + + """ + KORNIA_CHECK_IS_TENSOR(pts1) + KORNIA_CHECK_IS_TENSOR(pts2) + KORNIA_CHECK_IS_TENSOR(Fm) + + if (len(Fm.shape) < 3) or not Fm.shape[-2:] == (3, 3): + raise ValueError(f"Fm must be a (*, 3, 3) tensor. Got {Fm.shape}") + + if pts1.shape[-1] == 2: + pts1 = convert_points_to_homogeneous(pts1) + + F_t: Tensor = Fm.transpose(dim0=-2, dim1=-1) + line1_in_2: Tensor = pts1 @ F_t + + return point_line_distance(pts2, line1_in_2) + + +def right_to_left_epipolar_distance(pts1: Tensor, pts2: Tensor, Fm: Tensor) -> Tensor: + r"""Return one-sided epipolar distance for correspondences given the fundamental matrix. + + This method measures the distance from points in the left images to the epilines + of the corresponding points in the right images as they reflect in the left images. + + Args: + pts1: correspondences from the left images with shape + :math:`(*, N, 2 or 3)`. If they are not homogeneous, converted automatically. + pts2: correspondences from the right images with shape + :math:`(*, N, 2 or 3)`. If they are not homogeneous, converted automatically. + Fm: Fundamental matrices with shape :math:`(*, 3, 3)`. Called Fm to + avoid ambiguity with torch.nn.functional. + + Returns: + the computed Symmetrical distance with shape :math:`(*, N)`. + + """ + KORNIA_CHECK_IS_TENSOR(pts1) + KORNIA_CHECK_IS_TENSOR(pts2) + KORNIA_CHECK_IS_TENSOR(Fm) + + if (len(Fm.shape) < 3) or not Fm.shape[-2:] == (3, 3): + raise ValueError(f"Fm must be a (*, 3, 3) tensor. Got {Fm.shape}") + + if pts2.shape[-1] == 2: + pts2 = convert_points_to_homogeneous(pts2) + + line2_in_1: Tensor = pts2 @ Fm + + return point_line_distance(pts1, line2_in_1) diff --git a/kornia/geometry/epipolar/essential.py b/kornia/geometry/epipolar/essential.py new file mode 100644 index 0000000..3867612 --- /dev/null +++ b/kornia/geometry/epipolar/essential.py @@ -0,0 +1,765 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Module containing functionalities for the Essential matrix.""" + +from typing import Optional, Tuple + +import torch + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SAME_SHAPE, KORNIA_CHECK_SHAPE +from kornia.core.ops import eye_like, vec_like +from kornia.core.utils import _torch_svd_cast +from kornia.geometry.solvers.polynomial_solver import T_deg1, T_deg2, coefficient_map, multiplication_indices, signs + +from .numeric import cross_product_matrix, matrix_cofactor_tensor +from .projection import depth_from_point, projection_from_KRt +from .triangulation import triangulate_points + +__all__ = [ + "decompose_essential_matrix", + "decompose_essential_matrix_no_svd", + "essential_from_Rt", + "essential_from_fundamental", + "find_essential", + "motion_from_essential", + "motion_from_essential_choose_solution", + "relative_camera_motion", +] + + +def run_5point(points1: torch.Tensor, points2: torch.Tensor, weights: Optional[torch.Tensor] = None) -> torch.Tensor: + r"""Compute the essential matrix using the 5-point algorithm from Nister. + + The linear system is solved by Nister's 5-point algorithm [@nister2004efficient], + and the solver implemented referred to [@barath2020magsac++][@wei2023generalized][@wang2023vggsfm]. + + Args: + points1: A set of carlibrated points in the first image with a tensor shape :math:`(B, N, 2), N>=8`. + points2: A set of points in the second image with a tensor shape :math:`(B, N, 2), N>=8`. + weights: Not used, kept for compatibility. + + Returns: + the computed essential matrix with shape :math:`(B, 10, 3, 3)`. + + """ + KORNIA_CHECK_SHAPE(points1, ["B", "N", "2"]) + KORNIA_CHECK_SAME_SHAPE(points1, points2) + KORNIA_CHECK(points1.shape[1] >= 5, "Number of points should be >=5") + + batch_size, _, _ = points1.shape + x1, y1 = points1[..., 0:1], points1[..., 1:2] + x2, y2 = points2[..., 0:1], points2[..., 1:2] + ones = torch.ones_like(x1) + # build the equation system and find the null space. + # https://www.cc.gatech.edu/~afb/classes/CS4495-Fall2013/slides/CS4495-09-TwoViews-2.pdf + # [x * x', x * y', x, y * x', y * y', y, x', y', 1] + # BxNx9 + X = torch.cat([x1 * x2, x1 * y2, x1, y1 * x2, y1 * y2, y1, x2, y2, ones], dim=-1) + # use Nister's 5PC to solve essential matrix + E = null_to_Nister_solution(X, batch_size) + bad = torch.isnan(E).all(dim=(-1, -2)).all(dim=-1) # (B,) + if bad.any(): + eye3 = torch.eye(3, device=E.device, dtype=E.dtype).view(1, 1, 3, 3).expand(batch_size, 10, 3, 3) + E = torch.where(bad.view(batch_size, 1, 1, 1), eye3, E) + return E + + +def _multiply_deg_one_poly(a: torch.Tensor, b: torch.Tensor, T_deg1: torch.Tensor) -> torch.Tensor: + # a, b: (..., 4) + product_basis = a.unsqueeze(2) * b.unsqueeze(1) # (..., 4, 4) + product_vector = product_basis.flatten(start_dim=-2) # (..., 16) + return product_vector @ T_deg1 # (..., 10) + + +def _multiply_deg_two_one_poly(a: torch.Tensor, b: torch.Tensor, T_deg2: torch.Tensor) -> torch.Tensor: + # a: (..., 10), b: (..., 4) + product_basis = a.unsqueeze(2) * b.unsqueeze(1) # (..., 10, 4) + product_vector = product_basis.flatten(start_dim=-2) # (..., 40) + return product_vector @ T_deg2 # (..., 20) + + +def _determinant_to_polynomial_jit( + A: torch.Tensor, + multiplication_indices: torch.Tensor, + signs: torch.Tensor, + coefficient_map: torch.Tensor, +) -> torch.Tensor: + # A: (B, 3, 13) -> (B, 11) + B = A.shape[0] + A_flat = A.view(B, -1) # (B, 39) + + gathered_values = A_flat[:, multiplication_indices] # (B, 486, 3) + products = torch.prod(gathered_values, dim=-1) # (B, 486) + signed_products = products * signs # (B, 486) + + cs = torch.zeros(B, 11, device=A.device, dtype=A.dtype) + batch_coefficient_map = coefficient_map.unsqueeze(0).expand(B, -1) # (B, 486) + cs.scatter_add_(1, batch_coefficient_map, signed_products) + return cs + + +def _solve_2x2_tikhonov_safe(A: torch.Tensor, b: torch.Tensor, eps: float = 1e-12) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Solve (A)x=b for A (...,2,2), b (...,2,1) using Tikhonov regularization. + + Uses the following methods: + - direct inverse when det is OK + - otherwise solve normal equations (A^T A + λI)x = A^T b (λ from trace scale) + Never throws. Returns (x, bad) where bad marks ill-conditioned A. + """ + a = A[..., 0, 0] + bb = A[..., 0, 1] + c = A[..., 1, 0] + d = A[..., 1, 1] + + det = a * d - bb * c + det_abs = det.abs() + bad = (det_abs <= eps) | torch.isnan(det_abs) | torch.isinf(det_abs) + + # ---- direct inverse branch (but branchless via where) ---- + det_safe = torch.where(det_abs > eps, det, torch.ones_like(det) * eps) + inv_det = 1.0 / det_safe + + inv00 = d * inv_det + inv01 = (-bb) * inv_det + inv10 = (-c) * inv_det + inv11 = a * inv_det + + x0_dir = inv00 * b[..., 0, 0] + inv01 * b[..., 1, 0] + x1_dir = inv10 * b[..., 0, 0] + inv11 * b[..., 1, 0] + x_dir = torch.stack((x0_dir, x1_dir), dim=-1).unsqueeze(-1) # (...,2,1) + + # ---- fallback: normal equations with λI (always SPD if λ>0) ---- + # ATA = A^T A, ATb = A^T b + # ATA = [[a^2 + c^2, a*bb + c*d], + # [a*bb + c*d, bb^2 + d^2]] + ata00 = a * a + c * c + ata01 = a * bb + c * d + ata11 = bb * bb + d * d + + atb0 = a * b[..., 0, 0] + c * b[..., 1, 0] + atb1 = bb * b[..., 0, 0] + d * b[..., 1, 0] + + # λ from trace scale; ensure strictly positive even if A is zero + tr = ata00 + ata11 + lam = (tr * 1e-8).clamp_min(eps) + + m00 = ata00 + lam + m01 = ata01 + m10 = ata01 + m11 = ata11 + lam + + det_m = m00 * m11 - m01 * m10 + det_m_safe = det_m.abs().clamp_min(eps) + inv_det_m = 1.0 / det_m_safe + + invm00 = m11 * inv_det_m + invm01 = (-m01) * inv_det_m + invm10 = (-m10) * inv_det_m + invm11 = m00 * inv_det_m + + x0_fb = invm00 * atb0 + invm01 * atb1 + x1_fb = invm10 * atb0 + invm11 * atb1 + x_fb = torch.stack((x0_fb, x1_fb), dim=-1).unsqueeze(-1) # (...,2,1) + + # choose fallback only when bad; else direct + x = torch.where(bad.unsqueeze(-1).unsqueeze(-1), x_fb, x_dir) + + # if still non-finite, mark bad and zero it + nonfinite = torch.isnan(x).any(dim=(-2, -1)) | torch.isinf(x).any(dim=(-2, -1)) + bad = bad | nonfinite + x = torch.where(bad.unsqueeze(-1).unsqueeze(-1), torch.zeros_like(x), x) + + return x, bad + + +# --- fun_select inline, to keep it JIT-simple --- +def _fun_select(mat: torch.Tensor, i: int, j: int, ratio: int = 3) -> torch.Tensor: + return mat[:, ratio * j + i] + + +def _null_to_Nister_solution_script( + X: torch.Tensor, + batch_size: int, + T_deg1: torch.Tensor, + T_deg2: torch.Tensor, + multiplication_indices: torch.Tensor, + signs: torch.Tensor, + coefficient_map: torch.Tensor, + idx_ij: torch.Tensor, # kept for signature compatibility (unused) + i_idx: torch.Tensor, + j_idx: torch.Tensor, + idx3: torch.Tensor, + top_idx: torch.Tensor, + bot_idx: torch.Tensor, +) -> torch.Tensor: + original_dtype = X.dtype + + _, _, V = _torch_svd_cast(X) # V: (B, 9, 9) + null_ = V[:, :, -4:].contiguous() # (B, 9, 4) + nullSpace = V.transpose(-1, -2)[:, -4:, :] # (B, 4, 9) + + B = batch_size + device = X.device + dtype = X.dtype + + coeffs = torch.zeros(B, 10, 20, device=device, dtype=dtype) + + # (B,9,4) -> (B,3,3,4) with column-major fix + null_ij = null_.view(B, 3, 3, 4).transpose(1, 2).contiguous() # (B, 3, 3, 4) + + # ---- determinant constraint ---- + n00 = null_ij[:, 0, 0, :] + n01 = null_ij[:, 0, 1, :] + n02 = null_ij[:, 0, 2, :] + n10 = null_ij[:, 1, 0, :] + n11 = null_ij[:, 1, 1, :] + n12 = null_ij[:, 1, 2, :] + n20 = null_ij[:, 2, 0, :] + n21 = null_ij[:, 2, 1, :] + n22 = null_ij[:, 2, 2, :] + + # small reuse to reduce launches + p01_12 = _multiply_deg_one_poly(n01, n12, T_deg1) + p02_11 = _multiply_deg_one_poly(n02, n11, T_deg1) + p02_10 = _multiply_deg_one_poly(n02, n10, T_deg1) + p00_12 = _multiply_deg_one_poly(n00, n12, T_deg1) + p00_11 = _multiply_deg_one_poly(n00, n11, T_deg1) + p01_10 = _multiply_deg_one_poly(n01, n10, T_deg1) + + coeffs[:, 9] = ( + _multiply_deg_two_one_poly(p01_12 - p02_11, n20, T_deg2) + + _multiply_deg_two_one_poly(p02_10 - p00_12, n21, T_deg2) + + _multiply_deg_two_one_poly(p00_11 - p01_10, n22, T_deg2) + ) + + # ---- EE^T constraints ---- + a_data = null_ij[:, i_idx, :, :] # (B,9,3,4) + b_data = null_ij[:, j_idx, :, :] # (B,9,3,4) + prods = _multiply_deg_one_poly(a_data.reshape(-1, 4), b_data.reshape(-1, 4), T_deg1).view(B, 9, 3, 10) + D_blocks = prods.sum(dim=2).view(B, 3, 3, 10).contiguous() + + # trace removal + diag = D_blocks[:, idx3, idx3, :] # (B,3,10) + t = 0.5 * diag.sum(dim=1, keepdim=True) # (B,1,10) + D_blocks[:, idx3, idx3, :] = D_blocks[:, idx3, idx3, :] - t + + # first 9 rows of coeffs + D_for_i = D_blocks[:, i_idx, :, :] # (B,9,3,10) + Null_for_j = null_ij[:, :, j_idx, :].permute(0, 2, 1, 3).contiguous() # (B,9,3,4) + prods2 = _multiply_deg_two_one_poly(D_for_i.reshape(-1, 10), Null_for_j.reshape(-1, 4), T_deg2).view(B, 9, 3, 20) + coeffs[:, :9, :] = prods2.sum(dim=2) # (B,9,20) + + # ---- elimination: solve A10 * X = b_poly ---- + A10 = coeffs[:, :, :10] # (B, 10, 10) + b_poly = coeffs[:, :, 10:] # (B, 10, 10) + + # Prefer direct solve; if singular, add tiny damping (no batch compaction). + eye10 = torch.eye(10, device=device, dtype=dtype).unsqueeze(0).expand(B, 10, 10) + + # Try direct solve first + eliminated = torch.linalg.solve(A10, b_poly) # (B,10,10) + + # Detect NaN/Inf from singular solve and fix with damping solve + bad = torch.isnan(eliminated).any(dim=(-2, -1)) | torch.isinf(eliminated).any(dim=(-2, -1)) # (B,) + if bad.any(): + # Damped solve only for bad rows but WITHOUT compaction: + # build damped A = A10 + λI, where λ depends on scale + # (use per-batch scalar to avoid huge allocations) + diagA = torch.diagonal(A10, dim1=-2, dim2=-1).abs().mean(dim=-1) # (B,) + lam = (diagA * 1e-8 + 1e-8).to(dtype) # (B,) + A_damped = A10 + eye10 * lam.view(B, 1, 1) + eliminated_d = torch.linalg.solve(A_damped, b_poly) + eliminated = torch.where(bad.view(B, 1, 1), eliminated_d, eliminated) + + coeffs_ = torch.cat((A10, eliminated), dim=-1) # (B,10,20) + + # ---- build A (B,3,13) ---- + A = torch.zeros(B, 3, 13, device=device, dtype=dtype) + + A[:, :, 1:4] = coeffs_[:, top_idx, 10:13] + A[:, :, 0:3] = A[:, :, 0:3] - coeffs_[:, bot_idx, 10:13] + + A[:, :, 5:8] = coeffs_[:, top_idx, 13:16] + A[:, :, 4:7] = A[:, :, 4:7] - coeffs_[:, bot_idx, 13:16] + + A[:, :, 9:13] = coeffs_[:, top_idx, 16:20] + A[:, :, 8:12] = A[:, :, 8:12] - coeffs_[:, bot_idx, 16:20] + + # ---- determinant polynomial -> companion ---- + cs = _determinant_to_polynomial_jit(A, multiplication_indices, signs, coefficient_map) # (B,11) + + C = torch.zeros(B, 10, 10, device=device, dtype=dtype) + C[:, 0:9, 1:10] = torch.eye(9, device=device, dtype=dtype) + + cs_de = cs[:, -1].clamp_min(1e-8) + C[:, -1, :] = -cs[:, :-1] / cs_de.unsqueeze(-1) + + roots_eig = torch.linalg.eigvals(C) # (B,10), complex + roots = torch.real(roots_eig) + is_real = torch.abs(torch.imag(roots_eig)) < 1e-10 + + roots_unsqu = roots.unsqueeze(1) # (B,1,10) + + Bs = torch.stack( + ( + A[:, :3, :1] * (roots_unsqu**3) + + A[:, :3, 1:2] * roots_unsqu.square() + + A[:, :3, 2:3] * roots_unsqu + + A[:, :3, 3:4], + A[:, :3, 4:5] * (roots_unsqu**3) + + A[:, :3, 5:6] * roots_unsqu.square() + + A[:, :3, 6:7] * roots_unsqu + + A[:, :3, 7:8], + ), + dim=1, + ).transpose(1, -1) # (B,10,3,2) + + bs_vec = ( + ( + A[:, :3, 8:9] * (roots_unsqu**4) + + A[:, :3, 9:10] * (roots_unsqu**3) + + A[:, :3, 10:11] * roots_unsqu.square() + + A[:, :3, 11:12] * roots_unsqu + + A[:, :3, 12:13] + ) + .transpose(1, 2) + .unsqueeze(-1) + ) # (B,10,3,1) + + A2 = Bs[:, :, 0:2, 0:2] # (B,10,2,2) + b2 = bs_vec[:, :, 0:2, :] # (B,10,2,1) + + xzs, bad2 = _solve_2x2_tikhonov_safe(A2, b2, 1e-12) # never throws + + # ---- build Es ---- + xzs_sq = xzs.squeeze(-1) # (B,10,2) + x = -xzs_sq[:, :, 0] + y = -xzs_sq[:, :, 1] + z = roots + + N0 = nullSpace[:, 0, :].unsqueeze(1) # (B,1,9) + N1 = nullSpace[:, 1, :].unsqueeze(1) + N2 = nullSpace[:, 2, :].unsqueeze(1) + N3 = nullSpace[:, 3, :].unsqueeze(1) + + Es_vec = x.unsqueeze(-1) * N0 + y.unsqueeze(-1) * N1 + z.unsqueeze(-1) * N2 + N3 # (B,10,9) + inv_norm = torch.rsqrt(x * x + y * y + z * z + 1.0) + Es_vec = Es_vec * inv_norm.unsqueeze(-1) + + Es = Es_vec.view(B, 10, 3, 3).transpose(-1, -2) + # after Es is created (B,10,3,3) + if bad2.any(): + Es[bad2] = torch.nan + # mark complex roots as NaN (keeps shape, no compaction) + if is_real.logical_not().any(): + Es[~is_real] = torch.nan + + return Es.to(dtype=original_dtype) + + +# Indices for null_ij reshaping +IDX_IJ = torch.tensor([[0, 3, 6], [1, 4, 7], [2, 5, 8]], dtype=torch.long) +I_IDX = torch.tensor([0, 0, 0, 1, 1, 1, 2, 2, 2], dtype=torch.long) +J_IDX = torch.tensor([0, 1, 2, 0, 1, 2, 0, 1, 2], dtype=torch.long) +IDX3 = torch.tensor([0, 1, 2], dtype=torch.long) + +TOP_IDX = torch.tensor([4, 6, 8], dtype=torch.long) +BOT_IDX = torch.tensor([5, 7, 9], dtype=torch.long) + + +def fun_select(null_mat: torch.Tensor, i: int, j: int, ratio: int = 3) -> torch.Tensor: + return null_mat[:, ratio * j + i] + + +def null_to_Nister_solution(X: torch.Tensor, batch_size: int) -> torch.Tensor: + device = X.device + dtype_internal = X.dtype + + T1 = T_deg1.to(device=device, dtype=dtype_internal) + T2 = T_deg2.to(device=device, dtype=dtype_internal) + mult_idx = multiplication_indices.to(device=device) + sgns = signs.to(device=device, dtype=dtype_internal) + coeff_map = coefficient_map.to(device=device) + i_idx_dev = I_IDX.to(device=device) + j_idx_dev = J_IDX.to(device=device) + idx3_dev = IDX3.to(device=device) + top_idx_dev = TOP_IDX.to(device=device) + bot_idx_dev = BOT_IDX.to(device=device) + idx_ij_dev = IDX_IJ.to(device=device) + + return _null_to_Nister_solution_script( + X, + batch_size, + T1, + T2, + mult_idx, + sgns, + coeff_map, + idx_ij_dev, + i_idx_dev, + j_idx_dev, + idx3_dev, + top_idx_dev, + bot_idx_dev, + ) + + +def essential_from_fundamental(F_mat: torch.Tensor, K1: torch.Tensor, K2: torch.Tensor) -> torch.Tensor: + r"""Get Essential matrix from Fundamental and Camera matrices. + + Uses the method from Hartley/Zisserman 9.6 pag 257 (formula 9.12). + + Args: + F_mat: The fundamental matrix with shape of :math:`(*, 3, 3)`. + K1: The camera matrix from first camera with shape :math:`(*, 3, 3)`. + K2: The camera matrix from second camera with shape :math:`(*, 3, 3)`. + + Returns: + The essential matrix with shape :math:`(*, 3, 3)`. + + """ + KORNIA_CHECK_SHAPE(F_mat, ["*", "3", "3"]) + KORNIA_CHECK_SHAPE(K1, ["*", "3", "3"]) + KORNIA_CHECK_SHAPE(K2, ["*", "3", "3"]) + return K2.transpose(-2, -1) @ F_mat @ K1 + + +def decompose_essential_matrix(E_mat: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + r"""Decompose an essential matrix to possible rotations and translation. + + This function decomposes the essential matrix E using svd decomposition [96] + and give the possible solutions: :math:`R1, R2, t`. + + Args: + E_mat: The essential matrix in the form of :math:`(*, 3, 3)`. + + Returns: + A tuple containing the first and second possible rotation matrices and the translation vector. + The shape of the tensors with be same input :math:`[(*, 3, 3), (*, 3, 3), (*, 3, 1)]`. + + """ + KORNIA_CHECK_SHAPE(E_mat, ["*", "3", "3"]) + + # decompose matrix by its singular values + U, _, V = _torch_svd_cast(E_mat) + Vt = V.transpose(-2, -1) + + mask = torch.ones_like(E_mat) + mask[..., -1:] *= -1.0 # fill last column with negative values + + maskt = mask.transpose(-2, -1) + + # avoid singularities + U = torch.where((torch.det(U) < 0.0)[..., None, None], U * mask, U) + Vt = torch.where((torch.det(Vt) < 0.0)[..., None, None], Vt * maskt, Vt) + + W = cross_product_matrix(torch.tensor([[0.0, 0.0, 1.0]]).type_as(E_mat)) + W[..., 2, 2] += 1.0 + + # reconstruct rotations and retrieve translation vector + U_W_Vt = U @ W @ Vt + U_Wt_Vt = U @ W.transpose(-2, -1) @ Vt + + # return values + R1 = U_W_Vt + R2 = U_Wt_Vt + T = U[..., -1:] + return (R1, R2, T) + + +def decompose_essential_matrix_no_svd(E_mat: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + r"""Decompose the essential matrix to rotation and translation. + + Recover rotation and translation from essential matrices without SVD + reference: Horn, Berthold KP. Recovering baseline and orientation from essential matrix[J]. + J. Opt. Soc. Am, 1990, 110. + + Args: + E_mat: The essential matrix in the form of :math:`(*, 3, 3)`. + + Returns: + A tuple containing the first and second possible rotation matrices and the translation vector. + The shape of the tensors with be same input :math:`[(*, 3, 3), (*, 3, 3), (*, 3, 1)]`. + + """ + KORNIA_CHECK_SHAPE(E_mat, ["*", "3", "3"]) + if len(E_mat.shape) != 3: + E_mat = E_mat.view(-1, 3, 3) + + B = E_mat.shape[0] + + # Eq.18, choose the largest of the three possible pairwise cross-products + e1, e2, e3 = E_mat[..., 0], E_mat[..., 1], E_mat[..., 2] + + # sqrt(1/2 trace(EE^T)), B + scale_factor = torch.sqrt(0.5 * torch.diagonal(E_mat @ E_mat.transpose(-1, -2), dim1=-1, dim2=-2).sum(-1)) + + # B, 3, 3 + cross_products = torch.stack( + [torch.linalg.cross(e1, e2, dim=-1), torch.linalg.cross(e2, e3, dim=-1), torch.linalg.cross(e3, e1, dim=-1)], + dim=1, + ) + + # B, 3, 1 + norms = torch.norm(cross_products, dim=-1, keepdim=True) + + # B, to select which b1 + largest = torch.argmax(norms, dim=-2) + + # B, 3, 3 + e_cross_products = scale_factor[:, None, None] * cross_products / norms + + # broadcast the index + index_expanded = largest.unsqueeze(-1).expand(-1, -1, e_cross_products.size(-1)) + + # slice at dim=1, select for each batch one b (e1*e2 or e2*e3 or e3*e1), B, 1, 3 + b1 = torch.gather(e_cross_products, dim=1, index=index_expanded).squeeze(1) + # normalization + b1_ = b1 / torch.norm(b1, dim=-1, keepdim=True) + + # skew-symmetric matrix + B1 = torch.zeros((B, 3, 3), device=E_mat.device, dtype=E_mat.dtype) + t0, t1, t2 = b1[:, 0], b1[:, 1], b1[:, 2] + B1[:, 0, 1], B1[:, 1, 0] = -t2, t2 + B1[:, 0, 2], B1[:, 2, 0] = t1, -t1 + B1[:, 1, 2], B1[:, 2, 1] = -t0, t0 + + # the second translation and rotation + B2 = -B1 + b2 = -b1 + + # Eq.24, recover R + # (bb)R = Cofactors(E)^T - BE + R1 = (matrix_cofactor_tensor(E_mat) - B1 @ E_mat) / (b1 * b1).sum().unsqueeze(-1) + R2 = (matrix_cofactor_tensor(E_mat) - B2 @ E_mat) / (b2 * b2).sum().unsqueeze(-1) + + return (R1, R2, b1_.unsqueeze(-1)) + + +def essential_from_Rt(R1: torch.Tensor, t1: torch.Tensor, R2: torch.Tensor, t2: torch.Tensor) -> torch.Tensor: + r"""Get the Essential matrix from Camera motion (Rs and ts). + + Reference: Hartley/Zisserman 9.6 pag 257 (formula 9.12) + + Args: + R1: The first camera rotation matrix with shape :math:`(*, 3, 3)`. + t1: The first camera translation vector with shape :math:`(*, 3, 1)`. + R2: The second camera rotation matrix with shape :math:`(*, 3, 3)`. + t2: The second camera translation vector with shape :math:`(*, 3, 1)`. + + Returns: + The Essential matrix with the shape :math:`(*, 3, 3)`. + + """ + KORNIA_CHECK_SHAPE(R1, ["*", "3", "3"]) + KORNIA_CHECK_SHAPE(R2, ["*", "3", "3"]) + KORNIA_CHECK_SHAPE(t1, ["*", "3", "1"]) + KORNIA_CHECK_SHAPE(t2, ["*", "3", "1"]) + + # first compute the camera relative motion + R, t = relative_camera_motion(R1, t1, R2, t2) + + # get the cross product from relative translation vector + Tx = cross_product_matrix(t[..., 0]) + + return Tx @ R + + +def motion_from_essential(E_mat: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Get Motion (R's and t's ) from Essential matrix. + + Computes and return four possible poses exist for the decomposition of the Essential + matrix. The possible solutions are :math:`[R1,t], [R1,-t], [R2,t], [R2,-t]`. + + Args: + E_mat: The essential matrix in the form of :math:`(*, 3, 3)`. + + Returns: + The rotation and translation containing the four possible combination for the retrieved motion. + The tuple is as following :math:`[(*, 4, 3, 3), (*, 4, 3, 1)]`. + + """ + KORNIA_CHECK_SHAPE(E_mat, ["*", "3", "3"]) + + # decompose the essential matrix by its possible poses + R1, R2, t = decompose_essential_matrix(E_mat) + + # compbine and returns the four possible solutions + Rs = torch.stack([R1, R1, R2, R2], dim=-3) + Ts = torch.stack([t, -t, t, -t], dim=-3) + + return Rs, Ts + + +def motion_from_essential_choose_solution( + E_mat: torch.Tensor, + K1: torch.Tensor, + K2: torch.Tensor, + x1: torch.Tensor, + x2: torch.Tensor, + mask: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + r"""Recover the relative camera rotation and the translation from an estimated essential matrix. + + The method checks the corresponding points in two images and also returns the triangulated + 3d points. Internally uses :py:meth:`~kornia.geometry.epipolar.decompose_essential_matrix` and then chooses + the best solution based on the combination that gives more 3d points in front of the camera plane from + :py:meth:`~kornia.geometry.epipolar.triangulate_points`. + + Args: + E_mat: The essential matrix in the form of :math:`(*, 3, 3)`. + K1: The camera matrix from first camera with shape :math:`(*, 3, 3)`. + K2: The camera matrix from second camera with shape :math:`(*, 3, 3)`. + x1: The set of points seen from the first camera frame in the camera plane + coordinates with shape :math:`(*, N, 2)`. + x2: The set of points seen from the first camera frame in the camera plane + coordinates with shape :math:`(*, N, 2)`. + mask: A boolean mask which can be used to exclude some points from choosing + the best solution. This is useful for using this function with sets of points of + different cardinality (for instance after filtering with RANSAC) while keeping batch + semantics. Mask is of shape :math:`(*, N)`. + + Returns: + The rotation and translation plus the 3d triangulated points. + The tuple is as following :math:`[(*, 3, 3), (*, 3, 1), (*, N, 3)]`. + + """ + KORNIA_CHECK_SHAPE(E_mat, ["*", "3", "3"]) + KORNIA_CHECK_SHAPE(K1, ["*", "3", "3"]) + KORNIA_CHECK_SHAPE(K2, ["*", "3", "3"]) + KORNIA_CHECK_SHAPE(x1, ["*", "N", "2"]) + KORNIA_CHECK_SHAPE(x2, ["*", "N", "2"]) + KORNIA_CHECK(len(E_mat.shape[:-2]) == len(K1.shape[:-2]) == len(K2.shape[:-2])) + + if mask is not None: + KORNIA_CHECK_SHAPE(mask, ["*", "N"]) + KORNIA_CHECK(mask.shape == x1.shape[:-1]) + + unbatched = len(E_mat.shape) == 2 + + if unbatched: + # add a leading batch dimension. We will remove it at the end, before + # returning the results + E_mat = E_mat[None] + K1 = K1[None] + K2 = K2[None] + x1 = x1[None] + x2 = x2[None] + if mask is not None: + mask = mask[None] + + # compute four possible pose solutions + Rs, ts = motion_from_essential(E_mat) + + # set reference view pose and compute projection matrix + R1 = eye_like(3, E_mat) # Bx3x3 + t1 = vec_like(3, E_mat) # Bx3x1 + + # compute the projection matrices for first camera + R1 = R1[:, None].expand(-1, 4, -1, -1) + t1 = t1[:, None].expand(-1, 4, -1, -1) + K1 = K1[:, None].expand(-1, 4, -1, -1) + P1 = projection_from_KRt(K1, R1, t1) # 1x4x4x4 + + # compute the projection matrices for second camera + R2 = Rs + t2 = ts + K2 = K2[:, None].expand(-1, 4, -1, -1) + P2 = projection_from_KRt(K2, R2, t2) # Bx4x4x4 + + # triangulate the points + x1 = x1[:, None].expand(-1, 4, -1, -1) + x2 = x2[:, None].expand(-1, 4, -1, -1) + X = triangulate_points(P1, P2, x1, x2) # Bx4xNx3 + + # project points and compute their depth values + d1 = depth_from_point(R1, t1, X) + d2 = depth_from_point(R2, t2, X) + + # verify the point values that have a positive depth value + depth_mask = (d1 > 0.0) & (d2 > 0.0) + if mask is not None: + depth_mask &= mask.unsqueeze(1) + + mask_indices = torch.max(depth_mask.sum(-1), dim=-1, keepdim=True)[1] + + # get pose and points 3d and return + R_out = Rs[:, mask_indices][:, 0, 0] + t_out = ts[:, mask_indices][:, 0, 0] + points3d_out = X[:, mask_indices][:, 0, 0] + + if unbatched: + R_out = R_out[0] + t_out = t_out[0] + points3d_out = points3d_out[0] + + return R_out, t_out, points3d_out + + +def relative_camera_motion( + R1: torch.Tensor, t1: torch.Tensor, R2: torch.Tensor, t2: torch.Tensor +) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Compute the relative camera motion between two cameras. + + Given the motion parameters of two cameras, computes the motion parameters of the second + one assuming the first one to be at the origin. If :math:`T1` and :math:`T2` are the camera motions, + the computed relative motion is :math:`T = T_{2}T^{-1}_{1}`. + + Args: + R1: The first camera rotation matrix with shape :math:`(*, 3, 3)`. + t1: The first camera translation vector with shape :math:`(*, 3, 1)`. + R2: The second camera rotation matrix with shape :math:`(*, 3, 3)`. + t2: The second camera translation vector with shape :math:`(*, 3, 1)`. + + Returns: + A tuple with the relative rotation matrix and + translation vector with the shape of :math:`[(*, 3, 3), (*, 3, 1)]`. + + """ + KORNIA_CHECK_SHAPE(R1, ["*", "3", "3"]) + KORNIA_CHECK_SHAPE(R2, ["*", "3", "3"]) + KORNIA_CHECK_SHAPE(t1, ["*", "3", "1"]) + KORNIA_CHECK_SHAPE(t2, ["*", "3", "1"]) + + # compute first the relative rotation + R = R2 @ R1.transpose(-2, -1) + + # compute the relative translation vector + t = t2 - R @ t1 + + return R, t + + +def find_essential( + points1: torch.Tensor, points2: torch.Tensor, weights: Optional[torch.Tensor] = None +) -> torch.Tensor: + r"""Find essential matrices. + + Args: + points1: A set of points in the first image with a tensor shape :math:`(B, N, 2), N>=5`. + points2: A set of points in the second image with a tensor shape :math:`(B, N, 2), N>=5`. + weights: Tensor containing the weights per point correspondence with a shape of :math:`(B, N)`. + + Returns: + the computed essential matrices with shape :math:`(B, 10, 3, 3)`. + Note that all possible solutions are returned, i.e., 10 essential matrices for each image pair. + To choose the best one out of 10, try to check the one with the lowest Sampson distance. + + """ + E = run_5point(points1, points2, weights).to(points1.dtype) + return E diff --git a/kornia/geometry/epipolar/fundamental.py b/kornia/geometry/epipolar/fundamental.py new file mode 100644 index 0000000..58ad40f --- /dev/null +++ b/kornia/geometry/epipolar/fundamental.py @@ -0,0 +1,526 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Module containing the functionalities for computing the Fundamental Matrix.""" + +import math +from typing import Literal, Optional, Tuple + +import torch + +from kornia.core.check import KORNIA_CHECK_SAME_SHAPE, KORNIA_CHECK_SHAPE +from kornia.core.utils import _torch_svd_cast, safe_inverse_with_mask +from kornia.geometry.conversions import convert_points_from_homogeneous, convert_points_to_homogeneous +from kornia.geometry.solvers import solve_cubic + + +def normalize_points(points: torch.Tensor, eps: float = 1e-8) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Normalize points (isotropic). + + Computes the transformation matrix such that the two principal moments of the set of points + are equal to unity, forming an approximately symmetric circular cloud of points of radius 1 + about the origin. Reference: Hartley/Zisserman 4.4.4 pag.107 + + This operation is an essential step before applying the DLT algorithm in order to consider + the result as optimal. + + Args: + points: Tensor containing the points to be normalized with shape :math:`(B, N, 2)`. + eps: epsilon value to avoid numerical instabilities. + + Returns: + tuple containing the normalized points in the shape :math:`(B, N, 2)` and the transformation matrix + in the shape :math:`(B, 3, 3)`. + + """ + if points.ndim != 3: + raise AssertionError(points.shape) + if points.shape[-1] != 2: + raise AssertionError(points.shape) + + B, _N, _ = points.shape + device, dtype = points.device, points.dtype + + # Center at mean + x_mean = points.mean(dim=1, keepdim=True) # (B,1,2) + centered = points - x_mean # (B,N,2) + + # Mean Euclidean distance to origin (radius) + mean_radius = centered.norm(dim=-1, p=2).mean(dim=-1) # (B,) + + # Scale so that mean radius becomes sqrt(2) + scale = (math.sqrt(2.0)) / (mean_radius + eps) # (B,) + + # Apply similarity transform in-place-ish (broadcast scale) + points_norm = centered * scale.view(B, 1, 1) # (B,N,2) + + # Build transform matrix: + # T = [[s, 0, -s*mx], + # [0, s, -s*my], + # [0, 0, 1 ]] + transform = torch.zeros((B, 3, 3), device=device, dtype=dtype) + transform[..., 0, 0] = scale + transform[..., 1, 1] = scale + transform[..., 0, 2] = -scale * x_mean[..., 0, 0] + transform[..., 1, 2] = -scale * x_mean[..., 0, 1] + transform[..., 2, 2] = 1.0 + + return points_norm, transform + + +def normalize_transformation(M: torch.Tensor, eps: float = 1e-8) -> torch.Tensor: + r"""Normalize a given transformation matrix. + + The function trakes the transformation matrix and normalize so that the value in + the last row and column is one. + + Args: + M: The transformation to be normalized of any shape with a minimum size of 2x2. + eps: small value to avoid unstabilities during the backpropagation. + + Returns: + the normalized transformation matrix with same shape as the input. + + """ + if len(M.shape) < 2: + raise AssertionError(M.shape) + norm_val: torch.Tensor = M[..., -1:, -1:] + return torch.where(norm_val.abs() > eps, M / (norm_val + eps), M) + + +def _nullspace_via_eigh(A: torch.Tensor) -> torch.Tensor: + """Compute the nullspace of a matrix A using the eigh method. + + Args: + A: (..., 7, 9) + + Returns: + N: (..., 9, 2) where columns span the right nullspace of A + """ + AT = A.transpose(-2, -1) # (..., 9, 7) + G = AT @ A # (..., 9, 9) SPD + _evals, evecs = torch.linalg.eigh(G) # ascending eigenvalues + N = evecs[..., :, :2] # eigenvectors for 2 smallest evals + return N # orthonormal columns + + +def _F1F2_from_nullspace(N: torch.Tensor): + """Compute the F1 and F2 matrices from the nullspace of a matrix A. + + Args: + N: (..., 9, 2) where columns span the right nullspace of A + Returns: + F1: (..., 3, 3) + F2: (..., 3, 3) + """ + F1 = N[..., 0].view(-1, 3, 3) + F2 = N[..., 1].view(-1, 3, 3) + return F1, F2 + + +def _normalize_F(F: torch.Tensor, eps: float = 1e-12) -> torch.Tensor: + """Frobenius-normalize each 3x3 (keeps cubic coefficients well-scaled). + + Args: + F: (..., 3, 3) + eps: small value to avoid unstabilities. + + Returns: + F: (..., 3, 3) + """ + nrm = F.norm(dim=(-2, -1), p=1, keepdim=True).clamp_min(eps) + return F / nrm + + +# Reference: Adapted from the 'run_7point' function in opencv +# https://github.com/opencv/opencv/blob/4.x/modules/calib3d/src/fundam.cpp +def _isclose0(x: torch.Tensor, eps: float = 1e-12) -> torch.Tensor: + # torch.isclose(x, 0) but TorchScript/GPU-friendly (no scalar tensor creation) + return x.abs() <= eps + + +def run_7point(points1: torch.Tensor, points2: torch.Tensor) -> torch.Tensor: + r"""Compute the fundamental matrix using the 7-point algorithm. + + The 7-point algorithm computes the fundamental matrix from exactly 7 point correspondences. + Unlike the 8-point algorithm, this method can return up to 3 possible fundamental matrices + as solutions to the rank-2 constraint, which is formulated as a cubic equation. + + Reference: Hartley/Zisserman 11.1.2 pag.281 + + Args: + points1: A set of 7 points in the first image with shape :math:`(B, 7, 2)`. + points2: A set of 7 points in the second image with shape :math:`(B, 7, 2)`. + + Returns: + The computed fundamental matrices with shape :math:`(B, 3, 3, 3)`, containing up to 3 + candidate solutions per batch. Invalid solutions are zeroed out. + + """ + KORNIA_CHECK_SHAPE(points1, ["B", "7", "2"]) + KORNIA_CHECK_SHAPE(points2, ["B", "7", "2"]) + + B = points1.shape[0] + device = points1.device + dtype = points1.dtype + + points1_norm, transform1 = normalize_points(points1) + points2_norm, transform2 = normalize_points(points2) + + x1, y1 = torch.chunk(points1_norm, dim=-1, chunks=2) # (B,7,1) + x2, y2 = torch.chunk(points2_norm, dim=-1, chunks=2) # (B,7,1) + ones = torch.ones_like(x1) + + # (B,7,9) + X = torch.cat([x2 * x1, x2 * y1, x2, y2 * x1, y2 * y1, y2, x1, y1, ones], dim=-1) + + # nullspace basis -> (B,3,3) + f1, f2 = _F1F2_from_nullspace(_nullspace_via_eigh(X)) + f1 = _normalize_F(f1) + f2 = _normalize_F(f2) + + # --- cubic coeffs (keep your known-good inverse-based formula) --- + coeffs = torch.zeros((B, 4), device=device, dtype=dtype) + f1_det = torch.linalg.det(f1) + f2_det = torch.linalg.det(f2) + + inv_f1, _ = safe_inverse_with_mask(f1) + inv_f2, _ = safe_inverse_with_mask(f2) + + coeffs[:, 0] = f1_det + coeffs[:, 1] = torch.einsum("bii->b", f2 @ inv_f1) * f1_det + coeffs[:, 2] = torch.einsum("bii->b", f1 @ inv_f2) * f2_det + coeffs[:, 3] = f2_det + + roots = solve_cubic(coeffs) # (B,3) + + # same "valid_root_mask" logic as your working version + cnz = torch.count_nonzero(roots, dim=1) # (B,) + valid_root_mask = (cnz < 3) | (cnz > 1) # (B,) bool + + # --- compute lambda/mu for ALL batches (no compaction) --- + _lambda = roots.clone() + _mu = torch.ones_like(_lambda) + + # _s = f1[2,2] * root + f2[2,2] for each of 3 roots + f1_22 = f1[:, 2, 2].unsqueeze(1) # (B,1) + f2_22 = f2[:, 2, 2].unsqueeze(1) # (B,1) + _s = f1_22 * roots + f2_22 # (B,3) + + # avoid torch.isclose with scalar tensor creation + _s_non_zero_mask = ~_isclose0(_s, 1e-12) # (B,3) bool + + # mu = 1/s where s != 0, else mu stays 1 + mu_new = torch.where(_s_non_zero_mask, 1.0 / _s, _mu) + lam_new = torch.where(_s_non_zero_mask, _lambda * mu_new, _lambda) + + _mu = mu_new + _lambda = lam_new + + # --- form candidates for ALL batches: F = f1*lambda + f2*mu --- + f1_expanded = f1.unsqueeze(1).expand(B, 3, 3, 3) + f2_expanded = f2.unsqueeze(1).expand(B, 3, 3, 3) + + fmatrix = f1_expanded * _lambda[:, :, None, None] + f2_expanded * _mu[:, :, None, None] # (B,3,3,3) + + # --- enforce last element handling like your original --- + # If s != 0 set F[2,2]=1 else set to 0 (per-candidate) + # This is exactly what your boolean-indexing version did, but without advanced indexing. + f22 = torch.where(_s_non_zero_mask, torch.ones_like(_s, dtype=dtype), torch.zeros_like(_s, dtype=dtype)) + fmatrix[:, :, 2, 2] = f22 # (B,3) + + # --- apply batch validity mask (no compaction) --- + # If batch invalid -> zero all 3 candidates + fmatrix = torch.where(valid_root_mask.view(B, 1, 1, 1), fmatrix, torch.zeros_like(fmatrix)) + + # --- denormalize for ALL batches --- + # F = T2^T * F * T1 + fmatrix = (transform2.unsqueeze(1).transpose(-2, -1) @ fmatrix) @ transform1.unsqueeze(1) + + return normalize_transformation(fmatrix) + + +def run_8point( + points1: torch.Tensor, + points2: torch.Tensor, + weights: Optional[torch.Tensor] = None, + use_einsum_at_more_than_points: int = 512, +) -> torch.Tensor: + r"""Compute the fundamental matrix using (weighted) 8-point DLT, optimized. + + Args: + points1: (B, N, 2), N >= 8 + points2: (B, N, 2), N >= 8 + weights: optional (B, N) nonnegative weights + use_einsum_at_more_than_points: threshold for using einsum vs GEMM for large N + + Returns: + (B, 3, 3) fundamental matrices + """ + KORNIA_CHECK_SHAPE(points1, ["B", "N", "2"]) + KORNIA_CHECK_SHAPE(points2, ["B", "N", "2"]) + KORNIA_CHECK_SAME_SHAPE(points1, points2) + if points1.shape[1] < 8: + raise AssertionError(points1.shape) + if weights is not None: + KORNIA_CHECK_SHAPE(weights, ["B", "N"]) + if weights.shape[1] != points1.shape[1]: + raise AssertionError(weights.shape) + + # Hartley normalization (same as before) + pts1n, T1 = normalize_points(points1) + pts2n, T2 = normalize_points(points2) + + x1, y1 = torch.chunk(pts1n, dim=-1, chunks=2) # (B,N,1) + x2, y2 = torch.chunk(pts2n, dim=-1, chunks=2) # (B,N,1) + ones = torch.ones_like(x1) + + # Design matrix rows A_i = [x2*x1, x2*y1, x2, y2*x1, y2*y1, y2, x1, y1, 1] + # Shape: A ∈ (B, N, 9) + A = torch.cat([x2 * x1, x2 * y1, x2, y2 * x1, y2 * y1, y2, x1, y1, ones], dim=-1).squeeze(-2) + + B, N, _ = A.shape + + # Build normal matrix M = A^T W A (B,9,9) without forming NxN diagonals. + if weights is None: + if N < use_einsum_at_more_than_points: + # Use GEMM on tall A: (B,9,N) @ (B,N,9) + M = A.transpose(-2, -1).contiguous() @ A + else: + # Accumulate via einsum (saves bandwidth for huge N) + M = torch.einsum("bni,bnj->bij", A, A) + else: + w = weights.clamp_min(0) + if N < use_einsum_at_more_than_points: + # Row-scale by sqrt(w) then GEMM + Aw = A * w.unsqueeze(-1).sqrt() + M = Aw.transpose(-2, -1).contiguous() @ Aw + else: + # Weighted einsum + M = torch.einsum("bni,bnj,bn->bij", A, A, w) + + _evals, evecs = torch.linalg.eigh(M) # ascending order + h = evecs[..., 0] # (B,9), eigenvector for smallest λ + F_hat = h.view(B, 3, 3) + + # Enforce rank-2 with a 3x3 SVD + U, S, V = _torch_svd_cast(F_hat) + S_new = torch.zeros_like(S) + S_new[..., :-1] = S[..., :-1] + F_rank2 = U @ torch.diag_embed(S_new) @ V.mH + F = T2.transpose(-2, -1) @ (F_rank2 @ T1) + + return normalize_transformation(F) + + +def find_fundamental( + points1: torch.Tensor, + points2: torch.Tensor, + weights: Optional[torch.Tensor] = None, + method: Literal["8POINT", "7POINT"] = "8POINT", +) -> torch.Tensor: + r"""Find the fundamental matrix. + + Args: + points1: A set of points in the first image with a tensor shape :math:`(B, N, 2), N>=8`. + points2: A set of points in the second image with a tensor shape :math:`(B, N, 2), N>=8`. + weights: Tensor containing the weights per point correspondence with a shape of :math:`(B, N)`. + method: The method to use for computing the fundamental matrix. Supported methods are "7POINT" and "8POINT". + + Returns: + the computed fundamental matrix with shape :math:`(B, 3*m, 3)`, where `m` number of fundamental matrix. + + Raises: + ValueError: If an invalid method is provided. + + """ + if method.upper() == "7POINT": + result = run_7point(points1, points2) + elif method.upper() == "8POINT": + result = run_8point(points1, points2, weights) + else: + raise ValueError(f"Invalid method: {method}. Supported methods are '7POINT' and '8POINT'.") + return result + + +def compute_correspond_epilines(points: torch.Tensor, F_mat: torch.Tensor) -> torch.Tensor: + r"""Compute the corresponding epipolar line for a given set of points. + + Args: + points: tensor containing the set of points to project in the shape of :math:`(*, N, 2)` or :math:`(*, N, 3)`. + F_mat: the fundamental to use for projection the points in the shape of :math:`(*, 3, 3)`. + + Returns: + a tensor with shape :math:`(*, N, 3)` containing a vector of the epipolar + lines corresponding to the points to the other image. Each line is described as + :math:`ax + by + c = 0` and encoding the vectors as :math:`(a, b, c)`. + + """ + KORNIA_CHECK_SHAPE(points, ["*", "N", "DIM"]) + if points.shape[-1] == 2: + points_h: torch.Tensor = convert_points_to_homogeneous(points) + elif points.shape[-1] == 3: + points_h = points + else: + raise AssertionError(points.shape) + KORNIA_CHECK_SHAPE(F_mat, ["*", "3", "3"]) + # project points and retrieve lines components + points_h = torch.transpose(points_h, dim0=-2, dim1=-1) + a, b, c = torch.chunk(F_mat @ points_h, dim=-2, chunks=3) + + # compute normal and compose equation line + nu: torch.Tensor = a * a + b * b + nu = torch.where(nu > 0.0, 1.0 / torch.sqrt(nu), torch.ones_like(nu)) + + line = torch.cat([a * nu, b * nu, c * nu], dim=-2) # *x3xN + return torch.transpose(line, dim0=-2, dim1=-1) # *xNx3 + + +def get_perpendicular(lines: torch.Tensor, points: torch.Tensor) -> torch.Tensor: + r"""Compute the perpendicular to a line, through the point. + + Args: + lines: tensor containing the set of lines :math:`(*, N, 3)`. + points: tensor containing the set of points :math:`(*, N, 2)`. + + Returns: + a tensor with shape :math:`(*, N, 3)` containing a vector of the epipolar + perpendicular lines. Each line is described as + :math:`ax + by + c = 0` and encoding the vectors as :math:`(a, b, c)`. + + """ + KORNIA_CHECK_SHAPE(lines, ["*", "N", "3"]) + KORNIA_CHECK_SHAPE(points, ["*", "N", "two"]) + if points.shape[2] == 2: + points_h: torch.Tensor = convert_points_to_homogeneous(points) + elif points.shape[2] == 3: + points_h = points + else: + raise AssertionError(points.shape) + infinity_point = lines * torch.tensor([1, 1, 0], dtype=lines.dtype, device=lines.device).view(1, 1, 3) + perp: torch.Tensor = torch.linalg.cross(points_h, infinity_point, dim=2) + return perp + + +def get_closest_point_on_epipolar_line(pts1: torch.Tensor, pts2: torch.Tensor, Fm: torch.Tensor) -> torch.Tensor: + """Return closest point on the epipolar line to the correspondence, given the fundamental matrix. + + Args: + pts1: correspondences from the left images with shape :math:`(*, N, (2|3))`. If they are not homogeneous, + converted automatically. + pts2: correspondences from the right images with shape :math:`(*, N, (2|3))`. If they are not homogeneous, + converted automatically. + Fm: Fundamental matrices with shape :math:`(*, 3, 3)`. Called Fm to avoid ambiguity with torch.nn.functional. + + Returns: + point on epipolar line :math:`(*, N, 2)`. + + """ + if not isinstance(Fm, torch.Tensor): + raise TypeError(f"Fm type is not a torch.Tensor. Got {type(Fm)}") + if (len(Fm.shape) < 3) or not Fm.shape[-2:] == (3, 3): + raise ValueError(f"Fm must be a (*, 3, 3) tensor. Got {Fm.shape}") + if pts1.shape[-1] == 2: + pts1 = convert_points_to_homogeneous(pts1) + if pts2.shape[-1] == 2: + pts2 = convert_points_to_homogeneous(pts2) + line1in2 = compute_correspond_epilines(pts1, Fm) + perp = get_perpendicular(line1in2, pts2) + points1_in_2 = convert_points_from_homogeneous(torch.linalg.cross(line1in2, perp, dim=2)) + + return points1_in_2 + + +def fundamental_from_essential(E_mat: torch.Tensor, K1: torch.Tensor, K2: torch.Tensor) -> torch.Tensor: + r"""Get the Fundamental matrix from Essential and camera matrices. + + Uses the method from Hartley/Zisserman 9.6 pag 257 (formula 9.12). + + Args: + E_mat: The essential matrix with shape of :math:`(*, 3, 3)`. + K1: The camera matrix from first camera with shape :math:`(*, 3, 3)`. + K2: The camera matrix from second camera with shape :math:`(*, 3, 3)`. + + Returns: + The fundamental matrix with shape :math:`(*, 3, 3)`. + + """ + KORNIA_CHECK_SHAPE(E_mat, ["*", "3", "3"]) + KORNIA_CHECK_SHAPE(K1, ["*", "3", "3"]) + KORNIA_CHECK_SHAPE(K2, ["*", "3", "3"]) + if not len(E_mat.shape[:-2]) == len(K1.shape[:-2]) == len(K2.shape[:-2]): + raise AssertionError + + return (safe_inverse_with_mask(K2)[0]).transpose(-2, -1) @ E_mat @ (safe_inverse_with_mask(K1)[0]) + + +# adapted from: +# https://github.com/opencv/opencv_contrib/blob/master/modules/sfm/src/fundamental.cpp#L109 +# https://github.com/openMVG/openMVG/blob/160643be515007580086650f2ae7f1a42d32e9fb/src/openMVG/multiview/projection.cpp#L134 + + +def fundamental_from_projections(P1: torch.Tensor, P2: torch.Tensor) -> torch.Tensor: + r"""Get the Fundamental matrix from Projection matrices. + + Args: + P1: The projection matrix from first camera with shape :math:`(*, 3, 4)`. + P2: The projection matrix from second camera with shape :math:`(*, 3, 4)`. + + Returns: + The fundamental matrix with shape :math:`(*, 3, 3)`. + """ + KORNIA_CHECK_SHAPE(P1, ["*", "3", "4"]) + KORNIA_CHECK_SHAPE(P2, ["*", "3", "4"]) + if P1.shape[:-2] != P2.shape[:-2]: + raise AssertionError + + def vstack(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: + return torch.cat([x, y], dim=-2) + + input_dtype = P1.dtype + if input_dtype not in (torch.float32, torch.float64): + P1 = P1.to(torch.float32) + P2 = P2.to(torch.float32) + + X1 = P1[..., 1:, :] + X2 = vstack(P1[..., 2:3, :], P1[..., 0:1, :]) + X3 = P1[..., :2, :] + + Y1 = P2[..., 1:, :] + Y2 = vstack(P2[..., 2:3, :], P2[..., 0:1, :]) + Y3 = P2[..., :2, :] + + X1Y1, X2Y1, X3Y1 = vstack(X1, Y1), vstack(X2, Y1), vstack(X3, Y1) + X1Y2, X2Y2, X3Y2 = vstack(X1, Y2), vstack(X2, Y2), vstack(X3, Y2) + X1Y3, X2Y3, X3Y3 = vstack(X1, Y3), vstack(X2, Y3), vstack(X3, Y3) + + F_vec = torch.cat( + [ + X1Y1.det().reshape(-1, 1), + X2Y1.det().reshape(-1, 1), + X3Y1.det().reshape(-1, 1), + X1Y2.det().reshape(-1, 1), + X2Y2.det().reshape(-1, 1), + X3Y2.det().reshape(-1, 1), + X1Y3.det().reshape(-1, 1), + X2Y3.det().reshape(-1, 1), + X3Y3.det().reshape(-1, 1), + ], + dim=1, + ) + + return F_vec.view(*P1.shape[:-2], 3, 3).to(input_dtype) diff --git a/kornia/geometry/epipolar/numeric.py b/kornia/geometry/epipolar/numeric.py new file mode 100644 index 0000000..fb34e50 --- /dev/null +++ b/kornia/geometry/epipolar/numeric.py @@ -0,0 +1,66 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Module containing numerical functionalities for SfM.""" + +import torch + +from kornia.core.utils import _torch_inverse_cast + + +def cross_product_matrix(x: torch.Tensor) -> torch.Tensor: + r"""Return the cross_product_matrix symmetric matrix of a vector. + + Args: + x: The input vector to construct the matrix in the shape :math:`(*, 3)`. + + Returns: + The constructed cross_product_matrix symmetric matrix with shape :math:`(*, 3, 3)`. + + """ + if not x.shape[-1] == 3: + raise AssertionError(x.shape) + # get vector compononens + x0 = x[..., 0] + x1 = x[..., 1] + x2 = x[..., 2] + + # construct the matrix, reshape to 3x3 and return + zeros = torch.zeros_like(x0) + cross_product_matrix_flat = torch.stack([zeros, -x2, x1, x2, zeros, -x0, -x1, x0, zeros], dim=-1) + shape_ = x.shape[:-1] + (3, 3) + return cross_product_matrix_flat.view(*shape_) + + +def matrix_cofactor_tensor(matrix: torch.Tensor) -> torch.Tensor: + """Cofactor matrix, refer to the numpy doc. + + Args: + matrix: The input matrix in the shape :math:`(*, 3, 3)`. + + """ + det = torch.det(matrix) + singular_mask = det != 0 + if singular_mask.sum() != 0: + # B, 3, 3 + cofactor = _torch_inverse_cast(matrix[singular_mask]).transpose(-2, -1) * det[singular_mask][:, None, None] + # return cofactor matrix of the given matrix + returned_cofactor = torch.zeros_like(matrix) + returned_cofactor[singular_mask] = cofactor + return returned_cofactor + else: + raise Exception("all singular matrices") diff --git a/kornia/geometry/epipolar/projection.py b/kornia/geometry/epipolar/projection.py new file mode 100644 index 0000000..580f8e6 --- /dev/null +++ b/kornia/geometry/epipolar/projection.py @@ -0,0 +1,221 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Module for image projections.""" + +from typing import Tuple, Union + +import torch +from torch.linalg import qr as linalg_qr + +from kornia.core.check import KORNIA_CHECK_SHAPE +from kornia.core.ops import eye_like, vec_like +from kornia.core.utils import _torch_svd_cast + +from .numeric import cross_product_matrix + + +def intrinsics_like(focal: float, input: torch.Tensor) -> torch.Tensor: + r"""Return a 3x3 intrinsics matrix, with same size as the input. + + The center of projection will be based in the input image size. + + Args: + focal: the focal length for the camera matrix. + input: image tensor that will determine the batch size and image height + and width. It is assumed to be a tensor in the shape of :math:`(B, C, H, W)`. + + Returns: + The camera matrix with the shape of :math:`(B, 3, 3)`. + + """ + if len(input.shape) != 4: + raise AssertionError(input.shape) + if focal <= 0: + raise AssertionError(focal) + + _, _, H, W = input.shape + + intrinsics = eye_like(3, input) + intrinsics[..., 0, 0] *= focal + intrinsics[..., 1, 1] *= focal + intrinsics[..., 0, 2] += 1.0 * W / 2 + intrinsics[..., 1, 2] += 1.0 * H / 2 + return intrinsics + + +def random_intrinsics(low: Union[float, torch.Tensor], high: Union[float, torch.Tensor]) -> torch.Tensor: + r"""Generate a random camera matrix based on a given uniform distribution. + + Args: + low: lower range (inclusive). + high: upper range (exclusive). + + Returns: + the random camera matrix with the shape of :math:`(1, 3, 3)`. + + """ + sampler = torch.distributions.Uniform(low, high) + params = sampler.sample((4,)) + fx, fy, cx, cy = params[0], params[1], params[2], params[3] + camera_matrix = torch.tensor([[fx, 0.0, cx], [0.0, fy, cy], [0.0, 0.0, 1.0]], dtype=fx.dtype, device=fx.device) + return camera_matrix.unsqueeze(0) + + +def scale_intrinsics(camera_matrix: torch.Tensor, scale_factor: Union[float, torch.Tensor]) -> torch.Tensor: + r"""Scale a camera matrix containing the intrinsics. + + Applies the scaling factor to the focal length and center of projection. + + Args: + camera_matrix: the camera calibration matrix containing the intrinsic + parameters. The expected shape for the tensor is :math:`(B, 3, 3)`. + scale_factor: the scaling factor to be applied. + + Returns: + The scaled camera matrix with shame shape as input :math:`(B, 3, 3)`. + + """ + K_scale = camera_matrix.clone() + K_scale[..., 0, 0] *= scale_factor + K_scale[..., 1, 1] *= scale_factor + K_scale[..., 0, 2] *= scale_factor + K_scale[..., 1, 2] *= scale_factor + return K_scale + + +def projection_from_KRt(K: torch.Tensor, R: torch.Tensor, t: torch.Tensor) -> torch.Tensor: + r"""Get the projection matrix P from K, R and t. + + This function estimate the projection matrix by solving the following equation: :math:`P = K * [R|t]`. + + Args: + K: the camera matrix with the intrinsics with shape :math:`(B, 3, 3)`. + R: The rotation matrix with shape :math:`(B, 3, 3)`. + t: The translation vector with shape :math:`(B, 3, 1)`. + + Returns: + The projection matrix P with shape :math:`(B, 4, 4)`. + + """ + KORNIA_CHECK_SHAPE(K, ["*", "3", "3"]) + KORNIA_CHECK_SHAPE(R, ["*", "3", "3"]) + KORNIA_CHECK_SHAPE(t, ["*", "3", "1"]) + if not len(K.shape) == len(R.shape) == len(t.shape): + raise AssertionError + + Rt = torch.cat([R, t], dim=-1) # 3x4 + Rt_h = torch.nn.functional.pad(Rt, [0, 0, 0, 1], "constant", 0.0) # 4x4 + Rt_h[..., -1, -1] += 1.0 + + K_h = torch.nn.functional.pad(K, [0, 1, 0, 1], "constant", 0.0) # 4x4 + K_h[..., -1, -1] += 1.0 + + return K @ Rt + + +def KRt_from_projection(P: torch.Tensor, eps: float = 1e-6) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + r"""Decompose the Projection matrix into Camera-Matrix, Rotation Matrix and Translation vector. + + Args: + P: the projection matrix with shape :math:`(B, 3, 4)`. + eps: epsilon for numerical stability. + + Returns: + - The Camera matrix with shape :math:`(B, 3, 3)`. + - The Rotation matrix with shape :math:`(B, 3, 3)`. + - The Translation vector with shape :math:`(B, 3)`. + + """ + KORNIA_CHECK_SHAPE(P, ["*", "3", "4"]) + submat_3x3 = P[:, 0:3, 0:3] + last_column = P[:, 0:3, 3].unsqueeze(-1) + + # Trick to turn QR-decomposition into RQ-decomposition + reverse = torch.tensor([[0, 0, 1], [0, 1, 0], [1, 0, 0]], device=P.device, dtype=P.dtype).unsqueeze(0) + submat_3x3 = torch.matmul(reverse, submat_3x3).permute(0, 2, 1) + ortho_mat, upper_mat = linalg_qr(submat_3x3) + ortho_mat = torch.matmul(reverse, ortho_mat.permute(0, 2, 1)) + upper_mat = torch.matmul(reverse, torch.matmul(upper_mat.permute(0, 2, 1), reverse)) + + # Turning the `upper_mat's` diagonal elements to positive. + diagonals = torch.diagonal(upper_mat, dim1=-2, dim2=-1) + eps + signs = torch.sign(diagonals) + signs_mat = torch.diag_embed(signs) + + K = torch.matmul(upper_mat, signs_mat) + R = torch.matmul(signs_mat, ortho_mat) + t = torch.linalg.solve(K, last_column) + + return K, R, t + + +def depth_from_point(R: torch.Tensor, t: torch.Tensor, X: torch.Tensor) -> torch.Tensor: + r"""Return the depth of a point transformed by a rigid transform. + + Args: + R: The rotation matrix with shape :math:`(*, 3, 3)`. + t: The translation vector with shape :math:`(*, 3, 1)`. + X: The 3d points with shape :math:`(*, 3)`. + + Returns: + The depth value per point with shape :math:`(*, 1)`. + + """ + X_tmp = R @ X.transpose(-2, -1) + X_out = X_tmp[..., 2, :] + t[..., 2, :] + return X_out + + +# adapted from: +# https://github.com/opencv/opencv_contrib/blob/master/modules/sfm/src/fundamental.cpp#L61 +# https://github.com/mapillary/OpenSfM/blob/master/opensfm/multiview.py#L14 +def _nullspace(A: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Compute the null space of A. + + Return the smallest singular value and the corresponding vector. + """ + _, s, v = _torch_svd_cast(A) + return s[..., -1], v[..., -1] + + +def projections_from_fundamental(F_mat: torch.Tensor) -> torch.Tensor: + r"""Get the projection matrices from the Fundamental Matrix. + + Args: + F_mat: the fundamental matrix with the shape :math:`(B, 3, 3)`. + + Returns: + The projection matrices with shape :math:`(B, 3, 4, 2)`. + + """ + KORNIA_CHECK_SHAPE(F_mat, ["*", "3", "3"]) + + R1 = eye_like(3, F_mat) # Bx3x3 + t1 = vec_like(3, F_mat) # Bx3 + + Ft_mat = F_mat.transpose(-2, -1) + + _, e2 = _nullspace(Ft_mat) + + R2 = cross_product_matrix(e2) @ F_mat # Bx3x3 + t2 = e2[..., :, None] # Bx3x1 + + P1 = torch.cat([R1, t1], dim=-1) # Bx3x4 + P2 = torch.cat([R2, t2], dim=-1) # Bx3x4 + + return torch.stack([P1, P2], dim=-1) diff --git a/kornia/geometry/epipolar/scene.py b/kornia/geometry/epipolar/scene.py new file mode 100644 index 0000000..7c82d2c --- /dev/null +++ b/kornia/geometry/epipolar/scene.py @@ -0,0 +1,64 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""nn.Module to generate synthetic 3d scenes.""" + +import math +from typing import Dict + +import torch + +from kornia.geometry.conversions import axis_angle_to_rotation_matrix +from kornia.geometry.linalg import transform_points + +from .projection import projection_from_KRt, random_intrinsics + + +def generate_scene(num_views: int, num_points: int) -> Dict[str, torch.Tensor]: + """Generate 3d scene.""" + # Generate the 3d points + points3d = torch.rand(1, num_points, 3) # NxMx3 + + # Create random camera matrix + K = random_intrinsics(0.0, 100.0) # 1x3x3 + + # Create random rotation per view + ang = torch.rand(num_views, 1) * math.pi * 2.0 + + rvec = torch.rand(num_views, 3) + rvec = ang * rvec / torch.norm(rvec, dim=1, keepdim=True) # Nx3 + rot_mat = axis_angle_to_rotation_matrix(rvec) # Nx3x3 + # matches with cv2.Rodrigues -> yay ! + + # Create random translation per view + tx = torch.empty(num_views).uniform_(-0.5, 0.5) + ty = torch.empty(num_views).uniform_(-0.5, 0.5) + tz = torch.empty(num_views).uniform_(-1.0, 2.0) + tvec = torch.stack([tx, ty, tz], dim=1)[..., None] + + # Make sure the shape is in front of the camera + points3d_trans = (rot_mat @ points3d.transpose(-2, -1)) + tvec + min_dist = torch.min(points3d_trans[:, 2], dim=1)[0] + tvec[:, 2, 0] = torch.where(min_dist < 0, tz - min_dist + 1.0, tz) + + # compute projection matrices + P = projection_from_KRt(K, rot_mat, tvec) + + # project points3d and backproject to image plane + points2d = transform_points(P, points3d.expand(num_views, -1, -1)) + + return {"K": K, "R": rot_mat, "t": tvec, "P": P, "points3d": points3d, "points2d": points2d} diff --git a/kornia/geometry/epipolar/triangulation.py b/kornia/geometry/epipolar/triangulation.py new file mode 100644 index 0000000..872f582 --- /dev/null +++ b/kornia/geometry/epipolar/triangulation.py @@ -0,0 +1,198 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Module with the functionalities for triangulation.""" + +from __future__ import annotations + +import torch + +from kornia.core.check import KORNIA_CHECK_SHAPE +from kornia.core.utils import _normalize_to_float32_or_float64, _torch_svd_cast, is_mps_tensor_safe +from kornia.geometry.conversions import convert_points_from_homogeneous +from kornia.geometry.solvers import null_vector_3x4 + +# https://github.com/opencv/opencv_contrib/blob/master/modules/sfm/src/triangulation.cpp#L68 + +# cuSOLVER's batched symmetric eigenvalue solver crashes above this many 4x4 matrices +# in a single call (empirically observed at >=32 K on current CUDA/PyTorch versions). +_CUSOLVER_EIGH_BATCH_LIMIT: int = 28_000 + + +def _eigh_smallest_vec(M: torch.Tensor) -> torch.Tensor: + """Return the eigenvector for the smallest eigenvalue of each symmetric matrix. + + Handles cuSOLVER's batch-size limit by chunking when necessary. + + Args: + M: batch of symmetric PSD matrices, shape ``(N, k, k)``. + + Returns: + Eigenvectors of shape ``(N, k)``. + """ + N = M.shape[0] + if N <= _CUSOLVER_EIGH_BATCH_LIMIT: + _, V = torch.linalg.eigh(M) + return V[..., 0] + + parts = [ + torch.linalg.eigh(M[i : i + _CUSOLVER_EIGH_BATCH_LIMIT])[1][..., 0] + for i in range(0, N, _CUSOLVER_EIGH_BATCH_LIMIT) + ] + return torch.cat(parts, dim=0) + + +def triangulate_points( + P1: torch.Tensor, + P2: torch.Tensor, + points1: torch.Tensor, + points2: torch.Tensor, + solver: str = "eigh", +) -> torch.Tensor: + r"""Reconstructs a bunch of points by triangulation. + + Triangulates the 3d position of 2d correspondences between several images. + Reference: Internally it uses DLT formulation from Hartley/Zisserman 12.2 pag.312 + + The input points are assumed to be in homogeneous coordinate system and being inliers + correspondences. The method does not perform any robust estimation. + + Args: + P1: The projection matrix for the first camera with shape :math:`(*, 3, 4)`. + P2: The projection matrix for the second camera with shape :math:`(*, 3, 4)`. + points1: The set of points seen from the first camera frame in the camera plane + coordinates with shape :math:`(*, N, 2)`. + points2: The set of points seen from the second camera frame in the camera plane + coordinates with shape :math:`(*, N, 2)`. + solver: Back-end used to find the null vector of the :math:`4 \times 4` DLT + constraint matrix. One of: + + * ``"svd"`` — most numerically stable. Promotes to fp64 and uses a full + SVD (via :func:`~kornia.core.utils._torch_svd_cast`). Suitable when + maximum accuracy is required regardless of speed. + * ``"eigh"`` *(default)* — forms :math:`X^\top X` and finds the eigenvector + for its smallest eigenvalue via :func:`torch.linalg.eigh`. Algebraically + equivalent to the SVD solution; slightly less numerically stable because + forming :math:`X^\top X` squares the singular values. Typically **10-26x + faster** than ``"svd"`` on GPU for large batches. + * ``"cofactor"`` — solves two :math:`3 \times 4` sub-systems analytically + using :func:`~kornia.geometry.solvers.null_vector_3x4` (closed-form + cofactor expansion, no LAPACK call). The two solutions are averaged after + normalisation. This matches the full DLT solution when the constraint + system is exactly consistent, but is only an approximation in the noisy + inconsistent case. Fastest option for all batch sizes. + + Returns: + The reconstructed 3d points in the world frame with shape :math:`(*, N, 3)`. + + Example: + >>> P1 = torch.eye(3, 4)[None] # 1x3x4 + >>> P2 = torch.eye(3, 4)[None] + >>> pts1 = torch.rand(1, 5, 2) + >>> pts2 = torch.rand(1, 5, 2) + >>> pts3d = triangulate_points(P1, P2, pts1, pts2) + >>> pts3d.shape + torch.Size([1, 5, 3]) + + """ + KORNIA_CHECK_SHAPE(P1, ["*", "3", "4"]) + KORNIA_CHECK_SHAPE(P2, ["*", "3", "4"]) + KORNIA_CHECK_SHAPE(points1, ["*", "N", "2"]) + KORNIA_CHECK_SHAPE(points2, ["*", "N", "2"]) + + # Build the four DLT constraint rows (each (*, N, 4)) via vectorized broadcasting. + # P[..., r:r+1, :] broadcasts with points[..., c:c+1] → (*, N, 4). + row0 = points1[..., 0:1] * P1[..., 2:3, :] - P1[..., 0:1, :] # (*, N1, 4) + row1 = points1[..., 1:2] * P1[..., 2:3, :] - P1[..., 1:2, :] # (*, N1, 4) + row2 = points2[..., 0:1] * P2[..., 2:3, :] - P2[..., 0:1, :] # (*, N2, 4) + row3 = points2[..., 1:2] * P2[..., 2:3, :] - P2[..., 1:2, :] # (*, N2, 4) + # Unify N1 and N2: one may be 1 when points1/points2 are broadcast-compatible. + row0, row1, row2, row3 = torch.broadcast_tensors(row0, row1, row2, row3) + + if solver == "svd": + X = torch.stack([row0, row1, row2, row3], dim=-2) # (*, N, 4, 4) + # SVD: last right singular vector minimises ||Ax|| s.t. ||x||=1. + # _torch_svd_cast promotes to fp64 for numerical stability and returns V + # with singular vectors as columns; the last column corresponds to the + # smallest singular value. + _, _, V = _torch_svd_cast(X) + points3d_h = V[..., -1] # (*, N, 4) + + elif solver == "eigh": + X = torch.stack([row0, row1, row2, row3], dim=-2) # (*, N, 4, 4) + # Solve the homogeneous least-squares problem min ||Ax|| s.t. ||x||=1. + # The minimiser is the eigenvector of X^T X associated with its smallest + # eigenvalue. This is algebraically equivalent to the last right singular + # vector of X used in SVD-based DLT, though forming X^T X can be less + # numerically stable than a direct SVD. The result is defined up to sign, + # which is fine for homogeneous coordinates. + # The approach is valid in both the noise-free (rank-3) and the noisy + # inconsistent case, where the rows do not share an exact nullspace. + # Mirror _torch_svd_cast's promotion rules so numerical behaviour is + # comparable to the "svd" solver: fp32 → fp64 for stability, fp16/bf16 → + # fp32, fp64 stays, MPS capped at fp32 (no fp64 support there). + if is_mps_tensor_safe(X): + compute_dtype = torch.float32 + elif X.dtype == torch.float32: + compute_dtype = torch.float64 + else: + compute_dtype = _normalize_to_float32_or_float64(X.dtype) + batch_shape = X.shape[:-2] # (*, N) + X_cast = X.to(compute_dtype) + XTX = X_cast.mT @ X_cast # (*, N, 4, 4) symmetric PSD + flat = XTX.flatten(0, -3) # (M, 4, 4) + v_flat = _eigh_smallest_vec(flat).to(X.dtype) # (M, 4) + points3d_h = v_flat.reshape(*batch_shape, 4) # (*, N, 4) + + elif solver == "cofactor": + # Solve two 3x4 sub-systems analytically via cofactor expansion and + # average the sign-aligned normalised results. This matches the full + # DLT solution when the constraint system is exactly consistent + # (noise-free), but is only an approximation in the noisy case. + # null_vector_3x4 uses only arithmetic ops, so promote fp16/bf16 → fp32. + compute_dtype = _normalize_to_float32_or_float64(row0.dtype) + r0 = row0.to(compute_dtype) + r1 = row1.to(compute_dtype) + r2 = row2.to(compute_dtype) + r3 = row3.to(compute_dtype) + # Both sub-systems include row2 (from camera 2's x-projection equation), + # which carries the camera-2 translation and is therefore well-conditioned + # for any camera pair with a non-zero baseline in x. Using rows {0,1,2} + # and {1,2,3} rather than {0,1,2} and {0,1,3} avoids the degenerate case + # that arises when camera 2 has zero last-column entries in its y- and + # z-projection rows (e.g. [R|t] with t = (-T,0,0)). + A_012 = torch.stack([r0, r1, r2], dim=-2) # (*, N, 3, 4) + A_123 = torch.stack([r1, r2, r3], dim=-2) # (*, N, 3, 4) + h_012 = null_vector_3x4(A_012).to(row0.dtype) # (*, N, 4) + h_123 = null_vector_3x4(A_123).to(row0.dtype) # (*, N, 4) + n012 = h_012.norm(dim=-1, keepdim=True).clamp(min=1e-8) + n123 = h_123.norm(dim=-1, keepdim=True).clamp(min=1e-8) + v012 = h_012 / n012 + v123 = h_123 / n123 + # Null vectors are defined up to a global sign; align signs before + # averaging in homogeneous space to prevent cancellation when the two + # sub-system solutions point in opposite directions (which would yield a + # near-zero homogeneous vector and NaN after dehomogenisation). + dot = (v012 * v123).sum(dim=-1, keepdim=True) + v123 = torch.where(dot < 0, -v123, v123) + points3d_h = v012 + v123 # (*, N, 4) + + else: + raise NotImplementedError(f"Unknown solver '{solver}'. Choose from: 'svd', 'eigh', 'cofactor'.") + + points3d: torch.Tensor = convert_points_from_homogeneous(points3d_h) + return points3d diff --git a/kornia/geometry/grid.py b/kornia/geometry/grid.py new file mode 100644 index 0000000..5a338c7 --- /dev/null +++ b/kornia/geometry/grid.py @@ -0,0 +1,126 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from typing import Optional + +import torch + + +def create_meshgrid( + height: int, + width: int, + normalized_coordinates: bool = True, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + """Generate a coordinate grid for an image. + + When the flag ``normalized_coordinates`` is set to True, the grid is + normalized to be in the range :math:`[-1,1]` to be consistent with the pytorch + function :py:func:`torch.nn.functional.grid_sample`. + + Args: + height: the image height (rows). + width: the image width (cols). + normalized_coordinates: whether to normalize + coordinates in the range :math:`[-1,1]` in order to be consistent with the + PyTorch function :py:func:`torch.nn.functional.grid_sample`. + device: the device on which the grid will be generated. + dtype: the data type of the generated grid. + + Return: + grid tensor with shape :math:`(1, H, W, 2)`. + + Example: + >>> create_meshgrid(2, 2) + tensor([[[[-1., -1.], + [ 1., -1.]], + + [[-1., 1.], + [ 1., 1.]]]]) + + >>> create_meshgrid(2, 2, normalized_coordinates=False) + tensor([[[[0., 0.], + [1., 0.]], + + [[0., 1.], + [1., 1.]]]]) + + """ + xs: torch.Tensor = torch.linspace(0, width - 1, width, device=device, dtype=dtype) + ys: torch.Tensor = torch.linspace(0, height - 1, height, device=device, dtype=dtype) + # Fix TracerWarning + # Note: normalize_pixel_coordinates still gots TracerWarning since new width and height + # tensors will be generated. + # Below is the code using normalize_pixel_coordinates: + # base_grid: torch.Tensor = torch.stack(torch.meshgrid([xs, ys]), dim=2) + # if normalized_coordinates: + # base_grid = K.geometry.normalize_pixel_coordinates(base_grid, height, width) + # return torch.unsqueeze(base_grid.transpose(0, 1), dim=0) + if normalized_coordinates: + xs = (xs / (width - 1) - 0.5) * 2 + ys = (ys / (height - 1) - 0.5) * 2 + # generate grid by stacking coordinates + # TODO: torchscript doesn't like `torch_version_ge` + # if torch_version_ge(1, 13, 0): + # x, y = torch.meshgrid([xs, ys], indexing="xy") + # return torch.stack([x, y], -1).unsqueeze(0) # 1xHxWx2 + # TODO: remove after we drop support of old versions + base_grid: torch.Tensor = torch.stack(torch.meshgrid([xs, ys], indexing="ij"), dim=-1) # WxHx2 + return base_grid.permute(1, 0, 2).unsqueeze(0) # 1xHxWx2 + + +def create_meshgrid3d( + depth: int, + height: int, + width: int, + normalized_coordinates: bool = True, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + """Generate a coordinate grid for an image. + + When the flag ``normalized_coordinates`` is set to True, the grid is + normalized to be in the range :math:`[-1,1]` to be consistent with the pytorch + function :py:func:`torch.nn.functional.grid_sample`. + + Args: + depth: the image depth (channels). + height: the image height (rows). + width: the image width (cols). + normalized_coordinates: whether to normalize + coordinates in the range :math:`[-1,1]` in order to be consistent with the + PyTorch function :py:func:`torch.nn.functional.grid_sample`. + device: the device on which the grid will be generated. + dtype: the data type of the generated grid. + + Return: + grid tensor with shape :math:`(1, D, H, W, 3)`. + + """ + xs: torch.Tensor = torch.linspace(0, width - 1, width, device=device, dtype=dtype) + ys: torch.Tensor = torch.linspace(0, height - 1, height, device=device, dtype=dtype) + zs: torch.Tensor = torch.linspace(0, depth - 1, depth, device=device, dtype=dtype) + # Fix TracerWarning + if normalized_coordinates: + xs = (xs / (width - 1) - 0.5) * 2 + ys = (ys / (height - 1) - 0.5) * 2 + zs = (zs / (depth - 1) - 0.5) * 2 + # generate grid by stacking coordinates + base_grid = torch.stack(torch.meshgrid([zs, xs, ys], indexing="ij"), dim=-1) # DxWxHx3 + return base_grid.permute(0, 2, 1, 3).unsqueeze(0) # 1xDxHxWx3 diff --git a/kornia/geometry/homography.py b/kornia/geometry/homography.py new file mode 100644 index 0000000..f89c2d5 --- /dev/null +++ b/kornia/geometry/homography.py @@ -0,0 +1,406 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import warnings +from typing import Optional, Tuple + +import torch + +from kornia.core.check import KORNIA_CHECK_SHAPE +from kornia.core.utils import _extract_device_dtype, _torch_svd_cast, safe_inverse_with_mask, safe_solve_with_mask +from kornia.geometry.conversions import convert_points_from_homogeneous, convert_points_to_homogeneous +from kornia.geometry.epipolar import normalize_points +from kornia.geometry.linalg import transform_points + +TupleTensor = Tuple[torch.Tensor, torch.Tensor] + + +def oneway_transfer_error( + pts1: torch.Tensor, pts2: torch.Tensor, H: torch.Tensor, squared: bool = True, eps: float = 1e-8 +) -> torch.Tensor: + r"""Return transfer error in image 2 for correspondences given the homography matrix. + + Args: + pts1: correspondences from the left images with shape + (B, N, 2 or 3). If they are homogeneous, converted automatically. + pts2: correspondences from the right images with shape + (B, N, 2 or 3). If they are homogeneous, converted automatically. + H: Homographies with shape :math:`(B, 3, 3)`. + squared: if True (default), the squared distance is returned. + eps: Small constant for safe sqrt. + + Returns: + the computed distance with shape :math:`(B, N)`. + + """ + KORNIA_CHECK_SHAPE(H, ["B", "3", "3"]) + + if pts1.shape[-1] == 3: + x1y1 = convert_points_from_homogeneous(pts1) + x1 = x1y1[..., 0] + y1 = x1y1[..., 1] + else: + x1 = pts1[..., :, 0] + y1 = pts1[..., :, 1] + + if pts2.shape[-1] == 3: + u2v2 = convert_points_from_homogeneous(pts2) + u2 = u2v2[..., 0] + v2 = u2v2[..., 1] + else: + u2 = pts2[..., :, 0] + v2 = pts2[..., :, 1] + + # ---- Grab H entries and broadcast across N ---- + h00 = H[..., 0, 0][..., None] + h01 = H[..., 0, 1][..., None] + h02 = H[..., 0, 2][..., None] + h10 = H[..., 1, 0][..., None] + h11 = H[..., 1, 1][..., None] + h12 = H[..., 1, 2][..., None] + h20 = H[..., 2, 0][..., None] + h21 = H[..., 2, 1][..., None] + h22 = H[..., 2, 2][..., None] + + # From Hartley and Zisserman, Error in one image (4.6) + # dist = \sum_{i} ( d(x', Hx)**2) + # ---- Apply homography to pts1 (Euclidean) and dehomogenize ---- + # [x'; y'; w']^T = H @ [x1, y1, 1]^T + x_num = h00 * x1 + h01 * y1 + h02 + y_num = h10 * x1 + h11 * y1 + h12 + w_den = h20 * x1 + h21 * y1 + h22 + + u1in2 = x_num / (w_den + eps) + v1in2 = y_num / (w_den + eps) + + # ---- Squared transfer error in image 2 ---- + err2 = (u1in2 - u2).pow(2) + (v1in2 - v2).pow(2) + if squared: + return err2 + return (err2 + eps).sqrt() + + +def symmetric_transfer_error( + pts1: torch.Tensor, pts2: torch.Tensor, H: torch.Tensor, squared: bool = True, eps: float = 1e-8 +) -> torch.Tensor: + r"""Return Symmetric transfer error for correspondences given the homography matrix. + + Args: + pts1: correspondences from the left images with shape + (B, N, 2 or 3). If they are homogeneous, converted automatically. + pts2: correspondences from the right images with shape + (B, N, 2 or 3). If they are homogeneous, converted automatically. + H: Homographies with shape :math:`(B, 3, 3)`. + squared: if True (default), the squared distance is returned. + eps: Small constant for safe sqrt. + + Returns: + the computed distance with shape :math:`(B, N)`. + + """ + KORNIA_CHECK_SHAPE(H, ["B", "3", "3"]) + if pts1.size(-1) == 3: + pts1 = convert_points_from_homogeneous(pts1) + + if pts2.size(-1) == 3: + pts2 = convert_points_from_homogeneous(pts2) + + max_num = torch.finfo(pts1.dtype).max + # From Hartley and Zisserman, Symmetric transfer error (4.7) + # dist = \sum_{i} (d(x, H^-1 x')**2 + d(x', Hx)**2) + H_inv, good_H = safe_inverse_with_mask(H) + + there: torch.Tensor = oneway_transfer_error(pts1, pts2, H, True, eps) + back: torch.Tensor = oneway_transfer_error(pts2, pts1, H_inv, True, eps) + good_H_reshape: torch.Tensor = good_H.view(-1, 1).expand_as(there) + out = (there + back) * good_H_reshape.to(there.dtype) + max_num * (~good_H_reshape).to(there.dtype) + if squared: + return out + return (out + eps).sqrt() + + +def line_segment_transfer_error_one_way( + ls1: torch.Tensor, ls2: torch.Tensor, H: torch.Tensor, squared: bool = False +) -> torch.Tensor: + r"""Return transfer error in image 2 for line segment correspondences given the homography matrix. + + Line segment end points are reprojected into image 2, and point-to-line error is calculated w.r.t. line, + induced by line segment in image 2. See :cite:`homolines2001` for details. + + Args: + ls1: line segment correspondences from the left images with shape + (B, N, 2, 2). + ls2: line segment correspondences from the right images with shape + (B, N, 2, 2). + H: Homographies with shape :math:`(B, 3, 3)`. + squared: if True (default is False), the squared distance is returned. + + Returns: + the computed distance with shape :math:`(B, N)`. + + """ + KORNIA_CHECK_SHAPE(H, ["B", "3", "3"]) + KORNIA_CHECK_SHAPE(ls1, ["B", "N", "2", "2"]) + KORNIA_CHECK_SHAPE(ls2, ["B", "N", "2", "2"]) + B, N = ls1.shape[:2] + ps1, pe1 = torch.chunk(ls1, dim=2, chunks=2) + ps2, pe2 = torch.chunk(ls2, dim=2, chunks=2) + ps2_h = convert_points_to_homogeneous(ps2) + pe2_h = convert_points_to_homogeneous(pe2) + ln2 = torch.linalg.cross(ps2_h, pe2_h, dim=3) + ps1_in2 = convert_points_to_homogeneous(transform_points(H, ps1)) + pe1_in2 = convert_points_to_homogeneous(transform_points(H, pe1)) + er_st1 = (ln2 @ ps1_in2.transpose(-2, -1)).view(B, N).abs() + er_end1 = (ln2 @ pe1_in2.transpose(-2, -1)).view(B, N).abs() + error = 0.5 * (er_st1 + er_end1) + if squared: + error = error**2 + return error + + +def find_homography_dlt( + points1: torch.Tensor, points2: torch.Tensor, weights: Optional[torch.Tensor] = None, solver: str = "lu" +) -> torch.Tensor: + r"""Compute the homography matrix using the DLT formulation. + + The linear system is solved by using the Weighted Least Squares Solution for the 4 Points algorithm. + + Args: + points1: A set of points in the first image with a tensor shape :math:`(B, N, 2)`. + points2: A set of points in the second image with a tensor shape :math:`(B, N, 2)`. + weights: Tensor containing the weights per point correspondence with a shape of :math:`(B, N)`. + solver: variants: svd, lu. + + + Returns: + the computed homography matrix with shape :math:`(B, 3, 3)`. + + """ + if points1.shape != points2.shape: + raise AssertionError(points1.shape) + if points1.shape[1] < 4: + raise AssertionError(points1.shape) + KORNIA_CHECK_SHAPE(points1, ["B", "N", "2"]) + KORNIA_CHECK_SHAPE(points2, ["B", "N", "2"]) + + device, dtype = _extract_device_dtype([points1, points2]) + + eps: float = 1e-8 + points1_norm, transform1 = normalize_points(points1) + points2_norm, transform2 = normalize_points(points2) + + x1, y1 = torch.chunk(points1_norm, dim=-1, chunks=2) # BxNx1 + x2, y2 = torch.chunk(points2_norm, dim=-1, chunks=2) # BxNx1 + ones, zeros = torch.ones_like(x1), torch.zeros_like(x1) + + # DIAPO 11: https://www.uio.no/studier/emner/matnat/its/nedlagte-emner/UNIK4690/v16/forelesninger/lecture_4_3-estimating-homographies-from-feature-correspondences.pdf # noqa: E501 + ax = torch.cat([zeros, zeros, zeros, -x1, -y1, -ones, y2 * x1, y2 * y1, y2], dim=-1) + ay = torch.cat([x1, y1, ones, zeros, zeros, zeros, -x2 * x1, -x2 * y1, -x2], dim=-1) + A = torch.cat((ax, ay), dim=-1).reshape(ax.shape[0], -1, ax.shape[-1]) + + if weights is None: + # All points are equally important + A = A.transpose(-2, -1) @ A + else: + # We should use provided weights + if not (len(weights.shape) == 2 and weights.shape == points1.shape[:2]): + raise AssertionError(weights.shape) + w_full = weights.repeat_interleave(2, dim=1).unsqueeze(1) + A = (A.transpose(-2, -1) * w_full) @ A + + if solver == "svd": + try: + _, _, V = _torch_svd_cast(A) + except RuntimeError: + warnings.warn("SVD did not converge", RuntimeWarning, stacklevel=1) + return torch.empty((points1_norm.size(0), 3, 3), device=device, dtype=dtype) + H = V[..., -1].view(-1, 3, 3) + elif solver == "lu": + B = torch.ones(A.shape[0], A.shape[1], device=device, dtype=dtype) + sol, _, _ = safe_solve_with_mask(B, A) + H = sol.reshape(-1, 3, 3) + else: + raise NotImplementedError + H = safe_inverse_with_mask(transform2)[0] @ (H @ transform1) + H_norm = H / (H[..., -1:, -1:] + eps) + return H_norm + + +def find_homography_dlt_iterated( + points1: torch.Tensor, points2: torch.Tensor, weights: torch.Tensor, soft_inl_th: float = 3.0, n_iter: int = 5 +) -> torch.Tensor: + r"""Compute the homography matrix using the iteratively-reweighted least squares (IRWLS). + + The linear system is solved by using the Reweighted Least Squares Solution for the 4 Points algorithm. + + Args: + points1: A set of points in the first image with a tensor shape :math:`(B, N, 2)`. + points2: A set of points in the second image with a tensor shape :math:`(B, N, 2)`. + weights: Tensor containing the weights per point correspondence with a shape of :math:`(B, N)`. + Used for the first iteration of the IRWLS. + soft_inl_th: Soft inlier threshold used for weight calculation. + n_iter: number of iterations. + + Returns: + the computed homography matrix with shape :math:`(B, 3, 3)`. + + """ + H: torch.Tensor = find_homography_dlt(points1, points2, weights) + for _ in range(n_iter - 1): + errors: torch.Tensor = symmetric_transfer_error(points1, points2, H, False) + weights_new: torch.Tensor = torch.exp(-errors / (2.0 * (soft_inl_th**2))) + H = find_homography_dlt(points1, points2, weights_new) + return H + + +def sample_is_valid_for_homography(points1: torch.Tensor, points2: torch.Tensor) -> torch.Tensor: + """Implement oriented constraint check from :cite:`Marquez-Neila2015`. + + Analogous to https://github.com/opencv/opencv/blob/4.x/modules/calib3d/src/usac/degeneracy.cpp#L88 + + Args: + points1: A set of points in the first image with a tensor shape :math:`(B, 4, 2)`. + points2: A set of points in the second image with a tensor shape :math:`(B, 4, 2)`. + + Returns: + Mask with the minimal sample is good for homography estimation :math:`(B)`. + + """ + if points1.shape != points2.shape: + raise AssertionError(points1.shape) + # Triples to test: (0,1,2), (0,1,3), (0,2,3), (1,2,3) + idx_i = torch.tensor([0, 0, 0, 1], device=points1.device) + J = torch.tensor([1, 1, 2, 2], device=points1.device) + K = torch.tensor([2, 3, 3, 3], device=points1.device) + + # Gather the triples for both sets: shape (B, 4, 2) + p1_i, p1_j, p1_k = points1[:, idx_i], points1[:, J], points1[:, K] + p2_i, p2_j, p2_k = points2[:, idx_i], points2[:, J], points2[:, K] + + # 2D orientation (signed area) for each triple: + # orient(a,b,c) = cross2d(b-a, c-a) = (bx-ax)*(cy-ay) - (by-ay)*(cx-ax) + def _orient(a: torch.Tensor, b: torch.Tensor, c: torch.Tensor) -> torch.Tensor: + ab = b - a + ac = c - a + return ab[..., 0] * ac[..., 1] - ab[..., 1] * ac[..., 0] # shape (B, 4) + + left_sign = torch.sign(_orient(p1_i, p1_j, p1_k)) + right_sign = torch.sign(_orient(p2_i, p2_j, p2_k)) + + # Valid if all four orientation signs match across views + sample_is_valid = (left_sign == right_sign).all(dim=1) + return sample_is_valid + + +def find_homography_lines_dlt( + ls1: torch.Tensor, ls2: torch.Tensor, weights: Optional[torch.Tensor] = None +) -> torch.Tensor: + """Compute the homography matrix using the DLT formulation for line correspondences. + + See :cite:`homolines2001` for details. + + The linear system is solved by using the Weighted Least Squares Solution for the 4 Line correspondences algorithm. + + Args: + ls1: A set of line segments in the first image with a tensor shape :math:`(B, N, 2, 2)`. + ls2: A set of line segments in the second image with a tensor shape :math:`(B, N, 2, 2)`. + weights: Tensor containing the weights per point correspondence with a shape of :math:`(B, N)`. + + Returns: + the computed homography matrix with shape :math:`(B, 3, 3)`. + + """ + if len(ls1.shape) == 3: + ls1 = ls1[None] + if len(ls2.shape) == 3: + ls2 = ls2[None] + KORNIA_CHECK_SHAPE(ls1, ["B", "N", "2", "2"]) + KORNIA_CHECK_SHAPE(ls2, ["B", "N", "2", "2"]) + BS, N = ls1.shape[:2] + device, dtype = _extract_device_dtype([ls1, ls2]) + + points1 = ls1.reshape(BS, 2 * N, 2) + points2 = ls2.reshape(BS, 2 * N, 2) + + points1_norm, transform1 = normalize_points(points1) + points2_norm, transform2 = normalize_points(points2) + lst1, le1 = torch.chunk(points1_norm, dim=1, chunks=2) + lst2, le2 = torch.chunk(points2_norm, dim=1, chunks=2) + + xs1, ys1 = torch.chunk(lst1, dim=-1, chunks=2) # BxNx1 + xs2, ys2 = torch.chunk(lst2, dim=-1, chunks=2) # BxNx1 + xe1, ye1 = torch.chunk(le1, dim=-1, chunks=2) # BxNx1 + xe2, ye2 = torch.chunk(le2, dim=-1, chunks=2) # BxNx1 + + A = ys2 - ye2 + B = xe2 - xs2 + C = xs2 * ye2 - xe2 * ys2 + + eps: float = 1e-8 + + # http://diis.unizar.es/biblioteca/00/09/000902.pdf + ax = torch.cat([A * xs1, A * ys1, A, B * xs1, B * ys1, B, C * xs1, C * ys1, C], dim=-1) + ay = torch.cat([A * xe1, A * ye1, A, B * xe1, B * ye1, B, C * xe1, C * ye1, C], dim=-1) + A = torch.cat((ax, ay), dim=-1).reshape(ax.shape[0], -1, ax.shape[-1]) + + if weights is None: + # All points are equally important + A = A.transpose(-2, -1) @ A + else: + # We should use provided weights + if not ((len(weights.shape) == 2) and (weights.shape == ls1.shape[:2])): + raise AssertionError(weights.shape) + w_diag = torch.diag_embed(weights.unsqueeze(dim=-1).repeat(1, 1, 2).reshape(weights.shape[0], -1)) + A = A.transpose(-2, -1) @ w_diag @ A + + try: + _, _, V = _torch_svd_cast(A) + except RuntimeError: + warnings.warn("SVD did not converge", RuntimeWarning, stacklevel=1) + return torch.empty((points1_norm.size(0), 3, 3), device=device, dtype=dtype) + + H = V[..., -1].view(-1, 3, 3) + H = safe_inverse_with_mask(transform2)[0] @ (H @ transform1) + H_norm = H / (H[..., -1:, -1:] + eps) + return H_norm + + +def find_homography_lines_dlt_iterated( + ls1: torch.Tensor, ls2: torch.Tensor, weights: torch.Tensor, soft_inl_th: float = 4.0, n_iter: int = 5 +) -> torch.Tensor: + r"""Compute the homography matrix using the iteratively-reweighted least squares (IRWLS) from line segments. + + The linear system is solved by using the Reweighted Least Squares Solution for the 4 line segments algorithm. + + Args: + ls1: A set of line segments in the first image with a tensor shape :math:`(B, N, 2, 2)`. + ls2: A set of line segments in the second image with a tensor shape :math:`(B, N, 2, 2)`. + weights: Tensor containing the weights per point correspondence with a shape of :math:`(B, N)`. + Used for the first iteration of the IRWLS. + soft_inl_th: Soft inlier threshold used for weight calculation. + n_iter: number of iterations. + + Returns: + the computed homography matrix with shape :math:`(B, 3, 3)`. + + """ + H: torch.Tensor = find_homography_lines_dlt(ls1, ls2, weights) + for _ in range(n_iter - 1): + errors: torch.Tensor = line_segment_transfer_error_one_way(ls1, ls2, H, False) + weights_new: torch.Tensor = torch.exp(-errors / (2.0 * (soft_inl_th**2))) + H = find_homography_lines_dlt(ls1, ls2, weights_new) + return H diff --git a/kornia/geometry/keypoints.py b/kornia/geometry/keypoints.py new file mode 100644 index 0000000..8b69519 --- /dev/null +++ b/kornia/geometry/keypoints.py @@ -0,0 +1,415 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import List, Optional, Tuple, Union, cast + +import torch +from torch import Size + +from kornia.geometry import transform_points + +__all__ = ["Keypoints", "Keypoints3D"] + + +def _merge_keypoint_list(keypoints: List[torch.torch.Tensor]) -> torch.torch.Tensor: + raise NotImplementedError + + +class Keypoints: + """2D Keypoints containing Nx2 or BxNx2 points. + + Args: + keypoints: Raw tensor or a list of torch.Tensors with the Nx2 coordinates + raise_if_not_floating_point: will raise if the torch.Tensor isn't float + + """ + + def __init__( + self, keypoints: Union[torch.torch.Tensor, List[torch.torch.Tensor]], raise_if_not_floating_point: bool = True + ) -> None: + self._N: Optional[List[int]] = None + + if isinstance(keypoints, list): + keypoints, self._N = _merge_keypoint_list(keypoints) + + if not isinstance(keypoints, torch.torch.Tensor): + raise TypeError(f"Input keypoints is not a torch.Tensor. Got: {type(keypoints)}.") + + if not keypoints.is_floating_point(): + if raise_if_not_floating_point: + raise ValueError(f"Coordinates must be in floating point. Got {keypoints.dtype}") + + keypoints = keypoints.float() + + if len(keypoints.shape) == 0: + # Use reshape, so we don't end up creating a new tensor that does not depend on + # the inputs (and consequently confuses jit) + keypoints = keypoints.reshape((-1, 2)) + + if not (2 <= keypoints.ndim <= 3 and keypoints.shape[-1:] == (2,)): + raise ValueError(f"Keypoints shape must be (N, 2) or (B, N, 2). Got {keypoints.shape}.") + + self._is_batched = False if keypoints.ndim == 2 else True + + self._data = keypoints + + def __getitem__(self, key: Union[slice, int, torch.torch.Tensor]) -> "Keypoints": + new_obj = type(self)(self._data[key], False) + return new_obj + + def __setitem__(self, key: Union[slice, int, torch.torch.Tensor], value: "Keypoints") -> "Keypoints": + self._data[key] = value._data + return self + + @property + def shape(self) -> Union[Tuple[int, ...], Size]: + """Return the tensor shape used to store 2D keypoints. + + Returns: + Shape of :attr:`data`. The common layouts are :math:`(N, 2)` for + unbatched keypoints and :math:`(B, N, 2)` for batched keypoints, + where :math:`B` is the batch size, :math:`N` is the number of + keypoints, and the final dimension stores ``(x, y)`` coordinates. + """ + return self.data.shape + + @property + def data(self) -> torch.torch.Tensor: + """Return the raw 2D keypoint coordinate tensor. + + Returns: + Tensor storing keypoint coordinates in ``(..., 2)`` format, where + the last dimension contains ``x`` and ``y``. + """ + return self._data + + @property + def device(self) -> torch.device: + """Returns keypoints device.""" + return self._data.device + + @property + def dtype(self) -> torch.dtype: + """Returns keypoints dtype.""" + return self._data.dtype + + def index_put( + self, + indices: Union[Tuple[torch.torch.Tensor, ...], List[torch.torch.Tensor]], + values: Union[torch.torch.Tensor, "Keypoints"], + inplace: bool = False, + ) -> "Keypoints": + """Write keypoint coordinates at selected tensor indices. + + Args: + indices: Index tuple or list accepted by ``Tensor.index_put_`` for + the stored coordinate tensor. + values: Replacement coordinates, either as a raw tensor or another + :class:`Keypoints` object. + inplace: If ``True``, update this object in place. Otherwise, + clone the coordinates first and return a new wrapper. + + Returns: + :class:`Keypoints` object containing the updated coordinates. + """ + if inplace: + _data = self._data + else: + _data = self._data.clone() + + if isinstance(values, Keypoints): + _data.index_put_(indices, values.data) + else: + _data.index_put_(indices, values) + + if inplace: + return self + + obj = self.clone() + obj._data = _data + return obj + + def pad(self, padding_size: torch.torch.Tensor) -> "Keypoints": + """Pad a bounding keypoints. + + Args: + padding_size: (B, 4) + + """ + if not (len(padding_size.shape) == 2 and padding_size.size(1) == 4): + raise RuntimeError(f"Expected padding_size as (B, 4). Got {padding_size.shape}.") + self._data[..., 0] += padding_size[..., :1] # left padding + self._data[..., 1] += padding_size[..., 2:3] # top padding + return self + + def unpad(self, padding_size: torch.torch.Tensor) -> "Keypoints": + """Pad a bounding keypoints. + + Args: + padding_size: (B, 4) + + """ + if not (len(padding_size.shape) == 2 and padding_size.size(1) == 4): + raise RuntimeError(f"Expected padding_size as (B, 4). Got {padding_size.shape}.") + self._data[..., 0] -= padding_size[..., :1] # left padding + self._data[..., 1] -= padding_size[..., 2:3] # top padding + return self + + def transform_keypoints(self, M: torch.torch.Tensor, inplace: bool = False) -> "Keypoints": + r"""Apply a transformation matrix to the 2D keypoints. + + Args: + M: The transformation matrix to be applied, shape of :math:`(3, 3)` or :math:`(B, 3, 3)`. + inplace: do transform in-place and return self. + + Returns: + The transformed keypoints. + + """ + if not 2 <= M.ndim <= 3 or M.shape[-2:] != (3, 3): + raise ValueError(f"The transformation matrix shape must be (3, 3) or (B, 3, 3). Got {M.shape}.") + + transformed_boxes = transform_points(M, self._data) + if inplace: + self._data = transformed_boxes + return self + + return Keypoints(transformed_boxes, False) + + def transform_keypoints_(self, M: torch.torch.Tensor) -> "Keypoints": + """Inplace version of :func:`Keypoints.transform_keypoints`.""" + return self.transform_keypoints(M, inplace=True) + + @classmethod + def from_tensor(cls, keypoints: torch.torch.Tensor) -> "Keypoints": + """Validate and wrap a tensor of 2D keypoint coordinates. + + Args: + keypoints: Tensor in :math:`(N, 2)` or :math:`(B, N, 2)` format. + The last dimension stores ``(x, y)`` coordinates. + + Returns: + New :class:`Keypoints` instance containing the input coordinates. + """ + return cls(keypoints) + + def to_tensor(self, as_padded_sequence: bool = False) -> Union[torch.torch.Tensor, List[torch.torch.Tensor]]: + r"""Cast :class:`Keypoints` to a tensor. + + ``mode`` controls which 2D keypoints format should be use to represent keypoints in the tensor. + + Args: + as_padded_sequence: whether to keep the pads for a list of keypoints. This parameter is only valid + if the keypoints are from a keypoint list. + + Returns: + Keypoints tensor :math:`(B, N, 2)` + + """ + if as_padded_sequence: + raise NotImplementedError + return self._data + + def clone(self) -> "Keypoints": + """Create an independent copy of the 2D keypoints.""" + return Keypoints(self._data.clone(), False) + + def type(self, dtype: torch.dtype) -> "Keypoints": + """Cast stored keypoint coordinates to a target dtype. + + Args: + dtype: Destination floating-point dtype for the coordinate tensor. + + Returns: + ``self`` after converting the stored coordinates in place. + """ + self._data = self._data.type(dtype) + return self + + +class VideoKeypoints(Keypoints): + temporal_channel_size: int + + @classmethod + def from_tensor( + cls, boxes: Union[torch.torch.Tensor, List[torch.torch.Tensor]], validate_boxes: bool = True + ) -> "VideoKeypoints": + if isinstance(boxes, (list,)) or (boxes.dim() != 4 or boxes.shape[-1] != 2): + raise ValueError("Input box type is not yet supported. Please input an `BxTxNx2` tensor directly.") + + temporal_channel_size = boxes.size(1) + + # Due to some torch.jit.script bug (at least <= 1.9), you need to pass all arguments to __init__ when + # constructing the class from inside of a method. + out = cls(boxes.view(boxes.size(0) * boxes.size(1), -1, boxes.size(3))) + out.temporal_channel_size = temporal_channel_size + return out + + def to_tensor(self) -> torch.torch.Tensor: # type: ignore[override] + out = super().to_tensor(as_padded_sequence=False) + out = cast(torch.torch.Tensor, out) + return out.view(-1, self.temporal_channel_size, *out.shape[1:]) + + def transform_keypoints(self, M: torch.torch.Tensor, inplace: bool = False) -> "VideoKeypoints": + out = super().transform_keypoints(M, inplace=inplace) + if inplace: + return self + out = VideoKeypoints(out.data, False) + out.temporal_channel_size = self.temporal_channel_size + return out + + def clone(self) -> "VideoKeypoints": + out = VideoKeypoints(self._data.clone(), False) + out.temporal_channel_size = self.temporal_channel_size + return out + + +class Keypoints3D: + """3D Keypoints containing Nx3 or BxNx3 points. + + Args: + keypoints: Raw tensor or a list of torch.Tensors with the Nx3 coordinates + raise_if_not_floating_point: will raise if the torch.Tensor isn't float + + """ + + def __init__( + self, keypoints: Union[torch.torch.Tensor, List[torch.torch.Tensor]], raise_if_not_floating_point: bool = True + ) -> None: + self._N: Optional[List[int]] = None + + if isinstance(keypoints, list): + keypoints, self._N = _merge_keypoint_list(keypoints) + + if not isinstance(keypoints, torch.torch.Tensor): + raise TypeError(f"Input keypoints is not a torch.Tensor. Got: {type(keypoints)}.") + + if not keypoints.is_floating_point(): + if raise_if_not_floating_point: + raise ValueError(f"Coordinates must be in floating point. Got {keypoints.dtype}") + + keypoints = keypoints.float() + + if len(keypoints.shape) == 0: + # Use reshape, so we don't end up creating a new tensor that does not depend on + # the inputs (and consequently confuses jit) + keypoints = keypoints.reshape((-1, 3)) + + if not (2 <= keypoints.ndim <= 3 and keypoints.shape[-1:] == (3,)): + raise ValueError(f"Keypoints shape must be (N, 3) or (B, N, 3). Got {keypoints.shape}.") + + self._is_batched = False if keypoints.ndim == 2 else True + + self._data = keypoints + + def __getitem__(self, key: Union[slice, int, torch.Tensor]) -> "Keypoints3D": + new_obj = type(self)(self._data[key], False) + return new_obj + + def __setitem__(self, key: Union[slice, int, torch.Tensor], value: "Keypoints3D") -> "Keypoints3D": + self._data[key] = value._data + return self + + @property + def shape(self) -> Size: + """Return the tensor shape used to store 3D keypoints. + + Returns: + Shape of :attr:`data`. The common layouts are :math:`(N, 3)` and + :math:`(B, N, 3)`, where the final dimension stores ``(x, y, z)`` + coordinates. + """ + return self.data.shape + + @property + def data(self) -> torch.torch.Tensor: + """Return the raw 3D keypoint coordinate tensor. + + Returns: + Tensor storing coordinates in ``(..., 3)`` format, with the final + dimension ordered as ``(x, y, z)``. + """ + return self._data + + def pad(self, padding_size: torch.torch.Tensor) -> "Keypoints3D": + """Pad a bounding keypoints. + + Args: + padding_size: (B, 6) + + """ + raise NotImplementedError + + def unpad(self, padding_size: torch.torch.Tensor) -> "Keypoints3D": + """Pad a bounding keypoints. + + Args: + padding_size: (B, 6) + + """ + raise NotImplementedError + + def transform_keypoints(self, M: torch.Tensor, inplace: bool = False) -> "Keypoints3D": + r"""Apply a transformation matrix to the 2D keypoints. + + Args: + M: The transformation matrix to be applied, shape of :math:`(3, 3)` or :math:`(B, 3, 3)`. + inplace: do transform in-place and return self. + + Returns: + The transformed keypoints. + + """ + raise NotImplementedError + + def transform_keypoints_(self, M: torch.Tensor) -> "Keypoints3D": + """Inplace version of :func:`Keypoints.transform_keypoints`.""" + return self.transform_keypoints(M, inplace=True) + + @classmethod + def from_tensor(cls, keypoints: torch.Tensor) -> "Keypoints3D": + """Validate and wrap a tensor of 3D keypoint coordinates. + + Args: + keypoints: Tensor in :math:`(N, 3)` or :math:`(B, N, 3)` format, + where the last dimension stores ``(x, y, z)``. + + Returns: + New :class:`Keypoints3D` instance containing the input coordinates. + """ + return cls(keypoints) + + def to_tensor(self, as_padded_sequence: bool = False) -> Union[torch.torch.Tensor, List[torch.torch.Tensor]]: + r"""Cast :class:`Keypoints` to a tensor. + + ``mode`` controls which 2D keypoints format should be use to represent keypoints in the tensor. + + Args: + as_padded_sequence: whether to keep the pads for a list of keypoints. This parameter is only valid + if the keypoints are from a keypoint list. + + Returns: + Keypoints tensor :math:`(B, N, 3)` + + """ + if as_padded_sequence: + raise NotImplementedError + return self._data + + def clone(self) -> "Keypoints3D": + """Create an independent copy of the 3D keypoints.""" + return Keypoints3D(self._data.clone(), False) diff --git a/kornia/geometry/liegroup/__init__.py b/kornia/geometry/liegroup/__init__.py new file mode 100644 index 0000000..b98a6eb --- /dev/null +++ b/kornia/geometry/liegroup/__init__.py @@ -0,0 +1,28 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Geometry Lie Group — Lie group operations for Kornia. + +This subpackage provides SE2, SE3, SO2, and related Lie group classes and utilities. +""" + +from .se2 import Se2 +from .se3 import Se3 +from .so2 import So2 +from .so3 import So3 + +__all__ = ["Se2", "Se3", "So2", "So3"] diff --git a/kornia/geometry/liegroup/se2.py b/kornia/geometry/liegroup/se2.py new file mode 100644 index 0000000..5349d71 --- /dev/null +++ b/kornia/geometry/liegroup/se2.py @@ -0,0 +1,439 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# kornia.geometry.se2 module inspired by Sophus-sympy. +# https://github.com/strasdat/Sophus/blob/master/sympy/sophus/se2.py +from __future__ import annotations + +from typing import Optional, Union, overload + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SAME_DEVICES, KORNIA_CHECK_SHAPE, KORNIA_CHECK_TYPE +from kornia.geometry.liegroup.so2 import So2 +from kornia.geometry.vector import Vector2 + + +def _check_se2_r_t_shape(r: So2, t: torch.Tensor) -> None: + z_shape = r.z.shape + if ((len(z_shape) == 1) and (len(t.shape) == 2)) or ((len(z_shape) == 0) and len(t.shape) == 1): + # check_se2_t_shape + is_batch_shape = KORNIA_CHECK_SHAPE(t, ["B", "2"], raises=False) + is_single_shape = KORNIA_CHECK_SHAPE(t, ["2"], raises=False) + if not (is_batch_shape or is_single_shape): + raise ValueError(f"Invalid translation shape, we expect [B, 2], or [2] Got: {t.shape}") + else: + raise ValueError( + f"Invalid input, both the inputs should be either batched or unbatched. Got: {r.z.shape} and {t.shape}" + ) + + +class Se2(nn.Module): + r"""Base class to represent the Se2 group. + + The SE(2) is the group of rigid body transformations about the origin of two-dimensional Euclidean + space :math:`R^2` under the operation of composition. + See more: + + Example: + >>> so2 = So2.identity(1) + >>> t = torch.ones((1, 2)) + >>> se2 = Se2(so2, t) + >>> se2 + rotation: Parameter containing: + tensor([1.+0.j], requires_grad=True) + translation: Parameter containing: + tensor([[1., 1.]], requires_grad=True) + + """ + + def __init__(self, rotation: So2, translation: Vector2 | torch.Tensor) -> None: + """Construct the base class. + + Internally represented by a torch.complex number `z` and a translation 2-vector. + + Args: + rotation: So2 group encompassing a rotation. + translation: translation vector with the shape of :math:`(B, 2)`. + + Example: + >>> so2 = So2.identity(1) + >>> t = torch.ones((1, 2)) + >>> se2 = Se2(so2, t) + >>> se2 + rotation: Parameter containing: + tensor([1.+0.j], requires_grad=True) + translation: Parameter containing: + tensor([[1., 1.]], requires_grad=True) + + """ + super().__init__() + KORNIA_CHECK_TYPE(rotation, So2) + if not isinstance(translation, (Vector2, torch.Tensor)): + raise TypeError(f"translation type is {type(translation)}") + self._translation: Vector2 | nn.Parameter + self._rotation: So2 = rotation + if isinstance(translation, torch.Tensor): + _check_se2_r_t_shape(rotation, translation) # TODO remove + self._translation = nn.Parameter(translation) + else: + self._translation = translation + + def __repr__(self) -> str: + return f"rotation: {self.r}\ntranslation: {self.t}" + + def __getitem__(self, idx: int | slice) -> Se2: + return Se2(self._rotation[idx], self._translation[idx]) + + def _mul_se2(self, right: Se2) -> Se2: + so2 = self.so2 + t = self.t + _r = so2 * right.so2 + _t = t + so2 * right.t + return Se2(_r, _t) + + @overload + def __mul__(self, right: Se2) -> Se2: ... + + @overload + def __mul__(self, right: torch.Tensor) -> torch.Tensor: ... + + def __mul__(self, right: Se2 | torch.Tensor) -> Se2 | torch.Tensor: + """Compose two Se2 transformations. + + Args: + right: the other Se2 transformation. + + Return: + The resulting Se2 transformation. + + """ + so2 = self.so2 + t = self.t + if isinstance(right, Se2): + KORNIA_CHECK_TYPE(right, Se2) + return self._mul_se2(right) + elif isinstance(right, (Vector2, torch.Tensor)): + # _check_se2_r_t_shape(so2, risght) + return so2 * right + t + else: + raise TypeError(f"Unsupported type: {type(right)}") + + @property + def so2(self) -> So2: + """Return the underlying `rotation(So2)`.""" + return self._rotation + + @property + def r(self) -> So2: + """Return the underlying `rotation(So2)`.""" + return self._rotation + + @property + def t(self) -> Vector2 | nn.Parameter: + """Return the underlying translation vector of shape :math:`(B,2)`.""" + return self._translation + + @property + def rotation(self) -> So2: + """Return the underlying `rotation(So2)`.""" + return self._rotation + + @property + def translation(self) -> Vector2 | nn.Parameter: + """Return the underlying translation vector of shape :math:`(B,2)`.""" + return self._translation + + @staticmethod + def exp(v: torch.Tensor) -> Se2: + """Convert elements of lie algebra to elements of lie group. + + Args: + v: vector of shape :math:`(B, 3)`. + + Example: + >>> v = torch.ones((1, 3)) + >>> s = Se2.exp(v) + >>> s.r + Parameter containing: + tensor([0.5403+0.8415j], requires_grad=True) + >>> s.t + Parameter containing: + tensor([[0.3818, 1.3012]], requires_grad=True) + + """ + # check_v_shape + is_batch = KORNIA_CHECK_SHAPE(v, ["B", "3"], raises=False) + is_single = KORNIA_CHECK_SHAPE(v, ["3"], raises=False) + if not (is_batch or is_single): + raise ValueError(f"Invalid input shape, we expect [B, 3], [3] Got: {v.shape}") + theta = v[..., 2] + so2 = So2.exp(theta) + z = torch.tensor(0.0, device=v.device, dtype=v.dtype) + theta_nonzeros = theta != 0.0 + a = torch.where(theta_nonzeros, so2.z.imag / theta, z) + b = torch.where(theta_nonzeros, (1.0 - so2.z.real) / theta, z) + x = v[..., 0] + y = v[..., 1] + t = torch.stack((a * x - b * y, b * x + a * y), -1) + return Se2(so2, t) + + def log(self) -> torch.Tensor: + """Convert elements of lie group to elements of lie algebra. + + Example: + >>> v = torch.ones((1, 3)) + >>> s = Se2.exp(v).log() + >>> s + tensor([[1.0000, 1.0000, 1.0000]], grad_fn=) + + """ + theta = self.so2.log() + half_theta = 0.5 * theta + denom = self.so2.z.real - 1 + a = torch.where( + denom != 0, + -(half_theta * self.so2.z.imag) / denom, + torch.tensor(0.0, device=theta.device, dtype=theta.dtype), + ) + row0 = torch.stack((a, half_theta), -1) + row1 = torch.stack((-half_theta, a), -1) + V_inv = torch.stack((row0, row1), -2) + upsilon = V_inv @ self.t.data[..., None] + return torch.stack((upsilon[..., 0, 0], upsilon[..., 1, 0], theta), -1) + + @staticmethod + def hat(v: torch.Tensor) -> torch.Tensor: + """Convert elements from vector space to lie algebra. Returns matrix of shape :math:`(B, 3, 3)`. + + Args: + v: vector of shape:math:`(B, 3)`. + + Example: + >>> theta = torch.tensor(3.1415/2) + >>> So2.hat(theta) + tensor([[0.0000, 1.5707], + [1.5707, 0.0000]]) + + """ + # check_v_shape + is_batch = KORNIA_CHECK_SHAPE(v, ["B", "3"], raises=False) + is_single = KORNIA_CHECK_SHAPE(v, ["3"], raises=False) + if not (is_batch or is_single): + raise ValueError(f"Invalid input shape, we expect [B, 3], [3] Got: {v.shape}") + upsilon = torch.stack((v[..., 0], v[..., 1]), -1) + theta = v[..., 2] + col0 = torch.cat((So2.hat(theta), upsilon.unsqueeze(-2)), -2) + return F.pad(col0, (0, 1)) + + @staticmethod + def vee(omega: torch.Tensor) -> torch.Tensor: + """Convert elements from lie algebra to vector space. + + Args: + omega: 3x3-matrix representing lie algebra of shape :math:`(B, 3, 3)`. + + Returns: + vector of shape :math:`(B, 3)`. + + Example: + >>> v = torch.ones(3) + >>> omega_hat = Se2.hat(v) + >>> Se2.vee(omega_hat) + tensor([1., 1., 1.]) + + """ + # check_se2_omega_shape + is_batch = KORNIA_CHECK_SHAPE(omega, ["B", "3", "3"], raises=False) + is_single = KORNIA_CHECK_SHAPE(omega, ["3", "3"], raises=False) + if not (is_batch or is_single): + raise ValueError(f"Invalid input size, we expect [B, 3, 3] or [3, 3]. Got: {omega.shape}") + upsilon = omega[..., 2, :2] + theta = So2.vee(omega[..., :2, :2]) + return torch.cat((upsilon, theta[..., None]), -1) + + @classmethod + def identity( + cls, + batch_size: Optional[int] = None, + device: Union[None, str, torch.device] = None, + dtype: Union[torch.dtype, None] = None, + ) -> Se2: + """Create a Se2 group representing an identity rotation and zero translation. + + Args: + batch_size: the batch size of the underlying data. + device: device to place the result on. + dtype: dtype of the result. + + Example: + >>> s = Se2.identity(1) + >>> s.r + Parameter containing: + tensor([1.+0.j], requires_grad=True) + >>> s.t + x: tensor([0.]) + y: tensor([0.]) + + """ + t: torch.Tensor = torch.tensor([0.0, 0.0], device=device, dtype=dtype) + if batch_size is not None: + KORNIA_CHECK(batch_size >= 1, msg="batch_size must be positive") + t = t.repeat(batch_size, 1) + return cls(So2.identity(batch_size, device, dtype), Vector2(t)) + + def matrix(self) -> torch.Tensor: + """Return the matrix representation of shape :math:`(B, 3, 3)`. + + Example: + >>> s = Se2(So2.identity(1), torch.ones(1, 2)) + >>> s.matrix() + tensor([[[1., -0., 1.], + [0., 1., 1.], + [0., 0., 1.]]], grad_fn=) + + """ + rt = torch.cat((self.r.matrix(), self.t.data[..., None]), -1) + rt_3x3 = F.pad(rt, (0, 0, 0, 1)) # add last row torch.zeros + rt_3x3[..., -1, -1] = 1.0 + return rt_3x3 + + @classmethod + def from_matrix(cls, matrix: torch.Tensor) -> Se2: + """Create an Se2 group from a matrix. + + Args: + matrix: torch.Tensor of shape :math:`(B, 3, 3)`. + + Example: + >>> s = Se2.from_matrix(torch.eye(3).repeat(2, 1, 1)) + >>> s.r + Parameter containing: + tensor([1.+0.j, 1.+0.j], requires_grad=True) + >>> s.t + Parameter containing: + tensor([[0., 0.], + [0., 0.]], requires_grad=True) + + """ + # KORNIA_CHECK_SHAPE(matrix, ["B", "3", "3"]) # FIXME: resolve shape bugs. @edgarriba + r = So2.from_matrix(matrix[..., :2, :2]) + t = matrix[..., :2, -1] + return cls(r, t) + + def inverse(self) -> Se2: + """Return the inverse transformation. + + Example: + >>> s = Se2(So2.identity(1), torch.ones(1,2)) + >>> s_inv = s.inverse() + >>> s_inv.r + Parameter containing: + tensor([1.+0.j], requires_grad=True) + >>> s_inv.t + Parameter containing: + tensor([[-1., -1.]], requires_grad=True) + + """ + r_inv: So2 = self.r.inverse() + _t = -1 * self.t + if isinstance(_t, int): + raise TypeError("Unexpected integer from `-1 * translation`") + + return Se2(r_inv, r_inv * _t) + + @classmethod + def random( + cls, + batch_size: Optional[int] = None, + device: Union[None, str, torch.device] = None, + dtype: Union[torch.dtype, None] = None, + ) -> Se2: + """Create a Se2 group representing a random transformation. + + Args: + batch_size: the batch size of the underlying data. + device: device to place the result on. + dtype: dtype of the result. + + Example: + >>> s = Se2.random() + >>> s = Se2.random(batch_size=3) + + """ + r = So2.random(batch_size, device, dtype) + shape: tuple[int, ...] + if batch_size is None: + shape = (2,) + else: + KORNIA_CHECK(batch_size >= 1, msg="batch_size must be positive") + shape = (batch_size, 2) + return cls(r, Vector2(torch.rand(shape, device=device, dtype=dtype))) + + @classmethod + def trans(cls, x: torch.Tensor, y: torch.Tensor) -> Se2: + """Construct a translation only Se2 instance. + + Args: + x: the x-axis translation. + y: the y-axis translation. + + """ + KORNIA_CHECK(x.shape == y.shape) + KORNIA_CHECK_SAME_DEVICES([x, y]) + batch_size = x.shape[0] if len(x.shape) > 0 else None + rotation = So2.identity(batch_size, x.device, x.dtype) + return cls(rotation, torch.stack((x, y), -1)) + + @classmethod + def trans_x(cls, x: torch.Tensor) -> Se2: + """Construct a x-axis translation. + + Args: + x: the x-axis translation. + + """ + zs = torch.zeros_like(x) + return cls.trans(x, zs) + + @classmethod + def trans_y(cls, y: torch.Tensor) -> Se2: + """Construct a y-axis translation. + + Args: + y: the y-axis translation. + + """ + zs = torch.zeros_like(y) + return cls.trans(zs, y) + + def adjoint(self) -> torch.Tensor: + """Return the adjoint matrix of shape :math:`(B, 3, 3)`. + + Example: + >>> s = Se2.identity() + >>> s.adjoint() + tensor([[1., -0., 0.], + [0., 1., -0.], + [0., 0., 1.]], grad_fn=) + + """ + rt = self.matrix() + rt[..., 0:2, 2] = torch.stack((self.t.data[..., 1], -self.t.data[..., 0]), -1) + return rt diff --git a/kornia/geometry/liegroup/se3.py b/kornia/geometry/liegroup/se3.py new file mode 100644 index 0000000..bc6d65b --- /dev/null +++ b/kornia/geometry/liegroup/se3.py @@ -0,0 +1,497 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# kornia.geometry.so3 module inspired by Sophus-sympy. +# https://github.com/strasdat/Sophus/blob/master/sympy/sophus/se3.py +from __future__ import annotations + +from typing import Optional, Union + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SAME_DEVICES +from kornia.geometry.liegroup.so3 import So3 +from kornia.geometry.linalg import batched_dot_product +from kornia.geometry.quaternion import Quaternion +from kornia.geometry.vector import Vector3 + + +class Se3(nn.Module): + r"""Base class to represent the Se3 group. + + The SE(3) is the group of rigid body transformations about the origin of three-dimensional Euclidean + space :math:`R^3` under the operation of composition. + See more: https://ingmec.ual.es/~jlblanco/papers/jlblanco2010geometry3D_techrep.pdf + + Example: + >>> q = Quaternion.identity() + >>> s = Se3(q, torch.ones(3)) + >>> s.r + tensor([1., 0., 0., 0.]) + >>> s.t + Parameter containing: + tensor([1., 1., 1.], requires_grad=True) + + """ + + def __init__(self, rotation: Quaternion | So3, translation: Vector3 | torch.Tensor) -> None: + """Construct the base class. + + Internally represented by a unit quaternion `q` and a translation 3-vector. + + Args: + rotation: So3 group encompassing a rotation. + translation: Vector3 or translation torch.Tensor with the shape of :math:`(B, 3)`. + + Example: + >>> from kornia.geometry.quaternion import Quaternion + >>> q = Quaternion.identity(batch_size=1) + >>> s = Se3(q, torch.ones((1, 3))) + >>> s.r + tensor([[1., 0., 0., 0.]]) + >>> s.t + Parameter containing: + tensor([[1., 1., 1.]], requires_grad=True) + + """ + super().__init__() + # KORNIA_CHECK_TYPE(rotation, (Quaternion, So3)) + if not isinstance(rotation, (Quaternion, So3)): + raise TypeError(f"rotation type is {type(rotation)}") + # KORNIA_CHECK_TYPE(translation, (Vector3, torch.Tensor)) + if not isinstance(translation, (Vector3, torch.Tensor)): + raise TypeError(f"translation type is {type(translation)}") + # KORNIA_CHECK_SHAPE(t, ["B", "3"]) # FIXME: resolve shape bugs. @edgarriba + self._translation: Vector3 | nn.Parameter + self._rotation: So3 + if isinstance(translation, torch.Tensor): + self._translation = nn.Parameter(translation) + else: + self._translation = translation + if isinstance(rotation, Quaternion): + self._rotation = So3(rotation) + else: + self._rotation = rotation + + def __repr__(self) -> str: + return f"rotation: {self.r}\ntranslation: {self.t}" + + def __getitem__(self, idx: int | slice) -> Se3: + return Se3(self._rotation[idx], self._translation[idx]) + + def _mul_se3(self, right: Se3) -> Se3: + _r = self.r * right.r + _t = self.t + self.r * right.t + return Se3(_r, _t) + + def __mul__(self, right: Se3) -> Se3 | Vector3 | torch.Tensor: + """Compose two Se3 transformations. + + Args: + right: the other Se3 transformation. + + Return: + The resulting Se3 transformation. + + """ + so3 = self.so3 + t = self.t + if isinstance(right, Se3): + # https://github.com/strasdat/Sophus/blob/master/sympy/sophus/se3.py#L97 + return self._mul_se3(right) + elif isinstance(right, (Vector3, torch.Tensor)): + # KORNIA_CHECK_SHAPE(right, ["B", "N"]) # FIXME: resolve shape bugs. @edgarriba + return so3 * right + t.data + else: + raise TypeError(f"Unsupported type: {type(right)}") + + @property + def so3(self) -> So3: + """Return the underlying rotation(So3).""" + return self._rotation + + @property + def quaternion(self) -> Quaternion: + """Return the underlying rotation(Quaternion).""" + return self._rotation.q + + @property + def r(self) -> So3: + """Return the underlying rotation(So3).""" + return self._rotation + + @property + def t(self) -> Vector3 | torch.Tensor: + """Return the underlying translation vector of shape :math:`(B,3)`.""" + return self._translation + + @property + def rotation(self) -> So3: + """Return the underlying `rotation(So3)`.""" + return self._rotation + + @property + def translation(self) -> Vector3 | torch.Tensor: + """Return the underlying translation vector of shape :math:`(B,3)`.""" + return self._translation + + @staticmethod + def exp(v: torch.Tensor) -> Se3: + """Convert elements of lie algebra to elements of lie group. + + Args: + v: vector of shape :math:`(B, 6)`. + + Example: + >>> v = torch.zeros((1, 6)) + >>> s = Se3.exp(v) + >>> s.r + tensor([[1., 0., 0., 0.]]) + >>> s.t + Parameter containing: + tensor([[0., 0., 0.]], requires_grad=True) + + """ + # KORNIA_CHECK_SHAPE(v, ["B", "6"]) # FIXME: resolve shape bugs. @edgarriba + upsilon = v[..., :3] + omega = v[..., 3:] + omega_hat = So3.hat(omega) + omega_hat_sq = omega_hat @ omega_hat + theta = batched_dot_product(omega, omega).sqrt() + R = So3.exp(omega) + V = ( + torch.eye(3, device=v.device, dtype=v.dtype) + + ((1 - torch.cos(theta)) / (theta**2))[..., None, None] * omega_hat + + ((theta - torch.sin(theta)) / (theta**3))[..., None, None] * omega_hat_sq + ) + U = torch.where(theta[..., None] != 0.0, (upsilon[..., None, :] * V).sum(-1), upsilon) + return Se3(R, U) + + def log(self) -> torch.Tensor: + """Convert elements of lie group to elements of lie algebra. + + Example: + >>> from kornia.geometry.quaternion import Quaternion + >>> q = Quaternion.identity() + >>> Se3(q, torch.zeros(3)).log() + tensor([0., 0., 0., 0., 0., 0.]) + + """ + omega = self.r.log() + theta = batched_dot_product(omega, omega).clamp_min(1e-12).sqrt() + t = self.t.data + omega_hat = So3.hat(omega) + omega_hat_sq = omega_hat @ omega_hat + V_inv = ( + torch.eye(3, device=omega.device, dtype=omega.dtype) + - 0.5 * omega_hat + + ((1 - theta * torch.cos(theta / 2) / (2 * torch.sin(theta / 2))) / theta.pow(2))[..., None, None] + * omega_hat_sq + ) + t = torch.where(theta[..., None] != 0.0, (t[..., None, :] * V_inv).sum(-1), t) + return torch.cat((t, omega), -1) + + @staticmethod + def hat(v: torch.Tensor) -> torch.Tensor: + """Convert elements from vector space to lie algebra. + + Args: + v: vector of shape :math:`(B, 6)`. + + Returns: + matrix of shape :math:`(B, 4, 4)`. + + Example: + >>> v = torch.ones((1, 6)) + >>> m = Se3.hat(v) + >>> m + tensor([[[ 0., -1., 1., 1.], + [ 1., 0., -1., 1.], + [-1., 1., 0., 1.], + [ 0., 0., 0., 0.]]]) + + """ + # KORNIA_CHECK_SHAPE(v, ["B", "6"]) # FIXME: resolve shape bugs. @edgarriba + upsilon, omega = v[..., :3], v[..., 3:] + rt = torch.cat((So3.hat(omega), upsilon[..., None]), -1) + return F.pad(rt, (0, 0, 0, 1)) # add torch.zeros bottom + + @staticmethod + def vee(omega: torch.Tensor) -> torch.Tensor: + """Convert elements from lie algebra to vector space. + + Args: + omega: 4x4-matrix representing lie algebra of shape :math:`(B,4,4)`. + + Returns: + vector of shape :math:`(B,6)`. + + Example: + >>> v = torch.ones((1, 6)) + >>> omega_hat = Se3.hat(v) + >>> Se3.vee(omega_hat) + tensor([[1., 1., 1., 1., 1., 1.]]) + + """ + # KORNIA_CHECK_SHAPE(omega, ["B", "4", "4"]) # FIXME: resolve shape bugs. @edgarriba + head = omega[..., :3, -1] + tail = So3.vee(omega[..., :3, :3]) + return torch.cat((head, tail), -1) + + @classmethod + def identity( + cls, + batch_size: Optional[int] = None, + device: Union[None, str, torch.device] = None, + dtype: Union[torch.dtype, None] = None, + ) -> Se3: + """Create a Se3 group representing an identity rotation and zero translation. + + Args: + batch_size: the batch size of the underlying data. + device: device to place the result on. + dtype: dtype of the result. + + Example: + >>> s = Se3.identity() + >>> s.r + tensor([1., 0., 0., 0.]) + >>> s.t + x: 0.0 + y: 0.0 + z: 0.0 + + """ + t = torch.tensor([0.0, 0.0, 0.0], device=device, dtype=dtype) + if batch_size is not None: + t = t.repeat(batch_size, 1) + + return cls(So3.identity(batch_size, device, dtype), Vector3(t)) + + def matrix(self) -> torch.Tensor: + """Return the matrix representation of shape :math:`(B, 4, 4)`. + + Example: + >>> s = Se3(So3.identity(), torch.ones(3)) + >>> s.matrix() + tensor([[1., 0., 0., 1.], + [0., 1., 0., 1.], + [0., 0., 1., 1.], + [0., 0., 0., 1.]]) + + """ + rt = torch.cat((self.r.matrix(), self.t.data[..., None]), -1) + rt_4x4 = F.pad(rt, (0, 0, 0, 1)) # add last row torch.zeros + rt_4x4[..., -1, -1] = 1.0 + return rt_4x4 + + @classmethod + def from_matrix(cls, matrix: torch.Tensor) -> Se3: + """Create a Se3 group from a matrix. + + Args: + matrix: torch.Tensor of shape :math:`(B, 4, 4)`. + + Example: + >>> s = Se3.from_matrix(torch.eye(4)) + >>> s.r + tensor([1., 0., 0., 0.]) + >>> s.t + Parameter containing: + tensor([0., 0., 0.], requires_grad=True) + + """ + # KORNIA_CHECK_SHAPE(matrix, ["B", "4", "4"]) # FIXME: resolve shape bugs. @edgarriba + r = So3.from_matrix(matrix[..., :3, :3]) + t = matrix[..., :3, -1] + return cls(r, t) + + @classmethod + def from_qxyz(cls, qxyz: torch.Tensor) -> Se3: + """Create a Se3 group a quaternion and translation vector. + + Args: + qxyz: torch.Tensor of shape :math:`(B, 7)`. + + Example: + >>> qxyz = torch.tensor([1., 2., 3., 0., 0., 0., 1.]) + >>> s = Se3.from_qxyz(qxyz) + >>> s.r + tensor([1., 2., 3., 0.]) + >>> s.t + x: 0.0 + y: 0.0 + z: 1.0 + + """ + # KORNIA_CHECK_SHAPE(qxyz, ["B", "7"]) # FIXME: resolve shape bugs. @edgarriba + q, xyz = qxyz[..., :4], qxyz[..., 4:] + return cls(So3.from_wxyz(q), Vector3(xyz)) + + def inverse(self) -> Se3: + """Return the inverse transformation. + + Example: + >>> s = Se3(So3.identity(), torch.ones(3)) + >>> s_inv = s.inverse() + >>> s_inv.r + tensor([1., -0., -0., -0.]) + >>> s_inv.t + Parameter containing: + tensor([-1., -1., -1.], requires_grad=True) + + """ + r_inv = self.r.inverse() + _t = -1 * self.t + if isinstance(_t, int): + raise TypeError("Unexpected integer from `-1 * translation`") + + return Se3(r_inv, r_inv * _t) + + @classmethod + def random( + cls, + batch_size: Optional[int] = None, + device: Union[None, str, torch.device] = None, + dtype: Union[torch.dtype, None] = None, + ) -> Se3: + """Create a Se3 group representing a random transformation. + + Args: + batch_size: the batch size of the underlying data. + device: device to place the result on. + dtype: dtype of the result. + + Example: + >>> s = Se3.random() + >>> s = Se3.random(batch_size=3) + + """ + shape: tuple[int, ...] + if batch_size is None: + shape = () + else: + KORNIA_CHECK(batch_size >= 1, msg="batch_size must be positive") + shape = (batch_size,) + r = So3.random(batch_size, device, dtype) + t = Vector3.random(shape, device, dtype) + return cls(r, t) + + @classmethod + def rot_x(cls, x: torch.Tensor) -> Se3: + """Construct a x-axis rotation. + + Args: + x: the x-axis rotation angle. + + """ + zs = torch.zeros_like(x) + return cls(So3.rot_x(x), torch.stack((zs, zs, zs), -1)) + + @classmethod + def rot_y(cls, y: torch.Tensor) -> Se3: + """Construct a y-axis rotation. + + Args: + y: the y-axis rotation angle. + + """ + zs = torch.zeros_like(y) + return cls(So3.rot_y(y), torch.stack((zs, zs, zs), -1)) + + @classmethod + def rot_z(cls, z: torch.Tensor) -> Se3: + """Construct a z-axis rotation. + + Args: + z: the z-axis rotation angle. + + """ + zs = torch.zeros_like(z) + return cls(So3.rot_z(z), torch.stack((zs, zs, zs), -1)) + + @classmethod + def trans(cls, x: torch.Tensor, y: torch.Tensor, z: torch.Tensor) -> Se3: + """Construct a translation only Se3 instance. + + Args: + x: the x-axis translation. + y: the y-axis translation. + z: the z-axis translation. + + """ + KORNIA_CHECK(x.shape == y.shape) + KORNIA_CHECK(y.shape == z.shape) + KORNIA_CHECK_SAME_DEVICES([x, y, z]) + batch_size = x.shape[0] if len(x.shape) > 0 else None + rotation = So3.identity(batch_size, x.device, x.dtype) + return cls(rotation, torch.stack((x, y, z), -1)) + + @classmethod + def trans_x(cls, x: torch.Tensor) -> Se3: + """Construct a x-axis translation. + + Args: + x: the x-axis translation. + + """ + zs = torch.zeros_like(x) + return cls.trans(x, zs, zs) + + @classmethod + def trans_y(cls, y: torch.Tensor) -> Se3: + """Construct a y-axis translation. + + Args: + y: the y-axis translation. + + """ + zs = torch.zeros_like(y) + return cls.trans(zs, y, zs) + + @classmethod + def trans_z(cls, z: torch.Tensor) -> Se3: + """Construct a z-axis translation. + + Args: + z: the z-axis translation. + + """ + zs = torch.zeros_like(z) + return cls.trans(zs, zs, z) + + def adjoint(self) -> torch.Tensor: + """Return the adjoint matrix of shape :math:`(B, 6, 6)`. + + Example: + >>> s = Se3.identity() + >>> s.adjoint() + tensor([[1., 0., 0., 0., 0., 0.], + [0., 1., 0., 0., 0., 0.], + [0., 0., 1., 0., 0., 0.], + [0., 0., 0., 1., 0., 0.], + [0., 0., 0., 0., 1., 0.], + [0., 0., 0., 0., 0., 1.]]) + + """ + R = self.so3.matrix() + z = torch.zeros_like(R) + row0 = torch.cat((R, So3.hat(self.t) @ R), -1) + row1 = torch.cat((z, R), -1) + return torch.cat((row0, row1), -2) diff --git a/kornia/geometry/liegroup/so2.py b/kornia/geometry/liegroup/so2.py new file mode 100644 index 0000000..731f12b --- /dev/null +++ b/kornia/geometry/liegroup/so2.py @@ -0,0 +1,332 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# kornia.geometry.so2 module inspired by Sophus-sympy. +# https://github.com/strasdat/Sophus/blob/master/sympy/sophus/so2.py +from __future__ import annotations + +from typing import Optional, Union, overload + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE +from kornia.geometry.vector import Vector2 + + +class So2(nn.Module): + r"""Base class to represent the So2 group. + + The SO(2) is the group of all rotations about the origin of two-dimensional Euclidean space + :math:`R^2` under the operation of composition. + See more: https://en.wikipedia.org/wiki/Orthogonal_group#Special_orthogonal_group + + We internally represent the rotation by a torch.complex number. + + Example: + >>> real = torch.tensor([1.0]) + >>> imag = torch.tensor([2.0]) + >>> So2(torch.complex(real, imag)) + Parameter containing: + tensor([1.+2.j], requires_grad=True) + + """ + + def __init__(self, z: torch.Tensor) -> None: + """Construct the base class. + + Internally represented by torch.complex number `z`. + + Args: + z: Complex number with the shape of :math:`(B, 1)` or :math:`(B)`. + + Example: + >>> real = torch.tensor(1.0) + >>> imag = torch.tensor(2.0) + >>> So2(torch.complex(real, imag)).z + Parameter containing: + tensor(1.+2.j, requires_grad=True) + + """ + super().__init__() + KORNIA_CHECK_IS_TENSOR(z) + # check_so2_z_shape(z) + is_scalar = len(z.shape) == 0 + is_flat = KORNIA_CHECK_SHAPE(z, ["B"], raises=False) + is_column = KORNIA_CHECK_SHAPE(z, ["B", "1"], raises=False) + + if not (is_scalar or is_flat or is_column): + raise ValueError(f"Invalid input size, we expect [], [B], or [B, 1]. Got: {z.shape}") + self._z = nn.Parameter(z) + + def __repr__(self) -> str: + return f"{self.z}" + + def __getitem__(self, idx: int | slice) -> So2: + return So2(self._z[idx]) + + @overload + def __mul__(self, right: So2) -> So2: ... + + @overload + def __mul__(self, right: torch.Tensor) -> torch.Tensor: ... + + def __mul__(self, right: So2 | torch.Tensor) -> So2 | torch.Tensor: + """Perform a left-multiplication either rotation concatenation or point-transform. + + Args: + right: the other So2 transformation. + + Return: + The resulting So2 transformation. + + """ + z = self.z + if isinstance(right, So2): + return So2(z * right.z) + elif isinstance(right, (Vector2, torch.Tensor)): + if isinstance(right, torch.Tensor): + # check_so2_t_shape + is_batch_shape = KORNIA_CHECK_SHAPE(right, ["B", "2"], raises=False) + is_single_shape = KORNIA_CHECK_SHAPE(right, ["2"], raises=False) + if not (is_batch_shape or is_single_shape): + raise ValueError(f"Invalid translation shape, we expect [B, 2], or [2] Got: {right.shape}") + x = right.data[..., 0] + y = right.data[..., 1] + real = z.real + imag = z.imag + out = torch.stack((real * x - imag * y, imag * x + real * y), -1) + if isinstance(right, torch.Tensor): + return out + else: + return Vector2(out) + else: + raise TypeError(f"Not So2 or torch.Tensor type. Got: {type(right)}") + + @property + def z(self) -> torch.Tensor: + """Return the underlying data with shape :math:`(B, 1)`.""" + return self._z + + @staticmethod + def exp(theta: torch.Tensor) -> So2: + """Convert elements of lie algebra to elements of lie group. + + Args: + theta: angle in radians of shape :math:`(B, 1)` or :math:`(B)`. + + Example: + >>> v = torch.tensor([3.1415/2]) + >>> s = So2.exp(v) + >>> s + Parameter containing: + tensor([4.6329e-05+1.j], requires_grad=True) + + """ + # check_so2_theta_shape + is_scalar = len(theta.shape) == 0 + is_flat = KORNIA_CHECK_SHAPE(theta, ["B"], raises=False) + is_column = KORNIA_CHECK_SHAPE(theta, ["B", "1"], raises=False) + if not (is_scalar or is_flat or is_column): + raise ValueError(f"Invalid input size, we expect [], [B], or [B, 1]. Got: {theta.shape}") + return So2(torch.complex(torch.cos(theta), torch.sin(theta))) + + def log(self) -> torch.Tensor: + """Convert elements of lie group to elements of lie algebra. + + Example: + >>> real = torch.tensor([1.0]) + >>> imag = torch.tensor([3.0]) + >>> So2(torch.complex(real, imag)).log() + tensor([1.2490], grad_fn=) + + """ + return self.z.imag.atan2(self.z.real) + + @staticmethod + def hat(theta: torch.Tensor) -> torch.Tensor: + """Convert elements from vector space to lie algebra. Returns matrix of shape :math:`(B, 2, 2)`. + + Args: + theta: angle in radians of shape :math:`(B)`. + + Example: + >>> theta = torch.tensor(3.1415/2) + >>> So2.hat(theta) + tensor([[0.0000, 1.5707], + [1.5707, 0.0000]]) + + """ + # check_so2_theta_shape + is_scalar = len(theta.shape) == 0 + is_flat = KORNIA_CHECK_SHAPE(theta, ["B"], raises=False) + is_column = KORNIA_CHECK_SHAPE(theta, ["B", "1"], raises=False) + if not (is_scalar or is_flat or is_column): + raise ValueError(f"Invalid input size, we expect [], [B], or [B, 1]. Got: {theta.shape}") + z = torch.zeros_like(theta) + row0 = torch.stack((z, theta), -1) + row1 = torch.stack((theta, z), -1) + return torch.stack((row0, row1), -1) + + @staticmethod + def vee(omega: torch.Tensor) -> torch.Tensor: + """Convert elements from lie algebra to vector space. Returns vector of shape :math:`(B,)`. + + Args: + omega: 2x2-matrix representing lie algebra. + + Example: + >>> v = torch.ones(3) + >>> omega = So2.hat(v) + >>> So2.vee(omega) + tensor([1., 1., 1.]) + + """ + # check_so2_matrix_shape + is_batch = KORNIA_CHECK_SHAPE(omega, ["B", "2", "2"], raises=False) + is_single = KORNIA_CHECK_SHAPE(omega, ["2", "2"], raises=False) + if not (is_batch or is_single): + raise ValueError(f"Invalid input size, we expect [B, 2, 2] or [2, 2]. Got: {omega.shape}") + return omega[..., 0, 1] + + def matrix(self) -> torch.Tensor: + """Convert the torch.complex number to a rotation matrix of shape :math:`(B, 2, 2)`. + + Example: + >>> s = So2.identity() + >>> m = s.matrix() + >>> m + tensor([[1., -0.], + [0., 1.]], grad_fn=) + + """ + row0 = torch.stack((self.z.real, -self.z.imag), -1) + row1 = torch.stack((self.z.imag, self.z.real), -1) + return torch.stack((row0, row1), -2) + + @classmethod + def from_matrix(cls, matrix: torch.Tensor) -> So2: + """Create So2 from a rotation matrix. + + Args: + matrix: the rotation matrix to convert of shape :math:`(B, 2, 2)`. + + Example: + >>> m = torch.eye(2) + >>> s = So2.from_matrix(m) + >>> s.z + Parameter containing: + tensor(1.+0.j, requires_grad=True) + + """ + # check_so2_matrix_shape + is_batch = KORNIA_CHECK_SHAPE(matrix, ["B", "2", "2"], raises=False) + is_single = KORNIA_CHECK_SHAPE(matrix, ["2", "2"], raises=False) + if not (is_batch or is_single): + raise ValueError(f"Invalid input size, we expect [B, 2, 2] or [2, 2]. Got: {matrix.shape}") + # check_so2_matrix + KORNIA_CHECK_IS_TENSOR(matrix) + if len(matrix.shape) < 2 or matrix.shape[-2:] != (2, 2): + raise ValueError(f"Input size must be (*, 2, 2). Got {matrix.shape}") + mask_diag = torch.allclose(matrix[..., 0, 0], matrix[..., 1, 1]) + mask_off_diag = torch.allclose(matrix[..., 0, 1], -matrix[..., 1, 0]) + if not (mask_diag and mask_off_diag): + raise ValueError("Invalid SO2 rotation matrix: constraints m00==m11 and m01==-m10 not met.") + z = torch.complex(matrix[..., 0, 0], matrix[..., 1, 0]) + return cls(z) + + @classmethod + def identity( + cls, + batch_size: Optional[int] = None, + device: Union[None, str, torch.device] = None, + dtype: Union[None, torch.dtype] = None, + ) -> So2: + """Create a So2 group representing an identity rotation. + + Args: + batch_size: the batch size of the underlying data. + device: device to place the result on. + dtype: dtype of the result. + + Example: + >>> s = So2.identity(batch_size=2) + >>> s + Parameter containing: + tensor([1.+0.j, 1.+0.j], requires_grad=True) + + """ + real_data = torch.tensor(1.0, device=device, dtype=dtype) + imag_data = torch.tensor(0.0, device=device, dtype=dtype) + if batch_size is not None: + KORNIA_CHECK(batch_size >= 1, msg="batch_size must be positive") + real_data = real_data.repeat(batch_size) + imag_data = imag_data.repeat(batch_size) + return cls(torch.complex(real_data, imag_data)) + + def inverse(self) -> So2: + """Return the inverse transformation. + + Example: + >>> s = So2.identity() + >>> s.inverse().z + Parameter containing: + tensor(1.+0.j, requires_grad=True) + + """ + return So2(1 / self.z) + + @classmethod + def random( + cls, + batch_size: Optional[int] = None, + device: Union[None, str, torch.device] = None, + dtype: Union[None, torch.dtype] = None, + ) -> So2: + """Create a So2 group representing a random rotation. + + Args: + batch_size: the batch size of the underlying data. + device: device to place the result on. + dtype: dtype of the result. + + Example: + >>> s = So2.random() + >>> s = So2.random(batch_size=3) + + """ + if batch_size is not None: + KORNIA_CHECK(batch_size >= 1, msg="batch_size must be positive") + real_data = torch.rand((batch_size,), device=device, dtype=dtype) + imag_data = torch.rand((batch_size,), device=device, dtype=dtype) + else: + real_data = torch.rand((), device=device, dtype=dtype) + imag_data = torch.rand((), device=device, dtype=dtype) + return cls(torch.complex(real_data, imag_data)) + + def adjoint(self) -> torch.Tensor: + """Return the adjoint matrix of shape :math:`(B, 2, 2)`. + + Example: + >>> s = So2.identity() + >>> s.adjoint() + tensor([[1., -0.], + [0., 1.]], grad_fn=) + + """ + batch_size = len(self.z) if len(self.z.shape) > 0 else None + return self.identity(batch_size, self.z.device, self.z.real.dtype).matrix() diff --git a/kornia/geometry/liegroup/so3.py b/kornia/geometry/liegroup/so3.py new file mode 100644 index 0000000..77e79ae --- /dev/null +++ b/kornia/geometry/liegroup/so3.py @@ -0,0 +1,451 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# kornia.geometry.so3 module inspired by Sophus-sympy. +# https://github.com/strasdat/Sophus/blob/master/sympy/sophus/so3.py +from __future__ import annotations + +from typing import Optional, Union + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK_TYPE +from kornia.geometry.conversions import vector_to_skew_symmetric_matrix +from kornia.geometry.linalg import batched_dot_product +from kornia.geometry.quaternion import Quaternion +from kornia.geometry.vector import Vector3 + + +class So3(nn.Module): + r"""Base class to represent the So3 group. + + The SO(3) is the group of all rotations about the origin of three-dimensional Euclidean space + :math:`R^3` under the operation of composition. + See more: https://en.wikipedia.org/wiki/3D_rotation_group + + We internally represent the rotation by a unit quaternion. + + Example: + >>> q = Quaternion.identity() + >>> s = So3(q) + >>> s.q + tensor([1., 0., 0., 0.]) + + """ + + def __init__(self, q: Quaternion) -> None: + """Construct the base class. + + Internally represented by a unit quaternion `q`. + + Args: + q: Quaternion with the shape of :math:`(B, 4)`. + + Example: + >>> data = torch.ones((2, 4)) + >>> q = Quaternion(data) + >>> So3(q) + tensor([[1., 1., 1., 1.], + [1., 1., 1., 1.]]) + + """ + super().__init__() + KORNIA_CHECK_TYPE(q, Quaternion) + self._q = q + + def __repr__(self) -> str: + return f"{self.q}" + + def __getitem__(self, idx: int | slice) -> So3: + return So3(self._q[idx]) + + def __mul__(self, right: So3) -> So3: + """Compose two So3 transformations. + + Args: + right: the other So3 transformation. + + Return: + The resulting So3 transformation. + + """ + # https://github.com/strasdat/Sophus/blob/master/sympy/sophus/so3.py#L98 + if isinstance(right, So3): + return So3(self.q * right.q) + elif isinstance(right, (torch.Tensor, Vector3)): + # KORNIA_CHECK_SHAPE(right, ["B", "3"]) # FIXME: resolve shape bugs. @edgarriba + w = torch.zeros(*right.shape[:-1], 1, device=right.device, dtype=right.dtype) + quat = Quaternion(torch.cat((w, right.data), -1)) + out = (self.q * quat * self.q.conj()).vec + if isinstance(right, torch.Tensor): + return out + elif isinstance(right, Vector3): + return Vector3(out) + else: + raise TypeError(f"Not So3 or torch.Tensor type. Got: {type(right)}") + + @property + def q(self) -> Quaternion: + """Return the underlying data with shape :math:`(B,4)`.""" + return self._q + + @staticmethod + def exp(v: torch.Tensor) -> So3: + """Convert elements of lie algebra to elements of lie group. + + See more: https://vision.in.tum.de/_media/members/demmeln/nurlanov2021so3log.pdf + + Args: + v: vector of shape :math:`(B,3)`. + + Example: + >>> v = torch.zeros((2, 3)) + >>> s = So3.exp(v) + >>> s + tensor([[1., 0., 0., 0.], + [1., 0., 0., 0.]]) + + """ + # KORNIA_CHECK_SHAPE(v, ["B", "3"]) # FIXME: resolve shape bugs. @edgarriba + theta = v.norm(dim=-1, keepdim=True) + theta_half = 0.5 * theta + w = torch.cos(theta_half) + eps = torch.finfo(v.dtype).eps * 1e3 + small_mask = theta <= eps + b_large = torch.sin(theta_half) / theta + b_small = 0.5 - (theta * theta) / 48.0 + b = torch.where(small_mask, b_small, b_large) + xyz = b * v + q = torch.cat((w, xyz), dim=-1) + return So3(Quaternion(q)) + + def log(self) -> torch.Tensor: + """Convert elements of lie group to elements of lie algebra. + + Example: + >>> data = torch.ones((2, 4)) + >>> q = Quaternion(data) + >>> So3(q).log() + tensor([[0., 0., 0.], + [0., 0., 0.]]) + + """ + theta = batched_dot_product(self.q.vec, self.q.vec).sqrt() + # NOTE: this differs from https://github.com/strasdat/Sophus/blob/master/sympy/sophus/so3.py#L33 + omega = torch.where( + theta[..., None] != 0, + 2 * self.q.real[..., None].acos() * self.q.vec / theta[..., None], + 2 * self.q.vec / self.q.real[..., None], + ) + return omega + + @staticmethod + def hat(v: Vector3 | torch.Tensor) -> torch.Tensor: + """Convert elements from vector space to lie algebra. Returns matrix of shape :math:`(B,3,3)`. + + Args: + v: Vector3 or torch.Tensor of shape :math:`(B,3)`. + + Example: + >>> v = torch.ones((1,3)) + >>> m = So3.hat(v) + >>> m + tensor([[[ 0., -1., 1.], + [ 1., 0., -1.], + [-1., 1., 0.]]]) + + """ + # KORNIA_CHECK_SHAPE(v, ["B", "3"]) # FIXME: resolve shape bugs. @edgarriba + if isinstance(v, torch.Tensor): + # TODO: Figure out why mypy think `v` can be a Vector3 which didn't allow ellipsis on index + a, b, c = v[..., 0], v[..., 1], v[..., 2] # type: ignore[index] + else: + a, b, c = v.x, v.y, v.z + z = torch.zeros_like(a) + row0 = torch.stack((z, -c, b), -1) + row1 = torch.stack((c, z, -a), -1) + row2 = torch.stack((-b, a, z), -1) + return torch.stack((row0, row1, row2), -2) + + @staticmethod + def vee(omega: torch.Tensor) -> torch.Tensor: + r"""Convert elements from lie algebra to vector space. Returns vector of shape :math:`(B,3)`. + + .. math:: + omega = \begin{bmatrix} 0 & -c & b \\ + c & 0 & -a \\ + -b & a & 0\end{bmatrix} + + Args: + omega: 3x3-matrix representing lie algebra. + + Example: + >>> v = torch.ones((1,3)) + >>> omega = So3.hat(v) + >>> So3.vee(omega) + tensor([[1., 1., 1.]]) + + """ + # KORNIA_CHECK_SHAPE(omega, ["B", "3", "3"]) # FIXME: resolve shape bugs. @edgarriba + a, b, c = omega[..., 2, 1], omega[..., 0, 2], omega[..., 1, 0] + return torch.stack((a, b, c), -1) + + def matrix(self) -> torch.Tensor: + r"""Convert the quaternion to a rotation matrix of shape :math:`(B,3,3)`. + + The matrix is of the form: + + .. math:: + \begin{bmatrix} 1-2y^2-2z^2 & 2xy-2zw & 2xy+2yw \\ + 2xy+2zw & 1-2x^2-2z^2 & 2yz-2xw \\ + 2xz-2yw & 2yz+2xw & 1-2x^2-2y^2\end{bmatrix} + + Example: + >>> s = So3.identity() + >>> m = s.matrix() + >>> m + tensor([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) + + """ + w = self.q.w[..., None] + x, y, z = self.q.x[..., None], self.q.y[..., None], self.q.z[..., None] + q0 = 1 - 2 * y**2 - 2 * z**2 + q1 = 2 * x * y - 2 * z * w + q2 = 2 * x * z + 2 * y * w + row0 = torch.cat((q0, q1, q2), -1) + q0 = 2 * x * y + 2 * z * w + q1 = 1 - 2 * x**2 - 2 * z**2 + q2 = 2 * y * z - 2 * x * w + row1 = torch.cat((q0, q1, q2), -1) + q0 = 2 * x * z - 2 * y * w + q1 = 2 * y * z + 2 * x * w + q2 = 1 - 2 * x**2 - 2 * y**2 + row2 = torch.cat((q0, q1, q2), -1) + return torch.stack((row0, row1, row2), -2) + + @classmethod + def from_matrix(cls, matrix: torch.Tensor) -> So3: + """Create So3 from a rotation matrix. + + Args: + matrix: the rotation matrix to convert of shape :math:`(B,3,3)`. + + Example: + >>> m = torch.eye(3) + >>> s = So3.from_matrix(m) + >>> s + tensor([1., 0., 0., 0.]) + + """ + return cls(Quaternion.from_matrix(matrix)) + + @classmethod + def from_wxyz(cls, wxyz: torch.Tensor) -> So3: + """Create So3 from a torch.Tensor representing a quaternion. + + Args: + wxyz: the quaternion to convert of shape :math:`(B,4)`. + + Example: + >>> q = torch.tensor([1., 0., 0., 0.]) + >>> s = So3.from_wxyz(q) + >>> s + tensor([1., 0., 0., 0.]) + + """ + # KORNIA_CHECK_SHAPE(wxyz, ["B", "4"]) # FIXME: resolve shape bugs. @edgarriba + return cls(Quaternion(wxyz)) + + @classmethod + def identity( + cls, + batch_size: Optional[int] = None, + device: Union[None, str, torch.device] = None, + dtype: Union[None, torch.dtype] = None, + ) -> So3: + """Create a So3 group representing an identity rotation. + + Args: + batch_size: the batch size of the underlying data. + device: device to place the result on. + dtype: dtype of the result. + + Example: + >>> s = So3.identity() + >>> s + tensor([1., 0., 0., 0.]) + + >>> s = So3.identity(batch_size=2) + >>> s + tensor([[1., 0., 0., 0.], + [1., 0., 0., 0.]]) + + """ + return cls(Quaternion.identity(batch_size, device, dtype)) + + def inverse(self) -> So3: + """Return the inverse transformation. + + Example: + >>> s = So3.identity() + >>> s.inverse() + tensor([1., -0., -0., -0.]) + + """ + return So3(self.q.conj()) + + @classmethod + def random( + cls, + batch_size: Optional[int] = None, + device: Union[None, str, torch.device] = None, + dtype: Union[None, torch.dtype] = None, + ) -> So3: + """Create a So3 group representing a random rotation. + + Args: + batch_size: the batch size of the underlying data. + device: device to place the result on. + dtype: dtype of the result. + + Example: + >>> s = So3.random() + >>> s = So3.random(batch_size=3) + + """ + return cls(Quaternion.random(batch_size, device, dtype)) + + @classmethod + def rot_x(cls, x: torch.Tensor) -> So3: + """Construct a x-axis rotation. + + Args: + x: the x-axis rotation angle. + + """ + zs = torch.zeros_like(x) + return cls.exp(torch.stack((x, zs, zs), -1)) + + @classmethod + def rot_y(cls, y: torch.Tensor) -> So3: + """Construct a z-axis rotation. + + Args: + y: the y-axis rotation angle. + + """ + zs = torch.zeros_like(y) + return cls.exp(torch.stack((zs, y, zs), -1)) + + @classmethod + def rot_z(cls, z: torch.Tensor) -> So3: + """Construct a z-axis rotation. + + Args: + z: the z-axis rotation angle. + + """ + zs = torch.zeros_like(z) + return cls.exp(torch.stack((zs, zs, z), -1)) + + def adjoint(self) -> torch.Tensor: + """Return the adjoint matrix of shape :math:`(B, 3, 3)`. + + Example: + >>> s = So3.identity() + >>> s.adjoint() + tensor([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) + + """ + return self.matrix() + + @staticmethod + def right_jacobian(vec: torch.Tensor) -> torch.Tensor: + """Compute the right Jacobian of So3. + + Args: + vec: the input point of shape :math:`(B, 3)`. + + Example: + >>> vec = torch.tensor([1., 2., 3.]) + >>> So3.right_jacobian(vec) + tensor([[-0.0687, 0.5556, -0.0141], + [-0.2267, 0.1779, 0.6236], + [ 0.5074, 0.3629, 0.5890]]) + + """ + # KORNIA_CHECK_SHAPE(vec, ["B", "3"]) # FIXME: resolve shape bugs. @edgarriba + R_skew = vector_to_skew_symmetric_matrix(vec) + theta = vec.norm(dim=-1, keepdim=True)[..., None] + I = torch.eye(3, device=vec.device, dtype=vec.dtype) # noqa: E741 + Jr = ( + I + - ((1 - torch.cos(theta)) / theta**2) * R_skew + + ((theta - torch.sin(theta)) / theta**3) * (R_skew @ R_skew) + ) + return Jr + + @staticmethod + def Jr(vec: torch.Tensor) -> torch.Tensor: + """Alias for right jacobian. + + Args: + vec: the input point of shape :math:`(B, 3)`. + + """ + return So3.right_jacobian(vec) + + @staticmethod + def left_jacobian(vec: torch.Tensor) -> torch.Tensor: + """Compute the left Jacobian of So3. + + Args: + vec: the input point of shape :math:`(B, 3)`. + + Example: + >>> vec = torch.tensor([1., 2., 3.]) + >>> So3.left_jacobian(vec) + tensor([[-0.0687, -0.2267, 0.5074], + [ 0.5556, 0.1779, 0.3629], + [-0.0141, 0.6236, 0.5890]]) + + """ + # KORNIA_CHECK_SHAPE(vec, ["B", "3"]) # FIXME: resolve shape bugs. @edgarriba + R_skew = vector_to_skew_symmetric_matrix(vec) + theta = vec.norm(dim=-1, keepdim=True)[..., None] + I = torch.eye(3, device=vec.device, dtype=vec.dtype) # noqa: E741 + Jl = ( + I + + ((1 - torch.cos(theta)) / theta**2) * R_skew + + ((theta - torch.sin(theta)) / theta**3) * (R_skew @ R_skew) + ) + return Jl + + @staticmethod + def Jl(vec: torch.Tensor) -> torch.Tensor: + """Alias for left jacobian. + + Args: + vec: the input point of shape :math:`(B, 3)`. + + """ + return So3.left_jacobian(vec) diff --git a/kornia/geometry/linalg.py b/kornia/geometry/linalg.py new file mode 100644 index 0000000..7e14f4c --- /dev/null +++ b/kornia/geometry/linalg.py @@ -0,0 +1,303 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch + +from kornia.core.check import KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE +from kornia.geometry.conversions import convert_points_from_homogeneous, convert_points_to_homogeneous + +__all__ = [ + "batched_dot_product", + "batched_squared_norm", + "compose_transformations", + "euclidean_distance", + "inverse_transformation", + "point_line_distance", + "relative_transformation", + "squared_norm", + "transform_points", +] + + +def compose_transformations(trans_01: torch.Tensor, trans_12: torch.Tensor) -> torch.Tensor: + r"""Compose two homogeneous transformations. + + .. math:: + T_0^{2} = \begin{bmatrix} R_0^1 R_1^{2} & R_0^{1} t_1^{2} + t_0^{1} \ + \\mathbf{0} & 1\end{bmatrix} + + Args: + trans_01: tensor with the homogeneous transformation from + a reference frame 1 respect to a frame 0. The tensor has must have a + shape of :math:`(N, 4, 4)` or :math:`(4, 4)`. + trans_12: tensor with the homogeneous transformation from + a reference frame 2 respect to a frame 1. The tensor has must have a + shape of :math:`(N, 4, 4)` or :math:`(4, 4)`. + + Returns: + the transformation between the two frames with shape :math:`(N, 4, 4)` or :math:`(4, 4)`. + + Example:: + >>> trans_01 = torch.eye(4) # 4x4 + >>> trans_12 = torch.eye(4) # 4x4 + >>> trans_02 = compose_transformations(trans_01, trans_12) # 4x4 + + """ + KORNIA_CHECK_IS_TENSOR(trans_01) + KORNIA_CHECK_IS_TENSOR(trans_12) + + if not ((trans_01.dim() in (2, 3)) and (trans_01.shape[-2:] == (4, 4))): + raise ValueError(f"Input trans_01 must be a of the shape Nx4x4 or 4x4. Got {trans_01.shape}") + + if not ((trans_12.dim() in (2, 3)) and (trans_12.shape[-2:] == (4, 4))): + raise ValueError(f"Input trans_12 must be a of the shape Nx4x4 or 4x4. Got {trans_12.shape}") + + if trans_01.dim() != trans_12.dim(): + raise ValueError(f"Input number of dims must match. Got {trans_01.dim()} and {trans_12.dim()}") + + # unpack input data + rmat_01 = trans_01[..., :3, :3] + rmat_12 = trans_12[..., :3, :3] + tvec_01 = trans_01[..., :3, 3:] + tvec_12 = trans_12[..., :3, 3:] + + # compute the actual transforms composition + rmat_02 = torch.matmul(rmat_01, rmat_12) + tvec_02 = torch.matmul(rmat_01, tvec_12) + tvec_01 + + trans_02 = trans_01.new_zeros(trans_01.shape) + trans_02[..., :3, :3] = rmat_02 + trans_02[..., :3, 3:] = tvec_02 + trans_02[..., 3, 3] = 1.0 + return trans_02 + + +def inverse_transformation(trans_12: torch.Tensor) -> torch.Tensor: + r"""Invert a 4x4 homogeneous transformation. + + :math:`T_1^{2} = \begin{bmatrix} R_1 & t_1 \\ \mathbf{0} & 1 \end{bmatrix}` + + The inverse transformation is computed as follows: + + .. math:: + + T_2^{1} = (T_1^{2})^{-1} = \begin{bmatrix} R_1^T & -R_1^T t_1 \\ + \mathbf{0} & 1\end{bmatrix} + + Args: + trans_12: transformation tensor of shape :math:`(N, 4, 4)` or :math:`(4, 4)`. + + Returns: + tensor with inverted transformations with shape :math:`(N, 4, 4)` or :math:`(4, 4)`. + + Example: + >>> trans_12 = torch.rand(1, 4, 4) # Nx4x4 + >>> trans_21 = inverse_transformation(trans_12) # Nx4x4 + + """ + KORNIA_CHECK_IS_TENSOR(trans_12) + + if not ((trans_12.dim() in (2, 3)) and (trans_12.shape[-2:] == (4, 4))): + raise ValueError(f"Input size must be a Nx4x4 or 4x4. Got {trans_12.shape}") + # unpack input tensor + rmat_12 = trans_12[..., :3, :3] # Nx3x3 or 3x3 + tvec_12 = trans_12[..., :3, 3:4] # Nx3x1 or 3x1 + + # compute the actual inverse + rmat_21 = rmat_12.transpose(-1, -2) + tvec_21 = torch.matmul(-rmat_21, tvec_12) + + # pack to output tensor + trans_21 = trans_12.new_zeros(trans_12.shape) + trans_21[..., :3, :3].copy_(rmat_21) + trans_21[..., :3, 3:4].copy_(tvec_21) + trans_21[..., 3, 3] = 1.0 + return trans_21 + + +def relative_transformation(trans_01: torch.Tensor, trans_02: torch.Tensor) -> torch.Tensor: + r"""Compute the relative homogeneous transformation from a reference transformation. + + :math:`T_1^{0} = \begin{bmatrix} R_1 & t_1 \\ \mathbf{0} & 1 \end{bmatrix}` to destination :math:`T_2^{0} = + \begin{bmatrix} R_2 & t_2 \\ \mathbf{0} & 1 \end{bmatrix}`. + + The relative transformation is computed as follows: + + .. math:: + + T_1^{2} = (T_0^{1})^{-1} \cdot T_0^{2} + + Args: + trans_01: reference transformation tensor of shape :math:`(N, 4, 4)` or :math:`(4, 4)`. + trans_02: destination transformation tensor of shape :math:`(N, 4, 4)` or :math:`(4, 4)`. + + Returns: + the relative transformation between the transformations with shape :math:`(N, 4, 4)` or :math:`(4, 4)`. + + Example:: + >>> trans_01 = torch.eye(4) # 4x4 + >>> trans_02 = torch.eye(4) # 4x4 + >>> trans_12 = relative_transformation(trans_01, trans_02) # 4x4 + + """ + KORNIA_CHECK_IS_TENSOR(trans_01) + KORNIA_CHECK_IS_TENSOR(trans_02) + if not ((trans_01.dim() in (2, 3)) and (trans_01.shape[-2:] == (4, 4))): + raise ValueError(f"Input must be a of the shape Nx4x4 or 4x4. Got {trans_01.shape}") + if not ((trans_02.dim() in (2, 3)) and (trans_02.shape[-2:] == (4, 4))): + raise ValueError(f"Input must be a of the shape Nx4x4 or 4x4. Got {trans_02.shape}") + if not trans_01.dim() == trans_02.dim(): + raise ValueError(f"Input number of dims must match. Got {trans_01.dim()} and {trans_02.dim()}") + + rmat_01 = trans_01[..., :3, :3] + tvec_01 = trans_01[..., :3, 3:4] + rmat_02 = trans_02[..., :3, :3] + tvec_02 = trans_02[..., :3, 3:4] + rmat_10 = rmat_01.transpose(-1, -2) + rmat_12 = torch.matmul(rmat_10, rmat_02) + tvec_12 = torch.matmul(rmat_10, tvec_02 - tvec_01) + trans_12 = torch.zeros_like(trans_01) + trans_12[..., :3, :3] = rmat_12 + trans_12[..., :3, 3:4] = tvec_12 + trans_12[..., 3, 3] = 1.0 + + return trans_12 + + +def transform_points(trans_01: torch.Tensor, points_1: torch.Tensor) -> torch.Tensor: + r"""Apply transformations to a set of points. + + Args: + trans_01: tensor for transformations of shape + :math:`(B, D+1, D+1)`. + points_1: tensor of points of shape :math:`(B, N, D)`. + + Returns: + a tensor of N-dimensional points. + + Shape: + - Output: :math:`(B, N, D)` + + Examples: + >>> points_1 = torch.rand(2, 4, 3) # BxNx3 + >>> trans_01 = torch.eye(4).view(1, 4, 4) # Bx4x4 + >>> points_0 = transform_points(trans_01, points_1) # BxNx3 + + """ + KORNIA_CHECK_IS_TENSOR(trans_01) + KORNIA_CHECK_IS_TENSOR(points_1) + if not trans_01.shape[0] == points_1.shape[0] and trans_01.shape[0] != 1: + raise ValueError( + f"Input batch size must be the same for both tensors or 1. Got {trans_01.shape} and {points_1.shape}" + ) + if not trans_01.shape[-1] == (points_1.shape[-1] + 1): + raise ValueError(f"Last input dimensions must differ by one unit Got{trans_01} and {points_1}") + + # We reshape to BxNxD in case we get more dimensions, e.g., MxBxNxD + shape_inp = list(points_1.shape) + points_1 = points_1.reshape(-1, points_1.shape[-2], points_1.shape[-1]) + trans_01 = trans_01.reshape(-1, trans_01.shape[-2], trans_01.shape[-1]) + # We expand trans_01 to match the dimensions needed for bmm. repeats input division is cast + # to integer so onnx doesn't record the value as a tensor and get a device mismatch + trans_01 = torch.repeat_interleave(trans_01, repeats=int(points_1.shape[0] // trans_01.shape[0]), dim=0) + # to homogeneous + points_1_h = convert_points_to_homogeneous(points_1) # BxNxD+1 + # `torch.bmm` requires both operands to share a scalar type. Callers may pass coordinate + # tensors in a different dtype than the transform matrix (e.g. fp32 bbox + fp16 image), so + # harmonise at the bmm site and restore the caller's coordinate dtype on return. + points_dtype = points_1_h.dtype + # transform coordinates + points_0_h = torch.bmm(points_1_h.to(trans_01.dtype), trans_01.permute(0, 2, 1)) + # to euclidean + points_0 = convert_points_from_homogeneous(points_0_h) # BxNxD + # reshape to the input shape + shape_inp[-2] = points_0.shape[-2] + shape_inp[-1] = points_0.shape[-1] + points_0 = points_0.reshape(shape_inp) + return points_0.to(points_dtype) + + +def point_line_distance(point: torch.Tensor, line: torch.Tensor, eps: float = 1e-9) -> torch.Tensor: + r"""Return the distance from points to lines. + + Args: + point: (possibly homogeneous) points :math:`(*, N, 2 or 3)`. + line: lines coefficients :math:`(a, b, c)` with shape :math:`(*, N, 3)`, where :math:`ax + by + c = 0`. + eps: Small constant for safe sqrt. + + Returns: + the computed distance with shape :math:`(*, N)`. + + """ + KORNIA_CHECK_IS_TENSOR(point) + KORNIA_CHECK_IS_TENSOR(line) + + if point.shape[-1] not in (2, 3): + raise ValueError(f"pts must be a (*, 2 or 3) tensor. Got {point.shape}") + + if line.shape[-1] != 3: + raise ValueError(f"lines must be a (*, 3) tensor. Got {line.shape}") + + # Using in-place operations to improve performance + numerator = line[..., 0] * point[..., 0] + numerator += line[..., 1] * point[..., 1] + numerator += line[..., 2] + numerator.abs_() + + # Avoid computing norm multiple times by saving its value + denom_norm = (line[..., 0].square() + line[..., 1].square()).sqrt() + + return numerator / (denom_norm + eps) + + +def batched_dot_product(x: torch.Tensor, y: torch.Tensor, keepdim: bool = False) -> torch.Tensor: + """Return a batched version of .dot().""" + KORNIA_CHECK_SHAPE(x, ["*", "N"]) + KORNIA_CHECK_SHAPE(y, ["*", "N"]) + return (x * y).sum(-1, keepdim) + + +def batched_squared_norm(x: torch.Tensor, keepdim: bool = False) -> torch.Tensor: + """Return the squared norm of a vector.""" + return batched_dot_product(x, x, keepdim) + + +def euclidean_distance(x: torch.Tensor, y: torch.Tensor, keepdim: bool = False, eps: float = 1e-6) -> torch.Tensor: + """Compute the Euclidean distance between two set of n-dimensional points. + + More: https://en.wikipedia.org/wiki/Euclidean_distance + + Args: + x: first set of points of shape :math:`(*, N)`. + y: second set of points of shape :math:`(*, N)`. + keepdim: whether to keep the dimension after reduction. + eps: small value to have numerical stability. + + """ + KORNIA_CHECK_SHAPE(x, ["*", "N"]) + KORNIA_CHECK_SHAPE(y, ["*", "N"]) + + return (x - y).pow(2).sum(dim=-1, keepdim=keepdim).add_(eps).sqrt_() + + +# aliases +squared_norm = batched_squared_norm + +# TODO: +# - project_points: from opencv diff --git a/kornia/geometry/line.py b/kornia/geometry/line.py new file mode 100644 index 0000000..7d5530b --- /dev/null +++ b/kornia/geometry/line.py @@ -0,0 +1,290 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# kornia.geometry.line module inspired by Eigen::geometry::ParametrizedLine +# https://gitlab.com/libeigen/eigen/-/blob/master/Eigen/src/Geometry/ParametrizedLine.h +from typing import Iterator, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE +from kornia.core.utils import _torch_svd_cast +from kornia.geometry.linalg import batched_dot_product +from kornia.geometry.plane import Hyperplane + +__all__ = ["ParametrizedLine", "fit_line"] + + +class ParametrizedLine(nn.Module): + """Class that describes a parametrize line. + + A parametrized line is defined by an origin point :math:`o` and a unit + direction vector :math:`d` such that the line corresponds to the set + + .. math:: + + l(t) = o + t * d + """ + + def __init__(self, origin: torch.Tensor, direction: torch.Tensor) -> None: + """Initialize a parametrized line of direction and origin. + + Args: + origin: any point on the line of any dimension. + direction: the normalized vector direction of any dimension. + + Example: + >>> o = torch.tensor([0.0, 0.0]) + >>> d = torch.tensor([1.0, 1.0]) + >>> l = ParametrizedLine(o, d) + + """ + super().__init__() + self._origin = nn.Parameter(origin) + self._direction = nn.Parameter(direction) + + def __str__(self) -> str: + return f"Origin: {self.origin}\nDirection: {self.direction}" + + def __repr__(self) -> str: + return str(self) + + def __getitem__(self, idx: int) -> torch.Tensor: + return self.origin if idx == 0 else self.direction + + def __iter__(self) -> Iterator[torch.Tensor]: + yield from (self.origin, self.direction) + + @property + def origin(self) -> torch.Tensor: + """Return the line origin point.""" + return self._origin + + @property + def direction(self) -> torch.Tensor: + """Return the line direction vector.""" + return self._direction + + def dim(self) -> int: + """Return the dimension in which the line holds.""" + return self.direction.shape[-1] + + @classmethod + def through(cls, p0: torch.Tensor, p1: torch.Tensor) -> "ParametrizedLine": + """Construct a parametrized line going from a point :math:`p0` to :math:`p1`. + + Args: + p0: tensor with first point :math:`(B, D)` where `D` is the point dimension. + p1: tensor with second point :math:`(B, D)` where `D` is the point dimension. + + Example: + >>> p0 = torch.tensor([0.0, 0.0]) + >>> p1 = torch.tensor([1.0, 1.0]) + >>> l = ParametrizedLine.through(p0, p1) + + """ + return ParametrizedLine(p0, F.normalize((p1 - p0), p=2, dim=-1)) + + def point_at(self, t: Union[float, torch.Tensor]) -> torch.Tensor: + """Get the point at :math:`t` along this line. + + Args: + t: step along the line. + + Return: + tensor with the point. + + Example: + >>> p0 = torch.tensor([0.0, 0.0]) + >>> p1 = torch.tensor([1.0, 1.0]) + >>> l = ParametrizedLine.through(p0, p1) + >>> p2 = l.point_at(0.1) + + """ + return self.origin + self.direction * t + + def projection(self, point: torch.Tensor) -> torch.Tensor: + """Return the projection of a point onto the line. + + Args: + point: the point to be projected. + + """ + return self.origin + (self.direction @ (point - self.origin)) * self.direction + + def squared_distance(self, point: torch.Tensor) -> torch.Tensor: + """Return the squared distance of a point to its projection onte the line. + + Args: + point: the point to calculate the distance onto the line. + """ + d = point - self.origin + proj = torch.sum(d * self.direction, dim=-1) + sq_norm_d = torch.sum(d * d, dim=-1) + return sq_norm_d - proj * proj + + def distance(self, point: torch.Tensor) -> torch.Tensor: + """Return the distance of a point to its projections onto the line. + + Args: + point: the point to calculate the distance into the line. + """ + return self.squared_distance(point).sqrt() + + # TODO(edgar) implement the following: + # - intersection + # - intersection_parameter + # - intersection_point + + # TODO: add tests, and possibly return a mask + def intersect(self, plane: Hyperplane, eps: float = 1e-6) -> Tuple[torch.Tensor, torch.Tensor]: + """Return the intersection point between the line and a given plane. + + Args: + plane: the plane to compute the intersection point. + eps: epsilon for numerical stability. + + Return: + - the lambda value used to compute the look at point. + - the intersected point. + + """ + dot_prod = batched_dot_product(plane.normal.data, self.direction.data) + dot_prod_mask = dot_prod.abs() >= eps + + # TODO: add check for dot product + res_lambda = torch.where( + dot_prod_mask, + -(plane.offset + batched_dot_product(plane.normal.data, self.origin.data)) / dot_prod, + torch.empty_like(dot_prod), + ) + + res_point = self.point_at(res_lambda) + return res_lambda, res_point + + +def _fit_line_ols_2d(points: torch.Tensor) -> ParametrizedLine: + x = points[..., 0] + y = points[..., 1] + x_mean = x.mean(dim=-1, keepdim=True) + y_mean = y.mean(dim=-1, keepdim=True) + dx = x - x_mean + dy = y - y_mean + + denom = (dx * dx).sum(dim=-1, keepdim=True) # (B, 1) + slope = torch.where(denom > 1e-8, (dx * dy).sum(dim=-1, keepdim=True) / denom, torch.zeros_like(denom)) + + # For vertical lines, fallback to [0,1] direction + direction = torch.where( + denom > 1e-8, + torch.cat([torch.ones_like(slope), slope], dim=-1), + torch.tensor([0.0, 1.0], device=points.device).expand(points.shape[0], 2), + ) + + direction = direction / direction.norm(dim=-1, keepdim=True) + origin = torch.cat([x_mean, y_mean], dim=-1) + return ParametrizedLine(origin, direction) + + +def _fit_line_weighted_ols_2d(points: torch.Tensor, weights: torch.Tensor) -> ParametrizedLine: + x = points[..., 0] # (B, N) + y = points[..., 1] # (B, N) + + w_sum = weights.sum(dim=-1, keepdim=True) # (B, 1) + x_mean = (weights * x).sum(dim=-1, keepdim=True) / w_sum # (B, 1) + y_mean = (weights * y).sum(dim=-1, keepdim=True) / w_sum # (B, 1) + + dx = x - x_mean # (B, N) + dy = y - y_mean # (B, N) + + weighted_dx2 = weights * dx * dx + weighted_dxdy = weights * dx * dy + + denom = weighted_dx2.sum(dim=-1, keepdim=True) # (B, 1) + slope = weighted_dxdy.sum(dim=-1, keepdim=True) / denom # (B, 1) + + # Replace NaNs or infs from division by zero + slope = torch.where(torch.isfinite(slope), slope, torch.zeros_like(slope)) + + # direction = F.normalize([1, slope]) or [0,1] if vertical + is_vertical = denom <= 1e-8 + direction = torch.cat([torch.ones_like(slope), slope], dim=-1) # (B, 2) + replacement = torch.tensor([0.0, 1.0], device=points.device, dtype=points.dtype) + direction[is_vertical.squeeze(-1)] = replacement + + direction = direction / direction.norm(dim=-1, keepdim=True) + origin = torch.cat([x_mean, y_mean], dim=-1) + + return ParametrizedLine(origin, direction) + + +def fit_line(points: torch.Tensor, weights: Optional[torch.Tensor] = None) -> ParametrizedLine: + """Fit a line from a set of points. + + Args: + points: tensor containing a batch of sets of n-dimensional points. The expected + shape of the tensor is :math:`(B, N, D)`. + weights: weights to use to solve the equations system. The expected + shape of the tensor is :math:`(B, N)`. + + Return: + A tensor containing the direction of the fitted line of shape :math:`(B, D)`. + + Example: + >>> points = torch.rand(2, 10, 3) + >>> weights = torch.ones(2, 10) + >>> line = fit_line(points, weights) + >>> line.direction.shape + torch.Size([2, 3]) + """ + KORNIA_CHECK_IS_TENSOR(points, "points must be a tensor") + KORNIA_CHECK_SHAPE(points, ["B", "N", "D"]) + + _B, _N, D = points.shape + + # Fast path: use OLS for unweighted 2D case + if D == 2: + if weights is not None: + KORNIA_CHECK_IS_TENSOR(weights, "weights must be a tensor") + KORNIA_CHECK_SHAPE(weights, ["B", "N"]) + KORNIA_CHECK(points.shape[0] == weights.shape[0]) + return _fit_line_weighted_ols_2d(points, weights) + else: + return _fit_line_ols_2d(points) + + mean = points.mean(-2, True) + A = points - mean + + if weights is not None: + KORNIA_CHECK_IS_TENSOR(weights, "weights must be a tensor") + KORNIA_CHECK_SHAPE(weights, ["B", "N"]) + KORNIA_CHECK(points.shape[0] == weights.shape[0]) + A = A.transpose(-2, -1) @ torch.diag_embed(weights) @ A + else: + A = A.transpose(-2, -1) @ A + + # NOTE: not optimal for 2d points, but for now works for other dimensions + _, _, V = _torch_svd_cast(A) + V = V.transpose(-2, -1) + + # the first left eigenvector is the direction on the fitted line + direction = V[..., 0, :] # BxD + origin = mean[..., 0, :] # BxD + + return ParametrizedLine(origin, direction) diff --git a/kornia/geometry/plane.py b/kornia/geometry/plane.py new file mode 100644 index 0000000..defecad --- /dev/null +++ b/kornia/geometry/plane.py @@ -0,0 +1,236 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# kornia.geometry.plane module inspired by Eigen::geometry::Hyperplane +# https://gitlab.com/libeigen/eigen/-/blob/master/Eigen/src/Geometry/Hyperplane.h + +from typing import Optional + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SHAPE, KORNIA_CHECK_TYPE +from kornia.core.tensor_wrapper import _unwrap, _wrap +from kornia.core.utils import _torch_svd_cast +from kornia.geometry.linalg import batched_dot_product +from kornia.geometry.vector import Scalar, Vector3 + +__all__ = ["Hyperplane", "fit_plane"] + + +def normalized(v: torch.Tensor, eps: float = 1e-6) -> torch.Tensor: + norm_sq = (v * v).sum(dim=-1, keepdim=True) + eps + return v * norm_sq.rsqrt() + + +class Hyperplane(nn.Module): + """Represent a hyperplane in n-dimensional space. + + Args: + n: The normal vector of the hyperplane. + d: The scalar distance from the origin. + """ + + def __init__(self, n: Vector3, d: Scalar) -> None: + super().__init__() + KORNIA_CHECK_TYPE(n, Vector3) + KORNIA_CHECK_TYPE(d, Scalar) + # TODO: fix checkers + # KORNIA_CHECK_SHAPE(n, ["B", "*"]) + # KORNIA_CHECK_SHAPE(d, ["B"]) + self._n = n + self._d = d + + def __str__(self) -> str: + return f"Normal: {self.normal}\nOffset: {self.offset}" + + def __repr__(self) -> str: + return str(self) + + @property + def normal(self) -> Vector3: + """Return the vector perpendicular to the hyperplane. + + Returns: + :class:`~kornia.geometry.vector.Vector3` storing the normal + direction. For a 3D plane this is the vector :math:`n` in + :math:`n^T x + d = 0`. + """ + return self._n + + @property + def offset(self) -> Scalar: + """Return the scalar offset in the implicit plane equation. + + Returns: + :class:`~kornia.geometry.vector.Scalar` containing the ``d`` term + in :math:`n^T x + d = 0`. The value controls where the plane sits + relative to the origin for the stored normal direction. + """ + return self._d + + def abs_distance(self, p: Vector3) -> Scalar: + """Compute unsigned distances from points to the hyperplane. + + Args: + p: Point or batch of points wrapped as + :class:`~kornia.geometry.vector.Vector3`. The last coordinate + dimension represents ``(x, y, z)``. + + Returns: + :class:`~kornia.geometry.vector.Scalar` with non-negative distance + values. Leading batch dimensions follow the broadcasted point and + plane inputs. + """ + return Scalar(self.signed_distance(p).abs()) + + # https://gitlab.com/libeigen/eigen/-/blob/master/Eigen/src/Geometry/Hyperplane.h#L145 + # TODO: tests + def signed_distance(self, p: Vector3) -> Scalar: + """Compute signed distances from points to the hyperplane. + + The sign is determined by the stored normal vector. Points in the + normal direction have positive values; points on the opposite side have + negative values; points on the plane evaluate to zero. + + Args: + p: Point or batch of points as + :class:`~kornia.geometry.vector.Vector3`, or a compatible + tensor-like vector accepted by the dot-product routine. + + Returns: + :class:`~kornia.geometry.vector.Scalar` containing signed distance + values for each input point. + """ + KORNIA_CHECK(isinstance(p, Vector3 | torch.Tensor)) + return self.normal.dot(p) + self.offset + + # https://gitlab.com/libeigen/eigen/-/blob/master/Eigen/src/Geometry/Hyperplane.h#L154 + # TODO: tests + def projection(self, p: Vector3) -> Vector3: + """Project points onto the hyperplane along the normal direction. + + Args: + p: Point or batch of points wrapped as + :class:`~kornia.geometry.vector.Vector3`. + + Returns: + :class:`~kornia.geometry.vector.Vector3` containing the closest + point on the hyperplane for each input point, preserving leading + batch dimensions. + """ + dist = self.signed_distance(p) + if len(dist.shape) != len(self.normal): + # non batched plane project a batch of points + dist = dist[..., None] # Nx1 + # TODO: TypeError: bad operand type for unary -: 'Scalar' + return p - dist.data * self.normal + # TODO: make that Vector can subtract Scalar + # return p - self.signed_distance(p) * self.normal + + @classmethod + def from_vector(self, n: Vector3, e: Vector3) -> "Hyperplane": + """Create a hyperplane from a normal and one point on the plane. + + Args: + n: Normal vector :math:`n` defining the plane orientation. + e: Point :math:`e` that lies on the target plane. + + Returns: + :class:`Hyperplane` whose offset is chosen so that + :math:`n^T e + d = 0`. + """ + normal: Vector3 = n + offset = -normal.dot(e) + return Hyperplane(normal, Scalar(offset)) + + @classmethod + def through(cls, p0: torch.Tensor, p1: torch.Tensor, p2: Optional[torch.Tensor] = None) -> "Hyperplane": + """Construct a line-like 2D hyperplane or a 3D plane through points. + + Args: + p0: First point tensor, shaped ``(..., 2)`` for the 2D case or + ``(..., 3)`` for the 3D case. + p1: Second point tensor with the same shape as ``p0``. + p2: Optional third point tensor. If omitted, the method builds the + 2D line representation from ``p0`` and ``p1``. If provided, it + builds the 3D plane passing through all three points. + + Returns: + :class:`Hyperplane` with a normal and offset determined by the + provided point set. + """ + # 2d case + if p2 is None: + # TODO: improve tests + KORNIA_CHECK_SHAPE(p0, ["*", "2"]) + KORNIA_CHECK(p0.shape == p1.shape) + # TODO: implement `.unitOrthonormal` + normal2d = normalized(p1 - p0) + offset2d = -batched_dot_product(p0, normal2d) + return Hyperplane(_wrap(normal2d, Vector3), _wrap(offset2d, Scalar)) + # 3d case + KORNIA_CHECK_SHAPE(p0, ["*", "3"]) + KORNIA_CHECK(p0.shape == p1.shape) + KORNIA_CHECK(p1.shape == p2.shape) + v0, v1 = (p2 - p0), (p1 - p0) + normal = torch.linalg.cross(v0, v1, dim=-1) + norm = normal.norm(-1) + + # https://gitlab.com/libeigen/eigen/-/blob/master/Eigen/src/Geometry/Hyperplane.h#L108 + def compute_normal_svd(v0: torch.Tensor, v1: torch.Tensor) -> "Vector3": + # NOTE: for reason torch.TensorWrapper does not stack well + m = torch.stack((_unwrap(v0), _unwrap(v1)), -2) # Bx2x3 + _, _, V = _torch_svd_cast(m) # kornia solution lies in the last row + return _wrap(V[..., :, -1], Vector3) # Bx3 + + normal_mask = norm <= v0.norm(-1) * v1.norm(-1) * 1e-6 + normal = torch.where(normal_mask, compute_normal_svd(v0, v1).data, normal / (norm + 1e-6)) + offset = -batched_dot_product(p0, normal) + + return Hyperplane(_wrap(normal, Vector3), _wrap(offset, Scalar)) + + +# TODO: factor to avoid duplicated from line.py +# https://github.com/strasdat/Sophus/blob/23.04-beta/cpp/sophus/geometry/fit_plane.h +def fit_plane(points: Vector3) -> Hyperplane: + """Fit a plane from a set of points using SVD. + + Args: + points: tensor containing a batch of sets of n-dimensional points. The expected + shape of the tensor is :math:`(N, D)`. + + Return: + The computed hyperplane object. + + """ + # TODO: fix to support more type check here + # KORNIA_CHECK_SHAPE(points, ["N", "D"]) + if points.shape[-1] != 3: + raise TypeError("vector must be (*, 3)") + + mean = points.mean(-2, True) + points_centered = points - mean + + # NOTE: not optimal for 2d points, but for now works for other dimensions + _, _, V = _torch_svd_cast(points_centered) + + # the first left eigenvector is the direction on the fited line + direction = V[..., :, -1] # BxD + origin = mean[..., 0, :] # BxD + + return Hyperplane.from_vector(Vector3(direction), Vector3(origin)) diff --git a/kornia/geometry/pointcloud.py b/kornia/geometry/pointcloud.py new file mode 100644 index 0000000..7da0bd8 --- /dev/null +++ b/kornia/geometry/pointcloud.py @@ -0,0 +1,107 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import os + +import torch + + +def save_pointcloud_ply(filename: str, pointcloud: torch.Tensor) -> None: + r"""Save to disk a pointcloud in PLY format. + + Args: + filename: the path to save the pointcloud. + pointcloud: tensor containing the pointcloud to save. + The tensor must be in the shape of :math:`(*, 3)` where the last + component is assumed to be a 3d point coordinate :math:`(X, Y, Z)`. + """ + if not (isinstance(filename, str) and filename.lower().endswith(".ply")): + raise TypeError(f"Input filename must be a string with the .ply extension. Got {filename!r}") + + if not torch.is_tensor(pointcloud): + raise TypeError(f"Input pointcloud type is not a torch.Tensor. Got {type(pointcloud)}") + + if pointcloud.ndim < 2 or pointcloud.shape[-1] != 3: + raise TypeError(f"Input pointcloud must have shape (..., 3). Got {tuple(pointcloud.shape)}") + + # Flatten points + xyz = pointcloud.reshape(-1, 3) + + valid_mask = torch.isfinite(xyz).any(dim=1) + valid_points = xyz[valid_mask] + valid_count = valid_points.shape[0] + + with open(filename, "w", encoding="utf-8") as f: + # Write PLY header + f.writelines( + [ + "ply\n", + "format ascii 1.0\n", + "comment arraiy generated\n", + f"element vertex {valid_count}\n", + "property double x\n", + "property double y\n", + "property double z\n", + "end_header\n", + ] + ) + + if valid_count > 0: + # Move to CPU, convert to float64 for matching 'double' in header + arr = valid_points.detach().cpu().to(torch.float64) + # Write each row as space-separated floats + for x, y, z in arr.tolist(): + f.write(f"{x:.9g} {y:.9g} {z:.9g}\n") + + +def load_pointcloud_ply(filename: str, header_size: int = 8) -> torch.Tensor: + r"""Load from disk a pointcloud in PLY format. + + Args: + filename: the path to the pointcloud. + header_size: the number of header lines to skip. + + Return: + tensor containing the loaded points with shape :math:`(*, 3)` where + :math:`*` represents the number of points. + """ + if not (isinstance(filename, str) and filename.lower().endswith(".ply")): + raise TypeError(f"Input filename must be a string with the .ply extension. Got {filename!r}") + if not os.path.isfile(filename): + raise ValueError("Input filename is not an existing file.") + if not (isinstance(header_size, int) and header_size > 0): + raise TypeError(f"Input header_size must be a positive integer. Got {header_size}.") + + # Read all file bytes + with open(filename, "rb") as f: + # Skip header lines + for _ in range(header_size): + f.readline() + raw_data = f.read() + + # Decode once and split (faster than line-by-line parsing in Python) + text = raw_data.decode("utf-8", errors="ignore") + parts = text.split() + + # We only take the first 3 columns per point + if len(parts) % 3 != 0: + raise ValueError(f"Expected 3 columns per point, got a total of {len(parts)} values.") + + # Convert directly to a float32 tensor in one go + tensor = torch.tensor(list(map(float, parts[: (len(parts) // 3) * 3])), dtype=torch.float32).view(-1, 3) + return tensor diff --git a/kornia/geometry/pose.py b/kornia/geometry/pose.py new file mode 100644 index 0000000..d25c966 --- /dev/null +++ b/kornia/geometry/pose.py @@ -0,0 +1,244 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import uuid + +import torch + +from kornia.geometry.liegroup import Se2, Se3, So2, So3 +from kornia.geometry.quaternion import Quaternion + + +def check_matrix_shape(matrix: torch.Tensor, matrix_type: str = "R") -> None: + """Verify matrix shape based on type.""" + target_shapes = [] + if matrix_type == "R": + target_shapes = [[2, 2], [3, 3]] + elif matrix_type == "RT": + target_shapes = [[3, 3], [4, 4]] + if len(matrix.shape) > 3 or len(matrix.shape) < 2 or list(matrix.shape[-2:]) not in target_shapes: + raise ValueError( + f"{matrix_type} must be either {target_shapes[0]}x{target_shapes[0]} or \ + {target_shapes[1]}x{target_shapes[1]}, got {matrix.shape}" + ) + + +class NamedPose: + r"""Class to represent a named pose between two frames. + + Internally represented by either Se2 or Se3. + + Example: + >>> b_from_a = NamedPose(Se3.identity(), frame_src="frame_a", frame_dst="frame_b") + >>> b_from_a + NamedPose(dst_from_src=rotation: tensor([1., 0., 0., 0.]) + translation: x: 0.0 + y: 0.0 + z: 0.0, + frame_src: frame_a -> frame_dst: frame_b) + + """ + + def __init__(self, dst_from_src: Se2 | Se3, frame_src: str | None = None, frame_dst: str | None = None) -> None: + """Construct NamedPose. + + Args: + dst_from_src: Pose from source frame to destination frame. + frame_src: Name of frame a. + frame_dst: Name of frame b. + + """ + self._dst_from_src = dst_from_src + self._frame_src = frame_src or uuid.uuid4().hex + self._frame_dst = frame_dst or uuid.uuid4().hex + + def __repr__(self) -> str: + return ( + f"NamedPose(dst_from_src={self._dst_from_src},\n" + f"frame_src: {self._frame_src} -> frame_dst: {self._frame_dst})" + ) + + def __mul__(self, other: NamedPose) -> NamedPose: + """Compose two NamedPoses. + + Args: + other: NamedPose to compose with. + + Returns: + Composed NamedPose. + + Example: + >>> b_from_a = NamedPose(Se3.identity(), frame_src="frame_a", frame_dst="frame_b") + >>> c_from_b = NamedPose(Se3.identity(), frame_src="frame_b", frame_dst="frame_c") + >>> c_from_b * b_from_a + NamedPose(dst_from_src=rotation: tensor([1., 0., 0., 0.]) + translation: x: 0.0 + y: 0.0 + z: 0.0, + frame_src: frame_a -> frame_dst: frame_c) + + """ + if self._frame_src != other._frame_dst: + raise ValueError(f"Cannot compose {self} with {other}") + if isinstance(other.pose, Se2): + return NamedPose(self._dst_from_src._mul_se2(other.pose), other._frame_src, self._frame_dst) + elif isinstance(other.pose, Se3): + return NamedPose(self._dst_from_src._mul_se3(other.pose), other._frame_src, self._frame_dst) + else: + raise ValueError(f"Pose must be either Se2 or Se3, got {type(self._dst_from_src)}") + + @property + def pose(self) -> Se2 | Se3: + """Pose from source frame to destination frame .""" + return self._dst_from_src + + @property + def rotation(self) -> So3 | So2: + """Rotation part of the pose.""" + return self._dst_from_src.rotation + + @property + def translation(self) -> torch.Tensor: + """Translation part of the pose.""" + return self._dst_from_src.translation + + @property + def frame_src(self) -> str: + """Name of the source frame.""" + return self._frame_src + + @property + def frame_dst(self) -> str: + """Name of the destination frame.""" + return self._frame_dst + + @classmethod + def from_rt( + cls, + rotation: So3 | So2 | torch.Tensor | Quaternion, + translation: torch.Tensor, + frame_src: str | None = None, + frame_dst: str | None = None, + ) -> NamedPose | None: + """Construct NamedPose from rotation and translation. + + Args: + rotation: Rotation part of the pose. + translation: Translation part of the pose. + frame_src: Name of the source frame. + frame_dst: Name of the destination frame. + + Returns: + NamedPose constructed from rotation and translation. + + Example: + >>> b_from_a_rot = So3.identity() + >>> b_from_a_trans = torch.tensor([1., 2., 3.]) + >>> b_from_a = NamedPose.from_rt(b_from_a_rot, b_from_a_trans, frame_src="frame_a", frame_dst="frame_b") + >>> b_from_a + NamedPose(dst_from_src=rotation: tensor([1., 0., 0., 0.]) + translation: Parameter containing: + tensor([1., 2., 3.], requires_grad=True), + frame_src: frame_a -> frame_dst: frame_b) + + """ + if isinstance(rotation, (So3, Quaternion)): + return cls(Se3(rotation, translation), frame_src, frame_dst) + elif isinstance(rotation, So2): + return cls(Se2(rotation, translation), frame_src, frame_dst) + elif isinstance(rotation, torch.Tensor): + check_matrix_shape(rotation) + dim = rotation.shape[-1] + RT = torch.eye(dim + 1, device=rotation.device, dtype=rotation.dtype) + RT[..., :dim, :dim] = rotation + RT[..., :dim, dim] = translation + if dim == 2: + return cls(Se2.from_matrix(RT), frame_src, frame_dst) + elif dim == 3: + return cls(Se3.from_matrix(RT), frame_src, frame_dst) + else: + raise ValueError(f"R must be either So2, So3, Quaternion, or Tensor, got {type(rotation)}") + return None + + @classmethod + def from_matrix( + cls, matrix: torch.Tensor, frame_src: str | None = None, frame_dst: str | None = None + ) -> NamedPose | None: + """Construct NamedPose from a matrix. + + Args: + matrix: Matrix representation of the pose. + frame_src: Name of the source frame. + frame_dst: Name of the destination frame. + + Returns: + NamedPose constructed from a matrix. + + Example: + >>> b_from_a_matrix = Se3.identity().matrix() + >>> b_from_a = NamedPose.from_matrix(b_from_a_matrix, frame_src="frame_a", frame_dst="frame_b") + >>> b_from_a + NamedPose(dst_from_src=rotation: tensor([1., 0., 0., 0.]) + translation: Parameter containing: + tensor([0., 0., 0.], requires_grad=True), + frame_src: frame_a -> frame_dst: frame_b) + + """ + check_matrix_shape(matrix, matrix_type="RT") + dim = matrix.shape[-1] + if dim == 3: + return cls(Se2.from_matrix(matrix), frame_src, frame_dst) + elif dim == 4: + return cls(Se3.from_matrix(matrix), frame_src, frame_dst) + return None + + def inverse(self) -> NamedPose: + """Inverse of the NamedPose. + + Returns: + Inverse of the NamedPose. + + Example: + >>> b_from_a = NamedPose(Se3.identity(), frame_src="frame_a", frame_dst="frame_b") + >>> b_from_a.inverse() + NamedPose(dst_from_src=rotation: tensor([1., -0., -0., -0.]) + translation: x: 0.0 + y: 0.0 + z: 0.0, + frame_src: frame_b -> frame_dst: frame_a) + + """ + return NamedPose(self._dst_from_src.inverse(), self._frame_dst, self._frame_src) + + def transform_points(self, points_in_src: torch.Tensor) -> torch.Tensor: + """Transform points from source frame to destination frame. + + Args: + points_in_src: Points in source frame. + + Returns: + Points in destination frame. + + Example: + >>> b_from_a = NamedPose(Se3.identity(), frame_src="frame_a", frame_dst="frame_b") + >>> b_from_a.transform_points(torch.tensor([1., 2., 3.])) + tensor([1., 2., 3.]) + + """ + return self._dst_from_src * points_in_src diff --git a/kornia/geometry/quaternion.py b/kornia/geometry/quaternion.py new file mode 100644 index 0000000..bde277a --- /dev/null +++ b/kornia/geometry/quaternion.py @@ -0,0 +1,683 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# kornia.geometry.quaternion module inspired by Eigen, Sophus-sympy, and PyQuaternion. +# https://github.com/strasdat/Sophus/blob/master/sympy/sophus/quaternion.py +# https://github.com/KieranWynn/pyquaternion/blob/master/pyquaternion/quaternion.py +# https://gitlab.com/libeigen/eigen/-/blob/master/Eigen/src/Geometry/Quaternion.h +from math import pi +from typing import Any, Optional, Tuple, Union + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK_TYPE +from kornia.geometry.conversions import ( + axis_angle_to_quaternion, + euler_from_quaternion, + normalize_quaternion, + quaternion_from_euler, + quaternion_to_axis_angle, + quaternion_to_rotation_matrix, + rotation_matrix_to_quaternion, +) +from kornia.geometry.linalg import batched_dot_product + + +class Quaternion(nn.Module): + r"""Base class to represent a Quaternion. + + A quaternion is a four dimensional vector representation of a rotation transformation in 3d. + See more: https://en.wikipedia.org/wiki/Quaternion + + The general definition of a quaternion is given by: + + .. math:: + + Q = a + b \cdot \mathbf{i} + c \cdot \mathbf{j} + d \cdot \mathbf{k} + + Thus, we represent a rotation quaternion as a contiguous torch.Tensor structure to + perform rigid bodies transformations: + + .. math:: + + Q = \begin{bmatrix} q_w & q_x & q_y & q_z \end{bmatrix} + + Example: + >>> q = Quaternion.identity(batch_size=4) + >>> q.data + tensor([[1., 0., 0., 0.], + [1., 0., 0., 0.], + [1., 0., 0., 0.], + [1., 0., 0., 0.]]) + >>> q.real + tensor([1., 1., 1., 1.]) + >>> q.vec + tensor([[0., 0., 0.], + [0., 0., 0.], + [0., 0., 0.], + [0., 0., 0.]]) + + """ + + _data: Union[torch.Tensor, nn.Parameter] + + def __init__(self, data: Union[torch.Tensor, nn.Parameter]) -> None: + """Construct a quaternion from torch.Tensor or parameter data. + + Args: + data: torch.Tensor or parameter containing the quaternion data with the shape of :math:`(B, 4)`. + + Example: + >>> # Create with torch.tensor(no gradients tracked by default) + >>> data = torch.tensor([1., 0., 0., 0.]) + >>> q1 = Quaternion(data) + >>> # Create with parameter (gradients tracked) + >>> param_data = torch.nn.Parameter(torch.tensor([1., 0., 0., 0.])) + >>> q2 = Quaternion(param_data) + + """ + super().__init__() + if not isinstance(data, (torch.Tensor, nn.Parameter)): + raise TypeError(f"Expected torch.Tensor or nn.Parameter, got {type(data)}") + # KORNIA_CHECK_SHAPE(data, ["B", "4"]) # FIXME: resolve shape bugs. @edgarriba + self._data = data + + def to(self, *args: Any, **kwargs: Any) -> "Quaternion": + """Move and/or cast the quaternion data. + + Args: + *args: Arguments to pass to torch.Tensor.to() + **kwargs: Keyword arguments to pass to torch.Tensor.to() + + Returns: + A new Quaternion with converted data. + """ + return Quaternion(self._data.to(*args, **kwargs)) + + def _to_scalar_quaternion(self, value: Union[torch.Tensor, float]) -> "Quaternion": + """Convert a scalar, torch.Tensor, or numeric value to a scalar quaternion. + + A scalar quaternion has the form [real, 0, 0, 0] where real is the input value. + + Args: + value: The scalar, torch.Tensor, or numeric value to convert. + + Returns: + A Quaternion object representing the scalar quaternion. + """ + if isinstance(value, (int, float)): + value = torch.tensor(value, device=self.data.device, dtype=self.data.dtype) + elif isinstance(value, torch.Tensor): + value = value.to(device=self.data.device, dtype=self.data.dtype) + + # Broadcast value to match the shape of self.real + try: + target_shape = torch.broadcast_shapes(self.real.shape, value.shape) + except RuntimeError as e: + raise ValueError(f"Cannot broadcast shapes {self.real.shape} and {value.shape}") from e + + broadcasted = self.real.expand(target_shape) + value.expand(target_shape) + # Create scalar quaternion: [value, 0, 0, 0] + # Expand value to match the broadcasted shape, then add quaternion dimension + if value.dim() == 0: # scalar + # Expand to match the broadcasted shape + expanded_value = value.expand_as(broadcasted) + else: + # Use broadcasting to get the right shape + expanded_value = torch.broadcast_to(value, broadcasted.shape) + + # Create zeros for the imaginary part + zeros = torch.zeros_like(expanded_value).unsqueeze(-1).expand(*expanded_value.shape, 3) + + # Stack real and imaginary parts: [real, 0, 0, 0] + scalar_quat_data = torch.cat([expanded_value.unsqueeze(-1), zeros], dim=-1) + + return Quaternion(scalar_quat_data) + + def __repr__(self) -> str: + return f"{self.data}" + + def __getitem__(self, idx: Union[int, slice]) -> "Quaternion": + return Quaternion(self.data[idx]) + + def __neg__(self) -> "Quaternion": + """Inverts the sign of the quaternion data. + + Example: + >>> q = Quaternion.identity() + >>> -q.data + tensor([-1., -0., -0., -0.]) + + """ + return Quaternion(-self.data) + + def __add__(self, right: Union["Quaternion", torch.Tensor, float]) -> "Quaternion": + """Add a given quaternion, scalar, or torch.Tensor. + + Args: + right: the quaternion, scalar, or torch.Tensor to add. + + Example: + >>> q1 = Quaternion.identity() + >>> q2 = Quaternion(torch.tensor([2., 0., 1., 1.])) + >>> q3 = q1 + q2 + >>> q3.data + tensor([3., 0., 1., 1.]) + + """ + if isinstance(right, Quaternion): + return Quaternion(self.data + right.data) + else: + right_quat = self._to_scalar_quaternion(right) + return Quaternion(self.data + right_quat.data) + + def __sub__(self, right: Union["Quaternion", torch.Tensor, float]) -> "Quaternion": + """Subtract a given quaternion, scalar, or torch.Tensor. + + Args: + right: the quaternion, scalar, or torch.Tensor to subtract. + + Example: + >>> q1 = Quaternion(torch.tensor([2., 0., 1., 1.])) + >>> q2 = Quaternion.identity() + >>> q3 = q1 - q2 + >>> q3.data + tensor([1., 0., 1., 1.]) + + """ + if isinstance(right, Quaternion): + return Quaternion(self.data - right.data) + else: + right_quat = self._to_scalar_quaternion(right) + # For scalar operations, ensure we return a torch.Tensor to preserve gradients + result_data = self.data - right_quat.data + if isinstance(result_data, nn.Parameter): + result_data = result_data.data # Convert to torch.Tensor to preserve gradients + return Quaternion(result_data) + + def __mul__(self, right: Union["Quaternion", torch.Tensor, float]) -> "Quaternion": + # If right is a Quaternion, do quaternion multiplication + if isinstance(right, Quaternion): + new_real = self.real * right.real - batched_dot_product(self.vec, right.vec) + new_vec = ( + self.real[..., None] * right.vec + + right.real[..., None] * self.vec + + torch.linalg.cross(self.vec, right.vec, dim=-1) + ) + return Quaternion(torch.cat((new_real[..., None], new_vec), -1)) + + # If right is a scalar/torch.Tensor, convert to scalar quaternion and multiply + else: + right_quat = self._to_scalar_quaternion(right) + new_real = self.real * right_quat.real - batched_dot_product(self.vec, right_quat.vec) + new_vec = ( + self.real[..., None] * right_quat.vec + + right_quat.real[..., None] * self.vec + + torch.linalg.cross(self.vec, right_quat.vec, dim=-1) + ) + return Quaternion(torch.cat((new_real[..., None], new_vec), -1)) + + def __rmul__(self, left: Union[torch.Tensor, float]) -> "Quaternion": + """Right multiplication (left * self) where left is a scalar or torch.Tensor.""" + left_quat = self._to_scalar_quaternion(left) + new_real = left_quat.real * self.real - batched_dot_product(left_quat.vec, self.vec) + new_vec = ( + left_quat.real[..., None] * self.vec + + self.real[..., None] * left_quat.vec + + torch.linalg.cross(left_quat.vec, self.vec, dim=-1) + ) + return Quaternion(torch.cat((new_real[..., None], new_vec), -1)) + + def __div__(self, right: Union[torch.Tensor, "Quaternion", float]) -> "Quaternion": + if isinstance(right, Quaternion): + return self * right.inv() + else: + # For scalars/tensors, just divide the quaternion data directly + if isinstance(right, (int, float)): + right_tensor = torch.tensor(right, device=self.data.device, dtype=self.data.dtype) + else: + right_tensor = right.to(device=self.data.device, dtype=self.data.dtype) + + # For division by scalar, expand to [right, right, right, right] for element-wise division + if right_tensor.dim() == 0: # scalar + divisor = right_tensor.expand_as(self.data[..., 0]).unsqueeze(-1).expand_as(self.data) + else: + # Broadcast the torch.Tensor to match the quaternion dimensions + divisor = right_tensor.unsqueeze(-1).expand_as(self.data) + + # For scalar operations, ensure we return a torch.Tensor to preserve gradients + result_data = self.data / divisor + if isinstance(result_data, nn.Parameter): + result_data = result_data.data # Convert to torch.Tensor to preserve gradients + return Quaternion(result_data) + + def __truediv__(self, right: Union[torch.Tensor, "Quaternion", float]) -> "Quaternion": + return self.__div__(right) + + def __radd__(self, left: Union[torch.Tensor, float]) -> "Quaternion": + """Right addition (left + self) where left is a scalar or torch.Tensor.""" + left_quat = self._to_scalar_quaternion(left) + return left_quat + self + + def __rsub__(self, left: Union[torch.Tensor, float]) -> "Quaternion": + """Right subtraction (left - self) where left is a scalar or torch.Tensor.""" + left_quat = self._to_scalar_quaternion(left) + return left_quat - self + + def __rtruediv__(self, left: Union[torch.Tensor, float]) -> "Quaternion": + """Right division (left / self) where left is a scalar or torch.Tensor.""" + left_quat = self._to_scalar_quaternion(left) + return left_quat / self + + def __rdiv__(self, left: Union[torch.Tensor, float]) -> "Quaternion": + """Right division (left / self) where left is a scalar or torch.Tensor.""" + return self.__rtruediv__(left) + + def __pow__(self, t: float) -> "Quaternion": + """Return the power of a quaternion raised to exponent t. + + Args: + t: raised exponent. + + Example: + >>> q = Quaternion(torch.tensor([1., .5, 0., 0.])) + >>> q_pow = q**2 + + """ + theta = self.polar_angle[..., None] + vec_norm = self.vec.norm(dim=-1, keepdim=True) + n = torch.where(vec_norm != 0, self.vec / vec_norm, self.vec * 0) + w = (t * theta).cos() + xyz = (t * theta).sin() * n + return Quaternion(torch.cat((w, xyz), -1)) + + @property + def data(self) -> torch.Tensor: + """Return the underlying data with shape :math:`(B, 4)`.""" + return self._data + + @property + def coeffs(self) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Return a tuple with the underlying coefficients in WXYZ order.""" + return self.w, self.x, self.y, self.z + + @property + def real(self) -> torch.Tensor: + """Return the real part with shape :math:`(B,)`. + + Alias for + :func: `~kornia.geometry.quaternion.Quaternion.w` + """ + return self.w + + @property + def vec(self) -> torch.Tensor: + """Return the vector with the imaginary part with shape :math:`(B, 3)`.""" + return self.data[..., 1:] + + @property + def q(self) -> torch.Tensor: + """Return the underlying data with shape :math:`(B, 4)`. + + Alias for :func:`~kornia.geometry.quaternion.Quaternion.data` + """ + return self.data + + @property + def scalar(self) -> torch.Tensor: + """Return a scalar with the real with shape :math:`(B,)`. + + Alias for + :func: `~kornia.geometry.quaternion.Quaternion.w` + """ + return self.real + + @property + def w(self) -> torch.Tensor: + """Return the :math:`q_w` with shape :math:`(B,)`.""" + return self.data[..., 0] + + @property + def x(self) -> torch.Tensor: + """Return the :math:`q_x` with shape :math:`(B,)`.""" + return self.data[..., 1] + + @property + def y(self) -> torch.Tensor: + """Return the :math:`q_y` with shape :math:`(B,)`.""" + return self.data[..., 2] + + @property + def z(self) -> torch.Tensor: + """Return the :math:`q_z` with shape :math:`(B,)`.""" + return self.data[..., 3] + + @property + def shape(self) -> Tuple[int, ...]: + """Return the shape of the underlying data with shape :math:`(B, 4)`.""" + return tuple(self.data.shape) + + @property + def polar_angle(self) -> torch.Tensor: + """Return the polar angle with shape :math:`(B,1)`. + + Example: + >>> q = Quaternion.identity() + >>> q.polar_angle + tensor(0.) + + """ + return (self.scalar / self.norm()).acos() + + def matrix(self) -> torch.Tensor: + """Convert the quaternion to a rotation matrix of shape :math:`(B, 3, 3)`. + + Example: + >>> q = Quaternion.identity() + >>> m = q.matrix() + >>> m + tensor([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) + + """ + return quaternion_to_rotation_matrix(self.data) + + @classmethod + def from_matrix(cls, matrix: torch.Tensor) -> "Quaternion": + """Create a quaternion from a rotation matrix. + + Args: + matrix: the rotation matrix to convert of shape :math:`(B, 3, 3)`. + + Example: + >>> m = torch.eye(3)[None] + >>> q = Quaternion.from_matrix(m) + >>> q.data + tensor([[1., 0., 0., 0.]]) + + """ + return cls(rotation_matrix_to_quaternion(matrix)) + + @classmethod + def from_euler(cls, roll: torch.Tensor, pitch: torch.Tensor, yaw: torch.Tensor) -> "Quaternion": + """Create a quaternion from Euler angles. + + Args: + roll: the roll euler angle. + pitch: the pitch euler angle. + yaw: the yaw euler angle. + + Example: + >>> roll, pitch, yaw = torch.tensor(0), torch.tensor(1), torch.tensor(0) + >>> q = Quaternion.from_euler(roll, pitch, yaw) + >>> q.data + tensor([0.8776, 0.0000, 0.4794, 0.0000]) + + """ + w, x, y, z = quaternion_from_euler(roll=roll, pitch=pitch, yaw=yaw) + q = torch.stack((w, x, y, z), -1) + return cls(q) + + def to_euler(self) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Convert the quaternion to a triple of Euler angles (roll, pitch, yaw). + + Example: + >>> q = Quaternion(torch.tensor([2., 0., 1., 1.])) + >>> roll, pitch, yaw = q.to_euler() + >>> roll + tensor(2.0344) + >>> pitch + tensor(1.5708) + >>> yaw + tensor(2.2143) + + """ + return euler_from_quaternion(self.w, self.x, self.y, self.z) + + @classmethod + def from_axis_angle(cls, axis_angle: torch.Tensor) -> "Quaternion": + """Create a quaternion from axis-angle representation. + + Args: + axis_angle: rotation vector of shape :math:`(B, 3)`. + + Example: + >>> axis_angle = torch.tensor([[1., 0., 0.]]) + >>> q = Quaternion.from_axis_angle(axis_angle) + >>> q.data + tensor([[0.8776, 0.4794, 0.0000, 0.0000]]) + + """ + return cls(axis_angle_to_quaternion(axis_angle)) + + def to_axis_angle(self) -> torch.Tensor: + """Convert the quaternion to an axis-angle representation. + + Example: + >>> q = Quaternion.identity() + >>> axis_angle = q.to_axis_angle() + >>> axis_angle + tensor([0., 0., 0.]) + + """ + return quaternion_to_axis_angle(self.data) + + @classmethod + def identity( + cls, + batch_size: Optional[int] = None, + device: Union[None, str, torch.device] = None, + dtype: Union[torch.dtype, None] = None, + ) -> "Quaternion": + """Create a quaternion representing an identity rotation. + + Args: + batch_size: the batch size of the underlying data. + device: device to place the result on. + dtype: dtype of the result. + + Example: + >>> q = Quaternion.identity() + >>> q.data + tensor([1., 0., 0., 0.]) + + """ + data = torch.tensor([1.0, 0.0, 0.0, 0.0], device=device, dtype=dtype) + if batch_size is not None: + data = data.repeat(batch_size, 1) + return cls(data) + + @classmethod + def from_coeffs(cls, w: float, x: float, y: float, z: float) -> "Quaternion": + """Create a quaternion from the data coefficients. + + Args: + w: a float representing the :math:`q_w` component. + x: a float representing the :math:`q_x` component. + y: a float representing the :math:`q_y` component. + z: a float representing the :math:`q_z` component. + + Example: + >>> q = Quaternion.from_coeffs(1., 0., 0., 0.) + >>> q.data + tensor([1., 0., 0., 0.]) + + """ + return cls(torch.tensor([w, x, y, z])) + + # TODO: update signature + # def random(cls, shape: Optional[List] = None, device = None, dtype = None) -> 'Quaternion': + @classmethod + def random( + cls, + batch_size: Optional[int] = None, + device: Union[None, str, torch.device] = None, + dtype: Union[torch.dtype, None] = None, + ) -> "Quaternion": + """Create a random unit quaternion of shape :math:`(B, 4)`. + + Uniformly distributed across the rotation space as per: http://planning.cs.uiuc.edu/node198.html + + Args: + batch_size: the batch size of the underlying data. + device: device to place the result on. + dtype: dtype of the result. + + Example: + >>> q = Quaternion.random() + >>> q = Quaternion.random(batch_size=2) + + """ + rand_shape = (batch_size,) if batch_size is not None else () + + r1, r2, r3 = torch.rand((3, *rand_shape), device=device, dtype=dtype) + q1 = (1.0 - r1).sqrt() * ((2 * pi * r2).sin()) + q2 = (1.0 - r1).sqrt() * ((2 * pi * r2).cos()) + q3 = r1.sqrt() * (2 * pi * r3).sin() + q4 = r1.sqrt() * (2 * pi * r3).cos() + return cls(torch.stack((q1, q2, q3, q4), -1)) + + def slerp(self, q1: "Quaternion", t: float) -> "Quaternion": + """Return a unit quaternion spherically interpolated between quaternions self.q and q1. + + See more: https://en.wikipedia.org/wiki/Slerp + + Args: + q1: second quaternion to be interpolated between. + t: interpolation ratio, range [0-1] + + Example: + >>> q0 = Quaternion.identity() + >>> q1 = Quaternion(torch.tensor([1., .5, 0., 0.])) + >>> q2 = q0.slerp(q1, .3) + + """ + KORNIA_CHECK_TYPE(q1, Quaternion) + q0 = self.normalize() + q1 = q1.normalize() + return q0 * (q0.inv() * q1) ** t + + def norm(self, keepdim: bool = False) -> torch.Tensor: + """Compute the norm (magnitude) of the quaternion. + + Args: + keepdim: whether to retain the last dimension. + + Returns: + The norm of the quaternion(s) as a torch.Tensor. + + Example: + >>> q = Quaternion.identity() + >>> q.norm() + tensor(1.) + + """ + # p==2, dim|axis==-1, keepdim + return self.data.norm(2, -1, keepdim) + + def normalize(self) -> "Quaternion": + """Return a normalized (unit) quaternion. + + Returns: + The normalized quaternion. + + Example: + >>> q = Quaternion(torch.tensor([2., 1., 0., 0.])) + >>> q_norm = q.normalize() + + """ + return Quaternion(normalize_quaternion(self.data)) + + def conj(self) -> "Quaternion": + """Compute the conjugate of the quaternion. + + Returns: + The conjugate quaternion, with the vector part negated. + + Example: + >>> q = Quaternion(torch.tensor([1., 2., 3., 4.])) + >>> q_conj = q.conj() + + """ + return Quaternion(torch.cat((self.real[..., None], -self.vec), -1)) + + def inv(self) -> "Quaternion": + """Compute the inverse of the quaternion. + + Returns: + The inverse quaternion. + + Example: + >>> q = Quaternion.identity() + >>> q_inv = q.inv() + + """ + return self.conj() / self.squared_norm() + + def squared_norm(self) -> torch.Tensor: + """Compute the squared norm (magnitude) of the quaternion. + + Returns: + The squared norm of the quaternion(s) as a torch.Tensor. + + Example: + >>> q = Quaternion.identity() + >>> q.squared_norm() + tensor(1.) + + """ + return batched_dot_product(self.vec, self.vec) + self.real**2 + + +def average_quaternions(Q: "Quaternion", w: Optional[torch.Tensor] = None) -> "Quaternion": + """Compute (weighted) average of multiple quaternions. + + Args: + Q (Quaternion): quaternion object containing data of shape (M, 4). + w (torch.Tensor, optional): Weights of shape (M,). If None, uniform weights are used. + + + Returns: + Quaternion: averaged quaternion (shape (4,)), wrapped back in the Quaternion class. + """ + data = Q.data + KORNIA_CHECK_TYPE(Q, Quaternion) + + M = data.shape[0] + if w is None: + A = (data.T @ data) / M + else: + w = w.to(data.device, dtype=data.dtype) + if w.numel() != M: + raise ValueError(f"weights length {w.numel()} must match number of quaternions {M}") + w = w / w.sum() + A = data.T @ torch.diag(w) @ data + + orig_dtype = A.dtype + if A.dtype in (torch.float16, torch.bfloat16): + A = A.float() + eigenvalues, eigenvectors = torch.linalg.eigh(A) + # Use float32 eigenvalues for argmax to avoid half-precision rounding + # changing which eigenvector is selected when eigenvalues are close. + max_idx = torch.argmax(eigenvalues) + eigenvectors = eigenvectors.to(orig_dtype) + q_avg = eigenvectors[:, max_idx] + q_avg = q_avg / q_avg.norm() + + return Quaternion(q_avg.unsqueeze(0)) diff --git a/kornia/geometry/ransac.py b/kornia/geometry/ransac.py new file mode 100644 index 0000000..437f20f --- /dev/null +++ b/kornia/geometry/ransac.py @@ -0,0 +1,408 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Module containing RANSAC modules.""" + +import math +from functools import partial +from typing import Callable, Optional, Tuple + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK_SHAPE +from kornia.geometry.epipolar import find_essential, find_fundamental, sampson_epipolar_distance +from kornia.geometry.homography import ( + find_homography_dlt, + find_homography_dlt_iterated, + find_homography_lines_dlt, + find_homography_lines_dlt_iterated, + line_segment_transfer_error_one_way, + oneway_transfer_error, + sample_is_valid_for_homography, +) + +__all__ = ["RANSAC"] + + +class RANSAC(nn.Module): + """Module for robust geometry estimation with RANSAC. https://en.wikipedia.org/wiki/Random_sample_consensus. + + Args: + model_type: type of model to estimate: "homography", "fundamental", "fundamental_7pt", + "homography_from_linesegments". + inliers_threshold: threshold for the correspondence to be an inlier. + batch_size: number of generated samples at once. + max_iterations: maximum batches to generate. Actual number of models to try is ``batch_size * max_iterations``. + confidence: desired confidence of the result, used for the early stopping. + max_local_iterations: number of local optimization (polishing) iterations. + + """ + + def __init__( + self, + model_type: str = "homography", + inl_th: float = 2.0, + batch_size: int = 2048, + max_iter: int = 10, + confidence: float = 0.99, + max_lo_iters: int = 5, + score_type: str = "ransac", + prosac_sampling: bool = False, + seed: Optional[int] = None, + ) -> None: + """Initialize the RANSAC estimator. + + Args: + model_type: type of model to estimate: "homography", "fundamental", "fundamental_7pt", "essential", + "homography_from_linesegments". + inl_th: threshold for the correspondence to be an inlier. Internally is squared. + batch_size: number of generated samples at once. + max_iter: maximum batches to generate. Actual number of models to try is ``batch_size * max_iter``. + confidence: desired confidence of the result, used for the early stopping. + max_lo_iters: number of local optimization (polishing) iterations. + score_type: scoring method to use: "ransac" or "msac". + prosac_sampling: whether to use PROSAC sampling instead of random sampling. + seed: optional random seed for reproducible results. If None, uses global random state. + + """ + super().__init__() + self.supported_models = [ + "homography", + "fundamental", + "fundamental_7pt", + "homography_from_linesegments", + "essential", + ] + self.supported_scores = ["msac", "ransac"] + self.score_type = score_type + self.inl_th = inl_th + self.max_iter = max_iter + self.batch_size = batch_size + self.model_type = model_type + self.confidence = confidence + self.max_lo_iters = max_lo_iters + self.model_type = model_type + self.prosac_sampling = prosac_sampling + self.seed = seed + + self.error_fn: Callable[..., torch.Tensor] + self.minimal_solver: Callable[..., torch.Tensor] + self.polisher_solver: Callable[..., torch.Tensor] + + if model_type == "homography": + self.error_fn = oneway_transfer_error + self.minimal_solver = find_homography_dlt + self.polisher_solver = find_homography_dlt_iterated + self.minimal_sample_size = 4 + self.polisher_sample_size = 4 + elif model_type == "homography_from_linesegments": + self.error_fn = line_segment_transfer_error_one_way + self.minimal_solver = find_homography_lines_dlt + self.polisher_solver = find_homography_lines_dlt_iterated + self.minimal_sample_size = 4 + self.polisher_sample_size = 4 + elif model_type == "fundamental": + self.error_fn = sampson_epipolar_distance + self.minimal_solver = find_fundamental + self.minimal_sample_size = 8 + self.polisher_solver = find_fundamental + self.polisher_sample_size = 8 + elif model_type == "fundamental_7pt": + self.error_fn = sampson_epipolar_distance + self.minimal_solver = partial(find_fundamental, method="7POINT") + self.minimal_sample_size = 7 + self.polisher_solver = find_fundamental + self.polisher_sample_size = 8 + elif model_type == "essential": + self.error_fn = sampson_epipolar_distance + self.minimal_solver = find_essential + self.minimal_sample_size = 5 + self.polisher_solver = find_fundamental + self.polisher_sample_size = 8 + else: + raise NotImplementedError(f"{model_type} is unknown. Try one of {self.supported_models}") + + def sample( + self, sample_size: int, pop_size: int, batch_size: int, iteration: int, device: Optional[torch.device] = None + ) -> torch.Tensor: + """Minimal sampler, but unlike traditional RANSAC we sample in batches. + + Yields the benefit of the parallel processing, esp. on GPU. + + Args: + sample_size: number of samples to draw from the population. + pop_size: size of the population to sample from. + batch_size: number of sample sets to generate. + iteration: current iteration number (used for PROSAC sampling). + device: device to place the samples on. + + Returns: + Tensor of sampled indices with shape :math:`(batch_size, sample_size)`. + + """ + if device is None: + device = torch.device("cpu") + if self.seed is not None: + generator = torch.Generator(device=device) + generator.manual_seed(self.seed + iteration) + rand = torch.rand(batch_size, pop_size, device=device, generator=generator) + else: + rand = torch.rand(batch_size, pop_size, device=device) + _, out = rand.topk(k=sample_size, dim=1) + return out + + @staticmethod + def max_samples_by_conf(n_inl: int, num_tc: int, sample_size: int, conf: float) -> int: + """Update max_iter to stop iterations earlier https://en.wikipedia.org/wiki/Random_sample_consensus. + + Args: + n_inl: number of inliers. + num_tc: total number of correspondences. + sample_size: size of minimal sample. + conf: desired confidence level. + + Returns: + Maximum number of samples needed to achieve the desired confidence. + + """ + eps = 1e-9 + if num_tc <= sample_size: + return 1 + if n_inl == num_tc: + return 1 + if n_inl <= sample_size: + return 1 + if conf >= 1.0: + return 1 + if conf <= 0.0: + return 1 + # Proper RANSAC formula for sampling without replacement + # P(all samples are inliers) = (n_inl/num_tc) * ((n_inl-1)/(num_tc-1)) * ... + # ... * ((n_inl-sample_size+1)/(num_tc-sample_size+1)) + prob_inlier = 1.0 + for i in range(sample_size): + prob_inlier *= (n_inl - i) / (num_tc - i) + + return int(math.log(1.0 - conf) / min(-eps, math.log(max(eps, 1.0 - prob_inlier)))) + + def estimate_model_from_minsample(self, kp1: torch.Tensor, kp2: torch.Tensor) -> torch.Tensor: + """Estimate models from minimal samples. + + Args: + kp1: source keypoints with shape :math:`(batch_size, sample_size, 2)`. + kp2: target keypoints with shape :math:`(batch_size, sample_size, 2)`. + + Returns: + Estimated models tensor. + + """ + batch_size, sample_size = kp1.shape[:2] + H = self.minimal_solver(kp1, kp2, torch.ones(batch_size, sample_size, dtype=kp1.dtype, device=kp1.device)) + return H + + def verify( + self, kp1: torch.Tensor, kp2: torch.Tensor, models: torch.Tensor, inl_th: float + ) -> Tuple[torch.Tensor, torch.Tensor, float, float]: + """Verify models by computing inliers and selecting the best model. + + Args: + kp1: source keypoints. + kp2: target keypoints. + models: candidate models to verify. + inl_th: inlier threshold. + + Returns: + Tuple containing: + - Best model + - Inlier mask for the best model + - Score of the best model + + """ + if len(kp1.shape) == 2: + kp1 = kp1[None] + if len(kp2.shape) == 2: + kp2 = kp2[None] + batch_size = models.shape[0] + if self.model_type == "homography_from_linesegments": + errors = self.error_fn(kp1.expand(batch_size, -1, 2, 2), kp2.expand(batch_size, -1, 2, 2), models) + else: + errors = self.error_fn(kp1.expand(batch_size, -1, 2), kp2.expand(batch_size, -1, 2), models) + inl_mask = errors <= inl_th + score_ransac = inl_mask.sum(dim=1) + if self.score_type == "msac": + score = errors.shape[1] - errors.clamp(min=0.0, max=inl_th).sum(dim=1) + elif self.score_type == "ransac": + score = score_ransac + else: + raise ValueError(f"Unsupported score type: {self.score_type}") + best_model_idx = score.argmax() + best_model_score = score[best_model_idx].item() + num_inliers = score_ransac[best_model_idx].item() + model_best = models[best_model_idx].clone() + inliers_best = inl_mask[best_model_idx] + return model_best, inliers_best, best_model_score, num_inliers + + def remove_bad_samples(self, kp1: torch.Tensor, kp2: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Remove degenerate samples based on model-specific constraints. + + Args: + kp1: source keypoints. + kp2: target keypoints. + + Returns: + Tuple of filtered keypoints (kp1, kp2). + + """ + # ToDo: add (model-specific) verification of the samples, + # E.g. constraints on not to be a degenerate sample + if self.model_type == "homography": + mask = sample_is_valid_for_homography(kp1, kp2) + return kp1[mask], kp2[mask] + return kp1, kp2 + + def remove_bad_models(self, models: torch.Tensor) -> torch.Tensor: + """Remove degenerate models based on simple heuristics. + + Args: + models: candidate models to filter. + + Returns: + Filtered models tensor. + + """ + # Filter out NaN or Inf models + mask = torch.isfinite(models).all(dim=-1).all(dim=-1) + return models[mask] + + def polish_model(self, kp1: torch.Tensor, kp2: torch.Tensor, inliers: torch.Tensor) -> torch.Tensor: + """Polish the model using inliers through local optimization. + + Args: + kp1: source keypoints. + kp2: target keypoints. + inliers: boolean mask indicating inlier correspondences. + + Returns: + Polished model tensor. + + """ + # TODO: Replace this with MAGSAC++ polisher + kp1_inl = kp1[inliers][None] + kp2_inl = kp2[inliers][None] + num_inl = kp1_inl.size(1) + model = self.polisher_solver( + kp1_inl, kp2_inl, torch.ones(1, num_inl, dtype=kp1_inl.dtype, device=kp1_inl.device) + ) + return model + + def validate_inputs(self, kp1: torch.Tensor, kp2: torch.Tensor, weights: Optional[torch.Tensor] = None) -> None: + """Validate input tensors for shape and size requirements. + + Args: + kp1: source keypoints. + kp2: target keypoints. + weights: optional correspondence weights (not used currently). + + Raises: + ValueError: if input shapes are invalid or insufficient correspondences. + + """ + if self.model_type in ["homography", "fundamental"]: + KORNIA_CHECK_SHAPE(kp1, ["N", "2"]) + KORNIA_CHECK_SHAPE(kp2, ["N", "2"]) + if not (kp1.shape[0] == kp2.shape[0]) or (kp1.shape[0] < self.minimal_sample_size): + raise ValueError( + "kp1 and kp2 should be equal shape at least" + f" [{self.minimal_sample_size}, 2], got {kp1.shape}, {kp2.shape}" + ) + if self.model_type == "homography_from_linesegments": + KORNIA_CHECK_SHAPE(kp1, ["N", "2", "2"]) + KORNIA_CHECK_SHAPE(kp2, ["N", "2", "2"]) + if not (kp1.shape[0] == kp2.shape[0]) or (kp1.shape[0] < self.minimal_sample_size): + raise ValueError( + "kp1 and kp2 should be equal shape at least" + f" [{self.minimal_sample_size}, 2, 2], got {kp1.shape}," + f" {kp2.shape}" + ) + + def forward( + self, kp1: torch.Tensor, kp2: torch.Tensor, weights: Optional[torch.Tensor] = None + ) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Call main forward method to execute the RANSAC algorithm. + + Args: + kp1: source image keypoints :math:`(N, 2)`. + kp2: distance image keypoints :math:`(N, 2)`. + weights: optional correspondences weights. Not used now. + + Returns: + - Estimated model, shape of :math:`(1, 3, 3)`. + - The inlier/outlier mask, shape of :math:`(1, N)`, where N is number of input correspondences. + + """ + self.validate_inputs(kp1, kp2, weights) + best_score_total: float = float(self.minimal_sample_size) + num_tc: int = len(kp1) + best_model_total = torch.zeros(3, 3, dtype=kp1.dtype, device=kp1.device) + inliers_best_total: torch.Tensor = torch.zeros(num_tc, 1, device=kp1.device, dtype=torch.bool) + for i in range(self.max_iter): + # Sample minimal samples in batch to estimate models + idxs = self.sample(self.minimal_sample_size, num_tc, self.batch_size, i, kp1.device) + kp1_sampled = kp1[idxs] + kp2_sampled = kp2[idxs] + kp1_sampled, kp2_sampled = self.remove_bad_samples(kp1_sampled, kp2_sampled) + if len(kp1_sampled) == 0: + continue + # Estimate models + models = self.estimate_model_from_minsample(kp1_sampled, kp2_sampled) + if self.model_type in ["essential", "fundamental_7pt"]: + models = models.reshape(-1, 3, 3) + models = self.remove_bad_models(models) + if (models is None) or (len(models) == 0): + continue + # Score the models and select the best one + model, inliers, model_score, num_inliers = self.verify(kp1, kp2, models, self.inl_th**2) + # Store far-the-best model and (optionally) do a local optimization + if (model_score > best_score_total) and num_inliers >= self.polisher_sample_size: + # Local optimization + for _ in range(self.max_lo_iters): + model_lo = self.polish_model(kp1, kp2, inliers) + if (model_lo is None) or (len(model_lo) == 0): + continue + _, inliers_lo, score_lo, num_inliers_lo = self.verify(kp1, kp2, model_lo, self.inl_th**2) + if (score_lo > model_score) and (num_inliers_lo >= self.polisher_sample_size): + model = model_lo.clone()[0] + inliers = inliers_lo.clone() + model_score = score_lo + else: + break + # Now storing the best model + best_model_total = model.clone() + inliers_best_total = inliers.clone() + best_score_total = model_score + + # Should we already stop? + new_max_iter = self.max_samples_by_conf( + int(best_score_total), num_tc, self.minimal_sample_size, self.confidence + ) + + # Stop estimation, if the model is very good + if (i + 1) * self.batch_size >= new_max_iter: + break + # local optimization with all inliers for better precision + return best_model_total, inliers_best_total diff --git a/kornia/geometry/ray.py b/kornia/geometry/ray.py new file mode 100644 index 0000000..a8fbbe1 --- /dev/null +++ b/kornia/geometry/ray.py @@ -0,0 +1,22 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from kornia.geometry.line import ParametrizedLine + +# NOTE: for now lets make an alias +# TODO: add more functionality and semantics for graphics stuff +Ray = ParametrizedLine diff --git a/kornia/geometry/solvers/__init__.py b/kornia/geometry/solvers/__init__.py new file mode 100644 index 0000000..641c1fb --- /dev/null +++ b/kornia/geometry/solvers/__init__.py @@ -0,0 +1,41 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Geometry Solvers — Polynomial and geometric solvers for Kornia. + +This subpackage provides polynomial solvers and related utilities for geometry tasks. +""" + +from .homogeneous import null_vector_3x4 +from .polynomial_solver import ( + determinant_to_polynomial, + multiply_deg_one_poly, + multiply_deg_two_one_poly, + solve_cubic, + solve_quadratic, + solve_quartic, +) + +__all__ = [ + "determinant_to_polynomial", + "multiply_deg_one_poly", + "multiply_deg_two_one_poly", + "null_vector_3x4", + "solve_cubic", + "solve_quadratic", + "solve_quartic", +] diff --git a/kornia/geometry/solvers/homogeneous.py b/kornia/geometry/solvers/homogeneous.py new file mode 100644 index 0000000..c2a38f6 --- /dev/null +++ b/kornia/geometry/solvers/homogeneous.py @@ -0,0 +1,168 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Closed-form solvers for homogeneous linear systems.""" + +from __future__ import annotations + +import torch + +from kornia.core.check import KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE + + +def _det3( + a0: torch.Tensor, + a1: torch.Tensor, + a2: torch.Tensor, + b0: torch.Tensor, + b1: torch.Tensor, + b2: torch.Tensor, + c0: torch.Tensor, + c1: torch.Tensor, + c2: torch.Tensor, +) -> torch.Tensor: + """Compute a batch of 3x3 determinants via Sarrus' rule. + + Given three rows (each split into their three scalar components), returns:: + + | a0 a1 a2 | + | b0 b1 b2 | + | c0 c1 c2 | + + All inputs must be broadcastable to the same shape. + + Args: + a0: first element of the first row. + a1: second element of the first row. + a2: third element of the first row. + b0: first element of the second row. + b1: second element of the second row. + b2: third element of the second row. + c0: first element of the third row. + c1: second element of the third row. + c2: third element of the third row. + + Returns: + Scalar determinant (or batch of scalars) with the broadcasted shape. + """ + return a0 * (b1 * c2 - b2 * c1) - a1 * (b0 * c2 - b2 * c0) + a2 * (b0 * c1 - b1 * c0) + + +def null_vector_3x4(A: torch.Tensor) -> torch.Tensor: + r"""Return the null vector of a rank-3 matrix of shape :math:`(*, 3, 4)`. + + The null vector :math:`\mathbf{v} \in \mathbb{R}^4` satisfies + :math:`A\,\mathbf{v} = \mathbf{0}`. For a matrix of rank 3 this solution + is unique up to scale. + + The computation uses the **4-D cross-product** (cofactor expansion): + each component of :math:`\mathbf{v}` is a :math:`3 \times 3` determinant of + the submatrix obtained by dropping the corresponding column of :math:`A`. + This is equivalent to computing the last right singular vector of :math:`A` + via SVD but replaces the SVD with 48 scalar multiplications and 20 + additions — no LAPACK or cuSOLVER call is made. + + .. math:: + + v_j = (-1)^j \det\!\bigl(A_{[0,1,2],\,\widehat{j}}\bigr), \quad j = 0, 1, 2, 3 + + where :math:`A_{[0,1,2],\,\widehat{j}}` denotes the :math:`3 \times 3` + submatrix formed by deleting column :math:`j`. + + .. note:: + + The returned vector is **not** normalised. Divide by its norm if a + unit null vector is required. + + .. note:: + + The function is only correct when :math:`A` has rank exactly 3. For + rank-deficient inputs (rank < 3) the result is the zero vector. + + Args: + A: matrix of shape :math:`(*, 3, 4)`. + + Returns: + Null vector of shape :math:`(*, 4)`. + + Raises: + TypeCheckError: if ``A`` is not a :class:`torch.Tensor`. + ShapeError: if the last two dimensions of ``A`` are not ``(3, 4)``. + + Example: + >>> A = torch.tensor([[[1., 0., 0., 0.], + ... [0., 1., 0., 0.], + ... [0., 0., 1., 0.]]]) # null vector is [0,0,0,1] + >>> v = null_vector_3x4(A) # shape (1, 4) + >>> (A @ v.unsqueeze(-1)).squeeze(-1) # should be near zero + tensor([[0., 0., 0.]]) + + """ + KORNIA_CHECK_IS_TENSOR(A) + KORNIA_CHECK_SHAPE(A, ["*", "3", "4"]) + + a = A[..., 0, :] # (*, 4) + b = A[..., 1, :] # (*, 4) + c = A[..., 2, :] # (*, 4) + + # Each component of the null vector is a signed 3x3 cofactor determinant. + v0 = _det3( + a[..., 1], + a[..., 2], + a[..., 3], + b[..., 1], + b[..., 2], + b[..., 3], + c[..., 1], + c[..., 2], + c[..., 3], + ) + v1 = -_det3( + a[..., 0], + a[..., 2], + a[..., 3], + b[..., 0], + b[..., 2], + b[..., 3], + c[..., 0], + c[..., 2], + c[..., 3], + ) + v2 = _det3( + a[..., 0], + a[..., 1], + a[..., 3], + b[..., 0], + b[..., 1], + b[..., 3], + c[..., 0], + c[..., 1], + c[..., 3], + ) + v3 = -_det3( + a[..., 0], + a[..., 1], + a[..., 2], + b[..., 0], + b[..., 1], + b[..., 2], + c[..., 0], + c[..., 1], + c[..., 2], + ) + + return torch.stack([v0, v1, v2, v3], dim=-1) diff --git a/kornia/geometry/solvers/polynomial_solver.py b/kornia/geometry/solvers/polynomial_solver.py new file mode 100644 index 0000000..ef593d7 --- /dev/null +++ b/kornia/geometry/solvers/polynomial_solver.py @@ -0,0 +1,1925 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""nn.Module containing the functionalities for computing the real roots of polynomial equation.""" + +import math + +import torch + +from kornia.core.check import KORNIA_CHECK_SHAPE + + +# Reference : https://github.com/opencv/opencv/blob/4.x/modules/calib3d/src/polynom_solver.cpp +def solve_quadratic(coeffs: torch.Tensor) -> torch.Tensor: + r"""Solve given quadratic equation. + + The function takes the coefficients of quadratic equation and returns the real roots. + + .. math:: coeffs[0]x^2 + coeffs[1]x + coeffs[2] = 0 + + Args: + coeffs : The coefficients of quadratic equation :`(B, 3)` + + Returns: + A torch.Tensor of shape `(B, 2)` containing the real roots to the quadratic equation. + + Example: + >>> coeffs = torch.tensor([[1., 4., 4.]]) + >>> roots = solve_quadratic(coeffs) + + .. note:: + In cases where a quadratic polynomial has only one real root, the output will be in the format + [real_root, 0]. And for the torch.complex roots should be represented as 0. This is done to maintain + a consistent output shape for all cases. + + """ + KORNIA_CHECK_SHAPE(coeffs, ["B", "3"]) + + # Coefficients of quadratic equation + a = coeffs[:, 0] # coefficient of x^2 + b = coeffs[:, 1] # coefficient of x + c = coeffs[:, 2] # constant term + + # Calculate discriminant + delta = b * b - 4 * a * c + + # Create masks for negative and zero discriminant + mask_negative = delta < 0 + mask_zero = delta == 0 + + # Calculate 1/(2*a) for efficient computation + inv_2a = 0.5 / a + + # Initialize solutions torch.Tensor + solutions = torch.zeros((coeffs.shape[0], 2), device=coeffs.device, dtype=coeffs.dtype) + + # Handle cases with zero discriminant + if torch.any(mask_zero): + solutions[mask_zero, 0] = -b[mask_zero] * inv_2a[mask_zero] + solutions[mask_zero, 1] = solutions[mask_zero, 0] + + # Negative discriminant cases are automatically handled since solutions is initialized with torch.zeros. + + sqrt_delta = torch.sqrt(delta) + + # Handle cases with non-negative discriminant + mask = torch.bitwise_and(~mask_negative, ~mask_zero) + if torch.any(mask): + solutions[mask, 0] = (-b[mask] + sqrt_delta[mask]) * inv_2a[mask] + solutions[mask, 1] = (-b[mask] - sqrt_delta[mask]) * inv_2a[mask] + + return solutions + + +def solve_cubic(coeffs: torch.Tensor) -> torch.Tensor: + r"""Solve given cubic equation. + + The function takes the coefficients of cubic equation and returns + the real roots. + + .. math:: coeffs[0]x^3 + coeffs[1]x^2 + coeffs[2]x + coeffs[3] = 0 + + Args: + coeffs : The coefficients cubic equation : `(B, 4)` + + Returns: + A torch.Tensor of shape `(B, 3)` containing the real roots to the cubic equation. + + Example: + >>> coeffs = torch.tensor([[32., 3., -11., -6.]]) + >>> roots = solve_cubic(coeffs) + + .. note:: + In cases where a cubic polynomial has only one or two real roots, the output for the non-real + roots should be represented as 0. Thus, the output for a single real root should be in the + format [real_root, 0, 0], and for two real roots, it should be [real_root_1, real_root_2, 0]. + + """ + KORNIA_CHECK_SHAPE(coeffs, ["B", "4"]) + + _PI = torch.tensor(math.pi, device=coeffs.device, dtype=coeffs.dtype) + + # Coefficients of cubic equation + a = coeffs[:, 0] # coefficient of x^3 + b = coeffs[:, 1] # coefficient of x^2 + c = coeffs[:, 2] # coefficient of x + d = coeffs[:, 3] # constant term + + solutions = torch.zeros((len(coeffs), 3), device=a.device, dtype=a.dtype) + + mask_a_zero = a == 0 + mask_b_zero = b == 0 + mask_c_zero = c == 0 + + # Zero order cases are automatically handled since solutions is initialized with torch.zeros. + # No need for explicit handling of mask_zero_order as solutions already contains torch.zeros by default. + + mask_first_order = mask_a_zero & mask_b_zero & ~mask_c_zero + mask_second_order = mask_a_zero & ~mask_b_zero & ~mask_c_zero + + if torch.any(mask_second_order): + solutions[mask_second_order, 0:2] = solve_quadratic(coeffs[mask_second_order, 1:]) + + if torch.any(mask_first_order): + solutions[mask_first_order, 0] = torch.tensor(1.0, device=a.device, dtype=a.dtype) + + # Normalized form x^3 + a2 * x^2 + a1 * x + a0 = 0 + inv_a = 1.0 / a[~mask_a_zero] + b_a = inv_a * b[~mask_a_zero] + b_a2 = b_a * b_a + + c_a = inv_a * c[~mask_a_zero] + d_a = inv_a * d[~mask_a_zero] + + # Solve the cubic equation + Q = (3 * c_a - b_a2) / 9 + R = (9 * b_a * c_a - 27 * d_a - 2 * b_a * b_a2) / 54 + Q3 = Q * Q * Q + D = Q3 + R * R + b_a_3 = (1.0 / 3.0) * b_a + + a_Q_zero = torch.ones_like(a) + a_R_zero = torch.ones_like(a) + a_D_zero = torch.ones_like(a) + + a_Q_zero[~mask_a_zero] = Q + a_R_zero[~mask_a_zero] = R + a_D_zero[~mask_a_zero] = D + + # Q == 0 + mask_Q_zero = (Q == 0) & (R != 0) + mask_Q_zero_solutions = (a_Q_zero == 0) & (a_R_zero != 0) + + if torch.any(mask_Q_zero): + x0_Q_zero = torch.pow(2 * R[mask_Q_zero], 1 / 3) - b_a_3[mask_Q_zero] + solutions[mask_Q_zero_solutions, 0] = x0_Q_zero + + mask_QR_zero = (Q == 0) & (R == 0) + mask_QR_zero_solutions = (a_Q_zero == 0) & (a_R_zero == 0) + + if torch.any(mask_QR_zero): + solutions[mask_QR_zero_solutions] = torch.stack( + [-b_a_3[mask_QR_zero], -b_a_3[mask_QR_zero], -b_a_3[mask_QR_zero]], dim=1 + ) + + # D <= 0 + mask_D_zero = (D <= 0) & (Q != 0) + mask_D_zero_solutions = (a_D_zero <= 0) & (a_Q_zero != 0) + + if torch.any(mask_D_zero): + theta_D_zero = torch.acos(R[mask_D_zero] / torch.sqrt(-Q3[mask_D_zero])) + sqrt_Q_D_zero = torch.sqrt(-Q[mask_D_zero]) + x0_D_zero = 2 * sqrt_Q_D_zero * torch.cos(theta_D_zero / 3.0) - b_a_3[mask_D_zero] + x1_D_zero = 2 * sqrt_Q_D_zero * torch.cos((theta_D_zero + 2 * _PI) / 3.0) - b_a_3[mask_D_zero] + x2_D_zero = 2 * sqrt_Q_D_zero * torch.cos((theta_D_zero + 4 * _PI) / 3.0) - b_a_3[mask_D_zero] + solutions[mask_D_zero_solutions] = torch.stack([x0_D_zero, x1_D_zero, x2_D_zero], dim=1) + + a_D_positive = torch.zeros_like(a) + a_D_positive[~mask_a_zero] = D + # D > 0 + mask_D_positive_solution = (a_D_positive > 0) & (a_Q_zero != 0) + mask_D_positive = (D > 0) & (Q != 0) + if torch.any(mask_D_positive): + AD = torch.zeros_like(R) + BD = torch.zeros_like(R) + R_abs = torch.abs(R) + mask_R_positive = R_abs > 1e-16 + if torch.any(mask_R_positive): + AD[mask_R_positive] = torch.pow(R_abs[mask_R_positive] + torch.sqrt(D[mask_R_positive]), 1 / 3) + mask_R_positive_ = R < 0 + + if torch.any(mask_R_positive_): + AD[mask_R_positive_] = -AD[mask_R_positive_] + + BD[mask_R_positive] = -Q[mask_R_positive] / AD[mask_R_positive] + x0_D_positive = AD[mask_D_positive] + BD[mask_D_positive] - b_a_3[mask_D_positive] + solutions[mask_D_positive_solution, 0] = x0_D_positive + + return solutions + + +def solve_quartic(coeffs: torch.Tensor) -> torch.Tensor: + r"""Solve given quartic equation. + + The function takes the coefficients of quartic equation and returns + the real roots. + + .. math:: coeffs[0]x^4 + coeffs[1]x^3 + coeffs[2]x^2 + coeffs[3]x + coeffs[4] = 0 + + Args: + coeffs : The coefficients quartic equation : `(B, 5)` + + Returns: + A torch.Tensor of shape `(B, 4)` containing the real roots to the quartic equation. + + Example: + >>> coeffs = torch.tensor([[1., -10., 35., -50., 24.]]) + >>> roots = solve_quartic(coeffs) + + .. note:: + In cases where a quartic polynomial has fewer than four real roots, the remaining entries + in the output are set to 0. Similarly, any non-real (complex) roots are represented as 0. + This is done to maintain a consistent output shape for all cases. + """ + KORNIA_CHECK_SHAPE(coeffs, ["B", "5"]) + + # Coefficients + a, b, c, d, e = coeffs.unbind(dim=-1) + + solutions = torch.zeros((len(coeffs), 4), device=coeffs.device, dtype=coeffs.dtype) + + # Numerical tolerances + zero_tol = 1e-6 if coeffs.dtype == torch.float32 else 1e-12 + + # Cubic fallback for a approx 0 + mask_a_zero = torch.abs(a) < zero_tol + mask_quartic = ~mask_a_zero + + if torch.any(mask_a_zero): + solutions[mask_a_zero, 0:3] = solve_cubic(coeffs[mask_a_zero, 1:]) + + if not torch.any(mask_quartic): + return solutions + + # Normalized coefficients: x^4 + A*x^3 + B*x^2 + C*x + D = 0 + a_q = a[mask_quartic] + inv_a = 1.0 / a_q + + A = b[mask_quartic] * inv_a + B = c[mask_quartic] * inv_a + C = d[mask_quartic] * inv_a + D = e[mask_quartic] * inv_a + + # Resolvent cubic coefficients + rc_a = torch.ones_like(A) + rc_b = -B + rc_c = A * C - 4.0 * D + rc_d = -1.0 * (A * A * D - 4.0 * B * D + C * C) + + cubic_coeffs = torch.stack([rc_a, rc_b, rc_c, rc_d], dim=1) + + # Solve cubic (Ferrari's method) + y_roots = solve_cubic(cubic_coeffs) + + # Robust Root Selection: Pick y that maximizes R^2 + A_sq = A * A + R_sq_candidates = 0.25 * A_sq.unsqueeze(-1) - B.unsqueeze(-1) + y_roots + + best_idx = torch.argmax(R_sq_candidates, dim=-1, keepdim=True) + y = torch.gather(y_roots, -1, best_idx).squeeze(-1) + R_sq = torch.gather(R_sq_candidates, -1, best_idx).squeeze(-1) + + R = torch.sqrt(torch.clamp(R_sq, min=0.0)) + + # Compute E term + mask_R_small = torch.abs(R) < zero_tol + mask_R_large = ~mask_R_small + E = torch.zeros_like(R) + + if torch.any(mask_R_large): + numerator = A[mask_R_large] * y[mask_R_large] - 2.0 * C[mask_R_large] + denominator = 4.0 * R[mask_R_large] + E[mask_R_large] = numerator / denominator + + # Fallback for R approx 0 + if torch.any(mask_R_small): + radicand = 0.25 * y[mask_R_small] * y[mask_R_small] - D[mask_R_small] + E[mask_R_small] = torch.sqrt(torch.clamp(radicand, min=0.0)) + + # Solve two resulting quadratic equations + # Quad 1: x^2 + (A/2 - R)x + (y/2 - E) = 0 + q1_a = torch.ones_like(A) + q1_b = 0.5 * A - R + q1_c = 0.5 * y - E + + # Quad 2: x^2 + (A/2 + R)x + (y/2 + E) = 0 + q2_a = torch.ones_like(A) + q2_b = 0.5 * A + R + q2_c = 0.5 * y + E + + roots1 = solve_quadratic(torch.stack([q1_a, q1_b, q1_c], dim=1)) + roots2 = solve_quadratic(torch.stack([q2_a, q2_b, q2_c], dim=1)) + + solutions[mask_quartic, 0:2] = roots1 + solutions[mask_quartic, 2:4] = roots2 + + return solutions + + +# Reference +# https://github.com/danini/graph-cut-ransac/blob/master/src/pygcransac/include/ +# estimators/solver_essential_matrix_five_point_nister.h#L108 + + +T_deg1 = torch.zeros(16, 10) +T_deg1[0, 0] = 1 # x * x → x^2 +T_deg1[1, 1] = 1 # x * y +T_deg1[4, 1] = 1 # y * x +T_deg1[2, 2] = 1 # x * z +T_deg1[8, 2] = 1 # z * x +T_deg1[3, 3] = 1 # x * 1 +T_deg1[12, 3] = 1 # 1 * x +T_deg1[5, 4] = 1 # y * y +T_deg1[6, 5] = 1 # y * z +T_deg1[9, 5] = 1 # z * y +T_deg1[7, 6] = 1 # y * 1 +T_deg1[13, 6] = 1 # 1 * y +T_deg1[10, 7] = 1 # z * z +T_deg1[11, 8] = 1 # z * 1 +T_deg1[14, 8] = 1 # 1 * z +T_deg1[15, 9] = 1 # 1 * 1 + + +def multiply_deg_one_poly(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: + r"""Multiply two polynomials of the first order [@nister2004efficient]. + + Args: + a: a first order polynomial for variables :math:`(x,y,z,1)`. + b: a first order polynomial for variables :math:`(x,y,z,1)`. + + Returns: + degree 2 poly with the order :math:`(x^2, x*y, x*z, x, y^2, y*z, y, z^2, z, 1)`. + + """ + global T_deg1 # noqa: PLW0603 + if T_deg1.device != a.device or T_deg1.dtype != a.dtype: + T_deg1 = T_deg1.to(device=a.device, dtype=a.dtype) + return (a.unsqueeze(2) * b.unsqueeze(1)).flatten(start_dim=-2) @ T_deg1 + + +# Reference +# https://github.com/danini/graph-cut-ransac/blob/aae1f40c2e10e31fd2191bac601c53a189673f60/src/pygcransac/ +# include/estimators/solver_essential_matrix_five_point_nister.h#L156 + +T_deg2 = torch.zeros(40, 20) +T_deg2[0, 0] = 1 # (0*4+0) +T_deg2[17, 1] = 1 # (4*4+1) +T_deg2[1, 2] = 1 # (0*4+1) +T_deg2[4, 2] = 1 # (1*4+0) +T_deg2[5, 3] = 1 # (1*4+1) +T_deg2[16, 3] = 1 # (4*4+0) +T_deg2[2, 4] = 1 # (0*4+2) +T_deg2[8, 4] = 1 # (2*4+0) +T_deg2[3, 5] = 1 # (0*4+3) +T_deg2[12, 5] = 1 # (3*4+0) +T_deg2[18, 6] = 1 # (4*4+2) +T_deg2[21, 6] = 1 # (5*4+1) +T_deg2[19, 7] = 1 # (4*4+3) +T_deg2[25, 7] = 1 # (6*4+1) +T_deg2[6, 8] = 1 # (1*4+2) +T_deg2[9, 8] = 1 # (2*4+1) +T_deg2[20, 8] = 1 # (5*4+0) +T_deg2[7, 9] = 1 # (1*4+3) +T_deg2[13, 9] = 1 # (3*4+1) +T_deg2[24, 9] = 1 # (6*4+0) +T_deg2[10, 10] = 1 # (2*4+2) +T_deg2[28, 10] = 1 # (7*4+0) +T_deg2[11, 11] = 1 # (2*4+3) +T_deg2[14, 11] = 1 # (3*4+2) +T_deg2[32, 11] = 1 # (8*4+0) +T_deg2[15, 12] = 1 # (3*4+3) +T_deg2[36, 12] = 1 # (9*4+0) +T_deg2[22, 13] = 1 # (5*4+2) +T_deg2[29, 13] = 1 # (7*4+1) +T_deg2[23, 14] = 1 # (5*4+3) +T_deg2[26, 14] = 1 # (6*4+2) +T_deg2[33, 14] = 1 # (8*4+1) +T_deg2[27, 15] = 1 # (6*4+3) +T_deg2[37, 15] = 1 # (9*4+1) +T_deg2[30, 16] = 1 # (7*4+2) +T_deg2[31, 17] = 1 # (7*4+3) +T_deg2[34, 17] = 1 # (8*4+2) +T_deg2[35, 18] = 1 # (8*4+3) +T_deg2[38, 18] = 1 # (9*4+2) +T_deg2[39, 19] = 1 # (9*4+3) + + +def multiply_deg_two_one_poly(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: + r"""Multiply two polynomials a and b of degrees two and one [@nister2004efficient]. + + Args: + a: a second degree poly for variables :math:`(x^2, x*y, x*z, x, y^2, y*z, y, z^2, z, 1)`. + b: a first degree poly for variables :math:`(x y z 1)`. + + Returns: + a third degree poly for variables, + :math:`(x^3, y^3, x^2*y, x*y^2, x^2*z, x^2, y^2*z, y^2, + x*y*z, x*y, x*z^2, x*z, x, y*z^2, y*z, y, z^3, z^2, z, 1)`. + + """ + global T_deg2 # noqa: PLW0603 + if T_deg2.device != a.device or T_deg2.dtype != a.dtype: + T_deg2 = T_deg2.to(device=a.device, dtype=a.dtype) + product_basis = a.unsqueeze(2) * b.unsqueeze(1) + product_vector = product_basis.flatten(start_dim=-2) + return product_vector @ T_deg2 + + +# Compute degree 10 poly representing determinant (equation 14 in the paper) +# https://github.com/danini/graph-cut-ransac/blob/aae1f40c2e10e31fd2191bac601c53a189673f60/src/pygcransac/ +# include/estimators/solver_essential_matrix_five_point_nister.h#L368C5-L368C82 + +multiplication_indices = torch.tensor( + [ + [12, 16, 33], + [12, 20, 29], + [3, 33, 25], + [7, 29, 25], + [3, 20, 38], + [7, 16, 38], + [11, 16, 33], + [11, 20, 29], + [12, 15, 33], + [12, 16, 32], + [12, 19, 29], + [12, 20, 28], + [2, 33, 25], + [3, 32, 25], + [3, 33, 24], + [6, 29, 25], + [7, 28, 25], + [7, 29, 24], + [2, 20, 38], + [3, 19, 38], + [3, 20, 37], + [6, 16, 38], + [7, 15, 38], + [7, 16, 37], + [10, 16, 33], + [10, 20, 29], + [11, 15, 33], + [11, 16, 32], + [11, 19, 29], + [11, 20, 28], + [14, 12, 33], + [12, 15, 32], + [12, 16, 31], + [12, 18, 29], + [12, 19, 28], + [12, 20, 27], + [1, 33, 25], + [2, 32, 25], + [2, 33, 24], + [3, 31, 25], + [3, 32, 24], + [3, 33, 23], + [5, 29, 25], + [6, 28, 25], + [6, 29, 24], + [7, 27, 25], + [7, 28, 24], + [7, 29, 23], + [1, 20, 38], + [2, 19, 38], + [2, 20, 37], + [3, 18, 38], + [3, 19, 37], + [3, 20, 36], + [5, 16, 38], + [6, 15, 38], + [6, 16, 37], + [7, 14, 38], + [7, 15, 37], + [7, 16, 36], + [3, 20, 35], + [3, 22, 33], + [7, 16, 35], + [7, 22, 29], + [9, 16, 33], + [9, 20, 29], + [10, 15, 33], + [10, 16, 32], + [10, 19, 29], + [10, 20, 28], + [13, 12, 33], + [11, 14, 33], + [11, 15, 32], + [11, 16, 31], + [11, 18, 29], + [11, 19, 28], + [11, 20, 27], + [14, 12, 32], + [12, 15, 31], + [12, 16, 30], + [12, 17, 29], + [12, 18, 28], + [12, 19, 27], + [12, 20, 26], + [0, 33, 25], + [1, 32, 25], + [1, 33, 24], + [2, 31, 25], + [2, 32, 24], + [2, 33, 23], + [3, 30, 25], + [3, 31, 24], + [3, 32, 23], + [4, 29, 25], + [5, 28, 25], + [5, 29, 24], + [6, 27, 25], + [6, 28, 24], + [6, 29, 23], + [7, 26, 25], + [7, 27, 24], + [7, 28, 23], + [0, 20, 38], + [1, 19, 38], + [1, 20, 37], + [2, 18, 38], + [2, 19, 37], + [2, 20, 36], + [3, 17, 38], + [3, 18, 37], + [3, 19, 36], + [4, 16, 38], + [5, 15, 38], + [5, 16, 37], + [6, 14, 38], + [6, 15, 37], + [6, 16, 36], + [7, 13, 38], + [7, 14, 37], + [7, 15, 36], + [2, 20, 35], + [2, 22, 33], + [3, 19, 35], + [3, 20, 34], + [3, 21, 33], + [3, 22, 32], + [6, 16, 35], + [6, 22, 29], + [7, 15, 35], + [7, 16, 34], + [7, 21, 29], + [7, 22, 28], + [8, 16, 33], + [8, 20, 29], + [9, 15, 33], + [9, 16, 32], + [9, 19, 29], + [9, 20, 28], + [10, 14, 33], + [10, 15, 32], + [10, 16, 31], + [10, 18, 29], + [10, 19, 28], + [10, 20, 27], + [13, 11, 33], + [13, 12, 32], + [11, 14, 32], + [11, 15, 31], + [11, 16, 30], + [11, 17, 29], + [11, 18, 28], + [11, 19, 27], + [11, 20, 26], + [14, 12, 31], + [12, 15, 30], + [12, 17, 28], + [12, 18, 27], + [12, 19, 26], + [0, 32, 25], + [0, 33, 24], + [1, 31, 25], + [1, 32, 24], + [1, 33, 23], + [2, 30, 25], + [2, 31, 24], + [2, 32, 23], + [3, 30, 24], + [3, 31, 23], + [4, 28, 25], + [4, 29, 24], + [5, 27, 25], + [5, 28, 24], + [5, 29, 23], + [6, 26, 25], + [6, 27, 24], + [6, 28, 23], + [7, 26, 24], + [7, 27, 23], + [0, 19, 38], + [0, 20, 37], + [1, 18, 38], + [1, 19, 37], + [1, 20, 36], + [2, 17, 38], + [2, 18, 37], + [2, 19, 36], + [3, 17, 37], + [3, 18, 36], + [4, 15, 38], + [4, 16, 37], + [5, 14, 38], + [5, 15, 37], + [5, 16, 36], + [6, 13, 38], + [6, 14, 37], + [6, 15, 36], + [7, 13, 37], + [7, 14, 36], + [1, 20, 35], + [1, 22, 33], + [2, 19, 35], + [2, 20, 34], + [2, 21, 33], + [2, 22, 32], + [3, 18, 35], + [3, 19, 34], + [3, 21, 32], + [3, 22, 31], + [5, 16, 35], + [5, 22, 29], + [6, 15, 35], + [6, 16, 34], + [6, 21, 29], + [6, 22, 28], + [7, 14, 35], + [7, 15, 34], + [7, 21, 28], + [7, 22, 27], + [8, 15, 33], + [8, 16, 32], + [8, 19, 29], + [8, 20, 28], + [9, 14, 33], + [9, 15, 32], + [9, 16, 31], + [9, 18, 29], + [9, 19, 28], + [9, 20, 27], + [10, 13, 33], + [10, 14, 32], + [10, 15, 31], + [10, 16, 30], + [10, 17, 29], + [10, 18, 28], + [10, 19, 27], + [10, 20, 26], + [13, 11, 32], + [13, 12, 31], + [11, 14, 31], + [11, 15, 30], + [11, 17, 28], + [11, 18, 27], + [11, 19, 26], + [14, 12, 30], + [12, 17, 27], + [12, 18, 26], + [0, 31, 25], + [0, 32, 24], + [0, 33, 23], + [1, 30, 25], + [1, 31, 24], + [1, 32, 23], + [2, 30, 24], + [2, 31, 23], + [3, 30, 23], + [4, 27, 25], + [4, 28, 24], + [4, 29, 23], + [5, 26, 25], + [5, 27, 24], + [5, 28, 23], + [6, 26, 24], + [6, 27, 23], + [7, 26, 23], + [0, 18, 38], + [0, 19, 37], + [0, 20, 36], + [1, 17, 38], + [1, 18, 37], + [1, 19, 36], + [2, 17, 37], + [2, 18, 36], + [3, 17, 36], + [4, 14, 38], + [4, 15, 37], + [4, 16, 36], + [5, 13, 38], + [5, 14, 37], + [5, 15, 36], + [6, 13, 37], + [6, 14, 36], + [7, 13, 36], + [0, 20, 35], + [0, 22, 33], + [1, 19, 35], + [1, 20, 34], + [1, 21, 33], + [1, 22, 32], + [2, 18, 35], + [2, 19, 34], + [2, 21, 32], + [2, 22, 31], + [3, 17, 35], + [3, 18, 34], + [3, 21, 31], + [3, 22, 30], + [4, 16, 35], + [4, 22, 29], + [5, 15, 35], + [5, 16, 34], + [5, 21, 29], + [5, 22, 28], + [6, 14, 35], + [6, 15, 34], + [6, 21, 28], + [6, 22, 27], + [7, 13, 35], + [7, 14, 34], + [7, 21, 27], + [7, 22, 26], + [8, 14, 33], + [8, 15, 32], + [8, 16, 31], + [8, 18, 29], + [8, 19, 28], + [8, 20, 27], + [9, 13, 33], + [9, 14, 32], + [9, 15, 31], + [9, 16, 30], + [9, 17, 29], + [9, 18, 28], + [9, 19, 27], + [9, 20, 26], + [10, 13, 32], + [10, 14, 31], + [10, 15, 30], + [10, 17, 28], + [10, 18, 27], + [10, 19, 26], + [13, 11, 31], + [13, 12, 30], + [11, 14, 30], + [11, 17, 27], + [11, 18, 26], + [12, 17, 26], + [0, 30, 25], + [0, 31, 24], + [0, 32, 23], + [1, 30, 24], + [1, 31, 23], + [2, 30, 23], + [4, 26, 25], + [4, 27, 24], + [4, 28, 23], + [5, 26, 24], + [5, 27, 23], + [6, 26, 23], + [0, 17, 38], + [0, 18, 37], + [0, 19, 36], + [1, 17, 37], + [1, 18, 36], + [2, 17, 36], + [4, 13, 38], + [4, 14, 37], + [4, 15, 36], + [5, 13, 37], + [5, 14, 36], + [6, 13, 36], + [0, 19, 35], + [0, 20, 34], + [0, 21, 33], + [0, 22, 32], + [1, 18, 35], + [1, 19, 34], + [1, 21, 32], + [1, 22, 31], + [2, 17, 35], + [2, 18, 34], + [2, 21, 31], + [2, 22, 30], + [3, 17, 34], + [3, 21, 30], + [4, 15, 35], + [4, 16, 34], + [4, 21, 29], + [4, 22, 28], + [5, 14, 35], + [5, 15, 34], + [5, 21, 28], + [5, 22, 27], + [6, 13, 35], + [6, 14, 34], + [6, 21, 27], + [6, 22, 26], + [7, 13, 34], + [7, 21, 26], + [8, 13, 33], + [8, 14, 32], + [8, 15, 31], + [8, 16, 30], + [8, 17, 29], + [8, 18, 28], + [8, 19, 27], + [8, 20, 26], + [9, 13, 32], + [9, 14, 31], + [9, 15, 30], + [9, 17, 28], + [9, 18, 27], + [9, 19, 26], + [10, 13, 31], + [10, 14, 30], + [10, 17, 27], + [10, 18, 26], + [13, 11, 30], + [11, 17, 26], + [0, 30, 24], + [0, 31, 23], + [1, 30, 23], + [4, 26, 24], + [4, 27, 23], + [5, 26, 23], + [0, 17, 37], + [0, 18, 36], + [1, 17, 36], + [4, 13, 37], + [4, 14, 36], + [5, 13, 36], + [0, 18, 35], + [0, 19, 34], + [0, 21, 32], + [0, 22, 31], + [1, 17, 35], + [1, 18, 34], + [1, 21, 31], + [1, 22, 30], + [2, 17, 34], + [2, 21, 30], + [4, 14, 35], + [4, 15, 34], + [4, 21, 28], + [4, 22, 27], + [5, 13, 35], + [5, 14, 34], + [5, 21, 27], + [5, 22, 26], + [6, 13, 34], + [6, 21, 26], + [8, 13, 32], + [8, 14, 31], + [8, 15, 30], + [8, 17, 28], + [8, 18, 27], + [8, 19, 26], + [9, 13, 31], + [9, 14, 30], + [9, 17, 27], + [9, 18, 26], + [10, 13, 30], + [10, 17, 26], + [0, 30, 23], + [4, 26, 23], + [0, 17, 36], + [4, 13, 36], + [0, 17, 35], + [0, 18, 34], + [0, 21, 31], + [0, 22, 30], + [1, 17, 34], + [1, 21, 30], + [4, 13, 35], + [4, 14, 34], + [4, 21, 27], + [4, 22, 26], + [5, 13, 34], + [5, 21, 26], + [8, 13, 31], + [8, 14, 30], + [8, 17, 27], + [8, 18, 26], + [9, 13, 30], + [9, 17, 26], + [0, 17, 34], + [0, 21, 30], + [4, 13, 34], + [4, 21, 26], + [8, 13, 30], + [8, 17, 26], + ], + dtype=torch.int64, +) + + +signs = torch.tensor( + [ + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + 1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + 1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + 1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + 1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + 1.0, + -1.0, + 1.0, + -1.0, + -1.0, + 1.0, + 1.0, + -1.0, + ], + dtype=torch.float32, +) + + +coefficient_map = torch.tensor( + [ + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 10, + 10, + 10, + 10, + 10, + 10, + ], + dtype=torch.int64, +) + + +def determinant_to_polynomial( + A: torch.Tensor, +) -> torch.Tensor: + r"""Represent the determinant by the 10th polynomial, used for 5PC solver [@nister2004efficient]. + + Args: + A: torch.Tensor :math:`(*, 3, 13)`. + + Returns: + a degree 10 poly, representing determinant (Eqn. 14 in the paper). + + """ + B, device, dtype = A.shape[0], A.device, A.dtype + global multiplication_indices, signs, coefficient_map # noqa: PLW0603 + + multiplication_indices = multiplication_indices.to(device) + signs = signs.to(device, dtype) + coefficient_map = coefficient_map.to(device) + + A_flat = A.view(B, -1) + gathered_values = A_flat[:, multiplication_indices] + products = torch.prod(gathered_values, dim=-1) + signed_products = products * signs + + cs = torch.zeros(B, 11, device=device, dtype=dtype) + batch_coefficient_map = coefficient_map.repeat(B, 1) + cs.scatter_add_(dim=1, index=batch_coefficient_map, src=signed_products) + return cs diff --git a/kornia/geometry/subpix/__init__.py b/kornia/geometry/subpix/__init__.py new file mode 100644 index 0000000..b6e74c9 --- /dev/null +++ b/kornia/geometry/subpix/__init__.py @@ -0,0 +1,61 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Geometry Subpix — Subpixel operations for Kornia. + +This subpackage provides subpixel localization and softmax utilities for geometry tasks. +""" + +from __future__ import annotations + +from .dsnt import render_gaussian2d, spatial_expectation2d, spatial_softmax2d +from .nms import NonMaximaSuppression2d, NonMaximaSuppression3d, nms2d, nms3d, nms3d_minmax +from .spatial_soft_argmax import ( + AdaptiveQuadInterp3d, + ConvQuadInterp3d, + ConvSoftArgmax2d, + ConvSoftArgmax3d, + IterativeQuadInterp3d, + SpatialSoftArgmax2d, + conv_quad_interp3d, + conv_soft_argmax2d, + conv_soft_argmax3d, + iterative_quad_interp3d, + spatial_soft_argmax2d, +) + +__all__ = [ + "AdaptiveQuadInterp3d", + "ConvQuadInterp3d", + "ConvSoftArgmax2d", + "ConvSoftArgmax3d", + "IterativeQuadInterp3d", + "NonMaximaSuppression2d", + "NonMaximaSuppression3d", + "SpatialSoftArgmax2d", + "conv_quad_interp3d", + "conv_soft_argmax2d", + "conv_soft_argmax3d", + "iterative_quad_interp3d", + "nms2d", + "nms3d", + "nms3d_minmax", + "render_gaussian2d", + "spatial_expectation2d", + "spatial_soft_argmax2d", + "spatial_softmax2d", +] diff --git a/kornia/geometry/subpix/dsnt.py b/kornia/geometry/subpix/dsnt.py new file mode 100644 index 0000000..6909def --- /dev/null +++ b/kornia/geometry/subpix/dsnt.py @@ -0,0 +1,178 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +r"""Implementation of "differentiable spatial to numerical" (soft-argmax) operations. + +As described in the paper "Numerical Coordinate Regression with Convolutional Neural Networks" by Nibali et al. +""" + +from __future__ import annotations + +from typing import Optional + +import torch +import torch.nn.functional as F + +from kornia.core.check import KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE +from kornia.geometry.grid import create_meshgrid + + +def _validate_batched_image_tensor_input(tensor: torch.Tensor) -> None: + KORNIA_CHECK_IS_TENSOR(tensor) + KORNIA_CHECK_SHAPE(tensor, ["B", "C", "H", "W"]) + + +def spatial_softmax2d(input: torch.Tensor, temperature: Optional[torch.Tensor] = None) -> torch.Tensor: + r"""Apply the Softmax function over features in each image channel. + + Note that this function behaves differently to :py:class:`torch.nn.Softmax2d`, which + instead applies Softmax over features at each spatial location. + + Args: + input: the input torch.Tensor with shape :math:`(B, N, H, W)`. + temperature: factor to apply to input, adjusting the "smoothness" of the output distribution. + + Returns: + a 2D probability distribution per image channel with shape :math:`(B, N, H, W)`. + + Examples: + >>> heatmaps = torch.tensor([[[ + ... [0., 0., 0.], + ... [0., 0., 0.], + ... [0., 1., 2.]]]]) + >>> spatial_softmax2d(heatmaps) + tensor([[[[0.0585, 0.0585, 0.0585], + [0.0585, 0.0585, 0.0585], + [0.0585, 0.1589, 0.4319]]]]) + + """ + _validate_batched_image_tensor_input(input) + + batch_size, channels, height, width = input.shape + if temperature is None: + temperature = torch.tensor(1.0) + temperature = temperature.to(device=input.device, dtype=input.dtype) + x = input.view(batch_size, channels, -1) + + x_soft = F.softmax(x * temperature, dim=-1) + + return x_soft.view(batch_size, channels, height, width) + + +def spatial_expectation2d(input: torch.Tensor, normalized_coordinates: bool = True) -> torch.Tensor: + r"""Compute the expectation of coordinate values using spatial probabilities. + + The input heatmap is assumed to represent a valid spatial probability distribution, + which can be achieved using :func:`~kornia.geometry.subpixel.spatial_softmax2d`. + + Args: + input: the input torch.Tensor representing dense spatial probabilities with shape :math:`(B, N, H, W)`. + normalized_coordinates: whether to return the coordinates normalized in the range + of :math:`[-1, 1]`. Otherwise, it will return the coordinates in the range of the input shape. + + Returns: + expected value of the 2D coordinates with shape :math:`(B, N, 2)`. Output order of the coordinates is (x, y). + + Examples: + >>> heatmaps = torch.tensor([[[ + ... [0., 0., 0.], + ... [0., 0., 0.], + ... [0., 1., 0.]]]]) + >>> spatial_expectation2d(heatmaps, False) + tensor([[[1., 2.]]]) + + """ + _validate_batched_image_tensor_input(input) + + batch_size, channels, height, width = input.shape + + # Create coordinates grid. + grid = create_meshgrid(height, width, normalized_coordinates, input.device) + grid = grid.to(input.dtype) + + pos_x = grid[..., 0].reshape(-1) + pos_y = grid[..., 1].reshape(-1) + + input_flat = input.view(batch_size, channels, -1) + + # Compute the expectation of the coordinates. + expected_y = torch.sum(pos_y * input_flat, -1, keepdim=True) + expected_x = torch.sum(pos_x * input_flat, -1, keepdim=True) + + output = torch.cat([expected_x, expected_y], -1) + + return output.view(batch_size, channels, 2) # BxNx2 + + +def render_gaussian2d( + mean: torch.Tensor, std: torch.Tensor, size: tuple[int, int], normalized_coordinates: bool = True +) -> torch.Tensor: + r"""Render the PDF of a 2D Gaussian distribution. + + Args: + mean: the mean location of the Gaussian to render, :math:`(\mu_x, \mu_y)`. Shape: :math:`(*, 2)`. + std: the standard deviation of the Gaussian to render, :math:`(\sigma_x, \sigma_y)`. + Shape :math:`(*, 2)`. Should be able to be broadcast with `mean`. + size: the (height, width) of the output image. + normalized_coordinates: whether ``mean`` and ``std`` are assumed to use coordinates normalized + in the range of :math:`[-1, 1]`. Otherwise, coordinates are assumed to be in the range of the output shape. + + Returns: + torch.Tensor including rendered points with shape :math:`(*, H, W)`. + + """ + if not (std.dtype == mean.dtype and std.device == mean.device): + raise TypeError("Expected inputs to have the same dtype and device") + + height, width = size + dtype = mean.dtype + device = mean.device + + # Create coordinates vectors. + if normalized_coordinates: + xs = torch.linspace(-1, 1, width, device=device, dtype=dtype) + ys = torch.linspace(-1, 1, height, device=device, dtype=dtype) + else: + xs = torch.linspace(0, width - 1, width, device=device, dtype=dtype) + ys = torch.linspace(0, height - 1, height, device=device, dtype=dtype) + + mu_x = mean[..., 0].unsqueeze(-1) + mu_y = mean[..., 1].unsqueeze(-1) + sigma_x = std[..., 0].unsqueeze(-1) + sigma_y = std[..., 1].unsqueeze(-1) + + # Gaussian PDF = exp(-(x - \mu)^2 / (2 \sigma^2)) + # = exp(dists * ks), + # where dists = (x - \mu)^2 and ks = -1 / (2 \sigma^2) + + # dists <- (x - \mu)^2 + dist_x_sq = (xs - mu_x) ** 2 + dist_y_sq = (ys - mu_y) ** 2 + + # ks <- -1 / (2 \sigma^2) + k_x = -0.5 * torch.reciprocal(sigma_x**2) + k_y = -0.5 * torch.reciprocal(sigma_y**2) + + # Assemble the 2D Gaussian. + gauss_x = torch.exp(dist_x_sq * k_x) + gauss_y = torch.exp(dist_y_sq * k_y) + + # Rescale so that values sum to one. + gauss_x = gauss_x / (gauss_x.sum(dim=-1, keepdim=True) + 1e-8) + gauss_y = gauss_y / (gauss_y.sum(dim=-1, keepdim=True) + 1e-8) + + return gauss_y.unsqueeze(-1) * gauss_x.unsqueeze(-2) diff --git a/kornia/geometry/subpix/nms.py b/kornia/geometry/subpix/nms.py new file mode 100644 index 0000000..fb16e7c --- /dev/null +++ b/kornia/geometry/subpix/nms.py @@ -0,0 +1,434 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +import torch.nn.functional as F +from torch import nn + + +def _get_nms_kernel2d(kx: int, ky: int) -> torch.Tensor: + """Return neigh2channels conv kernel.""" + numel: int = ky * kx + center: int = numel // 2 + weight = torch.eye(numel) + weight[center, center] = 0 + return weight.view(numel, 1, ky, kx) + + +def _get_nms_kernel3d(kd: int, ky: int, kx: int) -> torch.Tensor: + """Return neigh2channels conv kernel.""" + numel: int = kd * ky * kx + center: int = numel // 2 + weight = torch.eye(numel) + weight[center, center] = 0 + return weight.view(numel, 1, kd, ky, kx) + + +class NonMaximaSuppression2d(nn.Module): + r"""Apply non maxima suppression to filter. + + Flag `minima_are_also_good` is useful, when you want to detect both maxima and minima, e.g. for DoG + """ + + kernel: torch.Tensor + + def __init__(self, kernel_size: tuple[int, int]) -> None: + super().__init__() + self.kernel_size: tuple[int, int] = kernel_size + self.padding: tuple[int, int, int, int] = self._compute_zero_padding2d(kernel_size) + self.register_buffer("kernel", _get_nms_kernel2d(*kernel_size)) + + @staticmethod + def _compute_zero_padding2d(kernel_size: tuple[int, int]) -> tuple[int, int, int, int]: + # TODO: This method is duplicated with some utility function on kornia.filters + if not isinstance(kernel_size, tuple): + raise AssertionError(type(kernel_size)) + if len(kernel_size) != 2: + raise AssertionError(kernel_size) + + def pad(x: int) -> int: + return (x - 1) // 2 # zero padding function + + ky, kx = kernel_size # we assume a cubic kernel + return (pad(ky), pad(kx), pad(ky), pad(kx)) + + def forward(self, x: torch.Tensor, mask_only: bool = False) -> torch.Tensor: + """Keep only strict local maxima in a 2D response map. + + Each spatial location is compared with the surrounding values inside + ``self.kernel_size``. Locations that are not strictly larger than their + neighbors are suppressed. This is commonly used to turn dense corner or + keypoint response maps into sparse candidate locations. + + Args: + x: Response tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is the batch size, :math:`C` is the number of + response channels, :math:`H` is height, and :math:`W` is width. + mask_only: If ``True``, return the boolean maxima mask. If + ``False``, return ``x`` masked by local-maxima positions. + + Returns: + If ``mask_only`` is ``True``, a boolean tensor with shape + :math:`(B, C, H, W)`. Otherwise, a tensor with the same shape and + dtype as ``x`` where non-maxima have been set to zero. + """ + if len(x.shape) != 4: + raise AssertionError(x.shape) + B, CH, H, W = x.size() + + if self.kernel_size == (3, 3): + # 8-comparison explicit path: no extra memory for conv kernel. + left = slice(0, -2) + center = slice(1, -1) + right = slice(2, None) + mask = torch.zeros(B, CH, H, W, device=x.device, dtype=torch.bool) + ct = x[..., center, center] + mask[..., 1:-1, 1:-1] = ( + (ct > x[..., left, left]) + & (ct > x[..., left, center]) + & (ct > x[..., left, right]) + & (ct > x[..., center, left]) + & (ct > x[..., center, right]) + & (ct > x[..., right, left]) + & (ct > x[..., right, center]) + & (ct > x[..., right, right]) + ) + elif self.kernel_size == (5, 5): + # 24-comparison explicit path for 5x5 neighbourhood. + c2 = slice(0, -4) + c1 = slice(1, -3) + c0 = slice(2, -2) + p1 = slice(3, -1) + p2 = slice(4, None) + mask = torch.zeros(B, CH, H, W, device=x.device, dtype=torch.bool) + ct = x[..., c0, c0] + mask[..., 2:-2, 2:-2] = ( + (ct > x[..., c2, c2]) + & (ct > x[..., c2, c1]) + & (ct > x[..., c2, c0]) + & (ct > x[..., c2, p1]) + & (ct > x[..., c2, p2]) + & (ct > x[..., c1, c2]) + & (ct > x[..., c1, c1]) + & (ct > x[..., c1, c0]) + & (ct > x[..., c1, p1]) + & (ct > x[..., c1, p2]) + & (ct > x[..., c0, c2]) + & (ct > x[..., c0, c1]) + & (ct > x[..., c0, p1]) + & (ct > x[..., c0, p2]) + & (ct > x[..., p1, c2]) + & (ct > x[..., p1, c1]) + & (ct > x[..., p1, c0]) + & (ct > x[..., p1, p1]) + & (ct > x[..., p1, p2]) + & (ct > x[..., p2, c2]) + & (ct > x[..., p2, c1]) + & (ct > x[..., p2, c0]) + & (ct > x[..., p2, p1]) + & (ct > x[..., p2, p2]) + ) + elif self.kernel_size == (7, 7): + # 48-comparison explicit path for 7x7 neighbourhood. + c3 = slice(0, -6) + c2 = slice(1, -5) + c1 = slice(2, -4) + c0 = slice(3, -3) + p1 = slice(4, -2) + p2 = slice(5, -1) + p3 = slice(6, None) + mask = torch.zeros(B, CH, H, W, device=x.device, dtype=torch.bool) + ct = x[..., c0, c0] + mask[..., 3:-3, 3:-3] = ( + (ct > x[..., c3, c3]) + & (ct > x[..., c3, c2]) + & (ct > x[..., c3, c1]) + & (ct > x[..., c3, c0]) + & (ct > x[..., c3, p1]) + & (ct > x[..., c3, p2]) + & (ct > x[..., c3, p3]) + & (ct > x[..., c2, c3]) + & (ct > x[..., c2, c2]) + & (ct > x[..., c2, c1]) + & (ct > x[..., c2, c0]) + & (ct > x[..., c2, p1]) + & (ct > x[..., c2, p2]) + & (ct > x[..., c2, p3]) + & (ct > x[..., c1, c3]) + & (ct > x[..., c1, c2]) + & (ct > x[..., c1, c1]) + & (ct > x[..., c1, c0]) + & (ct > x[..., c1, p1]) + & (ct > x[..., c1, p2]) + & (ct > x[..., c1, p3]) + & (ct > x[..., c0, c3]) + & (ct > x[..., c0, c2]) + & (ct > x[..., c0, c1]) + & (ct > x[..., c0, p1]) + & (ct > x[..., c0, p2]) + & (ct > x[..., c0, p3]) + & (ct > x[..., p1, c3]) + & (ct > x[..., p1, c2]) + & (ct > x[..., p1, c1]) + & (ct > x[..., p1, c0]) + & (ct > x[..., p1, p1]) + & (ct > x[..., p1, p2]) + & (ct > x[..., p1, p3]) + & (ct > x[..., p2, c3]) + & (ct > x[..., p2, c2]) + & (ct > x[..., p2, c1]) + & (ct > x[..., p2, c0]) + & (ct > x[..., p2, p1]) + & (ct > x[..., p2, p2]) + & (ct > x[..., p2, p3]) + & (ct > x[..., p3, c3]) + & (ct > x[..., p3, c2]) + & (ct > x[..., p3, c1]) + & (ct > x[..., p3, c0]) + & (ct > x[..., p3, p1]) + & (ct > x[..., p3, p2]) + & (ct > x[..., p3, p3]) + ) + else: + # General path: conv2d maps every neighbour into its own channel. + x_padded = F.pad(x, list(self.padding)[::-1], mode="replicate") + B, CH, HP, WP = x_padded.size() + neighborhood = F.conv2d(x_padded.view(B * CH, 1, HP, WP), self.kernel.to(x.device, x.dtype), stride=1).view( + B, CH, -1, H, W + ) + max_non_center = neighborhood.max(dim=2)[0] + mask = x > max_non_center + + if mask_only: + return mask + return x * (mask.to(x.dtype)) + + +class NonMaximaSuppression3d(nn.Module): + r"""Apply non maxima suppression to filter.""" + + def __init__(self, kernel_size: tuple[int, int, int]) -> None: + super().__init__() + self.kernel_size: tuple[int, int, int] = kernel_size + self.padding: tuple[int, int, int, int, int, int] = self._compute_zero_padding3d(kernel_size) + self.kernel = _get_nms_kernel3d(*kernel_size) + + @staticmethod + def _compute_zero_padding3d(kernel_size: tuple[int, int, int]) -> tuple[int, int, int, int, int, int]: + # TODO: This method is duplicated with some utility function on kornia.filters + if not isinstance(kernel_size, tuple): + raise AssertionError(type(kernel_size)) + if len(kernel_size) != 3: + raise AssertionError(kernel_size) + + def pad(x: int) -> int: + return (x - 1) // 2 # zero padding function + + kd, ky, kx = kernel_size # we assume a cubic kernel + return (kd, kd, ky, ky, kx, kx) + + def forward(self, x: torch.Tensor, mask_only: bool = False) -> torch.Tensor: + """Keep only strict local maxima in a 3D response volume. + + Each voxel is compared with its neighbors across depth, height, and + width. This is used by scale-space detectors to keep responses that are + locally maximal both in image position and in scale/depth. + + Args: + x: Response tensor with shape :math:`(B, C, D, H, W)`, where + :math:`B` is batch size, :math:`C` is channel count, + :math:`D` is depth or scale level, :math:`H` is height, and + :math:`W` is width. + mask_only: If ``True``, return only the maxima mask; otherwise + return suppressed responses. + + Returns: + Boolean maxima mask with shape :math:`(B, C, D, H, W)` when + ``mask_only`` is ``True``. Otherwise, an NMS-filtered tensor with + the same shape and dtype as ``x``. + """ + if len(x.shape) != 5: + raise AssertionError(x.shape) + # find local maximum values + B, CH, D, H, W = x.size() + if self.kernel_size == (3, 3, 3): + # 26-comparison explicit path: strict local maximum, works on CPU and CUDA. + # Using integer slice literals (not slice objects) makes this torch.jit.script-friendly, + # which fuses the ops and runs ~13x faster on CUDA than the eager path. + mask = torch.zeros(B, CH, D, H, W, device=x.device, dtype=torch.bool) + ct = x[..., 1:-1, 1:-1, 1:-1] + mask[..., 1:-1, 1:-1, 1:-1] = ( + (ct > x[..., 0:-2, 0:-2, 0:-2]) + & (ct > x[..., 0:-2, 0:-2, 1:-1]) + & (ct > x[..., 0:-2, 0:-2, 2:]) + & (ct > x[..., 0:-2, 1:-1, 0:-2]) + & (ct > x[..., 0:-2, 1:-1, 1:-1]) + & (ct > x[..., 0:-2, 1:-1, 2:]) + & (ct > x[..., 0:-2, 2:, 0:-2]) + & (ct > x[..., 0:-2, 2:, 1:-1]) + & (ct > x[..., 0:-2, 2:, 2:]) + & (ct > x[..., 1:-1, 0:-2, 0:-2]) + & (ct > x[..., 1:-1, 0:-2, 1:-1]) + & (ct > x[..., 1:-1, 0:-2, 2:]) + & (ct > x[..., 1:-1, 1:-1, 0:-2]) + & (ct > x[..., 1:-1, 1:-1, 2:]) + & (ct > x[..., 1:-1, 2:, 0:-2]) + & (ct > x[..., 1:-1, 2:, 1:-1]) + & (ct > x[..., 1:-1, 2:, 2:]) + & (ct > x[..., 2:, 0:-2, 0:-2]) + & (ct > x[..., 2:, 0:-2, 1:-1]) + & (ct > x[..., 2:, 0:-2, 2:]) + & (ct > x[..., 2:, 1:-1, 0:-2]) + & (ct > x[..., 2:, 1:-1, 1:-1]) + & (ct > x[..., 2:, 1:-1, 2:]) + & (ct > x[..., 2:, 2:, 0:-2]) + & (ct > x[..., 2:, 2:, 1:-1]) + & (ct > x[..., 2:, 2:, 2:]) + ) + else: + max_non_center = ( + F.conv3d( + F.pad(x, list(self.padding)[::-1], mode="replicate"), + self.kernel.repeat(CH, 1, 1, 1, 1).to(x.device, x.dtype), + stride=1, + groups=CH, + ) + .view(B, CH, -1, D, H, W) + .max(dim=2, keepdim=False)[0] + ) + mask = x > max_non_center + if mask_only: + return mask + return x * (mask.to(x.dtype)) + + +# functional api + + +def nms2d(input: torch.Tensor, kernel_size: tuple[int, int], mask_only: bool = False) -> torch.Tensor: + r"""Apply non maxima suppression to filter. + + See :class:`~kornia.geometry.subpix.NonMaximaSuppression2d` for details. + """ + return NonMaximaSuppression2d(kernel_size)(input, mask_only) + + +def nms3d(input: torch.Tensor, kernel_size: tuple[int, int, int], mask_only: bool = False) -> torch.Tensor: + r"""Apply non maxima suppression to filter. + + See + :class: `~kornia.feature.NonMaximaSuppression3d` for details. + """ + return NonMaximaSuppression3d(kernel_size)(input, mask_only) + + +def nms3d_minmax(input: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + r"""Compute both local-maxima and local-minima NMS masks for a 3-D scale-space tensor in one pass. + + Equivalent to calling ``nms3d(input, (3,3,3), mask_only=True)`` and + ``nms3d(-input, (3,3,3), mask_only=True)`` separately, but only traverses + the 26-neighbour comparisons once, halving the NMS cost. + + Uses integer slice literals (not Python loops or slice objects) so the 52 + comparison-and-reduction ops are visible to the compiler at trace time, + allowing full fusion into a minimal number of kernels. + + Args: + input: 5-D tensor of shape :math:`(B, C, D, H, W)`. + + Returns: + A pair ``(max_mask, min_mask)`` of bool tensors with the same shape as + *input*. ``max_mask[..., d, h, w]`` is ``True`` when the voxel is + strictly greater than all 26 neighbours; ``min_mask`` is the same for + strict local minima. + + Example: + >>> x = torch.randn(1, 1, 5, 10, 10) + >>> max_mask, min_mask = nms3d_minmax(x) + >>> max_mask.shape + torch.Size([1, 1, 5, 10, 10]) + + """ + if input.dim() != 5: + raise AssertionError(input.shape) + B, CH, D, H, W = input.shape + max_mask = torch.zeros(B, CH, D, H, W, device=input.device, dtype=torch.bool) + min_mask = torch.zeros(B, CH, D, H, W, device=input.device, dtype=torch.bool) + ct = input[..., 1:-1, 1:-1, 1:-1] + # 26 explicit comparisons with integer slice literals — no Python loop so the + # compiler sees all ops at trace time and can fuse them into a single kernel. + is_max = ( + (ct > input[..., 0:-2, 0:-2, 0:-2]) + & (ct > input[..., 0:-2, 0:-2, 1:-1]) + & (ct > input[..., 0:-2, 0:-2, 2:]) + & (ct > input[..., 0:-2, 1:-1, 0:-2]) + & (ct > input[..., 0:-2, 1:-1, 1:-1]) + & (ct > input[..., 0:-2, 1:-1, 2:]) + & (ct > input[..., 0:-2, 2:, 0:-2]) + & (ct > input[..., 0:-2, 2:, 1:-1]) + & (ct > input[..., 0:-2, 2:, 2:]) + & (ct > input[..., 1:-1, 0:-2, 0:-2]) + & (ct > input[..., 1:-1, 0:-2, 1:-1]) + & (ct > input[..., 1:-1, 0:-2, 2:]) + & (ct > input[..., 1:-1, 1:-1, 0:-2]) + & (ct > input[..., 1:-1, 1:-1, 2:]) + & (ct > input[..., 1:-1, 2:, 0:-2]) + & (ct > input[..., 1:-1, 2:, 1:-1]) + & (ct > input[..., 1:-1, 2:, 2:]) + & (ct > input[..., 2:, 0:-2, 0:-2]) + & (ct > input[..., 2:, 0:-2, 1:-1]) + & (ct > input[..., 2:, 0:-2, 2:]) + & (ct > input[..., 2:, 1:-1, 0:-2]) + & (ct > input[..., 2:, 1:-1, 1:-1]) + & (ct > input[..., 2:, 1:-1, 2:]) + & (ct > input[..., 2:, 2:, 0:-2]) + & (ct > input[..., 2:, 2:, 1:-1]) + & (ct > input[..., 2:, 2:, 2:]) + ) + is_min = ( + (ct < input[..., 0:-2, 0:-2, 0:-2]) + & (ct < input[..., 0:-2, 0:-2, 1:-1]) + & (ct < input[..., 0:-2, 0:-2, 2:]) + & (ct < input[..., 0:-2, 1:-1, 0:-2]) + & (ct < input[..., 0:-2, 1:-1, 1:-1]) + & (ct < input[..., 0:-2, 1:-1, 2:]) + & (ct < input[..., 0:-2, 2:, 0:-2]) + & (ct < input[..., 0:-2, 2:, 1:-1]) + & (ct < input[..., 0:-2, 2:, 2:]) + & (ct < input[..., 1:-1, 0:-2, 0:-2]) + & (ct < input[..., 1:-1, 0:-2, 1:-1]) + & (ct < input[..., 1:-1, 0:-2, 2:]) + & (ct < input[..., 1:-1, 1:-1, 0:-2]) + & (ct < input[..., 1:-1, 1:-1, 2:]) + & (ct < input[..., 1:-1, 2:, 0:-2]) + & (ct < input[..., 1:-1, 2:, 1:-1]) + & (ct < input[..., 1:-1, 2:, 2:]) + & (ct < input[..., 2:, 0:-2, 0:-2]) + & (ct < input[..., 2:, 0:-2, 1:-1]) + & (ct < input[..., 2:, 0:-2, 2:]) + & (ct < input[..., 2:, 1:-1, 0:-2]) + & (ct < input[..., 2:, 1:-1, 1:-1]) + & (ct < input[..., 2:, 1:-1, 2:]) + & (ct < input[..., 2:, 2:, 0:-2]) + & (ct < input[..., 2:, 2:, 1:-1]) + & (ct < input[..., 2:, 2:, 2:]) + ) + max_mask[..., 1:-1, 1:-1, 1:-1] = is_max + min_mask[..., 1:-1, 1:-1, 1:-1] = is_min + return max_mask, min_mask diff --git a/kornia/geometry/subpix/spatial_soft_argmax.py b/kornia/geometry/subpix/spatial_soft_argmax.py new file mode 100644 index 0000000..2c6e333 --- /dev/null +++ b/kornia/geometry/subpix/spatial_soft_argmax.py @@ -0,0 +1,1387 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.geometry.conversions import normalize_pixel_coordinates, normalize_pixel_coordinates3d +from kornia.geometry.grid import create_meshgrid, create_meshgrid3d + +from .dsnt import spatial_expectation2d, spatial_softmax2d +from .nms import nms3d + +# Flat offsets for gathering the full 3x3x3 neighbourhood of a voxel. +# Layout: patch[k] = inp[bc, d+dd, h+dh, w+dw] where +# k = (dd+1)*9 + (dh+1)*3 + (dw+1), center k=13. +# Defined once at module level to avoid per-call allocation. +_PATCH_DD = torch.tensor( + [-1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1], + dtype=torch.long, +) +_PATCH_DH = torch.tensor( + [-1, -1, -1, 0, 0, 0, 1, 1, 1, -1, -1, -1, 0, 0, 0, 1, 1, 1, -1, -1, -1, 0, 0, 0, 1, 1, 1], + dtype=torch.long, +) +_PATCH_DW = torch.tensor( + [-1, 0, 1, -1, 0, 1, -1, 0, 1, -1, 0, 1, -1, 0, 1, -1, 0, 1, -1, 0, 1, -1, 0, 1, -1, 0, 1], + dtype=torch.long, +) + + +def _get_window_grid_kernel2d(h: int, w: int, device: Optional[torch.device] = None) -> torch.Tensor: + r"""Generate a kernel to with window coordinates, residual to window center. + + Args: + h: kernel height. + w: kernel width. + device: device, on which generate. + + Returns: + conv_kernel [2x1xhxw] + + """ + if device is None: + device = torch.device("cpu") + window_grid2d = create_meshgrid(h, w, False, device=device) + window_grid2d = normalize_pixel_coordinates(window_grid2d, h, w) + conv_kernel = window_grid2d.permute(3, 0, 1, 2) + return conv_kernel + + +def _get_center_kernel2d(h: int, w: int, device: Optional[torch.device] = None) -> torch.Tensor: + r"""Generate a kernel to return center coordinates, when applied with F.conv2d to 2d coordinates grid. + + Args: + h: kernel height. + w: kernel width. + device: device, on which generate. + + Returns: + conv_kernel [2x2xhxw]. + + """ + if device is None: + device = torch.device("cpu") + center_kernel = torch.zeros(2, 2, h, w, device=device) + + # If the size is odd, we have one pixel for center, if even - 2 + if h % 2 != 0: + h_i1 = h // 2 + h_i2 = (h // 2) + 1 + else: + h_i1 = (h // 2) - 1 + h_i2 = (h // 2) + 1 + if w % 2 != 0: + w_i1 = w // 2 + w_i2 = (w // 2) + 1 + else: + w_i1 = (w // 2) - 1 + w_i2 = (w // 2) + 1 + center_kernel[(0, 1), (0, 1), h_i1:h_i2, w_i1:w_i2] = 1.0 / float((h_i2 - h_i1) * (w_i2 - w_i1)) + return center_kernel + + +def _get_center_kernel3d(d: int, h: int, w: int, device: Optional[torch.device] = None) -> torch.Tensor: + r"""Generate a kernel to return center coordinates, when applied with F.conv2d to 3d coordinates grid. + + Args: + d: kernel depth. + h: kernel height. + w: kernel width. + device: device, on which generate. + + Returns: + conv_kernel [3x3xdxhxw]. + + """ + if device is None: + torch.device("cpu") + center_kernel = torch.zeros(3, 3, d, h, w, device=device) + # If the size is odd, we have one pixel for center, if even - 2 + if h % 2 != 0: + h_i1 = h // 2 + h_i2 = (h // 2) + 1 + else: + h_i1 = (h // 2) - 1 + h_i2 = (h // 2) + 1 + if w % 2 != 0: + w_i1 = w // 2 + w_i2 = (w // 2) + 1 + else: + w_i1 = (w // 2) - 1 + w_i2 = (w // 2) + 1 + if d % 2 != 0: + d_i1 = d // 2 + d_i2 = (d // 2) + 1 + else: + d_i1 = (d // 2) - 1 + d_i2 = (d // 2) + 1 + center_num = float((h_i2 - h_i1) * (w_i2 - w_i1) * (d_i2 - d_i1)) + center_kernel[(0, 1, 2), (0, 1, 2), d_i1:d_i2, h_i1:h_i2, w_i1:w_i2] = 1.0 / center_num + return center_kernel + + +def _get_window_grid_kernel3d(d: int, h: int, w: int, device: Optional[torch.device] = None) -> torch.Tensor: + r"""Generate a kernel to return coordinates, residual to window center. + + Args: + d: kernel depth. + h: kernel height. + w: kernel width. + device: device, on which generate. + + Returns: + conv_kernel [3x1xdxhxw] + + """ + if device is None: + device = torch.device("cpu") + grid2d = create_meshgrid(h, w, True, device=device) + if d > 1: + z = torch.linspace(-1, 1, d, device=device).view(d, 1, 1, 1) + else: # only onr channel with index == 0 + z = torch.zeros(1, 1, 1, 1, device=device) + grid3d = torch.cat([z.repeat(1, h, w, 1).contiguous(), grid2d.repeat(d, 1, 1, 1)], 3) + conv_kernel = grid3d.permute(3, 0, 1, 2).unsqueeze(1) + return conv_kernel + + +class ConvSoftArgmax2d(nn.Module): + r"""nn.Module that calculates soft argmax 2d per window. + + See + :func: `~kornia.geometry.subpix.conv_soft_argmax2d` for details. + """ + + def __init__( + self, + kernel_size: tuple[int, int] = (3, 3), + stride: tuple[int, int] = (1, 1), + padding: tuple[int, int] = (1, 1), + temperature: torch.Tensor | float = 1.0, + normalized_coordinates: bool = True, + eps: float = 1e-8, + output_value: bool = False, + ) -> None: + super().__init__() + self.kernel_size = kernel_size + self.stride = stride + self.padding = padding + self.temperature = temperature + self.normalized_coordinates = normalized_coordinates + self.eps = eps + self.output_value = output_value + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}" + f"(kernel_size={self.kernel_size}, " + f"stride={self.stride}, " + f"padding={self.padding}, " + f"temperature={self.temperature}, " + f"normalized_coordinates={self.normalized_coordinates}, " + f"eps={self.eps}, " + f"output_value={self.output_value})" + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + """Estimate local 2D coordinates from heatmap windows. + + The operation applies soft-argmax inside sliding windows. Instead of + choosing the hard maximum location, it computes an expectation over + window coordinates, which gives differentiable subpixel coordinates. + + Args: + x: Heatmap tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is batch size, :math:`C` is the number of heatmap + channels, :math:`H` is height, and :math:`W` is width. + + Returns: + Coordinates from :func:`conv_soft_argmax2d`. If + ``self.output_value`` is ``True``, returns ``(coords, values)``, + where ``values`` are the corresponding window responses. + """ + return conv_soft_argmax2d( + x, + self.kernel_size, + self.stride, + self.padding, + self.temperature, + self.normalized_coordinates, + self.eps, + self.output_value, + ) + + +class ConvSoftArgmax3d(nn.Module): + r"""nn.Module that calculates soft argmax 3d per window. + + See + :func: `~kornia.geometry.subpix.conv_soft_argmax3d` for details. + """ + + def __init__( + self, + kernel_size: tuple[int, int, int] = (3, 3, 3), + stride: tuple[int, int, int] = (1, 1, 1), + padding: tuple[int, int, int] = (1, 1, 1), + temperature: torch.Tensor | float = 1.0, + normalized_coordinates: bool = False, + eps: float = 1e-8, + output_value: bool = True, + strict_maxima_bonus: float = 0.0, + ) -> None: + super().__init__() + self.kernel_size = kernel_size + self.stride = stride + self.padding = padding + self.temperature = temperature + self.normalized_coordinates = normalized_coordinates + self.eps = eps + self.output_value = output_value + self.strict_maxima_bonus = strict_maxima_bonus + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}" + f"(kernel_size={self.kernel_size}, " + f"stride={self.stride}, " + f"padding={self.padding}, " + f"temperature={self.temperature}, " + f"normalized_coordinates={self.normalized_coordinates}, " + f"eps={self.eps}, " + f"strict_maxima_bonus={self.strict_maxima_bonus}, " + f"output_value={self.output_value})" + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + """Estimate local 3D coordinates from response-volume windows. + + Args: + x: Response tensor with shape :math:`(B, C, D, H, W)`, where + :math:`B` is batch size, :math:`C` is channel count, + :math:`D` is depth or scale level, :math:`H` is height, and + :math:`W` is width. + + Returns: + Coordinates from :func:`conv_soft_argmax3d`. If + ``self.output_value`` is ``True``, returns ``(coords, values)``; + otherwise returns only coordinates. + """ + return conv_soft_argmax3d( + x, + self.kernel_size, + self.stride, + self.padding, + self.temperature, + self.normalized_coordinates, + self.eps, + self.output_value, + self.strict_maxima_bonus, + ) + + +def conv_soft_argmax2d( + input: torch.Tensor, + kernel_size: tuple[int, int] = (3, 3), + stride: tuple[int, int] = (1, 1), + padding: tuple[int, int] = (1, 1), + temperature: torch.Tensor | float = 1.0, + normalized_coordinates: bool = True, + eps: float = 1e-8, + output_value: bool = False, +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + r"""Compute the convolutional spatial Soft-Argmax 2D over the windows of a given heatmap. + + .. math:: + ij(X) = \frac{\sum{(i,j)} * exp(x / T) \in X} {\sum{exp(x / T) \in X}} + + .. math:: + val(X) = \frac{\sum{x * exp(x / T) \in X}} {\sum{exp(x / T) \in X}} + + where :math:`T` is temperature. + + Args: + input: the given heatmap with shape :math:`(N, C, H_{in}, W_{in})`. + kernel_size: the size of the window. + stride: the stride of the window. + padding: input zero padding. + temperature: factor to apply to input. + normalized_coordinates: whether to return the coordinates normalized in the range of :math:`[-1, 1]`. + Otherwise, it will return the coordinates in the range of the input shape. + eps: small value to avoid zero division. + output_value: if True, val is output, if False, only ij. + + Returns: + Function has two outputs - argmax coordinates and the softmaxpooled heatmap values themselves. + On each window, the function computed returns with shapes :math:`(N, C, 2, H_{out}, + W_{out})`, :math:`(N, C, H_{out}, W_{out})`, + + where + + .. math:: + H_{out} = \left\lfloor\frac{H_{in} + 2 \times \text{padding}[0] - + (\text{kernel\_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor + + .. math:: + W_{out} = \left\lfloor\frac{W_{in} + 2 \times \text{padding}[1] - + (\text{kernel\_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor + + Examples: + >>> input = torch.randn(20, 16, 50, 32) + >>> nms_coords, nms_val = conv_soft_argmax2d(input, (3,3), (2,2), (1,1), output_value=True) + + """ + if not torch.is_tensor(input): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(input)}") + + if not len(input.shape) == 4: + raise ValueError(f"Invalid input shape, we expect BxCxHxW. Got: {input.shape}") + + if temperature <= 0: + raise ValueError(f"Temperature should be positive float or torch.Tensor. Got: {temperature}") + + b, c, h, w = input.shape + ky, kx = kernel_size + device: torch.device = input.device + dtype: torch.dtype = input.dtype + input = input.view(b * c, 1, h, w) + + center_kernel: torch.Tensor = _get_center_kernel2d(ky, kx, device).to(dtype) + window_kernel: torch.Tensor = _get_window_grid_kernel2d(ky, kx, device).to(dtype) + + # applies exponential normalization trick + # https://timvieira.github.io/blog/post/2014/02/11/exp-F.normalize-trick/ + # https://github.com/pytorch/pytorch/blob/bcb0bb7e0e03b386ad837015faba6b4b16e3bfb9/aten/src/ATen/native/SoftMax.cpp#L44 + x_max = input.amax(dim=(-2, -1), keepdim=True) # faster than F.adaptive_max_pool2d(input, (1,1)) + + # max is detached to prevent undesired backprop loops in the graph + x_exp = ((input - x_max.detach()) / temperature).exp() + + # F.avg_pool2d(.., divisor_override = 1.0) - proper way for sum pool in PyTorch 1.2. + # Not available yet in version 1.0, so let's do manually + pool_coef: float = float(kx * ky) + + # F.softmax denominator + den = pool_coef * F.avg_pool2d(x_exp, kernel_size, stride=stride, padding=padding) + eps + + x_softmaxpool = pool_coef * F.avg_pool2d(x_exp * input, kernel_size, stride=stride, padding=padding) / den + x_softmaxpool = x_softmaxpool.view(b, c, x_softmaxpool.size(2), x_softmaxpool.size(3)) + + # We need to output also coordinates + # Pooled window center coordinates + grid_global: torch.Tensor = create_meshgrid(h, w, False, device).to(dtype).permute(0, 3, 1, 2) + + grid_global_pooled = F.conv2d(grid_global, center_kernel, stride=stride, padding=padding) + + # Coordinates of maxima residual to window center + # prepare kernel + coords_max: torch.Tensor = F.conv2d(x_exp, window_kernel, stride=stride, padding=padding) + + coords_max = coords_max / den.expand_as(coords_max) + coords_max = coords_max + grid_global_pooled.expand_as(coords_max) + # [:,:, 0, ...] is x + # [:,:, 1, ...] is y + + if normalized_coordinates: + coords_max = normalize_pixel_coordinates(coords_max.permute(0, 2, 3, 1), h, w) + coords_max = coords_max.permute(0, 3, 1, 2) + + # Back B*C -> (b, c) + coords_max = coords_max.view(b, c, 2, coords_max.size(2), coords_max.size(3)) + + if output_value: + return coords_max, x_softmaxpool + return coords_max + + +def conv_soft_argmax3d( + input: torch.Tensor, + kernel_size: tuple[int, int, int] = (3, 3, 3), + stride: tuple[int, int, int] = (1, 1, 1), + padding: tuple[int, int, int] = (1, 1, 1), + temperature: torch.Tensor | float = 1.0, + normalized_coordinates: bool = False, + eps: float = 1e-8, + output_value: bool = True, + strict_maxima_bonus: float = 0.0, +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + r"""Compute the convolutional spatial Soft-Argmax 3D over the windows of a given heatmap. + + .. math:: + ijk(X) = \frac{\sum{(i,j,k)} * exp(x / T) \in X} {\sum{exp(x / T) \in X}} + + .. math:: + val(X) = \frac{\sum{x * exp(x / T) \in X}} {\sum{exp(x / T) \in X}} + + where ``T`` is temperature. + + Args: + input: the given heatmap with shape :math:`(N, C, D_{in}, H_{in}, W_{in})`. + kernel_size: size of the window. + stride: stride of the window. + padding: input zero padding. + temperature: factor to apply to input. + normalized_coordinates: whether to return the coordinates normalized in the range of :math:[-1, 1]`. + Otherwise, it will return the coordinates in the range of the input shape. + eps: small value to avoid zero division. + output_value: if True, val is output, if False, only ij. + strict_maxima_bonus: pixels, which are strict maxima will score (1 + strict_maxima_bonus) * value. + This is needed for mimic behavior of strict NMS in classic local features + + Returns: + Function has two outputs - argmax coordinates and the softmaxpooled heatmap values themselves. + On each window, the function computed returns with shapes :math:`(N, C, 3, D_{out}, H_{out}, W_{out})`, + :math:`(N, C, D_{out}, H_{out}, W_{out})`, + + where + + .. math:: + D_{out} = \left\lfloor\frac{D_{in} + 2 \times \text{padding}[0] - + (\text{kernel\_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor + + .. math:: + H_{out} = \left\lfloor\frac{H_{in} + 2 \times \text{padding}[1] - + (\text{kernel\_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor + + .. math:: + W_{out} = \left\lfloor\frac{W_{in} + 2 \times \text{padding}[2] - + (\text{kernel\_size}[2] - 1) - 1}{\text{stride}[2]} + 1\right\rfloor + + Examples: + >>> input = torch.randn(20, 16, 3, 50, 32) + >>> nms_coords, nms_val = conv_soft_argmax3d(input, (3, 3, 3), (1, 2, 2), (0, 1, 1)) + + """ + if not torch.is_tensor(input): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(input)}") + + if not len(input.shape) == 5: + raise ValueError(f"Invalid input shape, we expect BxCxDxHxW. Got: {input.shape}") + + if temperature <= 0: + raise ValueError(f"Temperature should be positive float or torch.Tensor. Got: {temperature}") + + b, c, d, h, w = input.shape + kz, ky, kx = kernel_size + device: torch.device = input.device + dtype: torch.dtype = input.dtype + input = input.view(b * c, 1, d, h, w) + + center_kernel: torch.Tensor = _get_center_kernel3d(kz, ky, kx, device).to(dtype) + window_kernel: torch.Tensor = _get_window_grid_kernel3d(kz, ky, kx, device).to(dtype) + + # applies exponential normalization trick + # https://timvieira.github.io/blog/post/2014/02/11/exp-F.normalize-trick/ + # https://github.com/pytorch/pytorch/blob/bcb0bb7e0e03b386ad837015faba6b4b16e3bfb9/aten/src/ATen/native/SoftMax.cpp#L44 + x_max = input.amax(dim=(-3, -2, -1), keepdim=True) # faster than F.adaptive_max_pool3d(input, (1,1,1)) + + # max is detached to prevent undesired backprop loops in the graph + x_exp = ((input - x_max.detach()) / temperature).exp() + + pool_coef: float = float(kx * ky * kz) + + # F.softmax denominator + den = pool_coef * F.avg_pool3d(x_exp.view_as(input), kernel_size, stride=stride, padding=padding) + eps + + # We need to output also coordinates + # Pooled window center coordinates + grid_global: torch.Tensor = create_meshgrid3d(d, h, w, False, device=device).to(dtype).permute(0, 4, 1, 2, 3) + + grid_global_pooled = F.conv3d(grid_global, center_kernel, stride=stride, padding=padding) + + # Coordinates of maxima residual to window center + # prepare kernel + coords_max: torch.Tensor = F.conv3d(x_exp, window_kernel, stride=stride, padding=padding) + + coords_max = coords_max / den.expand_as(coords_max) + coords_max = coords_max + grid_global_pooled.expand_as(coords_max) + # [:,:, 0, ...] is depth (scale) + # [:,:, 1, ...] is x + # [:,:, 2, ...] is y + + if normalized_coordinates: + coords_max = normalize_pixel_coordinates3d(coords_max.permute(0, 2, 3, 4, 1), d, h, w) + coords_max = coords_max.permute(0, 4, 1, 2, 3) + + # Back B*C -> (b, c) + coords_max = coords_max.view(b, c, 3, coords_max.size(2), coords_max.size(3), coords_max.size(4)) + + if not output_value: + return coords_max + + x_softmaxpool = ( + pool_coef * F.avg_pool3d(x_exp.view(input.size()) * input, kernel_size, stride=stride, padding=padding) / den + ) + if strict_maxima_bonus > 0: + in_levels: int = input.size(2) + out_levels: int = x_softmaxpool.size(2) + skip_levels: int = (in_levels - out_levels) // 2 + strict_maxima: torch.Tensor = F.avg_pool3d(nms3d(input, kernel_size), 1, stride, 0) + strict_maxima = strict_maxima[:, :, skip_levels : out_levels - skip_levels] + x_softmaxpool *= 1.0 + strict_maxima_bonus * strict_maxima + x_softmaxpool = x_softmaxpool.view(b, c, x_softmaxpool.size(2), x_softmaxpool.size(3), x_softmaxpool.size(4)) + return coords_max, x_softmaxpool + + +def spatial_soft_argmax2d( + input: torch.Tensor, temperature: Optional[torch.Tensor] = None, normalized_coordinates: bool = True +) -> torch.Tensor: + r"""Compute the Spatial Soft-Argmax 2D of a given input heatmap. + + Args: + input: the given heatmap with shape :math:`(B, N, H, W)`. + temperature: factor to apply to input. + normalized_coordinates: whether to return the coordinates normalized in the range of :math:`[-1, 1]`. + Otherwise, it will return the coordinates in the range of the input shape. + + Returns: + the index of the maximum 2d coordinates of the give map :math:`(B, N, 2)`. + The output order is x-coord and y-coord. + + Examples: + >>> input = torch.tensor([[[ + ... [0., 0., 0.], + ... [0., 10., 0.], + ... [0., 0., 0.]]]]) + >>> spatial_soft_argmax2d(input, normalized_coordinates=False) + tensor([[[1.0000, 1.0000]]]) + + """ + if temperature is None: + temperature = torch.tensor(1.0) + input_soft: torch.Tensor = spatial_softmax2d(input, temperature) + output: torch.Tensor = spatial_expectation2d(input_soft, normalized_coordinates) + return output + + +class SpatialSoftArgmax2d(nn.Module): + r"""Compute the Spatial Soft-Argmax 2D of a given heatmap. + + See :func:`~kornia.geometry.subpix.spatial_soft_argmax2d` for details. + """ + + def __init__(self, temperature: Optional[torch.Tensor] = None, normalized_coordinates: bool = True) -> None: + super().__init__() + if temperature is None: + temperature = torch.tensor(1.0) + self.temperature: torch.Tensor = temperature + self.normalized_coordinates: bool = normalized_coordinates + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}" + f"temperature={self.temperature}, " + f"normalized_coordinates={self.normalized_coordinates})" + ) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Compute one differentiable 2D coordinate per channel map. + + The full spatial map is converted into a probability distribution and + the coordinate expectation is returned. This is a smooth alternative to + taking the hard argmax of each heatmap. + + Args: + input: Heatmap tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is batch size, :math:`C` is channel count, + :math:`H` is height, and :math:`W` is width. + + Returns: + Coordinate tensor from :func:`spatial_soft_argmax2d`. Coordinate + values are normalized or pixel-based according to + ``self.normalized_coordinates``. + """ + return spatial_soft_argmax2d(input, self.temperature, self.normalized_coordinates) + + +def _solve_cramer_sym3x3( + dxx: torch.Tensor, + dyy: torch.Tensor, + dss: torch.Tensor, + dxy: torch.Tensor, + dxs: torch.Tensor, + dys: torch.Tensor, + r0: torch.Tensor, + r1: torch.Tensor, + r2: torch.Tensor, + eps: float = 1e-7, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Solve H * [sx, sy, ss]^T = [r0, r1, r2]^T via Cramer's rule. + + H is symmetric: H = [[dxx, dxy, dxs], [dxy, dyy, dys], [dxs, dys, dss]]. + All inputs are batched 1-D tensors of length N. + + Args: + dxx: diagonal Hessian element (d²/dx²). + dyy: diagonal Hessian element (d²/dy²). + dss: diagonal Hessian element (d²/ds²). + dxy: off-diagonal Hessian element (d²/dxdy). + dxs: off-diagonal Hessian element (d²/dxds). + dys: off-diagonal Hessian element (d²/dyds). + r0: right-hand side component along x. + r1: right-hand side component along y. + r2: right-hand side component along s. + eps: determinant magnitude below which the system is treated as singular. + Near-singular systems can produce numerically unstable (huge) shifts. + + Returns: + (sx, sy, ss, solved) where ``solved`` is a bool mask for well-conditioned + systems (``|det| > eps``). Outputs for unsolved entries are numerically + meaningless and should be discarded by the caller. + """ + cf00 = dyy * dss - dys * dys # cofactor M00 + cf01 = dxy * dss - dys * dxs # cofactor M01 + cf02 = dxy * dys - dyy * dxs # cofactor M02 + det = dxx * cf00 - dxy * cf01 + dxs * cf02 + solved = det.abs() > eps + # Avoid division by zero for singular/near-singular cases; outputs are discarded via solved. + safe_det = torch.where(solved, det, torch.ones_like(det)) + sx = (r0 * cf00 - dxy * (r1 * dss - dys * r2) + dxs * (r1 * dys - dyy * r2)) / safe_det + sy = (dxx * (r1 * dss - dys * r2) - r0 * cf01 + dxs * (dxy * r2 - r1 * dxs)) / safe_det + ss = (dxx * (dyy * r2 - r1 * dys) - dxy * (dxy * r2 - r1 * dxs) + r0 * cf02) / safe_det + return sx, sy, ss, solved + + +def conv_quad_interp3d( + input: torch.Tensor, + n_iters: int = 5, + strict_maxima_bonus: float = 10.0, + max_subpixel_shift: float = 0.6, + precomputed_nms_mask: Optional[torch.Tensor] = None, + dilation_radius: int = 1, + allow_scale_steps: bool = True, +) -> tuple[torch.Tensor, torch.Tensor]: + r"""Subpixel localization of 3D scale-space extrema via quadratic interpolation. + + For each NMS maximum the function fits a 3-D quadratic to the local + :math:`3 \times 3 \times 3` neighbourhood and solves for the sub-voxel + shift that maximises the fit. When the shift along any axis exceeds + ``max_subpixel_shift`` the integer centre is moved one step in that direction + and the solve is repeated — up to ``n_iters`` times. + + Unlike a naive iterative approach, all Hessian solves are precomputed once at + the start for every voxel that any keypoint could possibly visit (the + **dilated NMS neighbourhood**, an L\ :math:`\infty` ball of radius + ``dilation_radius`` around each maximum). The subsequent iteration loop contains + **no data-dependent Python control flow** and no GPU→CPU synchronisation, + making the function fully compatible with ``torch.compile`` / CUDA graphs. + + The ``dilation_radius`` controls the precompute footprint and should be set + to the maximum number of integer-centre moves expected per keypoint. With the + default ``max_subpixel_shift=0.6`` almost all keypoints converge within 1 move, + so the default ``dilation_radius=1`` (i.e. :math:`3^3 = 27` positions per + maximum) is sufficient. Use ``dilation_radius=2`` (:math:`5^3 = 125`) for + extra safety. Setting it equal to ``n_iters`` recovers the original behaviour + but is much slower on large images. + + Args: + input: response pyramid with shape :math:`(B, C, D, H, W)`. + n_iters: maximum number of localization iterations per keypoint. + strict_maxima_bonus: value added to ``y_max`` at NMS-maximum positions + so that strict maxima are preferred during top-K selection. + max_subpixel_shift: threshold above which the integer centre is + moved one step and another iteration is run. + precomputed_nms_mask: optional bool tensor of shape + :math:`(B, C, D, H, W)` — pass the result of + :func:`~kornia.geometry.subpix.nms3d` to skip the internal NMS call. + dilation_radius: L\ :math:`\infty` radius (in voxels) of the neighbourhood + around each NMS maximum where the Hessian solve is precomputed. + Keypoints that attempt to move farther than this are marked invalid. + allow_scale_steps: if ``True`` (default), the iterative shift is also + applied along the scale (depth) axis; set to ``False`` to keep the + keypoint on its original scale level. + + Returns: + Tuple ``(coords_max, y_max)``: + + * ``coords_max`` — shape :math:`(B, C, 3, D, H, W)`, refined + ``[scale, x(width), y(height)]`` coordinates for each NMS maximum; + non-maximum positions keep their grid coordinates. + * ``y_max`` — shape :math:`(B, C, D, H, W)`, quadratically corrected + response with optional strict-maxima bonus. + + Example: + >>> input = torch.randn(2, 3, 5, 64, 64) + >>> coords, vals = conv_quad_interp3d(input, n_iters=5) + >>> coords.shape + torch.Size([2, 3, 3, 5, 64, 64]) + >>> vals.shape + torch.Size([2, 3, 5, 64, 64]) + + """ + if not torch.is_tensor(input): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(input)}") + if input.ndim != 5: + raise ValueError(f"Invalid input shape, expected BxCxDxHxW. Got: {input.shape}") + + B, C, D, H, W = input.shape + device = input.device + dtype = input.dtype + BC = B * C + DHW = D * H * W + HW = H * W + + coords_max = torch.empty(B, C, 3, D, H, W, device=device, dtype=dtype) + coords_max[:, :, 0] = torch.arange(D, device=device, dtype=dtype).view(D, 1, 1) + coords_max[:, :, 1] = torch.arange(W, device=device, dtype=dtype).view(1, 1, W) + coords_max[:, :, 2] = torch.arange(H, device=device, dtype=dtype).view(1, H, 1) + y_max = input.clone() + + if D < 3 or H < 3 or W < 3: + return coords_max, y_max + + # ── Step 1: NMS maxima ──────────────────────────────────────────────────── + nms_mask = precomputed_nms_mask if precomputed_nms_mask is not None else nms3d(input, (3, 3, 3), True) + bc_idx, d_idx, h_idx, w_idx = torch.where(nms_mask.view(BC, D, H, W)) + N = bc_idx.shape[0] + # Note: no early-return for N==0 — empty tensors flow through all ops as no-ops. + + # ── Step 2: dilate NMS positions — L∞ ball of radius dilation_radius ──────── + # Generates all voxels a keypoint could visit across n_iters shift steps. + # With max_subpixel_shift=0.6, almost all keypoints converge in ≤1 integer move, + # so dilation_radius=1 (27 positions per max) is sufficient in practice. + r = dilation_radius + offs = torch.arange(-r, r + 1, device=device, dtype=torch.long) # (2r+1,) + od, oh, ow = torch.meshgrid(offs, offs, offs, indexing="ij") # each: (2r+1)³ + od = od.reshape(-1) # (K,) K = (2r+1)³ + oh = oh.reshape(-1) + ow = ow.reshape(-1) + K = od.shape[0] + + # Broadcast expand: (N,1) + (1,K) → (N*K,) + d_dil = (d_idx.unsqueeze(1) + od.unsqueeze(0)).reshape(-1) + h_dil = (h_idx.unsqueeze(1) + oh.unsqueeze(0)).reshape(-1) + w_dil = (w_idx.unsqueeze(1) + ow.unsqueeze(0)).reshape(-1) + bc_dil = bc_idx.unsqueeze(1).expand(-1, K).reshape(-1) + + # Keep only interior positions (Hessian needs ±1 neighbours in all dims). + keep = (d_dil >= 1) & (d_dil <= D - 2) & (h_dil >= 1) & (h_dil <= H - 2) & (w_dil >= 1) & (w_dil <= W - 2) + bc_u = bc_dil[keep] + d_u = d_dil[keep] + h_u = h_dil[keep] + w_u = w_dil[keep] + # Note: bc_u/d_u/h_u/w_u may contain duplicate positions (multiple NMS maxima sharing + # a dilated neighbour). We intentionally skip torch.unique here because: + # (a) torch.unique output size depends on VALUES not shapes → causes torch.compile + # to recompile whenever the unique-element count changes across images, producing + # multi-second spikes (e.g. 4-7 s) for images with different NMS density. + # (b) The keep-filter boolean-index above already forces a graph break at this point, + # so the code below runs in eager mode regardless — deduplication buys nothing. + # (c) Duplicate positions receive the same solve result (same 3x3x3 patch, deterministic + # quadratic system), so last-write-wins in the LUT is correct. + + # ── Step 3: gather 3x3x3 neighbourhood for all kept dilated positions ──── + inp_flat = input.view(-1) + patch_offsets = _PATCH_DD.to(device) * HW + _PATCH_DH.to(device) * W + _PATCH_DW.to(device) # (27,) + center_flat = bc_u * DHW + d_u * HW + h_u * W + w_u + patch = inp_flat[center_flat.unsqueeze(1) + patch_offsets.unsqueeze(0)] # (NU, 27) + + # Named patch elements. Flat index: k = (dd+1)*9 + (dh+1)*3 + (dw+1), center k=13. + c000 = patch[:, 13] + p_xm = patch[:, 12] + p_xp = patch[:, 14] + p_ym = patch[:, 10] + p_yp = patch[:, 16] + p_sm = patch[:, 4] + p_sp = patch[:, 22] + p_xm_ym = patch[:, 9] + p_xp_ym = patch[:, 11] + p_xm_yp = patch[:, 15] + p_xp_yp = patch[:, 17] + p_xm_sm = patch[:, 3] + p_xp_sm = patch[:, 5] + p_xm_sp = patch[:, 21] + p_xp_sp = patch[:, 23] + p_ym_sm = patch[:, 1] + p_yp_sm = patch[:, 7] + p_ym_sp = patch[:, 19] + p_yp_sp = patch[:, 25] + + # ── Step 4: compute gradients + Hessian + solve (all unique positions) ─── + gx = 0.5 * (p_xp - p_xm) + gy = 0.5 * (p_yp - p_ym) + gs = 0.5 * (p_sp - p_sm) + dxx = p_xp - 2.0 * c000 + p_xm + dyy = p_yp - 2.0 * c000 + p_ym + dss = p_sp - 2.0 * c000 + p_sm + dxy = 0.25 * (p_xp_yp - p_xm_yp - p_xp_ym + p_xm_ym) + dxs = 0.25 * (p_xp_sp - p_xm_sp - p_xp_sm + p_xm_sm) + dys = 0.25 * (p_yp_sp - p_ym_sp - p_yp_sm + p_ym_sm) + + sx_u, sy_u, ss_u, sol_u = _solve_cramer_sym3x3(dxx, dyy, dss, dxy, dxs, dys, -gx, -gy, -gs) + # Precompute gradient·shift for the response correction (avoids storing gx/gy/gs tables). + gds_u = gx * sx_u + gy * sy_u + gs * ss_u + + # ── Step 5: build a compact int32 lookup table ─────────────────────────── + # Maps flat(bc, d, h, w) → index into the NK-sized solution arrays + # (sx_u, sy_u, ss_u, gds_u, sol_u). -1 means "not precomputed here" — + # the keypoint has moved out of the dilated neighbourhood and is invalid. + # Duplicate positions use last-write-wins, which is correct since the same + # position always yields the same quadratic solution. + # + # Memory: 1 x BC*D*H*W x int32 (4 bytes/elem) + # vs 5 x BC*D*H*W x float32 + 1 x bool (21 bytes/elem previously) + # → ~5x reduction. For a 640x800x6 octave: 12 MB vs 62 MB. + NK = bc_u.shape[0] + lut = torch.full((BC * DHW,), -1, dtype=torch.int32, device=device) + lut[bc_u * DHW + d_u * HW + h_u * W + w_u] = torch.arange(NK, dtype=torch.int32, device=device) + + # ── Step 6: iterative lookup — no .any().item() sync ───────────────────── + d_cur = d_idx.clone() + h_cur = h_idx.clone() + w_cur = w_idx.clone() + valid = torch.ones(N, dtype=torch.bool, device=device) + shift_x = torch.zeros(N, device=device, dtype=dtype) + shift_y = torch.zeros(N, device=device, dtype=dtype) + shift_s = torch.zeros(N, device=device, dtype=dtype) + grad_dot_shift = torch.zeros(N, device=device, dtype=dtype) + + for _ in range(n_iters): + di = d_cur.clamp(1, D - 2) + hi = h_cur.clamp(1, H - 2) + wi = w_cur.clamp(1, W - 2) + + flat_q = bc_idx * DHW + di * HW + hi * W + wi # (N,) + lut_idx = lut[flat_q].long() # (N,) int64; -1 = not in precomputed range + in_lut = lut_idx >= 0 + idx_safe = lut_idx.clamp(min=0) # safe index (clamp -1 → 0 before gather) + + sx = sx_u[idx_safe] + sy = sy_u[idx_safe] + ss = ss_u[idx_safe] + gds = gds_u[idx_safe] + sol = sol_u[idx_safe] & in_lut + + valid = valid & sol + vf = valid.to(dtype) + sx = sx * vf + sy = sy * vf + ss = ss * vf + + shift_x = torch.where(valid, sx, shift_x) + shift_y = torch.where(valid, sy, shift_y) + shift_s = torch.where(valid, ss, shift_s) + grad_dot_shift = torch.where(valid, gds, grad_dot_shift) + + move_px = valid & (sx > max_subpixel_shift) + move_nx = valid & (sx < -max_subpixel_shift) + new_w = w_cur + move_px.long() - move_nx.long() + valid = valid & (new_w >= 1) & (new_w <= W - 2) + w_cur = new_w.clamp(0, W - 1) + + move_py = valid & (sy > max_subpixel_shift) + move_ny = valid & (sy < -max_subpixel_shift) + new_h = h_cur + move_py.long() - move_ny.long() + valid = valid & (new_h >= 1) & (new_h <= H - 2) + h_cur = new_h.clamp(0, H - 1) + + if allow_scale_steps: + move_ps = valid & (ss > max_subpixel_shift) + move_ns = valid & (ss < -max_subpixel_shift) + new_d = d_cur + move_ps.long() - move_ns.long() + valid = valid & (new_d >= 1) & (new_d <= D - 2) + d_cur = new_d.clamp(0, D - 1) + + valid = valid & (shift_x.abs() <= 1.5) & (shift_y.abs() <= 1.5) & (shift_s.abs() <= 1.5) + + # ── Write refined coordinates and corrected response ────────────────────── + b_idx = bc_idx // C + c_idx = bc_idx % C + + coords_max[b_idx, c_idx, 0, d_idx, h_idx, w_idx] = torch.where(valid, d_cur.to(dtype) + shift_s, d_idx.to(dtype)) + coords_max[b_idx, c_idx, 1, d_idx, h_idx, w_idx] = torch.where(valid, w_cur.to(dtype) + shift_x, w_idx.to(dtype)) + coords_max[b_idx, c_idx, 2, d_idx, h_idx, w_idx] = torch.where(valid, h_cur.to(dtype) + shift_y, h_idx.to(dtype)) + + val_correction = 0.5 * torch.where(valid, grad_dot_shift, torch.zeros_like(grad_dot_shift)) + # Use the final recentered position for val_center (h_cur/w_cur may have moved during iteration) + val_center = input.view(BC, D, H, W)[bc_idx, d_cur, h_cur, w_cur] + y_max[b_idx, c_idx, d_idx, h_idx, w_idx] = val_center + val_correction + if strict_maxima_bonus > 0: + y_max[b_idx, c_idx, d_idx, h_idx, w_idx] += strict_maxima_bonus * valid.to(dtype) + + return coords_max, y_max + + +class ConvQuadInterp3d(nn.Module): + r"""Subpixel localization of 3D scale-space extrema via quadratic interpolation. + + Wraps :func:`~kornia.geometry.subpix.conv_quad_interp3d`. The Hessian system + is solved once for each voxel in the dilated NMS neighbourhood (no dense + precomputation over the whole volume), then the shift chain is followed by + table lookup with no GPU→CPU synchronisation — making the module compatible + with ``torch.compile`` and CUDA graphs. + + Args: + n_iters: maximum localization iterations per keypoint. + strict_maxima_bonus: score bonus at NMS-maximum positions. + max_subpixel_shift: shift threshold that triggers integer centre move. + """ + + def __init__( + self, + n_iters: int = 5, + strict_maxima_bonus: float = 10.0, + max_subpixel_shift: float = 0.6, + dilation_radius: int = 1, + allow_scale_steps: bool = True, + ) -> None: + super().__init__() + self.n_iters = n_iters + self.strict_maxima_bonus = strict_maxima_bonus + self.max_subpixel_shift = max_subpixel_shift + self.dilation_radius = dilation_radius + self.allow_scale_steps = allow_scale_steps + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}(" + f"n_iters={self.n_iters}, " + f"strict_maxima_bonus={self.strict_maxima_bonus}, " + f"max_subpixel_shift={self.max_subpixel_shift}, " + f"dilation_radius={self.dilation_radius}, " + f"allow_scale_steps={self.allow_scale_steps})" + ) + + def forward( + self, x: torch.Tensor, precomputed_nms_mask: Optional[torch.Tensor] = None + ) -> tuple[torch.Tensor, torch.Tensor]: + """Refine 3D extrema with convolution-based quadratic interpolation. + + The method starts from local maxima in a scale-space response volume + and fits a local quadratic model around each candidate. The fitted + offset gives a subpixel position in scale, x, and y coordinates. + + Args: + x: Response tensor with shape :math:`(B, C, D, H, W)`, where + :math:`D` is the scale/depth dimension. + precomputed_nms_mask: Optional boolean NMS mask with the same shape + as ``x``. Passing it avoids recomputing local maxima. + + Returns: + Tuple ``(coords, values)``. ``coords`` stores refined coordinates + for scale, x, and y; ``values`` stores the corrected response + scores. + """ + return conv_quad_interp3d( + x, + self.n_iters, + self.strict_maxima_bonus, + self.max_subpixel_shift, + precomputed_nms_mask, + self.dilation_radius, + self.allow_scale_steps, + ) + + +def iterative_quad_interp3d( + input: torch.Tensor, + n_iters: int = 5, + strict_maxima_bonus: float = 10.0, + max_subpixel_shift: float = 0.6, + allow_scale_steps: bool = True, + precomputed_nms_mask: Optional[torch.Tensor] = None, + max_candidates: Optional[int] = None, +) -> tuple[torch.Tensor, torch.Tensor]: + r"""Iterative subpixel localization of 3D extrema via quadratic interpolation. + + Unlike :func:`conv_quad_interp3d`, which pre-computes the Hessian solve for all + voxels reachable from NMS maxima and then follows shifts by table lookup, this + function explicitly re-extracts the :math:`3 \times 3 \times 3` patch at each + NMS maximum and iterates up to ``n_iters`` times. When the estimated subpixel + shift along any spatial or scale axis exceeds ``max_subpixel_shift`` the integer + center is moved one step in that direction and the solve is repeated — matching + the localization loop from the HessAff / SIFT family of detectors. + + Args: + input: response pyramid with shape :math:`(B, C, D, H, W)`. + n_iters: maximum number of localization iterations per keypoint. + strict_maxima_bonus: value added to ``y_max`` at NMS-maximum positions so + that strict maxima are preferred when selecting the top-K keypoints. + max_subpixel_shift: if the estimated shift along any axis is larger than this + threshold the integer center is displaced and another iteration is run. + allow_scale_steps: if ``True`` (default), the iterative shift is also + applied along the scale (depth) axis; set to ``False`` to keep the + keypoint on its original scale level. + precomputed_nms_mask: optional bool tensor of shape + :math:`(B, C, D, H, W)` — pass the result of + :func:`~kornia.geometry.subpix.nms3d` to skip the internal NMS call. + max_candidates: if given, only the top-``max_candidates`` NMS maxima (ranked by + pre-refinement response) are processed. The rest keep their grid-coordinate + values. This is a **CPU speed-up knob**: for large images the number of 3-D + NMS maxima can be 10x-100x larger than the desired number of keypoints, + making the per-candidate gather+solve loop the dominant CPU cost. Setting + ``max_candidates = num_features * 5`` (say) dramatically reduces that work + at the cost of occasionally missing a feature whose response rank would have + improved after refinement. + + Returns: + A tuple ``(coords_max, y_max)`` where + + * ``coords_max`` has shape :math:`(B, C, 3, D, H, W)` and stores the refined + coordinates ``[scale, x, y]`` for every position in the input. + Non-NMS positions keep their original grid coordinates. + * ``y_max`` has shape :math:`(B, C, D, H, W)` and stores the quadratically + corrected response values (with the optional strict-maxima bonus added). + + Example: + >>> input = torch.randn(2, 3, 3, 8, 8) + >>> coords, vals = iterative_quad_interp3d(input, n_iters=5) + >>> coords.shape + torch.Size([2, 3, 3, 3, 8, 8]) + >>> vals.shape + torch.Size([2, 3, 3, 8, 8]) + + """ + if not torch.is_tensor(input): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(input)}") + if input.ndim != 5: + raise ValueError(f"Invalid input shape, expected BxCxDxHxW. Got: {input.shape}") + + B, C, D, H, W = input.shape + device = input.device + dtype = input.dtype + + coords_max = torch.empty(B, C, 3, D, H, W, device=device, dtype=dtype) + coords_max[:, :, 0] = torch.arange(D, device=device, dtype=dtype).view(D, 1, 1) + coords_max[:, :, 1] = torch.arange(W, device=device, dtype=dtype).view(1, 1, W) + coords_max[:, :, 2] = torch.arange(H, device=device, dtype=dtype).view(1, H, 1) + y_max = input.clone() + + if D < 3 or H < 3 or W < 3: + return coords_max, y_max + + inp = input.reshape(B * C, D, H, W) + DHW = D * H * W + HW = H * W + + nms_flat = (precomputed_nms_mask if precomputed_nms_mask is not None else nms3d(input, (3, 3, 3), True)).view( + B * C, D, H, W + ) + + bc_idx, d_idx, h_idx, w_idx = torch.where(nms_flat) + N = bc_idx.shape[0] + # Note: no early-return for N==0 — empty tensors flow through all ops as no-ops, + # avoiding a Python branch that would cause torch.compile to specialise and recompile + # when blurry octaves first yield zero NMS maxima. + + # ── CPU speed-up: pre-filter to top-max_candidates by pre-refinement response ── + # For large images the NMS can yield tens of thousands of candidates while only a + # few hundred features are ultimately needed. The per-candidate patch gather + # (random memory access into a multi-MB volume) is cache-miss dominated on CPU; + # reducing N here gives a proportional speedup of the iteration loop below. + if max_candidates is not None and N > max_candidates: + cand_vals = inp[bc_idx, d_idx, h_idx, w_idx] # (N,) pre-refinement responses + _, keep = torch.topk(cand_vals, k=max_candidates) + bc_idx = bc_idx[keep] + d_idx = d_idx[keep] + h_idx = h_idx[keep] + w_idx = w_idx[keep] + N = max_candidates + + patch_offsets = _PATCH_DD.to(device) * HW + _PATCH_DH.to(device) * W + _PATCH_DW.to(device) + + d_cur = d_idx.clone() + h_cur = h_idx.clone() + w_cur = w_idx.clone() + + valid = torch.ones(N, dtype=torch.bool, device=device) + + shift_x = torch.zeros(N, device=device, dtype=dtype) + shift_y = torch.zeros(N, device=device, dtype=dtype) + shift_s = torch.zeros(N, device=device, dtype=dtype) + grad_dot_shift = torch.zeros(N, device=device, dtype=dtype) + + inp_flat = inp.reshape(-1) + bc_base = bc_idx * DHW + + for _ in range(n_iters): + d_s = d_cur.clamp(1, D - 2) + h_s = h_cur.clamp(1, H - 2) + w_s = w_cur.clamp(1, W - 2) + + patch = inp_flat[(bc_base + d_s * HW + h_s * W + w_s).unsqueeze(1) + patch_offsets.unsqueeze(0)] + + c000 = patch[:, 13] + p_xm = patch[:, 12] + p_xp = patch[:, 14] + p_ym = patch[:, 10] + p_yp = patch[:, 16] + p_sm = patch[:, 4] + p_sp = patch[:, 22] + p_xm_ym = patch[:, 9] + p_xp_ym = patch[:, 11] + p_xm_yp = patch[:, 15] + p_xp_yp = patch[:, 17] + p_xm_sm = patch[:, 3] + p_xp_sm = patch[:, 5] + p_xm_sp = patch[:, 21] + p_xp_sp = patch[:, 23] + p_ym_sm = patch[:, 1] + p_yp_sm = patch[:, 7] + p_ym_sp = patch[:, 19] + p_yp_sp = patch[:, 25] + + gx = 0.5 * (p_xp - p_xm) + gy = 0.5 * (p_yp - p_ym) + gs = 0.5 * (p_sp - p_sm) + + dxx = p_xp - 2.0 * c000 + p_xm + dyy = p_yp - 2.0 * c000 + p_ym + dss = p_sp - 2.0 * c000 + p_sm + dxy = 0.25 * (p_xp_yp - p_xm_yp - p_xp_ym + p_xm_ym) + dxs = 0.25 * (p_xp_sp - p_xm_sp - p_xp_sm + p_xm_sm) + dys = 0.25 * (p_yp_sp - p_ym_sp - p_yp_sm + p_ym_sm) + + sx, sy, ss, solved = _solve_cramer_sym3x3(dxx, dyy, dss, dxy, dxs, dys, -gx, -gy, -gs) + valid = valid & solved + + valid_f = valid.to(dtype) + sx = sx * valid_f + sy = sy * valid_f + ss = ss * valid_f + + shift_x = torch.where(valid, sx, shift_x) + shift_y = torch.where(valid, sy, shift_y) + shift_s = torch.where(valid, ss, shift_s) + grad_dot_shift = torch.where(valid, gx * sx + gy * sy + gs * ss, grad_dot_shift) + + move_pos_x = valid & (sx > max_subpixel_shift) + move_neg_x = valid & (sx < -max_subpixel_shift) + new_w = w_cur + move_pos_x.long() - move_neg_x.long() + valid = valid & (new_w >= 1) & (new_w <= W - 2) + w_cur = new_w.clamp(0, W - 1) + + move_pos_y = valid & (sy > max_subpixel_shift) + move_neg_y = valid & (sy < -max_subpixel_shift) + new_h = h_cur + move_pos_y.long() - move_neg_y.long() + valid = valid & (new_h >= 1) & (new_h <= H - 2) + h_cur = new_h.clamp(0, H - 1) + + if allow_scale_steps: + move_pos_s = valid & (ss > max_subpixel_shift) + move_neg_s = valid & (ss < -max_subpixel_shift) + new_d = d_cur + move_pos_s.long() - move_neg_s.long() + valid = valid & (new_d >= 1) & (new_d <= D - 2) + d_cur = new_d.clamp(0, D - 1) + + valid = valid & (shift_x.abs() <= 1.5) & (shift_y.abs() <= 1.5) & (shift_s.abs() <= 1.5) + + b_idx = bc_idx // C + c_idx = bc_idx % C + + final_s = torch.where(valid, d_cur.to(dtype) + shift_s, d_idx.to(dtype)) + final_x = torch.where(valid, w_cur.to(dtype) + shift_x, w_idx.to(dtype)) + final_y = torch.where(valid, h_cur.to(dtype) + shift_y, h_idx.to(dtype)) + + coords_max[b_idx, c_idx, 0, d_idx, h_idx, w_idx] = final_s + coords_max[b_idx, c_idx, 1, d_idx, h_idx, w_idx] = final_x + coords_max[b_idx, c_idx, 2, d_idx, h_idx, w_idx] = final_y + + val_correction = 0.5 * torch.where(valid, grad_dot_shift, torch.zeros_like(grad_dot_shift)) + # Use the final recentered position for val_center (h_cur/w_cur may have moved during iteration) + val_center = inp[bc_idx, d_cur, h_cur, w_cur] + y_max[b_idx, c_idx, d_idx, h_idx, w_idx] = val_center + val_correction + + if strict_maxima_bonus > 0: + y_max[b_idx, c_idx, d_idx, h_idx, w_idx] += strict_maxima_bonus * valid.to(dtype) + + return coords_max, y_max + + +class IterativeQuadInterp3d(nn.Module): + r"""Iterative subpixel localization of 3D extrema via quadratic interpolation. + + See :func:`~kornia.geometry.subpix.iterative_quad_interp3d` for details. + """ + + def __init__( + self, + n_iters: int = 5, + strict_maxima_bonus: float = 10.0, + max_subpixel_shift: float = 0.6, + allow_scale_steps: bool = True, + max_candidates: Optional[int] = None, + ) -> None: + super().__init__() + self.n_iters = n_iters + self.strict_maxima_bonus = strict_maxima_bonus + self.max_subpixel_shift = max_subpixel_shift + self.allow_scale_steps = allow_scale_steps + self.max_candidates = max_candidates + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}(" + f"n_iters={self.n_iters}, " + f"strict_maxima_bonus={self.strict_maxima_bonus}, " + f"max_subpixel_shift={self.max_subpixel_shift}, " + f"allow_scale_steps={self.allow_scale_steps}, " + f"max_candidates={self.max_candidates})" + ) + + def forward( + self, + x: torch.Tensor, + precomputed_nms_mask: Optional[torch.Tensor] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Refine 3D extrema by iteratively fitting local quadratic patches. + + Args: + x: Response tensor with shape :math:`(B, C, D, H, W)`. + precomputed_nms_mask: Optional boolean local-maxima mask with the + same shape as ``x``. + + Returns: + Tuple ``(coords, values)`` from :func:`iterative_quad_interp3d`. + ``coords`` contains refined scale, x, and y locations, while + ``values`` contains their response scores. + """ + return iterative_quad_interp3d( + x, + self.n_iters, + self.strict_maxima_bonus, + self.max_subpixel_shift, + self.allow_scale_steps, + precomputed_nms_mask=precomputed_nms_mask, + max_candidates=self.max_candidates, + ) + + +class AdaptiveQuadInterp3d(nn.Module): + r"""Subpixel localization of 3D scale-space extrema with automatic backend selection. + + Wraps :func:`~kornia.geometry.subpix.conv_quad_interp3d` and + :func:`~kornia.geometry.subpix.iterative_quad_interp3d`, choosing the faster + backend based on the input device and the requested ``mode``. + + Benchmarks show: + + * **GPU** — :func:`conv_quad_interp3d` is 1.5-2x faster due to better + parallelism on the batched gather+solve. + * **CPU** — :func:`iterative_quad_interp3d` is faster for large images because + it processes only the NMS maxima directly without any dilation/dedup overhead. + + Args: + mode: backend selection strategy. + + * ``"patch"`` — always use :func:`iterative_quad_interp3d`. + * ``"conv"`` — always use :func:`conv_quad_interp3d`. + * ``"auto"`` — use ``"conv"`` when the input is on a CUDA device, + ``"patch"`` otherwise. + + n_iters: maximum localization iterations per keypoint. + strict_maxima_bonus: score bonus added at NMS-maximum positions. + max_subpixel_shift: integer-centre move threshold. + dilation_radius: L\ :math:`\infty` precompute radius for ``"conv"`` mode + (ignored in ``"patch"`` mode). + max_candidates: if set, only the top-``max_candidates`` NMS maxima by + pre-refinement response are processed in ``"patch"`` mode. Has no effect + in ``"conv"`` mode. Useful on CPU when the number of 3-D NMS maxima greatly + exceeds the desired number of keypoints (see :func:`iterative_quad_interp3d`). + + Example: + >>> inp = torch.randn(1, 1, 3, 64, 64) + >>> subpix = AdaptiveQuadInterp3d(mode="auto") + >>> coords, vals = subpix(inp) + >>> coords.shape + torch.Size([1, 1, 3, 3, 64, 64]) + >>> vals.shape + torch.Size([1, 1, 3, 64, 64]) + + """ + + MODES = ("patch", "conv", "auto") + + def __init__( + self, + mode: str = "auto", + n_iters: int = 5, + strict_maxima_bonus: float = 10.0, + max_subpixel_shift: float = 0.6, + dilation_radius: int = 1, + allow_scale_steps: bool = True, + max_candidates: Optional[int] = None, + ) -> None: + super().__init__() + if mode not in self.MODES: + raise ValueError(f"mode must be one of {self.MODES}, got '{mode}'") + self.mode = mode + self.n_iters = n_iters + self.strict_maxima_bonus = strict_maxima_bonus + self.max_subpixel_shift = max_subpixel_shift + self.dilation_radius = dilation_radius + self.allow_scale_steps = allow_scale_steps + self.max_candidates = max_candidates + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}(" + f"mode='{self.mode}', " + f"n_iters={self.n_iters}, " + f"strict_maxima_bonus={self.strict_maxima_bonus}, " + f"max_subpixel_shift={self.max_subpixel_shift}, " + f"dilation_radius={self.dilation_radius}, " + f"allow_scale_steps={self.allow_scale_steps}, " + f"max_candidates={self.max_candidates})" + ) + + def forward( + self, + x: torch.Tensor, + precomputed_nms_mask: Optional[torch.Tensor] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Refine 3D extrema with the selected interpolation backend. + + The backend is chosen according to ``self.mode`` (or device when + ``mode='auto'``), then delegated to either + :func:`conv_quad_interp3d` or :func:`iterative_quad_interp3d`. + + Args: + x: Response tensor with shape :math:`(B, C, D, H, W)`. + precomputed_nms_mask: Optional boolean local-maxima mask with the + same shape as ``x``. + + Returns: + Tuple ``(coords, values)`` containing refined scale-space + coordinates and corresponding response scores. + """ + use_conv = self.mode == "conv" or (self.mode == "auto" and x.is_cuda) + if use_conv: + return conv_quad_interp3d( + x, + self.n_iters, + self.strict_maxima_bonus, + self.max_subpixel_shift, + precomputed_nms_mask, + self.dilation_radius, + self.allow_scale_steps, + ) + return iterative_quad_interp3d( + x, + self.n_iters, + self.strict_maxima_bonus, + self.max_subpixel_shift, + self.allow_scale_steps, + precomputed_nms_mask=precomputed_nms_mask, + max_candidates=self.max_candidates, + ) diff --git a/kornia/geometry/transform/__init__.py b/kornia/geometry/transform/__init__.py new file mode 100644 index 0000000..c1d46df --- /dev/null +++ b/kornia/geometry/transform/__init__.py @@ -0,0 +1,32 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Geometry Transform — Geometric transformation operations for Kornia. + +This subpackage provides affine, crop, and other geometric transformation utilities. +""" + +from .affwarp import * +from .crop2d import * +from .crop3d import * +from .elastic_transform import * +from .flips import * +from .homography_warper import * +from .image_registrator import * +from .imgwarp import * +from .pyramid import * +from .thin_plate_spline import * diff --git a/kornia/geometry/transform/affwarp.py b/kornia/geometry/transform/affwarp.py new file mode 100644 index 0000000..58abc48 --- /dev/null +++ b/kornia/geometry/transform/affwarp.py @@ -0,0 +1,1168 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Optional, Tuple, Union + +import torch +from torch import nn + +from kornia.core.ops import eye_like +from kornia.core.utils import _extract_device_dtype +from kornia.filters import gaussian_blur2d + +from .imgwarp import get_affine_matrix2d, get_projective_transform, get_rotation_matrix2d, warp_affine, warp_affine3d + +__all__ = [ + "Affine", + "Rescale", + "Resize", + "Rotate", + "Scale", + "Shear", + "Translate", + "affine", + "affine3d", + "rescale", + "resize", + "resize_to_be_divisible", + "rotate", + "rotate3d", + "scale", + "shear", + "translate", +] + +# utilities to compute affine matrices + + +def _compute_tensor_center(tensor: torch.Tensor) -> torch.Tensor: + """Compute the center of tensor plane for (H, W), (C, H, W) and (B, C, H, W).""" + if not 2 <= len(tensor.shape) <= 4: + raise AssertionError(f"Must be a 3D tensor as HW, CHW and BCHW. Got {tensor.shape}.") + height, width = tensor.shape[-2:] + center_x: float = float(width - 1) / 2 + center_y: float = float(height - 1) / 2 + center: torch.Tensor = torch.tensor([center_x, center_y], device=tensor.device, dtype=tensor.dtype) + return center + + +def _compute_tensor_center3d(tensor: torch.Tensor) -> torch.Tensor: + """Compute the center of tensor plane for (D, H, W), (C, D, H, W) and (B, C, D, H, W).""" + if not 3 <= len(tensor.shape) <= 5: + raise AssertionError(f"Must be a 3D tensor as DHW, CDHW and BCDHW. Got {tensor.shape}.") + depth, height, width = tensor.shape[-3:] + center_x: float = float(width - 1) / 2 + center_y: float = float(height - 1) / 2 + center_z: float = float(depth - 1) / 2 + center: torch.Tensor = torch.tensor([center_x, center_y, center_z], device=tensor.device, dtype=tensor.dtype) + return center + + +def _compute_rotation_matrix(angle: torch.Tensor, center: torch.Tensor) -> torch.Tensor: + """Compute a pure affine rotation matrix.""" + scale: torch.Tensor = torch.ones_like(center) + matrix: torch.Tensor = get_rotation_matrix2d(center, angle, scale) + return matrix + + +def _compute_rotation_matrix3d( + yaw: torch.Tensor, pitch: torch.Tensor, roll: torch.Tensor, center: torch.Tensor +) -> torch.Tensor: + """Compute a pure affine rotation matrix.""" + if len(yaw.shape) == len(pitch.shape) == len(roll.shape) == 0: + yaw = yaw.unsqueeze(dim=0) + pitch = pitch.unsqueeze(dim=0) + roll = roll.unsqueeze(dim=0) + + if len(yaw.shape) == len(pitch.shape) == len(roll.shape) == 1: + yaw = yaw.unsqueeze(dim=1) + pitch = pitch.unsqueeze(dim=1) + roll = roll.unsqueeze(dim=1) + + if not (len(yaw.shape) == len(pitch.shape) == len(roll.shape) == 2): + raise AssertionError(f"Expected yaw, pitch, roll to be (B, 1). Got {yaw.shape}, {pitch.shape}, {roll.shape}.") + + angles: torch.Tensor = torch.cat([yaw, pitch, roll], dim=1) + scales: torch.Tensor = torch.ones_like(yaw) + matrix: torch.Tensor = get_projective_transform(center, angles, scales) + return matrix + + +def _compute_translation_matrix(translation: torch.Tensor) -> torch.Tensor: + """Compute affine matrix for translation.""" + matrix: torch.Tensor = eye_like(3, translation, shared_memory=False) + + dx, dy = torch.chunk(translation, chunks=2, dim=-1) + matrix[..., 0, 2:3] += dx + matrix[..., 1, 2:3] += dy + return matrix + + +def _compute_scaling_matrix(scale: torch.Tensor, center: torch.Tensor) -> torch.Tensor: + """Compute affine matrix for scaling.""" + angle: torch.Tensor = torch.zeros(scale.shape[:1], device=scale.device, dtype=scale.dtype) + matrix: torch.Tensor = get_rotation_matrix2d(center, angle, scale) + return matrix + + +def _compute_shear_matrix(shear: torch.Tensor) -> torch.Tensor: + """Compute affine matrix for shearing.""" + matrix: torch.Tensor = eye_like(3, shear, shared_memory=False) + + shx, shy = torch.chunk(shear, chunks=2, dim=-1) + matrix[..., 0, 1:2] += shx + matrix[..., 1, 0:1] += shy + return matrix + + +# based on: +# https://github.com/anibali/tvl/blob/master/src/tvl/transforms.py#L166 + + +def affine( + tensor: torch.Tensor, + matrix: torch.Tensor, + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = True, +) -> torch.Tensor: + r"""Apply an affine transformation to the image. + + .. image:: _static/img/warp_affine.png + + Args: + tensor: The image tensor to be warped in shapes of + :math:`(H, W)`, :math:`(D, H, W)` and :math:`(B, C, H, W)`. + matrix: The 2x3 affine transformation matrix. + mode: interpolation mode to calculate output values ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + ``'torch.zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: interpolation flag. + + Returns: + The warped image with the same shape as the input. + + Example: + >>> img = torch.rand(1, 2, 3, 5) + >>> aff = torch.eye(2, 3)[None] + >>> out = affine(img, aff) + >>> print(out.shape) + torch.Size([1, 2, 3, 5]) + + """ + # warping needs data in the shape of BCHW + is_unbatched: bool = tensor.dim() == 3 + if is_unbatched: + tensor = torch.unsqueeze(tensor, dim=0) + + # we enforce broadcasting since by default grid_sample it does not + # give support for that + if tensor.shape[0] == 1 and matrix.shape[0] != 1: + tensor = tensor.expand(matrix.shape[0], tensor.shape[1], tensor.shape[2], tensor.shape[3]) + + matrix = matrix.expand(tensor.shape[0], -1, -1) + + # warp the input tensor + height: int = tensor.shape[-2] + width: int = tensor.shape[-1] + warped: torch.Tensor = warp_affine(tensor, matrix, (height, width), mode, padding_mode, align_corners) + + # return in the original shape + if is_unbatched: + warped = torch.squeeze(warped, dim=0) + + return warped + + +def affine3d( + tensor: torch.Tensor, + matrix: torch.Tensor, + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = False, +) -> torch.Tensor: + r"""Apply an affine transformation to the 3d volume. + + Args: + tensor: The image tensor to be warped in shapes of + :math:`(D, H, W)`, :math:`(C, D, H, W)` and :math:`(B, C, D, H, W)`. + matrix: The affine transformation matrix with shape :math:`(B, 3, 4)`. + mode: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + `` 'torch.zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: interpolation flag. + + Returns: + The warped image. + + Example: + >>> img = torch.rand(1, 2, 4, 3, 5) + >>> aff = torch.eye(3, 4)[None] + >>> out = affine3d(img, aff) + >>> print(out.shape) + torch.Size([1, 2, 4, 3, 5]) + + """ + # warping needs data in the shape of BCDHW + is_unbatched: bool = tensor.dim() == 4 + if is_unbatched: + tensor = torch.unsqueeze(tensor, dim=0) + + # we enforce broadcasting since by default grid_sample it does not + # give support for that + matrix = matrix.expand(tensor.shape[0], -1, -1) + + # warp the input tensor + depth: int = tensor.shape[-3] + height: int = tensor.shape[-2] + width: int = tensor.shape[-1] + warped: torch.Tensor = warp_affine3d(tensor, matrix, (depth, height, width), mode, padding_mode, align_corners) + + # return in the original shape + if is_unbatched: + warped = torch.squeeze(warped, dim=0) + + return warped + + +# based on: +# https://github.com/anibali/tvl/blob/master/src/tvl/transforms.py#L185 + + +def rotate( + tensor: torch.Tensor, + angle: torch.Tensor, + center: Union[None, torch.Tensor] = None, + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = True, +) -> torch.Tensor: + r"""Rotate the tensor anti-clockwise about the center. + + .. image:: _static/img/rotate.png + + Args: + tensor: The image tensor to be warped in shapes of :math:`(B, C, H, W)`. + angle: The angle through which to rotate. The tensor + must have a shape of (B), where B is batch size. + center: The center through which to rotate. The tensor + must have a shape of (B, 2), where B is batch size and last + dimension contains cx and cy. + mode: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + ``'torch.zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: interpolation flag. + + Returns: + The rotated tensor with shape as input. + + .. note:: + See a working example `here `__. + + Example: + >>> img = torch.rand(1, 3, 4, 4) + >>> angle = torch.tensor([90.]) + >>> out = rotate(img, angle) + >>> print(out.shape) + torch.Size([1, 3, 4, 4]) + + """ + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Input tensor type is not a torch.Tensor. Got {type(tensor)}") + + if not isinstance(angle, torch.Tensor): + raise TypeError(f"Input angle type is not a torch.Tensor. Got {type(angle)}") + + if center is not None and not isinstance(center, torch.Tensor): + raise TypeError(f"Input center type is not a torch.Tensor. Got {type(center)}") + + if len(tensor.shape) not in (3, 4): + raise ValueError(f"Invalid tensor shape, we expect CxHxW or BxCxHxW. Got: {tensor.shape}") + + # compute the rotation center + if center is None: + center = _compute_tensor_center(tensor) + + # compute the rotation matrix + # TODO: add broadcasting to get_rotation_matrix2d for center + angle = angle.expand(tensor.shape[0]) + center = center.expand(tensor.shape[0], -1) + rotation_matrix: torch.Tensor = _compute_rotation_matrix(angle, center) + + # warp using the affine transform + return affine(tensor, rotation_matrix[..., :2, :3], mode, padding_mode, align_corners) + + +def rotate3d( + tensor: torch.Tensor, + yaw: torch.Tensor, + pitch: torch.Tensor, + roll: torch.Tensor, + center: Union[None, torch.Tensor] = None, + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = False, +) -> torch.Tensor: + r"""Rotate 3D the tensor anti-clockwise about the centre. + + Args: + tensor: The image tensor to be warped in shapes of :math:`(B, C, D, H, W)`. + yaw: The yaw angle through which to rotate. The tensor + must have a shape of (B), where B is batch size. + pitch: The pitch angle through which to rotate. The tensor + must have a shape of (B), where B is batch size. + roll: The roll angle through which to rotate. The tensor + must have a shape of (B), where B is batch size. + center: The center through which to rotate. The tensor + must have a shape of (B, 2), where B is batch size and last + dimension contains cx and cy. + mode: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + ``'torch.zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: interpolation flag. + + Returns: + torch.Tensor: The rotated tensor with shape as input. + + """ + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Input tensor type is not a torch.Tensor. Got {type(tensor)}") + + if not isinstance(yaw, torch.Tensor): + raise TypeError(f"yaw is not a torch.Tensor. Got {type(yaw)}") + + if not isinstance(pitch, torch.Tensor): + raise TypeError(f"pitch is not a torch.Tensor. Got {type(pitch)}") + + if not isinstance(roll, torch.Tensor): + raise TypeError(f"roll is not a torch.Tensor. Got {type(roll)}") + + if center is not None and not isinstance(center, torch.Tensor): + raise TypeError(f"Input center type is not a torch.Tensor. Got {type(center)}") + + if len(tensor.shape) not in (4, 5): + raise ValueError(f"Invalid tensor shape, we expect CxDxHxW or BxCxDxHxW. Got: {tensor.shape}") + + # compute the rotation center + if center is None: + center = _compute_tensor_center3d(tensor) + + # compute the rotation matrix + # TODO: add broadcasting to get_rotation_matrix2d for center + yaw = yaw.expand(tensor.shape[0]) + pitch = pitch.expand(tensor.shape[0]) + roll = roll.expand(tensor.shape[0]) + center = center.expand(tensor.shape[0], -1) + rotation_matrix: torch.Tensor = _compute_rotation_matrix3d(yaw, pitch, roll, center) + + # warp using the affine transform + return affine3d(tensor, rotation_matrix[..., :3, :4], mode, padding_mode, align_corners) + + +def translate( + tensor: torch.Tensor, + translation: torch.Tensor, + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = True, +) -> torch.Tensor: + r"""Translate the tensor in pixel units. + + .. image:: _static/img/translate.png + + Args: + tensor: The image tensor to be warped in shapes of :math:`(B, C, H, W)`. + translation: tensor containing the amount of pixels to + translate in the x and y direction. The tensor must have a shape of + (B, 2), where B is batch size, last dimension contains dx dy. + mode: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + ``'torch.zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: interpolation flag. + + Returns: + The translated tensor with shape as input. + + Example: + >>> img = torch.rand(1, 3, 4, 4) + >>> translation = torch.tensor([[1., 0.]]) + >>> out = translate(img, translation) + >>> print(out.shape) + torch.Size([1, 3, 4, 4]) + + """ + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Input tensor type is not a torch.Tensor. Got {type(tensor)}") + + if not isinstance(translation, torch.Tensor): + raise TypeError(f"Input translation type is not a torch.Tensor. Got {type(translation)}") + + if len(tensor.shape) not in (3, 4): + raise ValueError(f"Invalid tensor shape, we expect CxHxW or BxCxHxW. Got: {tensor.shape}") + + # compute the translation matrix + translation_matrix: torch.Tensor = _compute_translation_matrix(translation) + + # warp using the affine transform + return affine(tensor, translation_matrix[..., :2, :3], mode, padding_mode, align_corners) + + +def scale( + tensor: torch.Tensor, + scale_factor: torch.Tensor, + center: Union[None, torch.Tensor] = None, + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = True, +) -> torch.Tensor: + r"""Scale the tensor by a factor. + + .. image:: _static/img/scale.png + + Args: + tensor: The image tensor to be warped in shapes of :math:`(B, C, H, W)`. + scale_factor: The scale factor apply. The tensor + must have a shape of (B) or (B, 2), where B is batch size. + If (B), isotropic scaling will perform. + If (B, 2), x-y-direction specific scaling will perform. + center: The center through which to scale. The tensor + must have a shape of (B, 2), where B is batch size and last + dimension contains cx and cy. + mode: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + ``'torch.zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: interpolation flag. + + Returns: + The scaled tensor with the same shape as the input. + + Example: + >>> img = torch.rand(1, 3, 4, 4) + >>> scale_factor = torch.tensor([[2., 2.]]) + >>> out = scale(img, scale_factor) + >>> print(out.shape) + torch.Size([1, 3, 4, 4]) + + """ + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Input tensor type is not a torch.Tensor. Got {type(tensor)}") + + if not isinstance(scale_factor, torch.Tensor): + raise TypeError(f"Input scale_factor type is not a torch.Tensor. Got {type(scale_factor)}") + + if len(scale_factor.shape) == 1: + # convert isotropic scaling to x-y direction + scale_factor = scale_factor.repeat(1, 2) + + # compute the tensor center + if center is None: + center = _compute_tensor_center(tensor) + + # compute the rotation matrix + # TODO: add broadcasting to get_rotation_matrix2d for center + center = center.expand(tensor.shape[0], -1) + scale_factor = scale_factor.expand(tensor.shape[0], 2) + scaling_matrix: torch.Tensor = _compute_scaling_matrix(scale_factor, center) + + # warp using the affine transform + return affine(tensor, scaling_matrix[..., :2, :3], mode, padding_mode, align_corners) + + +def shear( + tensor: torch.Tensor, + shear: torch.Tensor, + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = False, +) -> torch.Tensor: + r"""Shear the tensor. + + .. image:: _static/img/shear.png + + Args: + tensor: The image tensor to be skewed with shape of :math:`(B, C, H, W)`. + shear: tensor containing the angle to shear + in the x and y direction. The tensor must have a shape of + (B, 2), where B is batch size, last dimension contains shx shy. + mode: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + ``'torch.zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: interpolation flag. + + Returns: + The skewed tensor with shape same as the input. + + Example: + >>> img = torch.rand(1, 3, 4, 4) + >>> shear_factor = torch.tensor([[0.5, 0.0]]) + >>> out = shear(img, shear_factor) + >>> print(out.shape) + torch.Size([1, 3, 4, 4]) + + """ + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Input tensor type is not a torch.Tensor. Got {type(tensor)}") + + if not isinstance(shear, torch.Tensor): + raise TypeError(f"Input shear type is not a torch.Tensor. Got {type(shear)}") + + if len(tensor.shape) not in (3, 4): + raise ValueError(f"Invalid tensor shape, we expect CxHxW or BxCxHxW. Got: {tensor.shape}") + + # compute the translation matrix + shear_matrix: torch.Tensor = _compute_shear_matrix(shear) + + # warp using the affine transform + return affine(tensor, shear_matrix[..., :2, :3], mode, padding_mode, align_corners) + + +def _side_to_image_size(side_size: int, aspect_ratio: float, side: str = "short") -> Tuple[int, int]: + if side not in ("short", "long", "vert", "horz"): + raise ValueError(f"side can be one of 'short', 'long', 'vert', and 'horz'. Got '{side}'") + if side == "vert": + return side_size, int(side_size * aspect_ratio) + if side == "horz": + return int(side_size / aspect_ratio), side_size + if (side == "short") ^ (aspect_ratio < 1.0): + return side_size, int(side_size * aspect_ratio) + return int(side_size / aspect_ratio), side_size + + +def resize( + input: torch.Tensor, + size: Union[int, Tuple[int, int]], + interpolation: str = "bilinear", + align_corners: Optional[bool] = None, + side: str = "short", + antialias: bool = False, +) -> torch.Tensor: + r"""Resize the input torch.Tensor to the given size. + + .. image:: _static/img/resize.png + + Args: + input: The image tensor to be skewed with shape of :math:`(..., H, W)`. + `...` means there can be any number of dimensions. + size: Desired output size. If size is a sequence like (h, w), + output size will be matched to this. If size is an int, smaller edge of the image will + be matched to this number. i.e, if height > width, then image will be rescaled + to (size * height / width, size) + interpolation: algorithm used for upsampling: ``'nearest'`` | ``'linear'`` | ``'bilinear'`` | + 'bicubic' | 'trilinear' | 'area'. + align_corners: interpolation flag. + side: Corresponding side if ``size`` is an integer. Can be one of ``'short'``, ``'long'``, ``'vert'``, + or ``'horz'``. + antialias: if True, then image will be filtered with Gaussian before downscaling. + No effect for upscaling. + + Returns: + The resized tensor with the shape as the specified size. + + Example: + >>> img = torch.rand(1, 3, 4, 4) + >>> out = resize(img, (6, 8)) + >>> print(out.shape) + torch.Size([1, 3, 6, 8]) + + """ + if not isinstance(input, torch.Tensor): + raise TypeError(f"Input tensor type is not a torch.Tensor. Got {type(input)}") + + if len(input.shape) < 2: + raise ValueError(f"Input tensor must have at least two dimensions. Got {len(input.shape)}") + + original_shape = input.shape + h, w = input.shape[-2:] + if isinstance(size, int): + aspect_ratio = w / h + size = _side_to_image_size(size, aspect_ratio, side) + if len(original_shape) == 2: + # (H, W) -> (1, 1, H, W) + input = input.unsqueeze(0).unsqueeze(0) + elif len(original_shape) == 3: + # (C, H, W) -> (1, C, H, W) + input = input.unsqueeze(0) + elif len(original_shape) > 4: + # Flatten all leading dims: (*, C, H, W) -> (B, C, H, W) + batch_size = 1 + for d in original_shape[:-3]: + batch_size *= d + input = input.reshape(batch_size, *original_shape[-3:]) + + factors = (h / size[0], w / size[1]) + antialias = antialias and (max(factors) > 1) + + if antialias: + sigmas = (max((factors[0] - 1.0) / 2.0, 0.001), max((factors[1] - 1.0) / 2.0, 0.001)) + + ks = int(max(2.0 * 2 * sigmas[0], 3)), int(max(2.0 * 2 * sigmas[1], 3)) + + ks = (ks[0] if ks[0] % 2 else ks[0] + 1, ks[1] if ks[1] % 2 else ks[1] + 1) + + input = gaussian_blur2d(input, ks, sigmas) + + output = torch.nn.functional.interpolate(input, size=size, mode=interpolation, align_corners=align_corners) + + if len(original_shape) == 2: + output = output[0, 0] + elif len(original_shape) == 3: + output = output[0] + elif len(original_shape) > 4: + output = output.reshape(*original_shape[:-2], size[0], size[1]) + + return output + + +def resize_to_be_divisible( + input: torch.Tensor, + divisible_factor: int, + interpolation: str = "bilinear", + align_corners: Optional[bool] = None, + side: str = "short", + antialias: bool = False, +) -> torch.Tensor: + """Resize the input tensor to be divisible by a certain factor. + + Args: + input (torch.Tensor): Input tensor to be resized. + divisible_factor (int): The factor to which the image should be divisible. + interpolation (str, optional): Interpolation flag. Defaults to "bilinear". + align_corners (Optional[bool], optional): + whether to align the corners of the input and output. Defaults to None. + side (str, optional): Side to resize. Defaults to "short". + antialias (bool, optional): + If True, then image will be filtered with Gaussian before downscaling. Defaults to False. + + Returns: + torch.Tensor: The resized tensor. + + """ + if isinstance(input, torch.Tensor) and len(input.shape) == 4: + height, width = input.shape[2], input.shape[3] + if isinstance(input, torch.Tensor) and len(input.shape) == 3: + height, width = input.shape[1], input.shape[2] + + height = round(height / divisible_factor) * divisible_factor + width = round(width / divisible_factor) * divisible_factor + return resize(input, (height, width), interpolation, align_corners, side, antialias) + + +def rescale( + input: torch.Tensor, + factor: Union[float, Tuple[float, float]], + interpolation: str = "bilinear", + align_corners: Optional[bool] = None, + antialias: bool = False, +) -> torch.Tensor: + r"""Rescale the input torch.Tensor with the given factor. + + .. image:: _static/img/rescale.png + + Args: + input: The image tensor to be scale with shape of :math:`(B, C, H, W)`. + factor: Desired scaling factor in each direction. If scalar, the value is used + for both the x- and y-direction. + interpolation: algorithm used for upsampling: ``'nearest'`` | ``'linear'`` | ``'bilinear'`` | + ``'bicubic'`` | ``'trilinear'`` | ``'area'``. + align_corners: interpolation flag. + side: Corresponding side if ``size`` is an integer. Can be one of ``'short'``, ``'long'``, ``'vert'``, + or ``'horz'``. + antialias: if True, then image will be filtered with Gaussian before downscaling. + No effect for upscaling. + + Returns: + The rescaled tensor with the shape as the specified size. + + Example: + >>> img = torch.rand(1, 3, 4, 4) + >>> out = rescale(img, (2, 3)) + >>> print(out.shape) + torch.Size([1, 3, 8, 12]) + + """ + if isinstance(factor, float): + factor_vert = factor_horz = factor + else: + factor_vert, factor_horz = factor + + height, width = input.size()[-2:] + size = (int(height * factor_vert), int(width * factor_horz)) + return resize(input, size, interpolation=interpolation, align_corners=align_corners, antialias=antialias) + + +class Resize(nn.Module): + r"""Resize the input torch.Tensor to the given size. + + Args: + size: Desired output size. If size is a sequence like (h, w), + output size will be matched to this. If size is an int, smaller edge of the image will + be matched to this number. i.e, if height > width, then image will be rescaled + to (size * height / width, size) + interpolation: algorithm used for upsampling: ``'nearest'`` | ``'linear'`` | ``'bilinear'`` | + 'bicubic' | 'trilinear' | 'area'. + align_corners: interpolation flag. + side: Corresponding side if ``size`` is an integer. Can be one of ``'short'``, ``'long'``, ``'vert'``, + or ``'horz'``. + antialias: if True, then image will be filtered with Gaussian before downscaling. + No effect for upscaling. + + Returns: + The resized tensor with the shape of the given size. + + Example: + >>> img = torch.rand(1, 3, 4, 4) + >>> out = Resize((6, 8))(img) + >>> print(out.shape) + torch.Size([1, 3, 6, 8]) + + .. raw:: html + + + + """ + + def __init__( + self, + size: Union[int, Tuple[int, int]], + interpolation: str = "bilinear", + align_corners: Optional[bool] = None, + side: str = "short", + antialias: bool = False, + ) -> None: + super().__init__() + self.size: Union[int, Tuple[int, int]] = size + self.interpolation: str = interpolation + self.align_corners: Optional[bool] = align_corners + self.side: str = side + self.antialias: bool = antialias + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Resize the spatial dimensions of an image or feature tensor. + + Args: + input: Tensor with image-like layout :math:`(*, C, H, W)`, where + ``*`` represents optional leading batch dimensions, + :math:`C` is channels, :math:`H` is height, and :math:`W` is + width. + + Returns: + Resized tensor produced by :func:`resize`. The output keeps the + same leading and channel dimensions as ``input``; the height and + width are determined by ``self.size`` and ``self.side``. + """ + return resize( + input, + self.size, + self.interpolation, + align_corners=self.align_corners, + side=self.side, + antialias=self.antialias, + ) + + +class Affine(nn.Module): + r"""Apply multiple elementary affine transforms simultaneously. + + Args: + angle: Angle in degrees for counter-clockwise rotation around the center. The tensor + must have a shape of (B), where B is the batch size. + translation: Amount of pixels for translation in x- and y-direction. The tensor must + have a shape of (B, 2), where B is the batch size and the last dimension contains dx and dy. + scale_factor: Factor for scaling. The tensor must have a shape of (B,2), where B is the + batch size and the last dimension contains scale factors for x and y direction. + shear: Factor for shearing in x- and y-direction around the center. The + tensor must have a shape of (B, 2), where B is the batch size and the last dimension + contains sx and sy. + center: Transformation center in pixels. The tensor must have a shape of (B, 2), where + B is the batch size and the last dimension contains cx and cy. Defaults to the center of image to be + transformed. + mode: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + ``'torch.zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: interpolation flag. + + Raises: + RuntimeError: If not one of ``angle``, ``translation``, ``scale_factor``, or ``shear`` is set. + + Returns: + The transformed tensor with same shape as input. + + Example: + >>> img = torch.rand(1, 2, 3, 5) + >>> angle = 90. * torch.rand(1) + >>> out = Affine(angle)(img) + >>> print(out.shape) + torch.Size([1, 2, 3, 5]) + + """ + + def __init__( + self, + angle: Optional[torch.Tensor] = None, + translation: Optional[torch.Tensor] = None, + scale_factor: Optional[torch.Tensor] = None, + shear: Optional[torch.Tensor] = None, + center: Optional[torch.Tensor] = None, + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = True, + ) -> None: + batch_sizes = [arg.size()[0] for arg in (angle, translation, scale_factor, shear) if arg is not None] + if not batch_sizes: + msg = ( + "Affine was created without any affine parameter. At least one of angle, translation, scale_factor, or " + "shear has to be set." + ) + raise RuntimeError(msg) + + batch_size = batch_sizes[0] + if not all(other == batch_size for other in batch_sizes[1:]): + raise RuntimeError(f"The batch sizes of the affine parameters mismatch: {batch_sizes}") + + self._batch_size = batch_size + + super().__init__() + device, dtype = _extract_device_dtype([angle, translation, scale_factor]) + + if angle is None: + angle = torch.zeros(batch_size, device=device, dtype=dtype) + self.angle = angle + + if translation is None: + translation = torch.zeros(batch_size, 2, device=device, dtype=dtype) + self.translation = translation + + if scale_factor is None: + scale_factor = torch.ones(batch_size, 2, device=device, dtype=dtype) + self.scale_factor = scale_factor + + self.shear = shear + self.center = center + self.mode = mode + self.padding_mode = padding_mode + self.align_corners = align_corners + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Warp the input with the affine transform configured in this module. + + The transform matrix is built from this module's stored angle, + translation, scale, optional shear, and center. The resulting matrix is + passed to :func:`affine` for sampling. + + Args: + input: Image tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is batch size, :math:`C` is channel count, + :math:`H` is height, and :math:`W` is width. + + Returns: + Affine-warped tensor with shape :math:`(B, C, H, W)`. Values + sampled outside the input image are handled by ``self.padding_mode``. + """ + if self.shear is None: + sx = sy = None + else: + sx, sy = self.shear[..., 0], self.shear[..., 1] + + if self.center is None: + center = _compute_tensor_center(input).expand(input.size()[0], -1) + else: + center = self.center + + matrix = get_affine_matrix2d(self.translation, center, self.scale_factor, -self.angle, sx=sx, sy=sy) + return affine(input, matrix[..., :2, :3], self.mode, self.padding_mode, self.align_corners) + + +class Rescale(nn.Module): + r"""Rescale the input torch.Tensor with the given factor. + + Args: + factor: Desired scaling factor in each direction. If scalar, the value is used + for both the x- and y-direction. + interpolation: algorithm used for upsampling: ``'nearest'`` | ``'linear'`` | ``'bilinear'`` | + ``'bicubic'`` | ``'trilinear'`` | ``'area'``. + align_corners: interpolation flag. + side: Corresponding side if ``size`` is an integer. Can be one of ``'short'``, ``'long'``, ``'vert'``, + or ``'horz'``. + antialias: if True, then image will be filtered with Gaussian before downscaling. + No effect for upscaling. + + Returns: + The rescaled tensor with the shape according to the given factor. + + Example: + >>> img = torch.rand(1, 3, 4, 4) + >>> out = Rescale((2, 3))(img) + >>> print(out.shape) + torch.Size([1, 3, 8, 12]) + + """ + + def __init__( + self, + factor: Union[float, Tuple[float, float]], + interpolation: str = "bilinear", + align_corners: bool = True, + antialias: bool = False, + ) -> None: + super().__init__() + self.factor: Union[float, Tuple[float, float]] = factor + self.interpolation: str = interpolation + self.align_corners: Optional[bool] = align_corners + self.antialias: bool = antialias + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Resize spatial dimensions by the stored scale factor. + + Args: + input: Tensor with shape :math:`(*, C, H, W)`, where ``*`` is any + leading batch shape, :math:`C` is channels, :math:`H` is + height, and :math:`W` is width. + + Returns: + Tensor with the same leading and channel dimensions as ``input``. + The output height and width are computed from ``self.factor``. + """ + return rescale( + input, self.factor, self.interpolation, align_corners=self.align_corners, antialias=self.antialias + ) + + +class Rotate(nn.Module): + r"""Rotate the tensor anti-clockwise about the centre. + + Args: + angle: The angle through which to rotate. The tensor + must have a shape of (B), where B is batch size. + center: The center through which to rotate. The tensor + must have a shape of (B, 2), where B is batch size and last + dimension contains cx and cy. + mode: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + ``'torch.zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: interpolation flag. + + Returns: + The rotated tensor with the same shape as the input. + + Example: + >>> img = torch.rand(1, 3, 4, 4) + >>> angle = torch.tensor([90.]) + >>> out = Rotate(angle)(img) + >>> print(out.shape) + torch.Size([1, 3, 4, 4]) + + """ + + def __init__( + self, + angle: torch.Tensor, + center: Union[None, torch.Tensor] = None, + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = True, + ) -> None: + super().__init__() + self.angle: torch.Tensor = angle + self.center: Union[None, torch.Tensor] = center + self.mode: str = mode + self.padding_mode: str = padding_mode + self.align_corners: bool = align_corners + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Rotate an image batch around the configured center. + + Args: + input: Image tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is batch size, :math:`C` is channels, :math:`H` is + height, and :math:`W` is width. + + Returns: + Rotated tensor with shape :math:`(B, C, H, W)`. The image extent is + kept fixed; newly exposed pixels are filled according to + ``self.padding_mode``. + """ + return rotate(input, self.angle, self.center, self.mode, self.padding_mode, self.align_corners) + + +class Translate(nn.Module): + r"""Translate the tensor in pixel units. + + Args: + translation: tensor containing the amount of pixels to + translate in the x and y direction. The tensor must have a shape of + (B, 2), where B is batch size, last dimension contains dx dy. + mode: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + ``'torch.zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: interpolation flag. + + Returns: + The translated tensor with the same shape as the input. + + Example: + >>> img = torch.rand(1, 3, 4, 4) + >>> translation = torch.tensor([[1., 0.]]) + >>> out = Translate(translation)(img) + >>> print(out.shape) + torch.Size([1, 3, 4, 4]) + + """ + + def __init__( + self, + translation: torch.Tensor, + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = True, + ) -> None: + super().__init__() + self.translation: torch.Tensor = translation + self.mode: str = mode + self.padding_mode: str = padding_mode + self.align_corners: bool = align_corners + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Shift an image batch by the stored pixel translation. + + Args: + input: Image tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is batch size, :math:`C` is channels, :math:`H` is + height, and :math:`W` is width. + + Returns: + Translated tensor with shape :math:`(B, C, H, W)`. Translation is + measured in pixels along the x and y image axes. + """ + return translate(input, self.translation, self.mode, self.padding_mode, self.align_corners) + + +class Scale(nn.Module): + r"""Scale the tensor by a factor. + + Args: + scale_factor: The scale factor apply. The tensor + must have a shape of (B) or (B, 2), where B is batch size. + If (B), isotropic scaling will perform. + If (B, 2), x-y-direction specific scaling will perform. + center: The center through which to scale. The tensor + must have a shape of (B, 2), where B is batch size and last + dimension contains cx and cy. + mode: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + ``'torch.zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: interpolation flag. + + Returns: + The scaled tensor with the same shape as the input. + + Example: + >>> img = torch.rand(1, 3, 4, 4) + >>> scale_factor = torch.tensor([[2., 2.]]) + >>> out = Scale(scale_factor)(img) + >>> print(out.shape) + torch.Size([1, 3, 4, 4]) + + """ + + def __init__( + self, + scale_factor: torch.Tensor, + center: Union[None, torch.Tensor] = None, + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = True, + ) -> None: + super().__init__() + self.scale_factor: torch.Tensor = scale_factor + self.center: Union[None, torch.Tensor] = center + self.mode: str = mode + self.padding_mode: str = padding_mode + self.align_corners: bool = align_corners + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Scale an image batch around the configured center point. + + Args: + input: Image tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is batch size, :math:`C` is channels, :math:`H` is + height, and :math:`W` is width. + + Returns: + Scaled tensor with the same shape as ``input``. The sampling grid + is built from ``self.scale_factor`` and ``self.center``. + """ + return scale(input, self.scale_factor, self.center, self.mode, self.padding_mode, self.align_corners) + + +class Shear(nn.Module): + r"""Shear the tensor. + + Args: + shear: tensor containing the angle to shear + in the x and y direction. The tensor must have a shape of + (B, 2), where B is batch size, last dimension contains shx shy. + mode: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + ``'torch.zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: interpolation flag. + + Returns: + The skewed tensor with the same shape as the input. + + Example: + >>> img = torch.rand(1, 3, 4, 4) + >>> shear_factor = torch.tensor([[0.5, 0.0]]) + >>> out = Shear(shear_factor)(img) + >>> print(out.shape) + torch.Size([1, 3, 4, 4]) + + """ + + def __init__( + self, shear: torch.Tensor, mode: str = "bilinear", padding_mode: str = "zeros", align_corners: bool = True + ) -> None: + super().__init__() + self.shear: torch.Tensor = shear + self.mode: str = mode + self.padding_mode: str = padding_mode + self.align_corners: bool = align_corners + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Skew an image batch with the configured x/y shear factors. + + Args: + input: Image tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is batch size, :math:`C` is channels, :math:`H` is + height, and :math:`W` is width. + + Returns: + Sheared tensor with shape :math:`(B, C, H, W)`. The output image + size is unchanged; pixels outside the sampled area use the + configured padding mode. + """ + return shear(input, self.shear, self.mode, self.padding_mode, self.align_corners) diff --git a/kornia/geometry/transform/crop2d.py b/kornia/geometry/transform/crop2d.py new file mode 100644 index 0000000..78cb583 --- /dev/null +++ b/kornia/geometry/transform/crop2d.py @@ -0,0 +1,493 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Optional, Tuple, Union + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.constants import Resample +from kornia.core.check import KORNIA_CHECK_SHAPE +from kornia.geometry.bbox import infer_bbox_shape, validate_bbox + +from .affwarp import resize +from .imgwarp import get_perspective_transform, warp_affine + +__all__ = [ + "CenterCrop2D", + "center_crop", + "crop_and_resize", + "crop_by_boxes", + "crop_by_indices", + "crop_by_transform_mat", +] + + +def crop_and_resize( + input_tensor: torch.Tensor, + boxes: torch.Tensor, + size: Tuple[int, int], + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = True, +) -> torch.Tensor: + r"""Extract crops from 2D images (4D torch.Tensor) and resize given a bounding box. + + Args: + input_tensor: the 2D image torch.Tensor with shape (B, C, H, W). + boxes : a torch.Tensor containing the coordinates of the bounding boxes to be extracted. + The torch.Tensor must have the shape of Bx4x2, where each box is defined in the following (clockwise) + order: top-left, top-right, bottom-right and bottom-left. The coordinates must be in the x, y order. + The coordinates would compose a rectangle with a shape of (N1, N2). + size: a tuple with the height and width that will be + used to resize the extracted patches. + mode: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + ``'torch.zeros'`` | ``'border'`` | 'reflection'. + align_corners: mode for grid_generation. + + Returns: + torch.Tensor: torch.tensor containing the patches with shape BxCxN1xN2. + + Example: + >>> input = torch.tensor([[[ + ... [1., 2., 3., 4.], + ... [5., 6., 7., 8.], + ... [9., 10., 11., 12.], + ... [13., 14., 15., 16.], + ... ]]]) + >>> boxes = torch.tensor([[ + ... [1., 1.], + ... [2., 1.], + ... [2., 2.], + ... [1., 2.], + ... ]]) # 1x4x2 + >>> crop_and_resize(input, boxes, (2, 2), mode='nearest', align_corners=True) + tensor([[[[ 6., 7.], + [10., 11.]]]]) + + """ + if not isinstance(input_tensor, torch.Tensor): + raise TypeError(f"Input torch.tensor type is not a torch.Tensor. Got {type(input_tensor)}") + + if not isinstance(boxes, torch.Tensor): + raise TypeError(f"Input boxes type is not a torch.Tensor. Got {type(boxes)}") + + if not isinstance(size, (tuple, list)) or len(size) != 2: + raise ValueError(f"Input size must be a tuple/list of length 2. Got {size}") + + if len(input_tensor.shape) != 4: + raise AssertionError(f"Only torch.Tensor with shape (B, C, H, W) supported. Got {input_tensor.shape}.") + + # unpack input data + dst_h, dst_w = size + + # [x, y] origin + # top-left, top-right, bottom-right, bottom-left + points_src = boxes.to(input_tensor) + + # [x, y] destination + # top-left, top-right, bottom-right, bottom-left + points_dst = torch.tensor( + [[[0, 0], [dst_w - 1, 0], [dst_w - 1, dst_h - 1], [0, dst_h - 1]]], + device=input_tensor.device, + dtype=input_tensor.dtype, + ).expand(points_src.shape[0], -1, -1) + + return crop_by_boxes(input_tensor, points_src, points_dst, mode, padding_mode, align_corners) + + +def center_crop( + input_tensor: torch.Tensor, + size: Tuple[int, int], + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = True, +) -> torch.Tensor: + r"""Crop the 2D images (4D torch.Tensor) from the center. + + Args: + input_tensor: the 2D image torch.Tensor with shape (B, C, H, W). + size: a tuple with the expected height and width + of the output patch. + mode: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + ``'torch.zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: mode for grid_generation. + + Returns: + the output torch.Tensor with patches. + + Examples: + >>> input = torch.tensor([[[ + ... [1., 2., 3., 4.], + ... [5., 6., 7., 8.], + ... [9., 10., 11., 12.], + ... [13., 14., 15., 16.], + ... ]]]) + >>> center_crop(input, (2, 4), mode='nearest', align_corners=True) + tensor([[[[ 5., 6., 7., 8.], + [ 9., 10., 11., 12.]]]]) + + """ + if not isinstance(input_tensor, torch.Tensor): + raise TypeError(f"Input torch.tensor type is not a torch.Tensor. Got {type(input_tensor)}") + + if not isinstance(size, (tuple, list)) or len(size) != 2: + raise ValueError(f"Input size must be a tuple/list of length 2. Got {size}") + + if len(input_tensor.shape) != 4: + raise AssertionError(f"Only torch.Tensor with shape (B, C, H, W) supported. Got {input_tensor.shape}.") + + # unpack input sizes + dst_h, dst_w = size + src_h, src_w = input_tensor.shape[-2:] + + # compute start/end offsets + dst_h_half: float = dst_h / 2 + dst_w_half: float = dst_w / 2 + src_h_half: float = src_h / 2 + src_w_half: float = src_w / 2 + + start_x: float = src_w_half - dst_w_half + start_y: float = src_h_half - dst_h_half + + end_x: float = start_x + dst_w - 1 + end_y: float = start_y + dst_h - 1 + + # [y, x] origin + # top-left, top-right, bottom-right, bottom-left + points_src: torch.Tensor = torch.tensor( + [[[start_x, start_y], [end_x, start_y], [end_x, end_y], [start_x, end_y]]], + device=input_tensor.device, + dtype=input_tensor.dtype, + ) + + # [y, x] destination + # top-left, top-right, bottom-right, bottom-left + points_dst: torch.Tensor = torch.tensor( + [[[0, 0], [dst_w - 1, 0], [dst_w - 1, dst_h - 1], [0, dst_h - 1]]], + device=input_tensor.device, + dtype=input_tensor.dtype, + ).expand(points_src.shape[0], -1, -1) + + return crop_by_boxes(input_tensor, points_src, points_dst, mode, padding_mode, align_corners) + + +def crop_by_boxes( + input_tensor: torch.Tensor, + src_box: torch.Tensor, + dst_box: torch.Tensor, + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = True, + validate_boxes: bool = True, +) -> torch.Tensor: + """Perform crop transform on 2D images (4D torch.Tensor) given two bounding boxes. + + Given an input torch.Tensor, this function selected the interested areas by the provided bounding boxes (src_box). + Then the selected areas would be fitted into the targeted bounding boxes (dst_box) by a perspective transformation. + So far, the ragged torch.Tensor is not supported by PyTorch right now. This function hereby requires + the bounding boxes in a batch must be rectangles with same width and height. + + Args: + input_tensor: the 2D image torch.Tensor with shape (B, C, H, W). + src_box: a torch.Tensor with shape (B, 4, 2) containing the coordinates of the bounding boxes + to be extracted. The torch.Tensor must have the shape of Bx4x2, where each box is defined + in the clockwise order: top-left, top-right, bottom-right and bottom-left. + The coordinates must be in x, y order. + dst_box: a torch.Tensor with shape (B, 4, 2) containing the coordinates of the bounding boxes + to be placed. The torch.Tensor must have the shape of Bx4x2, where each box is defined + in the clockwise order: top-left, top-right, bottom-right and bottom-left. + The coordinates must be in x, y order. + mode: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + ``'torch.zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: mode for grid_generation. + validate_boxes: flag to perform validation on boxes. + + Returns: + torch.Tensor: the output torch.tensor with patches. + + Examples: + >>> input = torch.arange(16, dtype=torch.float32).reshape((1, 1, 4, 4)) + >>> src_box = torch.tensor([[ + ... [1., 1.], + ... [2., 1.], + ... [2., 2.], + ... [1., 2.], + ... ]]) # 1x4x2 + >>> dst_box = torch.tensor([[ + ... [0., 0.], + ... [1., 0.], + ... [1., 1.], + ... [0., 1.], + ... ]]) # 1x4x2 + >>> crop_by_boxes(input, src_box, dst_box, align_corners=True) + tensor([[[[ 5.0000, 6.0000], + [ 9.0000, 10.0000]]]]) + + Note: + If the src_box is smaller than dst_box, the following error will be thrown. + RuntimeError: solve_cpu: For batch 0: U(2,2) is zero, singular U. + + """ + if validate_boxes: + validate_bbox(src_box) + validate_bbox(dst_box) + + if len(input_tensor.shape) != 4: + raise AssertionError(f"Only torch.Tensor with shape (B, C, H, W) supported. Got {input_tensor.shape}.") + + # compute transformation between points and warp + # Note: torch.Tensor.dtype must be float. "solve_cpu" not implemented for 'Long' + dst_trans_src: torch.Tensor = get_perspective_transform(src_box.to(input_tensor), dst_box.to(input_tensor)) + + bbox: Tuple[torch.Tensor, torch.Tensor] = infer_bbox_shape(dst_box) + if not ((bbox[0] == bbox[0][0]).all() and (bbox[1] == bbox[1][0]).all()): + raise AssertionError( + f"Cropping height, width and depth must be exact same in a batch. Got height {bbox[0]} and width {bbox[1]}." + ) + + h_out: int = int(bbox[0][0].item()) + w_out: int = int(bbox[1][0].item()) + + return crop_by_transform_mat( + input_tensor, dst_trans_src, (h_out, w_out), mode=mode, padding_mode=padding_mode, align_corners=align_corners + ) + + +def crop_by_transform_mat( + input_tensor: torch.Tensor, + transform: torch.Tensor, + out_size: Tuple[int, int], + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = True, +) -> torch.Tensor: + """Perform crop transform on 2D images (4D torch.Tensor) given a perspective transformation matrix. + + Args: + input_tensor: the 2D image torch.Tensor with shape (B, C, H, W). + transform: a perspective transformation matrix with shape (B, 3, 3). + out_size: size of the output image (height, width). + mode: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + padding_mode (str): padding mode for outside grid values + ``'torch.zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: mode for grid_generation. + + Returns: + the output torch.Tensor with patches. + + """ + # simulate broadcasting + dst_trans_src = torch.as_tensor( + transform.expand(input_tensor.shape[0], -1, -1), device=input_tensor.device, dtype=input_tensor.dtype + ) + + patches: torch.Tensor = warp_affine( + input_tensor, + dst_trans_src[:, :2, :], + out_size, + mode=mode, + padding_mode=padding_mode, + align_corners=align_corners, + ) + + return patches + + +def crop_by_indices( + input_tensor: torch.Tensor, + src_box: torch.Tensor, + size: Optional[Tuple[int, int]] = None, + interpolation: str = "bilinear", + align_corners: Optional[bool] = None, + antialias: bool = False, + shape_compensation: str = "resize", +) -> torch.Tensor: + """Crop tensors with naive indices. + + Args: + input_tensor: the 2D image torch.Tensor with shape (B, C, H, W). + src_box: a torch.Tensor with shape (B, 4, 2) containing the coordinates of the bounding boxes + to be extracted. The torch.Tensor must have the shape of Bx4x2, where each box is defined + in the clockwise order: top-left, top-right, bottom-right and bottom-left. + The coordinates must be in x, y order. + size: output size. An auto resize or F.pad will be performed according to ``shape_compensation`` + if the cropped slice sizes are not exactly align `size`. + If None, will auto-infer from src_box. + interpolation: algorithm used for upsampling: ``'nearest'`` | ``'linear'`` | ``'bilinear'`` | + 'bicubic' | 'trilinear' | 'area'. + align_corners: interpolation flag. + antialias: if True, then image will be filtered with Gaussian before downscaling. + No effect for upscaling. + shape_compensation: if the cropped slice sizes are not exactly align `size`, the image can either be padded + or resized. + + """ + KORNIA_CHECK_SHAPE(input_tensor, ["B", "C", "H", "W"]) + KORNIA_CHECK_SHAPE(src_box, ["B", "4", "2"]) + + B, C, _, _ = input_tensor.shape + src = torch.as_tensor(src_box, device=input_tensor.device, dtype=torch.long) + x1 = src[:, 0, 0] + x2 = src[:, 1, 0] + 1 + y1 = src[:, 0, 1] + y2 = src[:, 3, 1] + 1 + + if ( + len(x1.unique(sorted=False)) + == len(x2.unique(sorted=False)) + == len(y1.unique(sorted=False)) + == len(y2.unique(sorted=False)) + == 1 + ): + out = input_tensor[..., int(y1[0]) : int(y2[0]), int(x1[0]) : int(x2[0])] + if size is not None and out.shape[-2:] != size: + return resize( + out, size, interpolation=interpolation, align_corners=align_corners, side="short", antialias=antialias + ) + + if size is None: + h, w = infer_bbox_shape(src) + size = h.unique(sorted=False), w.unique(sorted=False) + out = torch.empty(B, C, *size, device=input_tensor.device, dtype=input_tensor.dtype) + # Find out the cropped shapes that need to be resized. + for i, _ in enumerate(out): + _out = input_tensor[i : i + 1, :, int(y1[i]) : int(y2[i]), int(x1[i]) : int(x2[i])] + if _out.shape[-2:] != size: + if shape_compensation == "resize": + out[i] = resize( + _out, + size, + interpolation=interpolation, + align_corners=align_corners, + side="short", + antialias=antialias, + ) + else: + out[i] = F.pad(_out, [0, size[1] - _out.shape[-1], 0, size[0] - _out.shape[-2]]) + else: + out[i] = _out + return out + + +class CenterCrop2D(nn.Module): + """Center crop the input torch.Tensor. + + Args: + size: Size (h, w) in pixels of the resized region or just one side. + align_corners: interpolation flag. + resample: Resampling mode. + cropping_mode: Cropping mode, "resample" or "slice". + + Note: + For JIT, the cropping mode must be "resample". + """ + + def __init__( + self, + size: Union[int, Tuple[int, int]], + align_corners: bool = True, + resample: Union[str, int, Resample] = Resample.BILINEAR.name, + cropping_mode: str = "slice", + ) -> None: + super().__init__() + if isinstance(size, tuple): + self.size = (size[0], size[1]) + elif isinstance(size, int): + self.size = (size, size) + else: + raise Exception(f"Invalid size type. Expected (int, tuple(int, int). Got: {type(size)}.") + + dst_h, dst_w = self.size + points_dst = torch.tensor([[[0, 0], [dst_w - 1, 0], [dst_w - 1, dst_h - 1], [0, dst_h - 1]]], dtype=torch.long) + self.register_buffer("points_src", points_dst.clone()) + self.register_buffer("points_dst", points_dst) + + self.flags = { + "resample": Resample.get(resample), + "cropping_mode": cropping_mode, + "align_corners": align_corners, + "size": self.size, + "padding_mode": "zeros", + } + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Crop the central region of each input image to ``self.size``. + + Depending on ``cropping_mode``, the crop is produced either by + perspective-resampling (subpixel/interpolated) or by direct slicing. + + Args: + input: Image batch with shape :math:`(B, C, H, W)`, where + :math:`B` is batch size, :math:`C` is channels, :math:`H` is + source height, and :math:`W` is source width. + + Returns: + Center-cropped tensor with shape :math:`(B, C, h, w)`, where + :math:`h` and :math:`w` are the requested crop height and width + stored in ``self.size``. + """ + batch_size = input.shape[0] + + dst_h, dst_w = self.size + src_h, src_w = input.shape[-2:] + + dst_h_half, dst_w_half = dst_h / 2, dst_w / 2 + src_h_half, src_w_half = src_h / 2, src_w / 2 + + start_x, start_y = int(src_w_half - dst_w_half), int(src_h_half - dst_h_half) + end_x, end_y = start_x + dst_w - 1, start_y + dst_h - 1 + + # [y, x] origin + # top-left, top-right, bottom-right, bottom-left + self.points_src[0, 0, 0] = start_x + self.points_src[0, 0, 1] = start_y + self.points_src[0, 1, 0] = end_x + self.points_src[0, 1, 1] = start_y + self.points_src[0, 2, 0] = end_x + self.points_src[0, 2, 1] = end_y + self.points_src[0, 3, 0] = start_x + self.points_src[0, 3, 1] = end_y + + if self.flags["cropping_mode"] == "resample": # uses bilinear interpolation to crop + transform = get_perspective_transform( + self.points_src.expand(batch_size, -1, -1).to(input), + self.points_dst.expand(batch_size, -1, -1).to(input), + ) + transform = transform.expand(batch_size, -1, -1) + + return crop_by_transform_mat( + input, + transform[:, :2, :], + self.size, + self.flags["resample"].name.lower(), # type:ignore + "zeros", + self.flags["align_corners"], # type:ignore + ) + + if self.flags["cropping_mode"] == "slice": # uses advanced slicing to crop + return crop_by_indices(input, self.points_src.expand(batch_size, -1, -1).to(input), self.flags["size"]) # type:ignore + + raise NotImplementedError(f"Not supported type: {self.flags['cropping_mode']}.") diff --git a/kornia/geometry/transform/crop3d.py b/kornia/geometry/transform/crop3d.py new file mode 100644 index 0000000..5269ef2 --- /dev/null +++ b/kornia/geometry/transform/crop3d.py @@ -0,0 +1,382 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Tuple + +import torch + +from kornia.geometry.bbox import infer_bbox_shape3d, validate_bbox3d + +from .imgwarp import get_perspective_transform3d, warp_affine3d + +__all__ = ["center_crop3d", "crop_and_resize3d", "crop_by_boxes3d", "crop_by_transform_mat3d"] + + +def crop_and_resize3d( + tensor: torch.Tensor, + boxes: torch.Tensor, + size: Tuple[int, int, int], + interpolation: str = "bilinear", + align_corners: bool = False, +) -> torch.Tensor: + r"""Extract crops from 3D volumes (5D tensor) and resize them. + + Args: + tensor: the 3D volume tensor with shape (B, C, D, H, W). + boxes: a tensor with shape (B, 8, 3) containing the coordinates of the bounding boxes + to be extracted. The tensor must have the shape of Bx8x3, where each box is defined in the clockwise + order: front-top-left, front-top-right, front-bottom-right, front-bottom-left, back-top-left, + back-top-right, back-bottom-right, back-bottom-left. The coordinates must be in x, y, z order. + size: a tuple with the height and width that will be + used to resize the extracted patches. + interpolation: Interpolation flag. + align_corners: mode for grid_generation. + + Returns: + tensor containing the patches with shape (Bx)CxN1xN2xN3. + + Example: + >>> input = torch.arange(64, dtype=torch.float32).view(1, 1, 4, 4, 4) + >>> input + tensor([[[[[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.], + [12., 13., 14., 15.]], + + [[16., 17., 18., 19.], + [20., 21., 22., 23.], + [24., 25., 26., 27.], + [28., 29., 30., 31.]], + + [[32., 33., 34., 35.], + [36., 37., 38., 39.], + [40., 41., 42., 43.], + [44., 45., 46., 47.]], + + [[48., 49., 50., 51.], + [52., 53., 54., 55.], + [56., 57., 58., 59.], + [60., 61., 62., 63.]]]]]) + >>> boxes = torch.tensor([[ + ... [1., 1., 1.], + ... [3., 1., 1.], + ... [3., 3., 1.], + ... [1., 3., 1.], + ... [1., 1., 2.], + ... [3., 1., 2.], + ... [3., 3., 2.], + ... [1., 3., 2.], + ... ]]) # 1x8x3 + >>> crop_and_resize3d(input, boxes, (2, 2, 2), align_corners=True) + tensor([[[[[21.0000, 23.0000], + [29.0000, 31.0000]], + + [[37.0000, 39.0000], + [45.0000, 47.0000]]]]]) + + """ + if not isinstance(tensor, (torch.Tensor)): + raise TypeError(f"Input tensor type is not a torch.Tensor. Got {type(tensor)}") + if not isinstance(boxes, (torch.Tensor)): + raise TypeError(f"Input boxes type is not a torch.Tensor. Got {type(boxes)}") + if not isinstance(size, (tuple, list)) or len(size) != 3: + raise ValueError(f"Input size must be a tuple/list of length 3. Got {size}") + if len(tensor.shape) != 5: + raise AssertionError(f"Only tensor with shape (B, C, D, H, W) supported. Got {tensor.shape}.") + # unpack input data + dst_d, dst_h, dst_w = size[0], size[1], size[2] + + # [x, y, z] origin + # from front to back + # top-left, top-right, bottom-right, bottom-left + points_src: torch.Tensor = boxes + + # [x, y, z] destination + # from front to back + # top-left, top-right, bottom-right, bottom-left + points_dst: torch.Tensor = torch.tensor( + [ + [ + [0, 0, 0], + [dst_w - 1, 0, 0], + [dst_w - 1, dst_h - 1, 0], + [0, dst_h - 1, 0], + [0, 0, dst_d - 1], + [dst_w - 1, 0, dst_d - 1], + [dst_w - 1, dst_h - 1, dst_d - 1], + [0, dst_h - 1, dst_d - 1], + ] + ], + dtype=tensor.dtype, + device=tensor.device, + ).expand(points_src.shape[0], -1, -1) + + return crop_by_boxes3d(tensor, points_src, points_dst, interpolation, align_corners) + + +def center_crop3d( + tensor: torch.Tensor, size: Tuple[int, int, int], interpolation: str = "bilinear", align_corners: bool = True +) -> torch.Tensor: + r"""Crop the 3D volumes (5D tensor) at the center. + + Args: + tensor: the 3D volume tensor with shape (B, C, D, H, W). + size: a tuple with the expected depth, height and width + of the output patch. + interpolation: Interpolation flag. + align_corners : mode for grid_generation. + + Returns: + the output tensor with patches. + + Examples: + >>> input = torch.arange(64, dtype=torch.float32).view(1, 1, 4, 4, 4) + >>> input + tensor([[[[[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.], + [12., 13., 14., 15.]], + + [[16., 17., 18., 19.], + [20., 21., 22., 23.], + [24., 25., 26., 27.], + [28., 29., 30., 31.]], + + [[32., 33., 34., 35.], + [36., 37., 38., 39.], + [40., 41., 42., 43.], + [44., 45., 46., 47.]], + + [[48., 49., 50., 51.], + [52., 53., 54., 55.], + [56., 57., 58., 59.], + [60., 61., 62., 63.]]]]]) + >>> center_crop3d(input, (2, 2, 2), align_corners=True) + tensor([[[[[21.0000, 22.0000], + [25.0000, 26.0000]], + + [[37.0000, 38.0000], + [41.0000, 42.0000]]]]]) + + """ + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Input tensor type is not a torch.Tensor. Got {type(tensor)}") + + if len(tensor.shape) != 5: + raise AssertionError(f"Only tensor with shape (B, C, D, H, W) supported. Got {tensor.shape}.") + + if not isinstance(size, (tuple, list)) or len(size) != 3: + raise ValueError(f"Input size must be a tuple/list of length 3. Got {size}") + + # unpack input sizes + dst_d, dst_h, dst_w = size + src_d, src_h, src_w = tensor.shape[-3:] + + # compute start/end offsets + dst_d_half = dst_d / 2 + dst_h_half = dst_h / 2 + dst_w_half = dst_w / 2 + src_d_half = src_d / 2 + src_h_half = src_h / 2 + src_w_half = src_w / 2 + + start_x = src_w_half - dst_w_half + start_y = src_h_half - dst_h_half + start_z = src_d_half - dst_d_half + + end_x = start_x + dst_w - 1 + end_y = start_y + dst_h - 1 + end_z = start_z + dst_d - 1 + # [x, y, z] origin + # top-left-front, top-right-front, bottom-right-front, bottom-left-front + # top-left-back, top-right-back, bottom-right-back, bottom-left-back + points_src: torch.Tensor = torch.tensor( + [ + [ + [start_x, start_y, start_z], + [end_x, start_y, start_z], + [end_x, end_y, start_z], + [start_x, end_y, start_z], + [start_x, start_y, end_z], + [end_x, start_y, end_z], + [end_x, end_y, end_z], + [start_x, end_y, end_z], + ] + ], + device=tensor.device, + ) + + # [x, y, z] destination + # top-left-front, top-right-front, bottom-right-front, bottom-left-front + # top-left-back, top-right-back, bottom-right-back, bottom-left-back + points_dst: torch.Tensor = torch.tensor( + [ + [ + [0, 0, 0], + [dst_w - 1, 0, 0], + [dst_w - 1, dst_h - 1, 0], + [0, dst_h - 1, 0], + [0, 0, dst_d - 1], + [dst_w - 1, 0, dst_d - 1], + [dst_w - 1, dst_h - 1, dst_d - 1], + [0, dst_h - 1, dst_d - 1], + ] + ], + device=tensor.device, + ).expand(points_src.shape[0], -1, -1) + + return crop_by_boxes3d( + tensor, points_src.to(tensor.dtype), points_dst.to(tensor.dtype), interpolation, align_corners + ) + + +def crop_by_boxes3d( + tensor: torch.Tensor, + src_box: torch.Tensor, + dst_box: torch.Tensor, + interpolation: str = "bilinear", + align_corners: bool = False, +) -> torch.Tensor: + """Perform crop transform on 3D volumes (5D tensor) by bounding boxes. + + Given an input tensor, this function selected the interested areas by the provided bounding boxes (src_box). + Then the selected areas would be fitted into the targeted bounding boxes (dst_box) by a perspective transformation. + So far, the ragged tensor is not supported by PyTorch right now. This function hereby requires the bounding boxes + in a batch must be rectangles with same width, height and depth. + + Args: + tensor : the 3D volume tensor with shape (B, C, D, H, W). + src_box : a tensor with shape (B, 8, 3) containing the coordinates of the bounding boxes + to be extracted. The tensor must have the shape of Bx8x3, where each box is defined in the clockwise + order: front-top-left, front-top-right, front-bottom-right, front-bottom-left, back-top-left, + back-top-right, back-bottom-right, back-bottom-left. The coordinates must be in x, y, z order. + dst_box: a tensor with shape (B, 8, 3) containing the coordinates of the bounding boxes + to be placed. The tensor must have the shape of Bx8x3, where each box is defined in the clockwise + order: front-top-left, front-top-right, front-bottom-right, front-bottom-left, back-top-left, + back-top-right, back-bottom-right, back-bottom-left. The coordinates must be in x, y, z order. + interpolation: Interpolation flag. + align_corners: mode for grid_generation. + + Returns: + the output tensor with patches. + + Examples: + >>> input = torch.tensor([[[ + ... [[ 0., 1., 2., 3.], + ... [ 4., 5., 6., 7.], + ... [ 8., 9., 10., 11.], + ... [12., 13., 14., 15.]], + ... [[16., 17., 18., 19.], + ... [20., 21., 22., 23.], + ... [24., 25., 26., 27.], + ... [28., 29., 30., 31.]], + ... [[32., 33., 34., 35.], + ... [36., 37., 38., 39.], + ... [40., 41., 42., 43.], + ... [44., 45., 46., 47.]]]]]) + >>> src_box = torch.tensor([[ + ... [1., 1., 1.], + ... [3., 1., 1.], + ... [3., 3., 1.], + ... [1., 3., 1.], + ... [1., 1., 2.], + ... [3., 1., 2.], + ... [3., 3., 2.], + ... [1., 3., 2.], + ... ]]) # 1x8x3 + >>> dst_box = torch.tensor([[ + ... [0., 0., 0.], + ... [2., 0., 0.], + ... [2., 2., 0.], + ... [0., 2., 0.], + ... [0., 0., 1.], + ... [2., 0., 1.], + ... [2., 2., 1.], + ... [0., 2., 1.], + ... ]]) # 1x8x3 + >>> crop_by_boxes3d(input, src_box, dst_box, interpolation='nearest', align_corners=True) + tensor([[[[[21., 22., 23.], + [25., 26., 27.], + [29., 30., 31.]], + + [[37., 38., 39.], + [41., 42., 43.], + [45., 46., 47.]]]]]) + + """ + validate_bbox3d(src_box) + validate_bbox3d(dst_box) + + if len(tensor.shape) != 5: + raise AssertionError(f"Only tensor with shape (B, C, D, H, W) supported. Got {tensor.shape}.") + + # compute transformation between points and warp + # Note: Tensor.dtype must be float. "solve_cpu" not implemented for 'Long' + dst_trans_src: torch.Tensor = get_perspective_transform3d(src_box.to(tensor.dtype), dst_box.to(tensor.dtype)) + # simulate broadcasting + dst_trans_src = dst_trans_src.expand(tensor.shape[0], -1, -1).type_as(tensor) + + bbox = infer_bbox_shape3d(dst_box) + if not ((bbox[0] == bbox[0][0]).all() and (bbox[1] == bbox[1][0]).all() and (bbox[2] == bbox[2][0]).all()): + raise AssertionError( + "Cropping height, width and depth must be exact same in a batch." + f"Got height {bbox[0]}, width {bbox[1]} and depth {bbox[2]}." + ) + + patches: torch.Tensor = crop_by_transform_mat3d( + tensor, + dst_trans_src, + (int(bbox[0][0].item()), int(bbox[1][0].item()), int(bbox[2][0].item())), + mode=interpolation, + align_corners=align_corners, + ) + + return patches + + +def crop_by_transform_mat3d( + tensor: torch.Tensor, + transform: torch.Tensor, + out_size: Tuple[int, int, int], + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = True, +) -> torch.Tensor: + """Perform crop transform on 3D volumes (5D tensor) given a perspective transformation matrix. + + Args: + tensor: the 2D image tensor with shape (B, C, H, W). + transform: a perspective transformation matrix with shape (B, 4, 4). + out_size: size of the output image (depth, height, width). + mode: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + ``'zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: mode for grid_generation. + + Returns: + the output tensor with patches. + + """ + # simulate broadcasting + dst_trans_src = transform.expand(tensor.shape[0], -1, -1) + + patches: torch.Tensor = warp_affine3d( + tensor, dst_trans_src[:, :3, :], out_size, flags=mode, padding_mode=padding_mode, align_corners=align_corners + ) + + return patches diff --git a/kornia/geometry/transform/elastic_transform.py b/kornia/geometry/transform/elastic_transform.py new file mode 100644 index 0000000..8702c9f --- /dev/null +++ b/kornia/geometry/transform/elastic_transform.py @@ -0,0 +1,121 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Tuple, Union + +import torch +import torch.nn.functional as F + +from kornia.core.check import KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE +from kornia.filters import filter2d +from kornia.filters.kernels import get_gaussian_kernel2d +from kornia.geometry.grid import create_meshgrid + +__all__ = ["elastic_transform2d"] + + +def elastic_transform2d( + image: torch.Tensor, + noise: torch.Tensor, + kernel_size: Tuple[int, int] = (63, 63), + sigma: Union[Tuple[float, float], torch.Tensor] = (32.0, 32.0), + alpha: Union[Tuple[float, float], torch.Tensor] = (1.0, 1.0), + align_corners: bool = False, + mode: str = "bilinear", + padding_mode: str = "zeros", +) -> torch.Tensor: + r"""Apply elastic transform of images as described in :cite:`Simard2003BestPF`. + + .. image:: _static/img/elastic_transform2d.png + + Args: + image: Input image to be transformed with shape :math:`(B, C, H, W)`. + noise: Noise image used to spatially transform the input image. Same + resolution as the input image with shape :math:`(B, 2, H, W)`. The coordinates order + it is expected to be in x-y. + kernel_size: the size of the Gaussian kernel. + sigma: The standard deviation of the Gaussian in the y and x directions, + respectively. Larger sigma results in smaller pixel displacements. + alpha : The scaling factor that controls the intensity of the deformation + in the y and x directions, respectively. + align_corners: Interpolation flag used by ```grid_sample```. + mode: Interpolation mode used by ```grid_sample```. Either ``'bilinear'`` or ``'nearest'``. + padding_mode: The padding used by ```grid_sample```. Either ``'torch.zeros'``, ``'border'`` or ``'refection'``. + + Returns: + the elastically transformed input image with shape :math:`(B,C,H,W)`. + + Example: + >>> image = torch.rand(1, 3, 5, 5) + >>> noise = torch.rand(1, 2, 5, 5, requires_grad=True) + >>> image_hat = elastic_transform2d(image, noise, (3, 3)) + >>> image_hat.mean().backward() + + >>> image = torch.rand(1, 3, 5, 5) + >>> noise = torch.rand(1, 2, 5, 5) + >>> sigma = torch.tensor([4., 4.], requires_grad=True) + >>> image_hat = elastic_transform2d(image, noise, (3, 3), sigma) + >>> image_hat.mean().backward() + + >>> image = torch.rand(1, 3, 5, 5) + >>> noise = torch.rand(1, 2, 5, 5) + >>> alpha = torch.tensor([16., 32.], requires_grad=True) + >>> image_hat = elastic_transform2d(image, noise, (3, 3), alpha=alpha) + >>> image_hat.mean().backward() + + """ + KORNIA_CHECK_IS_TENSOR(image) + KORNIA_CHECK_IS_TENSOR(noise) + KORNIA_CHECK_SHAPE(image, ["B", "C", "H", "W"]) + KORNIA_CHECK_SHAPE(noise, ["B", "C", "H", "W"]) + + device, dtype = image.device, image.dtype + # if isinstance(sigma, tuple): + # sigma_t = torch.tensor(sigma, device=device, dtype=dtype) + if isinstance(sigma, torch.Tensor): + sigma = sigma.expand(2)[None, ...] + # sigma = sigma.to(device=device, dtype=dtype) + + # Get Gaussian kernel for 'y' and 'x' displacement + kernel_x = get_gaussian_kernel2d(kernel_size, sigma) # _t[0].expand(2).unsqueeze(0)) + kernel_y = get_gaussian_kernel2d(kernel_size, sigma) # _t[1].expand(2).unsqueeze(0)) + + if isinstance(alpha, torch.Tensor): + alpha_x = alpha[0] + alpha_y = alpha[1] + else: + alpha_x = torch.tensor(alpha[0], device=device, dtype=dtype) + alpha_y = torch.tensor(alpha[1], device=device, dtype=dtype) + + # Convolve over a random displacement matrix and scale them with 'alpha' + disp_x = noise[:, :1] + disp_y = noise[:, 1:] + + disp_x = filter2d(disp_x, kernel=kernel_y, border_type="constant") * alpha_x + disp_y = filter2d(disp_y, kernel=kernel_x, border_type="constant") * alpha_y + + # torch.stack and F.normalize displacement + disp = torch.cat([disp_x, disp_y], 1).permute(0, 2, 3, 1) + + # Warp image based on displacement matrix + _, _, h, w = image.shape + grid = create_meshgrid(h, w, device=image.device).to(image.dtype) + warped = F.grid_sample( + image, (grid + disp).clamp(-1, 1), align_corners=align_corners, mode=mode, padding_mode=padding_mode + ) + + return warped diff --git a/kornia/geometry/transform/flips.py b/kornia/geometry/transform/flips.py new file mode 100644 index 0000000..f0ec4c2 --- /dev/null +++ b/kornia/geometry/transform/flips.py @@ -0,0 +1,198 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch +from torch import nn + +__all__ = ["Hflip", "Rot180", "Vflip", "hflip", "rot180", "vflip"] + + +class Vflip(nn.Module): + r"""Vertically flip a torch.Tensor image or a batch of torch.Tensor images. + + Input must be a torch.Tensor of shape (C, H, W) or a batch of tensors :math:`(*, C, H, W)`. + + Args: + input: input torch.Tensor. + + Returns: + The vertically flipped image torch.Tensor. + + Examples: + >>> vflip = Vflip() + >>> input = torch.tensor([[[ + ... [0., 0., 0.], + ... [0., 0., 0.], + ... [0., 1., 1.] + ... ]]]) + >>> vflip(input) + tensor([[[[0., 1., 1.], + [0., 0., 0.], + [0., 0., 0.]]]]) + + """ + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Flip the image content along the vertical image axis. + + Args: + input: Tensor containing one image or a batch of images, typically + shaped :math:`(*, C, H, W)`, where :math:`H` is height and + :math:`W` is width. + + Returns: + Tensor with the same shape as ``input`` whose rows are reversed + from top to bottom. + """ + return vflip(input) + + def __repr__(self) -> str: + return self.__class__.__name__ + + +class Hflip(nn.Module): + r"""Horizontally flip a torch.Tensor image or a batch of torch.Tensor images. + + Input must be a torch.Tensor of shape (C, H, W) or a batch of tensors :math:`(*, C, H, W)`. + + Args: + input: input torch.Tensor. + + Returns: + The horizontally flipped image torch.Tensor. + + Examples: + >>> hflip = Hflip() + >>> input = torch.tensor([[[ + ... [0., 0., 0.], + ... [0., 0., 0.], + ... [0., 1., 1.] + ... ]]]) + >>> hflip(input) + tensor([[[[0., 0., 0.], + [0., 0., 0.], + [1., 1., 0.]]]]) + + """ + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Flip the image content along the horizontal image axis. + + Args: + input: Tensor containing one image or a batch of images, typically + shaped :math:`(*, C, H, W)`, where :math:`H` is height and + :math:`W` is width. + + Returns: + Tensor with the same shape as ``input`` whose columns are reversed + from left to right. + """ + return hflip(input) + + def __repr__(self) -> str: + return self.__class__.__name__ + + +class Rot180(nn.Module): + r"""Rotate a torch.Tensor image or a batch of torch.Tensor images 180 degrees. + + Input must be a torch.Tensor of shape (C, H, W) or a batch of tensors :math:`(*, C, H, W)`. + + Args: + input: input torch.Tensor. + + Examples: + >>> rot180 = Rot180() + >>> input = torch.tensor([[[ + ... [0., 0., 0.], + ... [0., 0., 0.], + ... [0., 1., 1.] + ... ]]]) + >>> rot180(input) + tensor([[[[1., 1., 0.], + [0., 0., 0.], + [0., 0., 0.]]]]) + + """ + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Rotate image content by 180 degrees without changing tensor shape. + + Args: + input: Tensor containing one image or a batch of images, typically + shaped :math:`(*, C, H, W)`, where :math:`H` is height and + :math:`W` is width. + + Returns: + Tensor with the same shape as ``input`` and both spatial axes + reversed. + """ + return rot180(input) + + def __repr__(self) -> str: + return self.__class__.__name__ + + +def rot180(input: torch.Tensor) -> torch.Tensor: + r"""Rotate a torch.Tensor image or a batch of torch.Tensor images 180 degrees. + + .. image:: _static/img/rot180.png + + Input must be a torch.Tensor of shape (C, H, W) or a batch of tensors :math:`(*, C, H, W)`. + + Args: + input: input torch.Tensor. + + Returns: + The rotated image torch.Tensor. + + """ + return torch.flip(input, [-2, -1]) + + +def hflip(input: torch.Tensor) -> torch.Tensor: + r"""Horizontally flip a torch.Tensor image or a batch of torch.Tensor images. + + .. image:: _static/img/hflip.png + + Input must be a torch.Tensor of shape (C, H, W) or a batch of tensors :math:`(*, C, H, W)`. + + Args: + input: input torch.Tensor. + + Returns: + The horizontally flipped image torch.Tensor. + + """ + return input.flip(-1).contiguous() + + +def vflip(input: torch.Tensor) -> torch.Tensor: + r"""Vertically flip a torch.Tensor image or a batch of torch.Tensor images. + + .. image:: _static/img/vflip.png + + Input must be a torch.Tensor of shape (C, H, W) or a batch of tensors :math:`(*, C, H, W)`. + + Args: + input: input torch.Tensor. + + Returns: + The vertically flipped image torch.Tensor. + + """ + return input.flip(-2).contiguous() diff --git a/kornia/geometry/transform/homography_warper.py b/kornia/geometry/transform/homography_warper.py new file mode 100644 index 0000000..595b034 --- /dev/null +++ b/kornia/geometry/transform/homography_warper.py @@ -0,0 +1,186 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from abc import abstractmethod +from typing import Any, Optional + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.geometry.grid import create_meshgrid + +from .imgwarp import homography_warp, warp_grid + +__all__ = ["BaseWarper", "HomographyWarper"] + + +class BaseWarper(nn.Module): + """Provide a base class for homography-based image warping.""" + + def __init__(self, height: int, width: int, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.height = height + self.width = width + + @abstractmethod + def forward(self, patch_src: torch.Tensor, src_homo_dst: Optional[torch.Tensor] = None) -> torch.Tensor: + """Sample a source patch on this warper's destination grid. + + Args: + patch_src: Source image or patch tensor with shape + :math:`(B, C, H, W)`, where :math:`B` is batch size, + :math:`C` is channels, :math:`H` is source height, and + :math:`W` is source width. + src_homo_dst: Optional homography with shape :math:`(B, 3, 3)` + mapping destination coordinates into the source image. Concrete + implementations may also use a grid precomputed from this + matrix. + + Returns: + Tensor sampled in the destination frame managed by this warper. + """ + ... + + @abstractmethod + def precompute_warp_grid(self, src_homo_dst: torch.Tensor) -> None: + """Precompute a sampling grid for repeated homography warps. + + Args: + src_homo_dst: Homography tensor with shape :math:`(B, 3, 3)` used + to transform destination grid points into source coordinates. + """ + ... + + +class HomographyWarper(BaseWarper): + r"""Warp tensors by homographies. + + .. math:: + + X_{dst} = H_{src}^{\{dst\}} * X_{src} + + Args: + height: The height of the destination torch.Tensor. + width: The width of the destination torch.Tensor. + mode: interpolation mode to calculate output values ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + ``'torch.zeros'`` | ``'border'`` | ``'reflection'``. + normalized_coordinates: whether to use a grid with normalized coordinates. + align_corners: interpolation flag. + + """ + + _warped_grid: Optional[torch.Tensor] + + def __init__( + self, + height: int, + width: int, + mode: str = "bilinear", + padding_mode: str = "zeros", + normalized_coordinates: bool = True, + align_corners: bool = False, + ) -> None: + super().__init__(height, width) + self.mode = mode + self.padding_mode = padding_mode + self.normalized_coordinates = normalized_coordinates + self.align_corners = align_corners + # create base grid to compute the flow + self.grid = create_meshgrid(height, width, normalized_coordinates=normalized_coordinates) + + # initialice the warped destination grid + self._warped_grid = None + + def precompute_warp_grid(self, src_homo_dst: torch.Tensor) -> None: + r"""Compute and store internally the transformations of the points. + + Useful when the same homography/homographies are reused. + + Args: + src_homo_dst: Homography or homographies (stacked) to + transform all points in the grid. Shape of the homography + has to be :math:`(1, 3, 3)` or :math:`(N, 1, 3, 3)`. + The homography assumes normalized coordinates [-1, 1] if + normalized_coordinates is True. + + """ + self._warped_grid = warp_grid(self.grid, src_homo_dst) + + def forward(self, patch_src: torch.Tensor, src_homo_dst: Optional[torch.Tensor] = None) -> torch.Tensor: + r"""Warp a torch.Tensor from source into reference frame. + + Args: + patch_src: The torch.Tensor to warp. + src_homo_dst: The homography or torch.stack of + homographies from destination to source. The homography assumes + normalized coordinates [-1, 1] if normalized_coordinates is True. + + Return: + Patch sampled at locations from source to destination. + + Shape: + - Input: :math:`(N, C, H, W)` and :math:`(N, 3, 3)` + - Output: :math:`(N, C, H, W)` + + Example: + >>> input = torch.rand(1, 3, 32, 32) + >>> homography = torch.eye(3).view(1, 3, 3) + >>> warper = HomographyWarper(32, 32) + >>> # without precomputing the warp + >>> output = warper(input, homography) # NxCxHxW + >>> # precomputing the warp + >>> warper.precompute_warp_grid(homography) + >>> output = warper(input) # NxCxHxW + + """ + _warped_grid = self._warped_grid + if src_homo_dst is not None: + warped_patch = homography_warp( + patch_src, + src_homo_dst, + (self.height, self.width), + mode=self.mode, + padding_mode=self.padding_mode, + align_corners=self.align_corners, + normalized_coordinates=self.normalized_coordinates, + ) + elif _warped_grid is not None: + if not _warped_grid.device == patch_src.device: + raise TypeError( + "Patch and warped grid must be on the same device. Got" + f" patch.device: {patch_src.device} warped_grid.device: {_warped_grid.device}. Whether recall" + " precompute_warp_grid() with the correct device for the homograhy" + " or change the patch device." + ) + warped_patch = F.grid_sample( + patch_src, + _warped_grid, + mode=self.mode, + padding_mode=self.padding_mode, + align_corners=self.align_corners, + ) + else: + raise RuntimeError( + "Unknown warping. If homographies are not provided they must be preset" + " using the method: precompute_warp_grid()." + ) + + return warped_patch diff --git a/kornia/geometry/transform/image_registrator.py b/kornia/geometry/transform/image_registrator.py new file mode 100644 index 0000000..63c96ce --- /dev/null +++ b/kornia/geometry/transform/image_registrator.py @@ -0,0 +1,326 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from abc import abstractmethod +from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union + +import torch +import torch.nn.functional as F +from torch import nn, optim + +from kornia.geometry.conversions import angle_to_rotation_matrix, convert_affinematrix_to_homography + +from .homography_warper import BaseWarper, HomographyWarper +from .pyramid import build_pyramid + +__all__ = ["BaseModel", "Homography", "ImageRegistrator", "Similarity"] + + +class BaseModel(nn.Module): + """Provide an abstract base class for image registration models.""" + + @abstractmethod + def reset_model(self) -> None: + """Reset learnable registration parameters to the identity transform.""" + ... + + @abstractmethod + def forward(self) -> torch.Tensor: + """Return the transform that maps source coordinates toward target coordinates. + + Returns: + Transform matrix tensor for the current model state. Concrete + models return the matrix shape required by their warp function. + """ + ... + + @abstractmethod + def forward_inverse(self) -> torch.Tensor: + """Return the inverse mapping for the current registration transform. + + Returns: + Transform matrix tensor that maps target coordinates back toward + source coordinates. + """ + ... + + +class Homography(BaseModel): + r"""Homography geometric model to be used with ImageRegistrator for the optimization-based image registration.""" + + def __init__(self) -> None: + super().__init__() + self.model = nn.Parameter(torch.eye(3)) + self.reset_model() + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.model})" + + def reset_model(self) -> None: + """Initialize the model with identity transform.""" + torch.nn.init.eye_(self.model) + + def forward(self) -> torch.Tensor: + r"""Single-batch homography". + + Returns: + Homography matrix with shape :math:`(1, 3, 3)`. + + """ + return torch.unsqueeze(self.model / self.model[2, 2], dim=0) # 1x3x3 + + def forward_inverse(self) -> torch.Tensor: + r"""Interted Single-batch homography". + + Returns: + Homography martix with shape :math:`(1, 3, 3)`. + + """ + return torch.unsqueeze(torch.inverse(self.model), dim=0) + + +class Similarity(BaseModel): + """Similarity geometric model to be used with ImageRegistrator module for the optimization-based image registration. + + Args: + rotation: if True, the rotation is optimizable, else constant zero. + scale: if True, the scale is optimizable, else constant zero. + shift: if True, the shift is optimizable, else constant one. + + """ + + def __init__(self, rotation: bool = True, scale: bool = True, shift: bool = True) -> None: + super().__init__() + if rotation: + self.rot = nn.Parameter(torch.zeros(1)) + else: + self.register_buffer("rot", torch.zeros(1)) + if shift: + self.shift = nn.Parameter(torch.zeros(1, 2, 1)) + else: + self.register_buffer("shift", torch.zeros(1, 2, 1)) + if scale: + self.scale = nn.Parameter(torch.ones(1)) + else: + self.register_buffer("scale", torch.ones(1)) + self.reset_model() + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}(angle = {self.rot}, \n shift={self.shift}, \n scale={self.scale})" + ) + + def reset_model(self) -> None: + """Initialize the model with identity transform.""" + torch.nn.init.zeros_(self.rot) + torch.nn.init.zeros_(self.shift) + torch.nn.init.ones_(self.scale) + + def forward(self) -> torch.Tensor: + r"""Single-batch similarity transform". + + Returns: + Similarity with shape :math:`(1, 3, 3)` + + """ + rot = self.scale * angle_to_rotation_matrix(self.rot) + out = convert_affinematrix_to_homography(torch.cat([rot, self.shift], dim=2)) + return out + + def forward_inverse(self) -> torch.Tensor: + r"""Single-batch inverse similarity transform". + + Returns: + Similarity with shape :math:`(1, 3, 3)` + + """ + return torch.inverse(self.forward()) + + +class ImageRegistrator(nn.Module): + r"""nn.Module, which performs optimization-based image registration. + + Args: + model_type: Geometrical model for registration. Can be string or nn.Module. + optimizer: optimizer class used for the optimization. + loss_fn: torch loss function. + pyramid_levels: number of scale pyramid levels. + lr: learning rate for optimization. + num_iterations: maximum number of iterations. + tolerance: stop optimizing if loss difference is less. default 1e-4. + warper: if model_type is not string, one needs to provide warper object. + + Example: + >>> from kornia.geometry import ImageRegistrator + >>> img_src = torch.rand(1, 1, 32, 32) + >>> img_dst = torch.rand(1, 1, 32, 32) + >>> registrator = ImageRegistrator('similarity') + >>> homo = registrator.register(img_src, img_dst) + + """ + + # TODO: resolve better type, potentially using factory. + def __init__( + self, + model_type: Union[str, BaseModel] = "homography", + optimizer: Type[optim.Optimizer] = optim.Adam, + loss_fn: Callable[..., torch.Tensor] = F.l1_loss, + pyramid_levels: int = 5, + lr: float = 1e-3, + num_iterations: int = 100, + tolerance: float = 1e-4, + warper: Optional[Type[BaseWarper]] = None, + allow_shape_mismatch: bool = False, + ) -> None: + super().__init__() + self.known_models = ["homography", "similarity", "translation", "scale", "rotation"] + # We provide pre-defined combinations or allow user to supply model + # together with warper + if not isinstance(model_type, str): + if warper is None: + raise ValueError("You must supply warper together with custom model") + self.warper = warper + self.model = model_type + elif model_type.lower() == "homography": + self.warper = HomographyWarper + self.model = Homography() + elif model_type.lower() == "similarity": + self.warper = HomographyWarper + self.model = Similarity(True, True, True) + elif model_type.lower() == "translation": + self.warper = HomographyWarper + self.model = Similarity(False, False, True) + elif model_type.lower() == "rotation": + self.warper = HomographyWarper + self.model = Similarity(True, False, False) + elif model_type.lower() == "scale": + self.warper = HomographyWarper + self.model = Similarity(False, True, False) + else: + raise ValueError(f"{model_type} is not supported. Try {self.known_models}") + self.pyramid_levels = pyramid_levels + self.optimizer = optimizer + self.lr = lr + self.loss_fn = loss_fn + self.num_iterations = num_iterations + self.tolerance = tolerance + self.allow_shape_mismatch = allow_shape_mismatch + + def get_single_level_loss( + self, img_src: torch.Tensor, img_dst: torch.Tensor, transform_model: torch.Tensor + ) -> torch.Tensor: + """Warp img_src into img_dst with transform_model and returns loss.""" + # ToDo: Make possible registration of images of different shape + if img_src.shape != img_dst.shape: + raise ValueError( + "Cannot register images of different shapes " + f" {img_src.shape} {img_dst.shape:} " + ) + _height, _width = img_dst.shape[-2:] + warper = self.warper(_height, _width) + img_src_to_dst = warper(img_src, transform_model) + # compute and mask loss + loss = self.loss_fn(img_src_to_dst, img_dst, reduction="none") # 1xCxHxW + ones_tensor = warper(torch.ones_like(img_src), transform_model) + loss = loss.masked_select(ones_tensor > 0.9).mean() + return loss + + def reset_model(self) -> None: + """Call model reset function.""" + self.model.reset_model() + + def register( + self, + src_img: torch.Tensor, + dst_img: torch.Tensor, + verbose: bool = False, + output_intermediate_models: bool = False, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, List[torch.Tensor]]]: + r"""Estimate the transformation which warps src_img into dst_img by gradient descent. + + The shape of the tensors is not checked, because it may depend on the model, e.g. volume registration. + + Args: + src_img: Input image torch.Tensor. + dst_img: Input image torch.Tensor. + verbose: if True, outputs loss every 10 iterations. + output_intermediate_models: if True with intermediate models + + Returns: + the transformation between two images, shape depends on the model, + typically [1x3x3] torch.Tensor for string model_types. + + """ + self.reset_model() + if src_img.shape != dst_img.shape: + if not self.allow_shape_mismatch: + raise ValueError( + f"Cannot register images of different shapes {src_img.shape} {dst_img.shape}. " + "Consider setting `allow_shape_mismatch = True`" + ) + src_img = F.interpolate( + src_img, + size=dst_img.shape[-2:], + mode="bilinear", + align_corners=False, + ) + # ToDo: better parameter passing to optimizer + _opt_args: Dict[str, Any] = {} + _opt_args["lr"] = self.lr + opt = self.optimizer(self.model.parameters(), **_opt_args) + + # compute the gaussian pyramids + # [::-1] because we have to register from coarse to fine + img_src_pyr = build_pyramid(src_img, self.pyramid_levels)[::-1] + img_dst_pyr = build_pyramid(dst_img, self.pyramid_levels)[::-1] + prev_loss = 1e10 + aux_models = [] + if len(img_dst_pyr) != len(img_src_pyr): + raise ValueError("Cannot register images of different sizes") + for img_src_level, img_dst_level in zip(img_src_pyr, img_dst_pyr): + for i in range(self.num_iterations): + # compute gradient and update optimizer parameters + opt.zero_grad() + loss = self.get_single_level_loss(img_src_level, img_dst_level, self.model()) + loss += self.get_single_level_loss(img_dst_level, img_src_level, self.model.forward_inverse()) + current_loss = loss.item() + if abs(current_loss - prev_loss) < self.tolerance: + break + prev_loss = current_loss + loss.backward() + if verbose and (i % 10 == 0): + print(f"Loss = {current_loss:.4f}, iter={i}") + opt.step() + if output_intermediate_models: + aux_models.append(self.model().clone().detach()) + if output_intermediate_models: + return self.model(), aux_models + return self.model() + + def warp_src_into_dst(self, src_img: torch.Tensor) -> torch.Tensor: + r"""Warp src_img with estimated model.""" + _height, _width = src_img.shape[-2:] + warper = self.warper(_height, _width) + img_src_to_dst = warper(src_img, self.model()) + return img_src_to_dst + + def warp_dst_inro_src(self, dst_img: torch.Tensor) -> torch.Tensor: + r"""Warp src_img with inverted estimated model.""" + _height, _width = dst_img.shape[-2:] + warper = self.warper(_height, _width) + img_dst_to_src = warper(dst_img, self.model.forward_inverse()) + return img_dst_to_src diff --git a/kornia/geometry/transform/imgwarp.py b/kornia/geometry/transform/imgwarp.py new file mode 100644 index 0000000..5faacd6 --- /dev/null +++ b/kornia/geometry/transform/imgwarp.py @@ -0,0 +1,1468 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional + +import torch +import torch.nn.functional as F + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SHAPE +from kornia.core.ops import eye_like +from kornia.core.utils import _inverse_3x3_closed_form, _torch_inverse_cast, _torch_solve_cast +from kornia.geometry.conversions import ( + angle_to_rotation_matrix, + axis_angle_to_rotation_matrix, + convert_affinematrix_to_homography, + convert_affinematrix_to_homography3d, + normalize_homography, + normalize_homography3d, + normalize_pixel_coordinates, +) +from kornia.geometry.grid import create_meshgrid, create_meshgrid3d +from kornia.geometry.linalg import transform_points + +__all__ = [ + "get_affine_matrix2d", + "get_affine_matrix3d", + "get_perspective_transform", + "get_perspective_transform3d", + "get_projective_transform", + "get_rotation_matrix2d", + "get_shear_matrix2d", + "get_shear_matrix3d", + "get_translation_matrix2d", + "homography_warp", + "homography_warp3d", + "invert_affine_transform", + "projection_from_Rt", + "remap", + "warp_affine", + "warp_affine3d", + "warp_grid", + "warp_grid3d", + "warp_perspective", + "warp_perspective3d", +] + + +def warp_perspective( + src: torch.Tensor, + M: torch.Tensor, + dsize: tuple[int, int], + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = True, + fill_value: Optional[torch.Tensor] = None, # needed for jit +) -> torch.Tensor: + r"""Apply a perspective transformation to an image. + + The function warp_perspective transforms the source image using + the specified matrix: + + .. math:: + \text{dst} (x, y) = \text{src} \left( + \frac{M^{-1}_{11} x + M^{-1}_{12} y + M^{-1}_{13}}{M^{-1}_{31} x + M^{-1}_{32} y + M^{-1}_{33}} , + \frac{M^{-1}_{21} x + M^{-1}_{22} y + M^{-1}_{23}}{M^{-1}_{31} x + M^{-1}_{32} y + M^{-1}_{33}} + \right ) + + Args: + src: input image with shape :math:`(B, C, H, W)`. + M: transformation matrix with shape :math:`(B, 3, 3)`. + dsize: size of the output image (height, width). + mode: interpolation mode to calculate output values ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values ``'torch.zeros'`` | ``'border'`` | ``'reflection'`` + | ``'fill'``. + align_corners: interpolation flag. + fill_value: torch.Tensor of shape :math:`(3)` that fills the padding area. Only supported for RGB. + + Returns: + the warped input image :math:`(B, C, H, W)`. + + Example: + >>> img = torch.rand(1, 4, 5, 6) + >>> H = torch.eye(3)[None] + >>> out = warp_perspective(img, H, (4, 2), align_corners=True) + >>> print(out.shape) + torch.Size([1, 4, 4, 2]) + + .. note:: + This function is often used in conjunction with :func:`get_perspective_transform`. + + .. note:: + See a working example `here `_. + + """ + if not isinstance(src, torch.Tensor): + raise TypeError(f"Input src type is not a torch.Tensor. Got {type(src)}") + + if not isinstance(M, torch.Tensor): + raise TypeError(f"Input M type is not a torch.Tensor. Got {type(M)}") + + if not len(src.shape) == 4: + raise ValueError(f"Input src must be a BxCxHxW torch.Tensor. Got {src.shape}") + + if not (len(M.shape) == 3 and M.shape[-2:] == (3, 3)): + raise ValueError(f"Input M must be a Bx3x3 torch.Tensor. Got {M.shape}") + + # fill padding is only supported for 3 channels because we can't set fill_value default + # to None as this gives jit issues. + if fill_value is None: + fill_value = torch.zeros(3) + if padding_mode == "fill" and fill_value.shape != torch.Size([3]): + raise ValueError(f"Padding_tensor only supported for 3 channels. Got {fill_value.shape}") + + B, _, H, W = src.size() + h_out, w_out = dsize + + # we F.normalize the 3x3 transformation matrix and convert to 3x4 + dst_norm_trans_src_norm: torch.Tensor = normalize_homography(M, (H, W), (h_out, w_out)) # Bx3x3 + + src_norm_trans_dst_norm = _torch_inverse_cast(dst_norm_trans_src_norm) # Bx3x3 + + # this piece of code substitutes F.affine_grid since it does not support 3x3 + grid = ( + create_meshgrid(h_out, w_out, normalized_coordinates=True, device=src.device) + .to(src.dtype) + .expand(B, h_out, w_out, 2) + ) + grid = transform_points(src_norm_trans_dst_norm[:, None, None], grid) + + if padding_mode == "fill": + return _fill_and_warp(src, grid, align_corners=align_corners, mode=mode, fill_value=fill_value) + return F.grid_sample(src, grid, align_corners=align_corners, mode=mode, padding_mode=padding_mode) + + +def warp_affine( + src: torch.Tensor, + M: torch.Tensor, + dsize: tuple[int, int], + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = True, + fill_value: Optional[torch.Tensor] = None, +) -> torch.Tensor: + r"""Apply an affine transformation to a torch.Tensor. + + .. image:: _static/img/warp_affine.png + + The function warp_affine transforms the source torch.Tensor using + the specified matrix: + + .. math:: + \text{dst}(x, y) = \text{src} \left( M_{11} x + M_{12} y + M_{13} , + M_{21} x + M_{22} y + M_{23} \right ) + + Args: + src: input torch.Tensor of shape :math:`(B, C, H, W)`. + M: affine transformation of shape :math:`(B, 2, 3)`. + dsize: size of the output image (height, width). + mode: interpolation mode to calculate output values ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values ``'torch.zeros'`` | ``'border'`` | ``'reflection'`` + | ``'fill'``. + align_corners : mode for grid_generation. + fill_value: torch.Tensor of shape :math:`(C)` or :math:`(1)` that fills the padding area. + + Returns: + the warped torch.Tensor with shape :math:`(B, C, H, W)`. + + .. note:: + This function is often used in conjunction with :func:`get_rotation_matrix2d`, + :func:`get_shear_matrix2d`, :func:`get_affine_matrix2d`, :func:`invert_affine_transform`. + + .. note:: + See a working example `here `__. + + Example: + >>> img = torch.rand(1, 4, 5, 6) + >>> A = torch.eye(2, 3)[None] + >>> out = warp_affine(img, A, (4, 2), align_corners=True) + >>> print(out.shape) + torch.Size([1, 4, 4, 2]) + + """ + if not isinstance(src, torch.Tensor): + raise TypeError(f"Input src type is not a torch.Tensor. Got {type(src)}") + + if not isinstance(M, torch.Tensor): + raise TypeError(f"Input M type is not a torch.Tensor. Got {type(M)}") + + if not len(src.shape) == 4: + raise ValueError(f"Input src must be a BxCxHxW torch.Tensor. Got {src.shape}") + + if not (len(M.shape) == 3 or M.shape[-2:] == (2, 3)): + raise ValueError(f"Input M must be a Bx2x3 torch.Tensor. Got {M.shape}") + + B, C, H, W = src.size() + + M_3x3: torch.Tensor = convert_affinematrix_to_homography(M) + dst_norm_trans_src_norm: torch.Tensor = normalize_homography(M_3x3, (H, W), dsize) + + src_norm_trans_dst_norm = _torch_inverse_cast(dst_norm_trans_src_norm) + + grid = F.affine_grid(src_norm_trans_dst_norm[:, :2, :], [B, C, dsize[0], dsize[1]], align_corners=align_corners) + + if padding_mode == "fill": + if fill_value is None: + fill_value = torch.zeros(src.shape[1], device=src.device, dtype=src.dtype) + return _fill_and_warp(src, grid, align_corners=align_corners, mode=mode, fill_value=fill_value) + + return F.grid_sample(src, grid, align_corners=align_corners, mode=mode, padding_mode=padding_mode) + + +def _fill_and_warp( + src: torch.Tensor, grid: torch.Tensor, mode: str, align_corners: bool, fill_value: torch.Tensor +) -> torch.Tensor: + r"""Warp a mask of torch.ones, then multiply with fill_value and add to default warp. + + Args: + src: input torch.Tensor of shape :math:`(B, C, H, W)`. + grid: grid torch.Tensor from `transform_points`. + mode: interpolation mode to calculate output values ``'bilinear'`` | ``'nearest'``. + align_corners: interpolation flag. + fill_value: torch.Tensor of shape :math:`(C)` or :math:`(1)` that fills the padding area. + + Returns: + the warped and filled torch.Tensor with shape :math:`(B, C, H, W)`. + """ + ones_mask = torch.ones_like(src) + fill_value = fill_value.to(ones_mask) + + if fill_value.ndim == 0: + fill_value = fill_value.view(1, 1, 1, 1) + elif fill_value.ndim == 1: + fill_value = fill_value.view(1, -1, 1, 1) + + inv_ones_mask = 1 - F.grid_sample(ones_mask, grid, align_corners=align_corners, mode=mode, padding_mode="zeros") + + inv_color_mask = inv_ones_mask * fill_value + + return F.grid_sample(src, grid, align_corners=align_corners, mode=mode, padding_mode="zeros") + inv_color_mask + + +def warp_grid(grid: torch.Tensor, src_homo_dst: torch.Tensor) -> torch.Tensor: + r"""Compute the grid to warp the coordinates grid by the homography/ies. + + Args: + grid: Unwrapped grid of the shape :math:`(1, H, W, 2)`. + src_homo_dst: Homography or homographies (stacked) to + transform all points in the grid. Shape of the homography + has to be :math:`(1, 3, 3)` or :math:`(N, 1, 3, 3)`. + + Returns: + the transformed grid of shape :math:`(N, H, W, 2)`. + + """ + batch_size: int = src_homo_dst.size(0) + _, height, width, _ = grid.size() + # expand grid to match the input batch size + grid = grid.expand(batch_size, -1, -1, -1) # NxHxWx2 + if len(src_homo_dst.shape) == 3: # local homography case + src_homo_dst = src_homo_dst.view(batch_size, 1, 3, 3) # Nx1x3x3 + # perform the actual grid transformation, + # the grid is copied to input device and casted to the same type + flow: torch.Tensor = transform_points(src_homo_dst, grid.to(src_homo_dst)) # NxHxWx2 + return flow.view(batch_size, height, width, 2) # NxHxWx2 + + +def warp_grid3d(grid: torch.Tensor, src_homo_dst: torch.Tensor) -> torch.Tensor: + r"""Compute the grid to warp the coordinates grid by the homography/ies. + + Args: + grid: Unwrapped grid of the shape :math:`(1, D, H, W, 3)`. + src_homo_dst: Homography or homographies (stacked) to + transform all points in the grid. Shape of the homography + has to be :math:`(1, 4, 4)` or :math:`(N, 1, 4, 4)`. + + Returns: + the transformed grid of shape :math:`(N, H, W, 3)`. + + """ + batch_size: int = src_homo_dst.size(0) + _, depth, height, width, _ = grid.size() + # expand grid to match the input batch size + grid = grid.expand(batch_size, -1, -1, -1, -1) # NxDxHxWx3 + if len(src_homo_dst.shape) == 3: # local homography case + src_homo_dst = src_homo_dst.view(batch_size, 1, 4, 4) # Nx1x3x3 + # perform the actual grid transformation, + # the grid is copied to input device and casted to the same type + flow: torch.Tensor = transform_points(src_homo_dst, grid.to(src_homo_dst)) # NxDxHxWx3 + return flow.view(batch_size, depth, height, width, 3) # NxDxHxWx3 + + +# TODO: move to kornia.geometry.projective +# TODO: create the nn.Module -- TBD what inputs/outputs etc +# class PerspectiveTransform(nn.Module): +# def __init__(self) -> None: +# super().__init__() + + +def _unit_square_to_quad(points: torch.Tensor) -> torch.Tensor: + """Closed-form perspective matrix mapping the unit square to a quadrilateral. + + Maps the canonical unit-square corners ``[(0,0), (1,0), (1,1), (0,1)]`` to the + four points in ``points`` (in the same corner order). Implemented via Heckbert's + direct formulation (no linear solve) so it lowers cleanly to ONNX. + + Args: + points: ``(B, 4, 2)`` tensor of quadrilateral vertices. + + Returns: + ``(B, 3, 3)`` perspective transformation matrix. + """ + x0 = points[..., 0, 0] + y0 = points[..., 0, 1] + x1 = points[..., 1, 0] + y1 = points[..., 1, 1] + x2 = points[..., 2, 0] + y2 = points[..., 2, 1] + x3 = points[..., 3, 0] + y3 = points[..., 3, 1] + + dx1 = x1 - x2 + dx2 = x3 - x2 + sx = x0 - x1 + x2 - x3 + dy1 = y1 - y2 + dy2 = y3 - y2 + sy = y0 - y1 + y2 - y3 + + denom = dx1 * dy2 - dy1 * dx2 + a31 = (sx * dy2 - sy * dx2) / denom + a32 = (dx1 * sy - dy1 * sx) / denom + a11 = x1 - x0 + a31 * x1 + a12 = x3 - x0 + a32 * x3 + a13 = x0 + a21 = y1 - y0 + a31 * y1 + a22 = y3 - y0 + a32 * y3 + a23 = y0 + + one = torch.ones_like(x0) + + row0 = torch.stack([a11, a12, a13], dim=-1) + row1 = torch.stack([a21, a22, a23], dim=-1) + row2 = torch.stack([a31, a32, one], dim=-1) + return torch.stack([row0, row1, row2], dim=-2) + + +def _get_perspective_transform_closed_form(points_src: torch.Tensor, points_dst: torch.Tensor) -> torch.Tensor: + """ONNX-traceable perspective transform via two unit-square decompositions. + + Computes ``H = H_d @ inv(H_s)`` where ``H_s`` maps the unit square to + ``points_src`` and ``H_d`` maps it to ``points_dst``. Replaces the 8x8 + ``torch.linalg.solve`` in :func:`get_perspective_transform` with two 3x3 + Heckbert constructions and one 3x3 inverse — all closed-form ops that the + legacy ONNX exporter supports. + """ + h_s = _unit_square_to_quad(points_src) + h_d = _unit_square_to_quad(points_dst) + return h_d @ _inverse_3x3_closed_form(h_s) + + +def get_perspective_transform(points_src: torch.Tensor, points_dst: torch.Tensor) -> torch.Tensor: + r"""Calculate a perspective transform from four pairs of the corresponding points. + + The algorithm is a vanilla implementation of the Direct Linear transform (DLT). + See more: https://www.cs.cmu.edu/~16385/s17/Slides/10.2_2D_Alignment__DLT.pdf + + The function calculates the matrix of a perspective transform that maps from + the source to destination points: + + .. math:: + + \begin{bmatrix} + x^{'} \\ + y^{'} \\ + 1 \\ + \end{bmatrix} + = + \begin{bmatrix} + h_1 & h_2 & h_3 \\ + h_4 & h_5 & h_6 \\ + h_7 & h_8 & h_9 \\ + \end{bmatrix} + \cdot + \begin{bmatrix} + x \\ + y \\ + 1 \\ + \end{bmatrix} + + Args: + points_src: coordinates of quadrangle vertices in the source image with shape :math:`(B, 4, 2)`. + points_dst: coordinates of the corresponding quadrangle vertices in + the destination image with shape :math:`(B, 4, 2)`. + + Returns: + the perspective transformation with shape :math:`(B, 3, 3)`. + + .. note:: + This function is often used in conjunction with :func:`warp_perspective`. + + Example: + >>> x1 = torch.tensor([[[0., 0.], [1., 0.], [1., 1.], [0., 1.]]]) + >>> x2 = torch.tensor([[[1., 0.], [0., 0.], [0., 1.], [1., 1.]]]) + >>> x2_trans_x1 = get_perspective_transform(x1, x2) + + """ + KORNIA_CHECK_SHAPE(points_src, ["B", "4", "2"]) + KORNIA_CHECK_SHAPE(points_dst, ["B", "4", "2"]) + KORNIA_CHECK(points_src.shape == points_dst.shape, "Source data shape must match Destination data shape.") + KORNIA_CHECK(points_src.dtype == points_dst.dtype, "Source data type must match Destination data type.") + + # Under tracing (legacy torch.onnx.export, torch.jit.trace), use the closed-form + # Heckbert decomposition: the legacy ONNX tracer does not lower + # ``aten::linalg_solve``, but the closed-form path only uses arithmetic ops + # + a closed-form 3x3 inverse, both fully supported. ``torch.jit.is_tracing()`` + # is JIT-script-safe (unlike ``torch.onnx.is_in_onnx_export``, which contains + # an ``import`` statement that JIT script cannot compile). + if torch.jit.is_tracing(): + return _get_perspective_transform_closed_form(points_src, points_dst) + + # we build matrix A by using only 4 point correspondence. The linear + # system is solved with the least square method, so here + # we could even pass more correspondence + + # create the lhs torch.Tensor with shape # Bx8x8 + B: int = points_src.shape[0] # batch_size + + A = torch.empty(B, 8, 8, device=points_src.device, dtype=points_src.dtype) + + # we need to perform in batch + _zeros = torch.zeros(B, device=points_src.device, dtype=points_src.dtype) + _ones = torch.ones(B, device=points_src.device, dtype=points_src.dtype) + + for i in range(4): + x1, y1 = points_src[..., i, 0], points_src[..., i, 1] # Bx4 + x2, y2 = points_dst[..., i, 0], points_dst[..., i, 1] # Bx4 + + A[:, 2 * i] = torch.stack([x1, y1, _ones, _zeros, _zeros, _zeros, -x1 * x2, -y1 * x2], -1) + A[:, 2 * i + 1] = torch.stack([_zeros, _zeros, _zeros, x1, y1, _ones, -x1 * y2, -y1 * y2], -1) + + # the rhs torch.Tensor + b = points_dst.view(-1, 8, 1) + + # solve the system Ax = b + X: torch.Tensor = _torch_solve_cast(A, b) + + # create variable to return the Bx3x3 transform + M = torch.empty(B, 9, device=points_src.device, dtype=points_src.dtype) + M[..., :8] = X[..., 0] # Bx8 + M[..., -1].fill_(1) + + return M.view(-1, 3, 3) # Bx3x3 + + +# TODO: move to kornia.geometry.affine +def get_rotation_matrix2d(center: torch.Tensor, angle: torch.Tensor, scale: torch.Tensor) -> torch.Tensor: + r"""Calculate an affine matrix of 2D rotation. + + The function calculates the following matrix: + + .. math:: + \begin{bmatrix} + \alpha & \beta & (1 - \alpha) \cdot \text{x} + - \beta \cdot \text{y} \\ + -\beta & \alpha & \beta \cdot \text{x} + + (1 - \alpha) \cdot \text{y} + \end{bmatrix} + + where + + .. math:: + \alpha = \text{scale} \cdot torch.cos(\text{angle}) \\ + \beta = \text{scale} \cdot torch.sin(\text{angle}) + + The transformation maps the rotation center to itself + If this is not the target, adjust the shift. + + Args: + center: center of the rotation in the source image with shape :math:`(B, 2)`. + angle: rotation angle in degrees. Positive values mean + counter-clockwise rotation (the coordinate origin is assumed to + be the top-left corner) with shape :math:`(B)`. + scale: scale factor for x, y scaling with shape :math:`(B, 2)`. + + Returns: + the affine matrix of 2D rotation with shape :math:`(B, 2, 3)`. + + Example: + >>> center = torch.zeros(1, 2) + >>> scale = torch.ones((1, 2)) + >>> angle = 45. * torch.ones(1) + >>> get_rotation_matrix2d(center, angle, scale) + tensor([[[ 0.7071, 0.7071, 0.0000], + [-0.7071, 0.7071, 0.0000]]]) + + .. note:: + This function is often used in conjunction with :func:`warp_affine`. + + """ + if not isinstance(center, torch.Tensor): + raise TypeError(f"Input center type is not a torch.Tensor. Got {type(center)}") + + if not isinstance(angle, torch.Tensor): + raise TypeError(f"Input angle type is not a torch.Tensor. Got {type(angle)}") + + if not isinstance(scale, torch.Tensor): + raise TypeError(f"Input scale type is not a torch.Tensor. Got {type(scale)}") + + if not (len(center.shape) == 2 and center.shape[1] == 2): + raise ValueError(f"Input center must be a Bx2 torch.Tensor. Got {center.shape}") + + if not len(angle.shape) == 1: + raise ValueError(f"Input angle must be a B torch.Tensor. Got {angle.shape}") + + if not (len(scale.shape) == 2 and scale.shape[1] == 2): + raise ValueError(f"Input scale must be a Bx2 torch.Tensor. Got {scale.shape}") + + if not (center.shape[0] == angle.shape[0] == scale.shape[0]): + raise ValueError( + f"Inputs must have same batch size dimension. Got center {center.shape}, angle {angle.shape} and scale " + f"{scale.shape}" + ) + + if not (center.device == angle.device == scale.device) or not (center.dtype == angle.dtype == scale.dtype): + raise ValueError( + f"Inputs must have same device Got center ({center.device}, {center.dtype}), angle ({angle.device}, " + f"{angle.dtype}) and scale ({scale.device}, {scale.dtype})" + ) + + shift_m = eye_like(3, center) + shift_m[:, :2, 2] = center + + shift_m_inv = eye_like(3, center) + shift_m_inv[:, :2, 2] = -center + + scale_m = eye_like(3, center) + scale_m[:, 0, 0] *= scale[:, 0] + scale_m[:, 1, 1] *= scale[:, 1] + + rotat_m = eye_like(3, center) + rotat_m[:, :2, :2] = angle_to_rotation_matrix(angle) + + affine_m = shift_m @ rotat_m @ scale_m @ shift_m_inv + return affine_m[:, :2, :] # Bx2x3 + + +def remap( + image: torch.Tensor, + map_x: torch.Tensor, + map_y: torch.Tensor, + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: Optional[bool] = None, + normalized_coordinates: bool = False, +) -> torch.Tensor: + r"""Apply a generic geometrical transformation to an image torch.Tensor. + + .. image:: _static/img/remap.png + + The function remap transforms the source torch.Tensor using the specified map: + + .. math:: + \text{dst}(x, y) = \text{src}(map_x(x, y), map_y(x, y)) + + Args: + image: the torch.Tensor to remap with shape (B, C, H, W). + Where C is the number of channels. + map_x: the flow in the x-direction in pixel coordinates. + The torch.Tensor must be in the shape of (B, H, W). + map_y: the flow in the y-direction in pixel coordinates. + The torch.Tensor must be in the shape of (B, H, W). + mode: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + ``'torch.zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: mode for grid_generation. + normalized_coordinates: whether the input coordinates are + normalized in the range of [-1, 1]. + + Returns: + the warped torch.Tensor with same shape as the input grid maps. + + Example: + >>> import torch + >>> from kornia.geometry import create_meshgrid + >>> grid = create_meshgrid(2, 2, False) # 1x2x2x2 + >>> grid += 1 # apply offset in both directions + >>> input = torch.ones(1, 1, 2, 2) + >>> remap(input, grid[..., 0], grid[..., 1], align_corners=True) # 1x1x2x2 + tensor([[[[1., 0.], + [0., 0.]]]]) + + .. note:: + This function is often used in conjunction with :func:`kornia.geometry.create_meshgrid`. + + """ + KORNIA_CHECK_SHAPE(image, ["B", "C", "H", "W"]) + KORNIA_CHECK_SHAPE(map_x, ["B", "H", "W"]) + KORNIA_CHECK_SHAPE(map_y, ["B", "H", "W"]) + + batch_size, _, height, width = image.shape + + # grid_sample need the grid between -1/1 + map_xy: torch.Tensor = torch.stack([map_x, map_y], -1) + + # F.normalize coordinates if not already normalized + if not normalized_coordinates: + map_xy = normalize_pixel_coordinates(map_xy, height, width) + + # simulate broadcasting since grid_sample does not support it + map_xy = map_xy.expand(batch_size, -1, -1, -1) + + # Default to False if align_corners is None to avoid PyTorch warning + if align_corners is None: + align_corners = False + + # warp the image tensor and return + return F.grid_sample(image, map_xy, mode=mode, padding_mode=padding_mode, align_corners=align_corners) + + +def invert_affine_transform(matrix: torch.Tensor) -> torch.Tensor: + r"""Invert an affine transformation. + + The function computes an inverse affine transformation represented by + 2x3 matrix: + + .. math:: + \begin{bmatrix} + a_{11} & a_{12} & b_{1} \\ + a_{21} & a_{22} & b_{2} \\ + \end{bmatrix} + + The result is also a 2x3 matrix of the same type as M. + + Args: + matrix: original affine transform. The torch.Tensor must be + in the shape of :math:`(B, 2, 3)`. + + Return: + the reverse affine transform with shape :math:`(B, 2, 3)`. + + .. note:: + This function is often used in conjunction with :func:`warp_affine`. + + """ + if not isinstance(matrix, torch.Tensor): + raise TypeError(f"Input matrix type is not a torch.Tensor. Got {type(matrix)}") + + if not (len(matrix.shape) == 3 and matrix.shape[-2:] == (2, 3)): + raise ValueError(f"Input matrix must be a Bx2x3 torch.Tensor. Got {matrix.shape}") + + matrix_tmp: torch.Tensor = convert_affinematrix_to_homography(matrix) + matrix_inv: torch.Tensor = _torch_inverse_cast(matrix_tmp) + + return matrix_inv[..., :2, :3] + + +def get_affine_matrix2d( + translations: torch.Tensor, + center: torch.Tensor, + scale: torch.Tensor, + angle: torch.Tensor, + sx: Optional[torch.Tensor] = None, + sy: Optional[torch.Tensor] = None, +) -> torch.Tensor: + r"""Compose affine matrix from the components. + + Args: + translations: torch.Tensor containing the translation vector with shape :math:`(B, 2)`. + center: torch.Tensor containing the center vector with shape :math:`(B, 2)`. + scale: torch.Tensor containing the scale factor with shape :math:`(B, 2)`. + angle: torch.Tensor of angles in degrees :math:`(B)`. + sx: torch.Tensor containing the shear factor in the x-direction with shape :math:`(B)`. + sy: torch.Tensor containing the shear factor in the y-direction with shape :math:`(B)`. + + Returns: + the affine transformation matrix :math:`(B, 3, 3)`. + + .. note:: + This function is often used in conjunction with :func:`warp_affine`, :func:`warp_perspective`. + + """ + transform: torch.Tensor = get_rotation_matrix2d(center, -angle, scale) + transform[..., 2] += translations # tx/ty + + # F.pad transform to get Bx3x3 + transform_h = convert_affinematrix_to_homography(transform) + + if any(s is not None for s in [sx, sy]): + shear_mat = get_shear_matrix2d(center, sx, sy) + transform_h = transform_h @ shear_mat + + return transform_h + + +def get_translation_matrix2d(translations: torch.Tensor) -> torch.Tensor: + r"""Compose translation matrix from the components. + + Args: + translations: torch.Tensor containing the translation vector with shape :math:`(B, 2)`. + + Returns: + the affine transformation matrix :math:`(B, 3, 3)`. + + .. note:: + This function is often used in conjunction with :func:`warp_affine`, :func:`warp_perspective`. + + """ + transform: torch.Tensor = eye_like(3, translations)[:, :2, :] + transform[..., 2] += translations # tx/ty + + # F.pad transform to get Bx3x3 + transform_h = convert_affinematrix_to_homography(transform) + + return transform_h + + +def get_shear_matrix2d( + center: torch.Tensor, sx: Optional[torch.Tensor] = None, sy: Optional[torch.Tensor] = None +) -> torch.Tensor: + r"""Compose shear matrix Bx4x4 from the components. + + Note: Ordered shearing, shear x-axis then y-axis. + + .. math:: + \begin{bmatrix} + 1 & b \\ + a & ab + 1 \\ + \end{bmatrix} + + Args: + center: shearing center coordinates of (x, y). + sx: shearing angle along x axis in radiants. + sy: shearing angle along y axis in radiants + + Returns: + params to be passed to the affine transformation with shape :math:`(B, 3, 3)`. + + Examples: + >>> rng = torch.manual_seed(0) + >>> sx = torch.randn(1) + >>> sx + tensor([1.5410]) + >>> center = torch.tensor([[0., 0.]]) # Bx2 + >>> get_shear_matrix2d(center, sx=sx) + tensor([[[ 1.0000, -33.5468, 0.0000], + [ -0.0000, 1.0000, 0.0000], + [ 0.0000, 0.0000, 1.0000]]]) + + .. note:: + This function is often used in conjunction with :func:`warp_affine`, :func:`warp_perspective`. + + """ + sx = torch.zeros(center.size(0), device=center.device, dtype=center.dtype) if sx is None else sx + sy = torch.zeros(center.size(0), device=center.device, dtype=center.dtype) if sy is None else sy + + x, y = torch.split(center, 1, dim=-1) + x, y = x.view(-1), y.view(-1) + + sx_tan = torch.tan(sx) + sy_tan = torch.tan(sy) + ones_tensor = torch.ones_like(sx) + shear_mat = torch.stack( + [ones_tensor, -sx_tan, sx_tan * y, -sy_tan, ones_tensor + sx_tan * sy_tan, sy_tan * (x - sx_tan * y)], dim=-1 + ).view(-1, 2, 3) + + shear_mat = convert_affinematrix_to_homography(shear_mat) + return shear_mat + + +def get_affine_matrix3d( + translations: torch.Tensor, + center: torch.Tensor, + scale: torch.Tensor, + angles: torch.Tensor, + sxy: Optional[torch.Tensor] = None, + sxz: Optional[torch.Tensor] = None, + syx: Optional[torch.Tensor] = None, + syz: Optional[torch.Tensor] = None, + szx: Optional[torch.Tensor] = None, + szy: Optional[torch.Tensor] = None, +) -> torch.Tensor: + r"""Compose 3d affine matrix from the components. + + Args: + translations: torch.Tensor containing the translation vector (dx,dy,dz) with shape :math:`(B, 3)`. + center: torch.Tensor containing the center vector (x,y,z) with shape :math:`(B, 3)`. + scale: torch.Tensor containing the scale factor with shape :math:`(B)`. + angles: axis angle vector containing the rotation angles in degrees in the form + of (rx, ry, rz) with shape :math:`(B, 3)`. Internally it calls Rodrigues to compute + the rotation matrix from axis-angle. + sxy: torch.Tensor containing the shear factor in the xy-direction with shape :math:`(B)`. + sxz: torch.Tensor containing the shear factor in the xz-direction with shape :math:`(B)`. + syx: torch.Tensor containing the shear factor in the yx-direction with shape :math:`(B)`. + syz: torch.Tensor containing the shear factor in the yz-direction with shape :math:`(B)`. + szx: torch.Tensor containing the shear factor in the zx-direction with shape :math:`(B)`. + szy: torch.Tensor containing the shear factor in the zy-direction with shape :math:`(B)`. + + Returns: + the 3d affine transformation matrix :math:`(B, 3, 3)`. + + .. note:: + This function is often used in conjunction with :func:`warp_perspective`. + + """ + transform: torch.Tensor = get_projective_transform(center, -angles, scale) + transform[..., 3] += translations # tx/ty/tz + + # F.pad transform to get Bx3x3 + transform_h = convert_affinematrix_to_homography3d(transform) + if any(s is not None for s in [sxy, sxz, syx, syz, szx, szy]): + shear_mat = get_shear_matrix3d(center, sxy, sxz, syx, syz, szx, szy) + transform_h = transform_h @ shear_mat + + return transform_h + + +def get_shear_matrix3d( + center: torch.Tensor, + sxy: Optional[torch.Tensor] = None, + sxz: Optional[torch.Tensor] = None, + syx: Optional[torch.Tensor] = None, + syz: Optional[torch.Tensor] = None, + szx: Optional[torch.Tensor] = None, + szy: Optional[torch.Tensor] = None, +) -> torch.Tensor: + r"""Compose shear matrix Bx4x4 from the components. + + Note: Ordered shearing, shear x-axis then y-axis then z-axis. + + .. math:: + \begin{bmatrix} + 1 & o & r & oy + rz \\ + m & p & s & mx + py + sz -y \\ + n & q & t & nx + qy + tz -z \\ + 0 & 0 & 0 & 1 \\ + \end{bmatrix} + Where: + m = S_{xy} + n = S_{xz} + o = S_{yx} + p = S_{xy}S_{yx} + 1 + q = S_{xz}S_{yx} + S_{yz} + r = S_{zx} + S_{yx}S_{zy} + s = S_{xy}S_{zx} + (S_{xy}S_{yx} + 1)S_{zy} + t = S_{xz}S_{zx} + (S_{xz}S_{yx} + S_{yz})S_{zy} + 1 + + Params: + center: shearing center coordinates of (x, y, z). + sxy: shearing angle along x axis, towards y plane in radiants. + sxz: shearing angle along x axis, towards z plane in radiants. + syx: shearing angle along y axis, towards x plane in radiants. + syz: shearing angle along y axis, towards z plane in radiants. + szx: shearing angle along z axis, towards x plane in radiants. + szy: shearing angle along z axis, towards y plane in radiants. + + Returns: + params to be passed to the affine transformation. + + Examples: + >>> rng = torch.manual_seed(0) + >>> sxy, sxz, syx, syz = torch.randn(4, 1) + >>> sxy, sxz, syx, syz + (tensor([1.5410]), tensor([-0.2934]), tensor([-2.1788]), tensor([0.5684])) + >>> center = torch.tensor([[0., 0., 0.]]) # Bx3 + >>> get_shear_matrix3d(center, sxy=sxy, sxz=sxz, syx=syx, syz=syz) + tensor([[[ 1.0000, -1.4369, 0.0000, 0.0000], + [-33.5468, 49.2039, 0.0000, 0.0000], + [ 0.3022, -1.0729, 1.0000, 0.0000], + [ 0.0000, 0.0000, 0.0000, 1.0000]]]) + + .. note:: + This function is often used in conjunction with :func:`warp_perspective3d`. + + """ + sxy = torch.zeros(center.size(0), device=center.device, dtype=center.dtype) if sxy is None else sxy + sxz = torch.zeros(center.size(0), device=center.device, dtype=center.dtype) if sxz is None else sxz + syx = torch.zeros(center.size(0), device=center.device, dtype=center.dtype) if syx is None else syx + syz = torch.zeros(center.size(0), device=center.device, dtype=center.dtype) if syz is None else syz + szx = torch.zeros(center.size(0), device=center.device, dtype=center.dtype) if szx is None else szx + szy = torch.zeros(center.size(0), device=center.device, dtype=center.dtype) if szy is None else szy + + x, y, z = torch.split(center, 1, dim=-1) + x, y, z = x.view(-1), y.view(-1), z.view(-1) + # Prepare parameters + sxy_tan = torch.tan(sxy) + sxz_tan = torch.tan(sxz) + syx_tan = torch.tan(syx) + syz_tan = torch.tan(syz) + szx_tan = torch.tan(szx) + szy_tan = torch.tan(szy) + + # compute translation matrix + m00, m10, m20, m01, m11, m21, m02, m12, m22 = _compute_shear_matrix_3d( + sxy_tan, sxz_tan, syx_tan, syz_tan, szx_tan, szy_tan + ) + + m03 = m01 * y + m02 * z + m13 = m10 * x + m11 * y + m12 * z - y + m23 = m20 * x + m21 * y + m22 * z - z + + # shear matrix is implemented with negative values + sxy_tan, sxz_tan, syx_tan, syz_tan, szx_tan, szy_tan = -sxy_tan, -sxz_tan, -syx_tan, -syz_tan, -szx_tan, -szy_tan + m00, m10, m20, m01, m11, m21, m02, m12, m22 = _compute_shear_matrix_3d( + sxy_tan, sxz_tan, syx_tan, syz_tan, szx_tan, szy_tan + ) + + shear_mat = torch.stack([m00, m01, m02, m03, m10, m11, m12, m13, m20, m21, m22, m23], -1).view(-1, 3, 4) + shear_mat = convert_affinematrix_to_homography3d(shear_mat) + + return shear_mat + + +def _compute_shear_matrix_3d( + sxy_tan: torch.Tensor, + sxz_tan: torch.Tensor, + syx_tan: torch.Tensor, + syz_tan: torch.Tensor, + szx_tan: torch.Tensor, + szy_tan: torch.Tensor, +) -> tuple[torch.Tensor, ...]: + ones_tensor = torch.ones_like(sxy_tan) + + m00, m10, m20 = ones_tensor, sxy_tan, sxz_tan + m01, m11, m21 = syx_tan, sxy_tan * syx_tan + ones_tensor, sxz_tan * syx_tan + syz_tan + m02 = syx_tan * szy_tan + szx_tan + m12 = sxy_tan * szx_tan + szy_tan * m11 + m22 = sxz_tan * szx_tan + szy_tan * m21 + ones_tensor + return m00, m10, m20, m01, m11, m21, m02, m12, m22 + + +def warp_affine3d( + src: torch.Tensor, + M: torch.Tensor, + dsize: tuple[int, int, int], + flags: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = True, +) -> torch.Tensor: + r"""Apply a projective transformation a to 3d torch.Tensor. + + .. warning:: + This API signature it is experimental and might suffer some changes in the future. + + Args: + src : input torch.Tensor of shape :math:`(B, C, D, H, W)`. + M: projective transformation matrix of shape :math:`(B, 3, 4)`. + dsize: size of the output image (depth, height, width). + flags: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values + ``'torch.zeros'`` | ``'border'`` | ``'reflection'``. + align_corners : mode for grid_generation. + + Returns: + torch.Tensor: the warped 3d torch.tensor with shape :math:`(B, C, D, H, W)`. + + .. note:: + This function is often used in conjunction with :func:`get_perspective_transform3d`. + + """ + if len(src.shape) != 5: + raise AssertionError(src.shape) + if not (len(M.shape) == 3 and M.shape[-2:] == (3, 4)): + raise AssertionError(M.shape) + if len(dsize) != 3: + raise AssertionError(dsize) + B, C, D, H, W = src.size() + + size_src: tuple[int, int, int] = (D, H, W) + size_out: tuple[int, int, int] = dsize + + M_4x4 = convert_affinematrix_to_homography3d(M) # Bx4x4 + + # we need to F.normalize the transformation since grid sample needs -1/1 coordinates + dst_norm_trans_src_norm: torch.Tensor = normalize_homography3d(M_4x4, size_src, size_out) # Bx4x4 + + src_norm_trans_dst_norm = _torch_inverse_cast(dst_norm_trans_src_norm) + P_norm: torch.Tensor = src_norm_trans_dst_norm[:, :3] # Bx3x4 + + # compute meshgrid and apply to input + dsize_out: list[int] = [B, C, *list(size_out)] + grid = F.affine_grid(P_norm, dsize_out, align_corners=align_corners) + return F.grid_sample(src, grid, align_corners=align_corners, mode=flags, padding_mode=padding_mode) + + +def projection_from_Rt(rmat: torch.Tensor, tvec: torch.Tensor) -> torch.Tensor: + r"""Compute the projection matrix from Rotation and translation. + + .. warning:: + This API signature it is experimental and might suffer some changes in the future. + + Concatenates the batch of rotations and translations such that :math:`P = [R | t]`. + + Args: + rmat: the rotation matrix with shape :math:`(*, 3, 3)`. + tvec: the translation vector with shape :math:`(*, 3, 1)`. + + Returns: + the projection matrix with shape :math:`(*, 3, 4)`. + + """ + if not (len(rmat.shape) >= 2 and rmat.shape[-2:] == (3, 3)): + raise AssertionError(rmat.shape) + if not (len(tvec.shape) >= 2 and tvec.shape[-2:] == (3, 1)): + raise AssertionError(tvec.shape) + + return torch.cat([rmat, tvec], -1) # Bx3x4 + + +def get_projective_transform(center: torch.Tensor, angles: torch.Tensor, scales: torch.Tensor) -> torch.Tensor: + r"""Calculate the projection matrix for a 3D rotation. + + .. warning:: + This API signature it is experimental and might suffer some changes in the future. + + The function computes the projection matrix given the center and angles per axis. + + Args: + center: center of the rotation (x,y,z) in the source with shape :math:`(B, 3)`. + angles: axis angle vector containing the rotation angles in degrees in the form + of (rx, ry, rz) with shape :math:`(B, 3)`. Internally it calls Rodrigues to compute + the rotation matrix from axis-angle. + scales: scale factor for x-y-z-directions with shape :math:`(B, 3)`. + + Returns: + the projection matrix of 3D rotation with shape :math:`(B, 3, 4)`. + + .. note:: + This function is often used in conjunction with :func:`warp_affine3d`. + + """ + if not (len(center.shape) == 2 and center.shape[-1] == 3): + raise AssertionError(center.shape) + if not (len(angles.shape) == 2 and angles.shape[-1] == 3): + raise AssertionError(angles.shape) + if center.device != angles.device: + raise AssertionError(center.device, angles.device) + if center.dtype != angles.dtype: + raise AssertionError(center.dtype, angles.dtype) + + # create rotation matrix + axis_angle_rad: torch.Tensor = torch.torch.deg2rad(angles) + rmat: torch.Tensor = axis_angle_to_rotation_matrix(axis_angle_rad) # Bx3x3 + scaling_matrix: torch.Tensor = eye_like(3, rmat) + scaling_matrix = scaling_matrix * scales.unsqueeze(dim=1) + rmat = rmat @ scaling_matrix.to(rmat) + + # define matrix to move forth and back to origin + from_origin_mat = eye_like(4, rmat, shared_memory=False) # Bx4x4 + from_origin_mat[..., :3, -1] += center + + to_origin_mat = from_origin_mat.clone() + to_origin_mat = _torch_inverse_cast(from_origin_mat) + + # append translation with torch.zeros + proj_mat = projection_from_Rt(rmat, torch.zeros_like(center)[..., None]) # Bx3x4 + + # chain 4x4 transforms + proj_mat = convert_affinematrix_to_homography3d(proj_mat) # Bx4x4 + proj_mat = from_origin_mat @ proj_mat @ to_origin_mat + + return proj_mat[..., :3, :] # Bx3x4 + + +def get_perspective_transform3d(src: torch.Tensor, dst: torch.Tensor) -> torch.Tensor: + r"""Calculate a 3d perspective transform from four pairs of the corresponding points. + + The function calculates the matrix of a perspective transform so that: + + .. math:: + + \begin{bmatrix} + t_{i}x_{i}^{'} \\ + t_{i}y_{i}^{'} \\ + t_{i}z_{i}^{'} \\ + t_{i} \\ + \end{bmatrix} + = + \textbf{map_matrix} \cdot + \begin{bmatrix} + x_{i} \\ + y_{i} \\ + z_{i} \\ + 1 \\ + \end{bmatrix} + + where + + .. math:: + dst(i) = (x_{i}^{'},y_{i}^{'},z_{i}^{'}), src(i) = (x_{i}, y_{i}, z_{i}), i = 0,1,2,5,7 + + Concrete math is as below: + + .. math:: + + \[ u_i =\frac{c_{00} * x_i + c_{01} * y_i + c_{02} * z_i + c_{03}} + {c_{30} * x_i + c_{31} * y_i + c_{32} * z_i + c_{33}} \] + \[ v_i =\frac{c_{10} * x_i + c_{11} * y_i + c_{12} * z_i + c_{13}} + {c_{30} * x_i + c_{31} * y_i + c_{32} * z_i + c_{33}} \] + \[ w_i =\frac{c_{20} * x_i + c_{21} * y_i + c_{22} * z_i + c_{23}} + {c_{30} * x_i + c_{31} * y_i + c_{32} * z_i + c_{33}} \] + + .. math:: + + \begin{pmatrix} + x_0 & y_0 & z_0 & 1 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & -x_0*u_0 & -y_0*u_0 & -z_0 * u_0 \\ + x_1 & y_1 & z_1 & 1 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & -x_1*u_1 & -y_1*u_1 & -z_1 * u_1 \\ + x_2 & y_2 & z_2 & 1 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & -x_2*u_2 & -y_2*u_2 & -z_2 * u_2 \\ + x_5 & y_5 & z_5 & 1 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & -x_5*u_5 & -y_5*u_5 & -z_5 * u_5 \\ + x_7 & y_7 & z_7 & 1 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & -x_7*u_7 & -y_7*u_7 & -z_7 * u_7 \\ + 0 & 0 & 0 & 0 & x_0 & y_0 & z_0 & 1 & 0 & 0 & 0 & 0 & -x_0*v_0 & -y_0*v_0 & -z_0 * v_0 \\ + 0 & 0 & 0 & 0 & x_1 & y_1 & z_1 & 1 & 0 & 0 & 0 & 0 & -x_1*v_1 & -y_1*v_1 & -z_1 * v_1 \\ + 0 & 0 & 0 & 0 & x_2 & y_2 & z_2 & 1 & 0 & 0 & 0 & 0 & -x_2*v_2 & -y_2*v_2 & -z_2 * v_2 \\ + 0 & 0 & 0 & 0 & x_5 & y_5 & z_5 & 1 & 0 & 0 & 0 & 0 & -x_5*v_5 & -y_5*v_5 & -z_5 * v_5 \\ + 0 & 0 & 0 & 0 & x_7 & y_7 & z_7 & 1 & 0 & 0 & 0 & 0 & -x_7*v_7 & -y_7*v_7 & -z_7 * v_7 \\ + 0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & x_0 & y_0 & z_0 & 1 & -x_0*w_0 & -y_0*w_0 & -z_0 * w_0 \\ + 0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & x_1 & y_1 & z_1 & 1 & -x_1*w_1 & -y_1*w_1 & -z_1 * w_1 \\ + 0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & x_2 & y_2 & z_2 & 1 & -x_2*w_2 & -y_2*w_2 & -z_2 * w_2 \\ + 0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & x_5 & y_5 & z_5 & 1 & -x_5*w_5 & -y_5*w_5 & -z_5 * w_5 \\ + 0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & x_7 & y_7 & z_7 & 1 & -x_7*w_7 & -y_7*w_7 & -z_7 * w_7 \\ + \end{pmatrix} + + Args: + src: coordinates of quadrangle vertices in the source image with shape :math:`(B, 8, 3)`. + dst: coordinates of the corresponding quadrangle vertices in + the destination image with shape :math:`(B, 8, 3)`. + + Returns: + the perspective transformation with shape :math:`(B, 4, 4)`. + + .. note:: + This function is often used in conjunction with :func:`warp_perspective3d`. + + """ + if not isinstance(src, (torch.Tensor)): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(src)}") + + if not isinstance(dst, (torch.Tensor)): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(dst)}") + + if not src.shape[-2:] == (8, 3): + raise ValueError(f"Inputs must be a Bx8x3 torch.Tensor. Got {src.shape}") + + if not src.shape == dst.shape: + raise ValueError(f"Inputs must have the same shape. Got {dst.shape}") + + if not (src.shape[0] == dst.shape[0]): + raise ValueError(f"Inputs must have same batch size dimension. Expect {src.shape} but got {dst.shape}") + + if not (src.device == dst.device and src.dtype == dst.dtype): + raise AssertionError( + f"Expect `src` and `dst` to be in the same device (Got {src.dtype}, {dst.dtype}) " + f"with the same dtype (Got {src.dtype}, {dst.dtype})." + ) + + # we build matrix A by using only 4 point correspondence. The linear + # system is solved with the least square method, so here + # we could even pass more correspondence + p = [] + + # 000, 100, 110, 101, 011 + for i in [0, 1, 2, 5, 7]: + p.append(_build_perspective_param3d(src[:, i], dst[:, i], "x")) + p.append(_build_perspective_param3d(src[:, i], dst[:, i], "y")) + p.append(_build_perspective_param3d(src[:, i], dst[:, i], "z")) + + # A is Bx15x15 + A = torch.stack(p, 1) + + # b is a Bx15x1 + b = torch.stack( + [ + dst[:, 0:1, 0], + dst[:, 0:1, 1], + dst[:, 0:1, 2], + dst[:, 1:2, 0], + dst[:, 1:2, 1], + dst[:, 1:2, 2], + dst[:, 2:3, 0], + dst[:, 2:3, 1], + dst[:, 2:3, 2], + # dst[:, 3:4, 0], dst[:, 3:4, 1], dst[:, 3:4, 2], + # dst[:, 4:5, 0], dst[:, 4:5, 1], dst[:, 4:5, 2], + dst[:, 5:6, 0], + dst[:, 5:6, 1], + dst[:, 5:6, 2], + # dst[:, 6:7, 0], dst[:, 6:7, 1], dst[:, 6:7, 2], + dst[:, 7:8, 0], + dst[:, 7:8, 1], + dst[:, 7:8, 2], + ], + 1, + ) + + # solve the system Ax = b + X: torch.Tensor = _torch_solve_cast(A, b) + + # create variable to return + batch_size: int = src.shape[0] + M = torch.empty(batch_size, 16, device=src.device, dtype=src.dtype) + M[..., :15] = X[..., 0] + M[..., -1].fill_(1) + + return M.view(-1, 4, 4) # Bx4x4 + + +def _build_perspective_param3d(p: torch.Tensor, q: torch.Tensor, axis: str) -> torch.Tensor: + ones_tensor = torch.ones_like(p)[..., 0:1] + zeros_tensor = torch.zeros_like(p)[..., 0:1] + + if axis == "x": + return torch.cat( + [ + p[:, 0:1], + p[:, 1:2], + p[:, 2:3], + ones_tensor, + zeros_tensor, + zeros_tensor, + zeros_tensor, + zeros_tensor, + zeros_tensor, + zeros_tensor, + zeros_tensor, + zeros_tensor, + -p[:, 0:1] * q[:, 0:1], + -p[:, 1:2] * q[:, 0:1], + -p[:, 2:3] * q[:, 0:1], + ], + 1, + ) + + if axis == "y": + return torch.cat( + [ + zeros_tensor, + zeros_tensor, + zeros_tensor, + zeros_tensor, + p[:, 0:1], + p[:, 1:2], + p[:, 2:3], + ones_tensor, + zeros_tensor, + zeros_tensor, + zeros_tensor, + zeros_tensor, + -p[:, 0:1] * q[:, 1:2], + -p[:, 1:2] * q[:, 1:2], + -p[:, 2:3] * q[:, 1:2], + ], + 1, + ) + + if axis == "z": + return torch.cat( + [ + zeros_tensor, + zeros_tensor, + zeros_tensor, + zeros_tensor, + zeros_tensor, + zeros_tensor, + zeros_tensor, + zeros_tensor, + p[:, 0:1], + p[:, 1:2], + p[:, 2:3], + ones_tensor, + -p[:, 0:1] * q[:, 2:3], + -p[:, 1:2] * q[:, 2:3], + -p[:, 2:3] * q[:, 2:3], + ], + 1, + ) + + raise NotImplementedError(f"perspective params for axis `{axis}` is not implemented.") + + +def warp_perspective3d( + src: torch.Tensor, + M: torch.Tensor, + dsize: tuple[int, int, int], + flags: str = "bilinear", + border_mode: str = "zeros", + align_corners: bool = False, +) -> torch.Tensor: + r"""Apply a perspective transformation to an image. + + The function warp_perspective transforms the source image using + the specified matrix: + + .. math:: + \text{dst} (x, y) = \text{src} \left( + \frac{M_{11} x + M_{12} y + M_{13}}{M_{31} x + M_{32} y + M_{33}} , + \frac{M_{21} x + M_{22} y + M_{23}}{M_{31} x + M_{32} y + M_{33}} + \right ) + + Args: + src: input image with shape :math:`(B, C, D, H, W)`. + M: transformation matrix with shape :math:`(B, 4, 4)`. + dsize: size of the output image (height, width). + flags: interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'``. + border_mode: padding mode for outside grid values + ``'zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: interpolation flag. + + Returns: + the warped input image :math:`(B, C, D, H, W)`. + + .. note:: + This function is often used in conjunction with :func:`get_perspective_transform3d`. + + """ + if not isinstance(src, torch.Tensor): + raise TypeError(f"Input src type is not a torch.Tensor. Got {type(src)}") + + if not isinstance(M, torch.Tensor): + raise TypeError(f"Input M type is not a torch.Tensor. Got {type(M)}") + + if not len(src.shape) == 5: + raise ValueError(f"Input src must be a BxCxDxHxW torch.Tensor. Got {src.shape}") + + if not (len(M.shape) == 3 or M.shape[-2:] == (4, 4)): + raise ValueError(f"Input M must be a Bx4x4 torch.Tensor. Got {M.shape}") + + # launches the warper + d, h, w = src.shape[-3:] + return _transform_warp_impl3d(src, M, (d, h, w), dsize, flags, border_mode, align_corners) + + +def homography_warp( + patch_src: torch.Tensor, + src_homo_dst: torch.Tensor, + dsize: tuple[int, int], + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = False, + normalized_coordinates: bool = True, + normalized_homography: bool = True, +) -> torch.Tensor: + r"""Warp image patches or tensors by normalized 2D homographies. + + See :class:`~kornia.geometry.warp.HomographyWarper` for details. + + Args: + patch_src: The image or torch.Tensor to warp. Should be from source of shape :math:`(N, C, H, W)`. + src_homo_dst: The homography or torch.stack of homographies from destination to source of shape + :math:`(N, 3, 3)`. + dsize: + if homography normalized: The height and width of the image to warp. + if homography not normalized: size of the output image (height, width). + mode: interpolation mode to calculate output values ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values ``'zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: interpolation flag. + normalized_coordinates: Whether the homography assumes [-1, 1] normalized coordinates or not. + normalized_homography: show is homography normalized. + + Return: + Patch sampled at locations from source to destination. + + Example: + >>> input = torch.rand(1, 3, 32, 32) + >>> homography = torch.eye(3).view(1, 3, 3) + >>> output = homography_warp(input, homography, (32, 32)) + + Example: + >>> img = torch.rand(1, 4, 5, 6) + >>> H = torch.eye(3)[None] + >>> out = homography_warp(img, H, (4, 2), align_corners=True, normalized_homography=False) + >>> print(out.shape) + torch.Size([1, 4, 4, 2]) + + """ + if not src_homo_dst.device == patch_src.device: + raise TypeError( + f"Patch and homography must be on the same device. Got patch.device: {patch_src.device} " + f"src_H_dst.device: {src_homo_dst.device}." + ) + if normalized_homography: + height, width = dsize + grid = create_meshgrid( + height, width, normalized_coordinates=normalized_coordinates, device=patch_src.device, dtype=patch_src.dtype + ) + warped_grid = warp_grid(grid, src_homo_dst) + + return F.grid_sample(patch_src, warped_grid, mode=mode, padding_mode=padding_mode, align_corners=align_corners) + return warp_perspective( + patch_src, src_homo_dst, dsize, mode="bilinear", padding_mode=padding_mode, align_corners=True + ) + + +def _transform_warp_impl3d( + src: torch.Tensor, + dst_pix_trans_src_pix: torch.Tensor, + dsize_src: tuple[int, int, int], + dsize_dst: tuple[int, int, int], + grid_mode: str, + padding_mode: str, + align_corners: bool, +) -> torch.Tensor: + """Compute the transform in normalized coordinates and perform the warping.""" + dst_norm_trans_src_norm: torch.Tensor = normalize_homography3d(dst_pix_trans_src_pix, dsize_src, dsize_dst) + + src_norm_trans_dst_norm = _torch_inverse_cast(dst_norm_trans_src_norm) + return homography_warp3d(src, src_norm_trans_dst_norm, dsize_dst, grid_mode, padding_mode, align_corners, True) + + +def homography_warp3d( + patch_src: torch.Tensor, + src_homo_dst: torch.Tensor, + dsize: tuple[int, int, int], + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: bool = False, + normalized_coordinates: bool = True, +) -> torch.Tensor: + r"""Warp image patches or tensors by normalized 3D homographies. + + Args: + patch_src: The image or torch.Tensor to warp. Should be from source of shape :math:`(N, C, D, H, W)`. + src_homo_dst: The homography or torch.stack of homographies from destination to source of shape + :math:`(N, 4, 4)`. + dsize: The height and width of the image to warp. + mode: interpolation mode to calculate output values ``'bilinear'`` | ``'nearest'``. + padding_mode: padding mode for outside grid values ``'zeros'`` | ``'border'`` | ``'reflection'``. + align_corners: interpolation flag. + normalized_coordinates: Whether the homography assumes [-1, 1] normalized coordinates or not. + + Return: + Patch sampled at locations from source to destination. + + Example: + >>> input = torch.rand(1, 3, 32, 32) + >>> homography = torch.eye(3).view(1, 3, 3) + >>> output = homography_warp(input, homography, (32, 32)) + + """ + if not src_homo_dst.device == patch_src.device: + raise TypeError( + f"Patch and homography must be on the same device. Got patch.device: {patch_src.device} " + f"src_H_dst.device: {src_homo_dst.device}." + ) + + depth, height, width = dsize + grid = create_meshgrid3d( + depth, height, width, normalized_coordinates=normalized_coordinates, device=patch_src.device + ) + warped_grid = warp_grid3d(grid, src_homo_dst) + + return F.grid_sample(patch_src, warped_grid, mode=mode, padding_mode=padding_mode, align_corners=align_corners) diff --git a/kornia/geometry/transform/pyramid.py b/kornia/geometry/transform/pyramid.py new file mode 100644 index 0000000..a33ccde --- /dev/null +++ b/kornia/geometry/transform/pyramid.py @@ -0,0 +1,557 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import math + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SHAPE +from kornia.filters import filter2d, gaussian_blur2d + +__all__ = ["PyrDown", "PyrUp", "ScalePyramid", "build_laplacian_pyramid", "build_pyramid", "pyrdown", "pyrup"] + + +def _get_pyramid_gaussian_kernel() -> torch.Tensor: + """Return a pre-computed gaussian kernel.""" + return ( + torch.tensor( + [ + [ + [1.0, 4.0, 6.0, 4.0, 1.0], + [4.0, 16.0, 24.0, 16.0, 4.0], + [6.0, 24.0, 36.0, 24.0, 6.0], + [4.0, 16.0, 24.0, 16.0, 4.0], + [1.0, 4.0, 6.0, 4.0, 1.0], + ] + ] + ) + / 256.0 + ) + + +class PyrDown(nn.Module): + r"""Blur a torch.Tensor and downsamples it. + + Args: + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. + align_corners: interpolation flag. + factor: the downsampling factor + + Return: + the downsampled torch.Tensor. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output: :math:`(B, C, H / 2, W / 2)` + + Examples: + >>> input = torch.rand(1, 2, 4, 4) + >>> output = PyrDown()(input) # 1x2x2x2 + + """ + + def __init__(self, border_type: str = "reflect", align_corners: bool = False, factor: float = 2.0) -> None: + super().__init__() + self.border_type: str = border_type + self.align_corners: bool = align_corners + self.factor: float = factor + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Build the next lower-resolution Gaussian pyramid level. + + The input is smoothed before resizing so high-frequency content is not + directly subsampled. This is the standard pyramid-down step used by + many multi-scale image algorithms. + + Args: + input: Image tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is batch size, :math:`C` is channels, :math:`H` is + height, and :math:`W` is width. + + Returns: + Downsampled tensor from :func:`pyrdown`. The channel and batch + dimensions are preserved, while spatial dimensions are reduced + according to ``self.factor``. + """ + return pyrdown(input, self.border_type, self.align_corners, self.factor) + + +class PyrUp(nn.Module): + r"""Upsample a torch.Tensor and then blurs it. + + Args: + borde_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. + align_corners: interpolation flag. + + Return: + the upsampled torch.Tensor. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output: :math:`(B, C, H * 2, W * 2)` + + Examples: + >>> input = torch.rand(1, 2, 4, 4) + >>> output = PyrUp()(input) # 1x2x8x8 + + """ + + def __init__(self, border_type: str = "reflect", align_corners: bool = False) -> None: + super().__init__() + self.border_type: str = border_type + self.align_corners: bool = align_corners + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Build the next higher-resolution Gaussian pyramid level. + + The operation upsamples the image and applies pyramid smoothing so the + enlarged result is spatially consistent with Gaussian pyramid + reconstruction. + + Args: + input: Image tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is batch size, :math:`C` is channels, :math:`H` is + height, and :math:`W` is width. + + Returns: + Upsampled tensor from :func:`pyrup`, typically with doubled height + and width, preserving batch and channel dimensions. + """ + return pyrup(input, self.border_type, self.align_corners) + + +class ScalePyramid(nn.Module): + r"""Create an scale pyramid of image, usually used for local feature detection. + + Images are consequently smoothed with Gaussian blur and downscaled. + + Args: + n_levels: number of the levels in octave. + init_sigma: initial blur level. + min_size: the minimum size of the octave in pixels. + double_image: add 2x upscaled image as 1st level of pyramid. OpenCV SIFT does this. + + Returns: + 1st output: images + 2nd output: sigmas (coefficients for scale conversion) + 3rd output: pixelDists (coefficients for coordinate conversion) + + Shape: + - Input: :math:`(B, C, H, W)` + - Output 1st: :math:`[(B, C, NL, H, W), (B, C, NL, H/2, W/2), ...]` + - Output 2nd: :math:`[(B, NL), (B, NL), (B, NL), ...]` + - Output 3rd: :math:`[(B, NL), (B, NL), (B, NL), ...]` + + Examples: + >>> input = torch.rand(2, 4, 100, 100) + >>> sp, sigmas, pds = ScalePyramid(3, 15)(input) + + """ + + def __init__( + self, + n_levels: int = 3, + init_sigma: float = 1.6, + min_size: int = 15, + double_image: bool = False, + extra_levels: int = 3, + ) -> None: + super().__init__() + # 3 extra levels are needed for DoG nms. + self.n_levels = n_levels + self.extra_levels = extra_levels + self.init_sigma = init_sigma + self.min_size = min_size + self.border = min_size // 2 - 1 + self.sigma_step = 2 ** (1.0 / float(self.n_levels)) + self.double_image = double_image + self._precompute_gauss_kernels(n_levels, extra_levels, init_sigma, double_image) + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}(" + f"n_levels={self.n_levels}, " + f"init_sigma={self.init_sigma}, " + f"min_size={self.min_size}, " + f"extra_levels={self.extra_levels}, " + f"border={self.border}, " + f"sigma_step={self.sigma_step}, " + f"double_image={self.double_image})" + ) + + @staticmethod + def _make_gaussian_kernel1d(sigma: float, ksize: int) -> torch.Tensor: + """Normalized 1D Gaussian kernel as a float32 tensor.""" + x = torch.arange(ksize, dtype=torch.float64) - ksize // 2 + kernel = torch.exp(-0.5 * x**2 / sigma**2) + return (kernel / kernel.sum()).float() + + def _precompute_gauss_kernels( + self, n_levels: int, extra_levels: int, init_sigma: float, double_image: bool + ) -> None: + """Register precomputed 1D Gaussian kernels as buffers so they move with .to().""" + # Initial blur: from camera sigma (0.5 / 1.0) up to init_sigma + cur_sigma_init = 1.0 if double_image else 0.5 + if init_sigma > cur_sigma_init: + sigma = max(math.sqrt(init_sigma**2 - cur_sigma_init**2), 0.01) + ksize = self.get_kernel_size(sigma) + self.register_buffer("_gk_init", self._make_gaussian_kernel1d(sigma, ksize)) + else: + self.register_buffer("_gk_init", None) + + # Level-to-level kernels inside an octave. + # cur_sigma_oct resets to init_sigma at the start of every octave, so + # these delta_sigma values are the SAME for every octave — precompute once. + cur_s = init_sigma + for lvl in range(n_levels + extra_levels - 1): + delta = cur_s * math.sqrt(self.sigma_step**2 - 1.0) + ksize = self.get_kernel_size(delta) + self.register_buffer(f"_gk_{lvl}", self._make_gaussian_kernel1d(delta, ksize)) + cur_s *= self.sigma_step + + def _blur_fast(self, x: torch.Tensor, kernel: torch.Tensor) -> torch.Tensor: + """Separable Gaussian blur with a precomputed 1D kernel — no validation overhead.""" + ksize = kernel.shape[0] + pad = ksize // 2 + _B, C, _H, _W = x.shape + k = kernel.to(device=x.device, dtype=x.dtype) + # Depthwise separable: same kernel applied independently to every channel. + k_h = k.view(1, 1, 1, ksize).expand(C, 1, 1, ksize).contiguous() + tmp = F.conv2d(F.pad(x, (pad, pad, 0, 0), mode="reflect"), k_h, groups=C) + k_v = k.view(1, 1, ksize, 1).expand(C, 1, ksize, 1).contiguous() + return F.conv2d(F.pad(tmp, (0, 0, pad, pad), mode="reflect"), k_v, groups=C) + + def get_kernel_size(self, sigma: float) -> int: + """Choose an odd Gaussian kernel size for a given blur sigma. + + Args: + sigma: Gaussian standard deviation measured in pixels. + + Returns: + Odd integer kernel size large enough to cover the effective support + used by this scale-pyramid implementation. + """ + ksize = int(2.0 * 4.0 * sigma + 1.0) + + # matches OpenCV, but may cause padding problem for small images + # PyTorch does not allow to F.pad more than original size. + # Therefore there is a hack in forward function + + if ksize % 2 == 0: + ksize += 1 + return ksize + + def get_first_level(self, input: torch.Tensor) -> tuple[torch.Tensor, float, float]: + """Create the first image level and its scale metadata. + + The method optionally doubles image resolution, then applies initial + Gaussian blur so the first level reaches ``self.init_sigma``. + + Args: + input: Image tensor with shape :math:`(B, C, H, W)`. + + Returns: + Tuple ``(cur_level, cur_sigma, pixel_distance)``. ``cur_level`` is + the first image level, ``cur_sigma`` is its effective blur sigma, + and ``pixel_distance`` tells how one pixel in this level maps back + to the original input image. + """ + pixel_distance = 1.0 + cur_sigma = 0.5 + # Same as in OpenCV up to interpolation difference + if self.double_image: + x = F.interpolate(input, scale_factor=2.0, mode="bilinear", align_corners=True) + pixel_distance = 0.5 + cur_sigma *= 2.0 + else: + x = input + + if self.init_sigma > cur_sigma: + sigma = max(math.sqrt(self.init_sigma**2 - cur_sigma**2), 0.01) + ksize = self.get_kernel_size(sigma) + min_dim = min(x.size(2), x.size(3)) + if self._gk_init is not None and ksize <= min_dim: + cur_level = self._blur_fast(x, self._gk_init) + else: + ksize = min(ksize, min_dim if min_dim % 2 == 1 else min_dim - 1) + cur_level = gaussian_blur2d(x, (ksize, ksize), (sigma, sigma)) + cur_sigma = self.init_sigma + else: + cur_level = x + return cur_level, cur_sigma, pixel_distance + + def forward(self, x: torch.Tensor) -> tuple[list[torch.Tensor], list[torch.Tensor], list[torch.Tensor]]: + """Construct a Gaussian scale pyramid over several octaves. + + Each octave stores multiple progressively blurred versions of the same + image resolution. The next octave starts from a downsampled image, so + feature detectors can search both spatial location and scale. + + Args: + x: Input image tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is batch size, :math:`C` is channels, :math:`H` is + height, and :math:`W` is width. + + Returns: + Three lists with one entry per octave. ``pyr`` contains stacked + image levels, ``sigmas`` contains the absolute blur sigma for each + level, and ``pixel_dists`` contains the pixel spacing of each level + relative to the original image. + """ + bs, _, _, _ = x.size() + cur_level, cur_sigma, pixel_distance = self.get_first_level(x) + + sigmas = [torch.full((bs, self.n_levels + self.extra_levels), cur_sigma, device=x.device, dtype=x.dtype)] + pixel_dists = [ + torch.full((bs, self.n_levels + self.extra_levels), pixel_distance, device=x.device, dtype=x.dtype) + ] + pyr = [[cur_level]] + + while True: + # Build octave levels incrementally: each level is a Gaussian blur of + # the previous one. This matches VLFeat's level-to-level construction + # and avoids discrete truncation artefacts in DoG differences. + cur_sigma_oct = self.init_sigma # sigma of pyr[-1][0] in current octave pixels + for level_idx in range(1, self.n_levels + self.extra_levels): + kernel = getattr(self, f"_gk_{level_idx - 1}") + min_dim = min(pyr[-1][-1].size(2), pyr[-1][-1].size(3)) + if kernel.shape[0] <= min_dim: + new_level = self._blur_fast(pyr[-1][-1], kernel) + else: + # Tiny image: clamp kernel size and fall back to the validated path. + delta_sigma = cur_sigma_oct * math.sqrt(self.sigma_step**2 - 1.0) + ksize = min_dim if min_dim % 2 == 1 else min_dim - 1 + new_level = gaussian_blur2d(pyr[-1][-1], (ksize, ksize), (delta_sigma, delta_sigma)) + cur_sigma_oct *= self.sigma_step + pyr[-1].append(new_level) + sigmas[-1][:, level_idx] = cur_sigma_oct + pixel_dists[-1][:, level_idx] = pixel_distance + + # Seed next octave: bilinear 2x downscale of the (-extra_levels) level. + # Bilinear resampling is more accurate than integer-stride decimation. + _pyr = pyr[-1][-self.extra_levels] + H, W = _pyr.shape[2], _pyr.shape[3] + if min(H // 2, W // 2) <= self.min_size: + break + H_new, W_new = H // 2, W // 2 + nextOctaveFirstLevel = F.interpolate(_pyr, size=(H_new, W_new), mode="bilinear", align_corners=True) + pixel_distance *= 2.0 + pyr.append([nextOctaveFirstLevel]) + sigmas.append( + torch.full((bs, self.n_levels + self.extra_levels), self.init_sigma, device=x.device, dtype=x.dtype) + ) + pixel_dists.append( + torch.full((bs, self.n_levels + self.extra_levels), pixel_distance, device=x.device, dtype=x.dtype) + ) + + output_pyr = [torch.stack(i, 2) for i in pyr] + return output_pyr, sigmas, pixel_dists + + +def pyrdown( + input: torch.Tensor, border_type: str = "reflect", align_corners: bool = False, factor: float = 2.0 +) -> torch.Tensor: + r"""Blur a torch.Tensor and downsamples it. + + .. image:: _static/img/pyrdown.png + + Args: + input: the torch.Tensor to be downsampled. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. + align_corners: interpolation flag. + factor: the downsampling factor + + Return: + the downsampled torch.Tensor. + + Examples: + >>> input = torch.arange(16, dtype=torch.float32).reshape(1, 1, 4, 4) + >>> pyrdown(input, align_corners=True) + tensor([[[[ 3.7500, 5.2500], + [ 9.7500, 11.2500]]]]) + + """ + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + + kernel: torch.Tensor = _get_pyramid_gaussian_kernel() + _, _, height, width = input.shape + # blur image + x_blur: torch.Tensor = filter2d(input, kernel, border_type) + + # TODO: use kornia.geometry.resize/rescale + # downsample. + out: torch.Tensor = F.interpolate( + x_blur, + size=(int(float(height) / factor), int(float(width) // factor)), + mode="bilinear", + align_corners=align_corners, + ) + return out + + +def pyrup(input: torch.Tensor, border_type: str = "reflect", align_corners: bool = False) -> torch.Tensor: + r"""Upsample a torch.Tensor and then blurs it. + + .. image:: _static/img/pyrup.png + + Args: + input: the torch.Tensor to be downsampled. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, ``'replicate'`` or ``'circular'``. + align_corners: interpolation flag. + + Return: + the downsampled torch.Tensor. + + Examples: + >>> input = torch.arange(4, dtype=torch.float32).reshape(1, 1, 2, 2) + >>> pyrup(input, align_corners=True) + tensor([[[[0.7500, 0.8750, 1.1250, 1.2500], + [1.0000, 1.1250, 1.3750, 1.5000], + [1.5000, 1.6250, 1.8750, 2.0000], + [1.7500, 1.8750, 2.1250, 2.2500]]]]) + + """ + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + + kernel: torch.Tensor = _get_pyramid_gaussian_kernel() + # upsample torch.Tensor + _, _, height, width = input.shape + # TODO: use kornia.geometry.resize/rescale + x_up: torch.Tensor = F.interpolate( + input, size=(height * 2, width * 2), mode="bilinear", align_corners=align_corners + ) + + # blurs upsampled torch.Tensor + x_blur: torch.Tensor = filter2d(x_up, kernel, border_type) + return x_blur + + +def build_pyramid( + input: torch.Tensor, max_level: int, border_type: str = "reflect", align_corners: bool = False +) -> list[torch.Tensor]: + r"""Construct the Gaussian pyramid for a torch.Tensor image. + + .. image:: _static/img/build_pyramid.png + + The function constructs a vector of images and builds the Gaussian pyramid + by recursively applying pyrDown to the previously built pyramid layers. + + Args: + input : the torch.Tensor to be used to construct the pyramid. + max_level: 0-based index of the last (the smallest) pyramid layer. + It must be non-negative. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. + align_corners: interpolation flag. + + Shape: + - Input: :math:`(B, C, H, W)` + - Output :math:`[(B, C, H, W), (B, C, H/2, W/2), ...]` + + """ + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + KORNIA_CHECK( + isinstance(max_level, int) or max_level < 0, + f"Invalid max_level, it must be a positive integer. Got: {max_level}", + ) + + # create empty list and append the original image + pyramid: list[torch.Tensor] = [] + pyramid.append(input) + + # iterate and downsample + for _ in range(max_level - 1): + img_curr: torch.Tensor = pyramid[-1] + img_down: torch.Tensor = pyrdown(img_curr, border_type, align_corners) + pyramid.append(img_down) + + return pyramid + + +def is_powerof_two(x: int) -> bool: + # check if number x is a power of two + return bool(x) and (not (x & (x - 1))) + + +def find_next_powerof_two(x: int) -> int: + return 1 << (x - 1).bit_length() + + +def build_laplacian_pyramid( + input: torch.Tensor, max_level: int, border_type: str = "reflect", align_corners: bool = False +) -> list[torch.Tensor]: + r"""Construct the Laplacian pyramid for a torch.Tensor image. + + The function constructs a vector of images and builds the Laplacian pyramid + by recursively computing the difference after applying + pyrUp to the adjacent layer in its Gaussian pyramid. + + See :cite:`burt1987laplacian` for more details. + + Args: + input : the torch.Tensor to be used to construct the pyramid with shape :math:`(B, C, H, W)`. + max_level: 0-based index of the last (the smallest) pyramid layer. + It must be non-negative. + border_type: the padding mode to be applied before convolving. + The expected modes are: ``'constant'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. + align_corners: interpolation flag. + + Return: + Output: :math:`[(B, C, H, W), (B, C, H/2, W/2), ...]` + + """ + KORNIA_CHECK_SHAPE(input, ["B", "C", "H", "W"]) + KORNIA_CHECK( + isinstance(max_level, int) or max_level < 0, + f"Invalid max_level, it must be a positive integer. Got: {max_level}", + ) + + h = input.size()[2] + w = input.size()[3] + require_padding = not (is_powerof_two(w) or is_powerof_two(h)) + + if require_padding: + # in case of arbitrary shape torch.Tensor image need to be padded. + # Reference: https://stackoverflow.com/a/29967555 + padding = (0, find_next_powerof_two(w) - w, 0, find_next_powerof_two(h) - h) + input = F.pad(input, padding, "reflect") + + # create gaussian pyramid + gaussian_pyramid: list[torch.Tensor] = build_pyramid(input, max_level, border_type, align_corners) + # create empty list + laplacian_pyramid: list[torch.Tensor] = [] + + # iterate and compute difference of adjacent layers in a gaussian pyramid + for i in range(max_level - 1): + img_expand: torch.Tensor = pyrup(gaussian_pyramid[i + 1], border_type, align_corners) + laplacian: torch.Tensor = gaussian_pyramid[i] - img_expand + laplacian_pyramid.append(laplacian) + laplacian_pyramid.append(gaussian_pyramid[-1]) + return laplacian_pyramid diff --git a/kornia/geometry/transform/thin_plate_spline.py b/kornia/geometry/transform/thin_plate_spline.py new file mode 100644 index 0000000..76f202f --- /dev/null +++ b/kornia/geometry/transform/thin_plate_spline.py @@ -0,0 +1,259 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +from torch import nn + +from kornia.core.utils import _torch_solve_cast +from kornia.geometry.grid import create_meshgrid + +__all__ = ["get_tps_transform", "warp_image_tps", "warp_points_tps"] + +# utilities for computing thin plate spline transforms + + +def _pair_square_euclidean(tensor1: torch.Tensor, tensor2: torch.Tensor) -> torch.Tensor: + r"""Compute the pairwise squared euclidean distance matrices :math:`(B, N, M)` between two tensors. + + Tensors with shapes (B, N, C) and (B, M, C). + """ + # ||t1-t2||^2 = (t1-t2)^T(t1-t2) = t1^T*t1 + t2^T*t2 - 2*t1^T*t2 + t1_sq: torch.Tensor = tensor1.mul(tensor1).sum(dim=-1, keepdim=True) + t2_sq: torch.Tensor = tensor2.mul(tensor2).sum(dim=-1, keepdim=True).transpose(1, 2) + t1_t2: torch.Tensor = tensor1.matmul(tensor2.transpose(1, 2)) + square_dist: torch.Tensor = -2 * t1_t2 + t1_sq + t2_sq + square_dist = square_dist.clamp(min=0) # handle possible numerical errors + return square_dist + + +def _kernel_distance(squared_distances: torch.Tensor, eps: float = 1e-8) -> torch.Tensor: + r"""Compute the TPS kernel distance function: :math:`r^2 log(r)`, where `r` is the euclidean distance. + + Since + :math: `\log(r) = 1/2 \log(r^2)`, this function takes the squared distance matrix and calculates + :math: `0.5 r^2 log(r^2)`. + """ + # r^2 * log(r) = 1/2 * r^2 * log(r^2) + return 0.5 * squared_distances * squared_distances.add(eps).log() + + +def get_tps_transform(points_src: torch.Tensor, points_dst: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + r"""Compute the TPS transform parameters that warp source points to target points. + + The input to this function is a torch.Tensor of :math:`(x, y)` source points :math:`(B, N, 2)` and a corresponding + torch.Tensor of target :math:`(x, y)` points :math:`(B, N, 2)`. + + Args: + points_src: batch of source points :math:`(B, N, 2)` as :math:`(x, y)` coordinate vectors. + points_dst: batch of target points :math:`(B, N, 2)` as :math:`(x, y)` coordinate vectors. + + Returns: + :math:`(B, N, 2)` torch.Tensor of kernel weights and :math:`(B, 3, 2)` + torch.Tensor of affine weights. The last dimension contains the x-transform and y-transform weights + as separate columns. + + Example: + >>> points_src = torch.rand(1, 5, 2) + >>> points_dst = torch.rand(1, 5, 2) + >>> kernel_weights, affine_weights = get_tps_transform(points_src, points_dst) + + .. note:: + This function is often used in conjunction with :func:`warp_points_tps`, :func:`warp_image_tps`. + + """ + if not isinstance(points_src, torch.Tensor): + raise TypeError(f"Input points_src is not torch.Tensor. Got {type(points_src)}") + + if not isinstance(points_dst, torch.Tensor): + raise TypeError(f"Input points_dst is not torch.Tensor. Got {type(points_dst)}") + + if not len(points_src.shape) == 3: + raise ValueError(f"Invalid shape for points_src, expected BxNx2. Got {points_src.shape}") + + if not len(points_dst.shape) == 3: + raise ValueError(f"Invalid shape for points_dst, expected BxNx2. Got {points_dst.shape}") + + device, dtype = points_src.device, points_src.dtype + batch_size, num_points = points_src.shape[:2] + + # set up and solve linear system + # [K P] [w] = [dst] + # [P^T 0] [a] [ 0 ] + pair_distance: torch.Tensor = _pair_square_euclidean(points_src, points_dst) + k_matrix: torch.Tensor = _kernel_distance(pair_distance) + + zero_mat: torch.Tensor = torch.zeros(batch_size, 3, 3, device=device, dtype=dtype) + one_mat: torch.Tensor = torch.ones(batch_size, num_points, 1, device=device, dtype=dtype) + dest_with_zeros: torch.Tensor = torch.cat((points_dst, zero_mat[:, :, :2]), 1) + p_matrix: torch.Tensor = torch.cat((one_mat, points_src), -1) + p_matrix_t: torch.Tensor = torch.cat((p_matrix, zero_mat), 1).transpose(1, 2) + l_matrix: torch.Tensor = torch.cat((k_matrix, p_matrix), -1) + l_matrix = torch.cat((l_matrix, p_matrix_t), 1) + + weights = _torch_solve_cast(l_matrix, dest_with_zeros) + kernel_weights: torch.Tensor = weights[:, :-3] + affine_weights: torch.Tensor = weights[:, -3:] + + return kernel_weights, affine_weights + + +def warp_points_tps( + points_src: torch.Tensor, kernel_centers: torch.Tensor, kernel_weights: torch.Tensor, affine_weights: torch.Tensor +) -> torch.Tensor: + r"""Warp a torch.Tensor of coordinate points using the thin plate spline defined by arguments. + + The source points should be a :math:`(B, N, 2)` torch.Tensor of :math:`(x, y)` coordinates. The kernel centers are + a :math:`(B, K, 2)` torch.Tensor of :math:`(x, y)` coordinates. The kernel weights are a :math:`(B, K, 2)` + torch.Tensor, and the affine weights are a :math:`(B, 3, 2)` torch.Tensor. For the weight tensors, + torch.Tensor[..., 0] contains the weights for the x-transform and torch.Tensor[..., 1] the weights + for the y-transform. + + Args: + points_src: torch.Tensor of source points :math:`(B, N, 2)`. + kernel_centers: torch.Tensor of kernel center points :math:`(B, K, 2)`. + kernel_weights: torch.Tensor of kernl weights :math:`(B, K, 2)`. + affine_weights: torch.Tensor of affine weights :math:`(B, 3, 2)`. + + Returns: + The :math:`(B, N, 2)` torch.Tensor of warped source points, from applying the TPS transform. + + Example: + >>> points_src = torch.rand(1, 5, 2) + >>> points_dst = torch.rand(1, 5, 2) + >>> kernel_weights, affine_weights = get_tps_transform(points_src, points_dst) + >>> warped = warp_points_tps(points_src, points_dst, kernel_weights, affine_weights) + >>> warped_correct = torch.allclose(warped, points_dst) + + .. note:: + This function is often used in conjunction with :func:`get_tps_transform`. + + """ + if not isinstance(points_src, torch.Tensor): + raise TypeError(f"Input points_src is not torch.Tensor. Got {type(points_src)}") + + if not isinstance(kernel_centers, torch.Tensor): + raise TypeError(f"Input kernel_centers is not torch.Tensor. Got {type(kernel_centers)}") + + if not isinstance(kernel_weights, torch.Tensor): + raise TypeError(f"Input kernel_weights is not torch.Tensor. Got {type(kernel_weights)}") + + if not isinstance(affine_weights, torch.Tensor): + raise TypeError(f"Input affine_weights is not torch.Tensor. Got {type(affine_weights)}") + + if not len(points_src.shape) == 3: + raise ValueError(f"Invalid shape for points_src, expected BxNx2. Got {points_src.shape}") + + if not len(kernel_centers.shape) == 3: + raise ValueError(f"Invalid shape for kernel_centers, expected BxNx2. Got {kernel_centers.shape}") + + if not len(kernel_weights.shape) == 3: + raise ValueError(f"Invalid shape for kernel_weights, expected BxNx2. Got {kernel_weights.shape}") + + if not len(affine_weights.shape) == 3: + raise ValueError(f"Invalid shape for affine_weights, expected BxNx2. Got {affine_weights.shape}") + + # f_{x|y}(v) = a_0 + [a_x a_y].v + \sum_i w_i * U(||v-u_i||) + pair_distance: torch.Tensor = _pair_square_euclidean(points_src, kernel_centers) + k_matrix: torch.Tensor = _kernel_distance(pair_distance) + + # broadcast the kernel distance matrix against the x and y weights to compute the x and y + # transforms simultaneously + k_mul_kernel = k_matrix[..., None].mul(kernel_weights[:, None]).sum(-2) + points_mul_affine = points_src[..., None].mul(affine_weights[:, None, 1:]).sum(-2) + warped: torch.Tensor = k_mul_kernel + points_mul_affine + affine_weights[:, None, 0] + + return warped + + +def warp_image_tps( + image: torch.Tensor, + kernel_centers: torch.Tensor, + kernel_weights: torch.Tensor, + affine_weights: torch.Tensor, + align_corners: bool = False, + padding_mode: str = "zeros", +) -> torch.Tensor: + r"""Warp an image torch.Tensor according to the thin plate spline transform defined by arguments. + + .. image:: _static/img/warp_image_tps.png + + The transform is applied to each pixel coordinate in the output image to obtain a point in the input + image for interpolation of the output pixel. So the TPS parameters should correspond to a warp from + output space to input space. + + The input `image` is a :math:`(B, C, H, W)` torch.Tensor. The kernel centers, kernel weight and affine weights + are the same as in `warp_points_tps`. + + Args: + image: input image torch.Tensor :math:`(B, C, H, W)`. + kernel_centers: kernel center points :math:`(B, K, 2)`. + kernel_weights: torch.Tensor of kernl weights :math:`(B, K, 2)`. + affine_weights: torch.Tensor of affine weights :math:`(B, 3, 2)`. + align_corners: interpolation flag used by `grid_sample`. + padding_mode: padding flag used by `grid_sample`. + + Returns: + warped image torch.Tensor :math:`(B, C, H, W)`. + + Example: + >>> points_src = torch.rand(1, 5, 2) + >>> points_dst = torch.rand(1, 5, 2) + >>> image = torch.rand(1, 3, 32, 32) + >>> # note that we are getting the reverse transform: dst -> src + >>> kernel_weights, affine_weights = get_tps_transform(points_dst, points_src) + >>> warped_image = warp_image_tps(image, points_src, kernel_weights, affine_weights) + + .. note:: + This function is often used in conjunction with :func:`get_tps_transform`. + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input image is not torch.Tensor. Got {type(image)}") + + if not isinstance(kernel_centers, torch.Tensor): + raise TypeError(f"Input kernel_centers is not torch.Tensor. Got {type(kernel_centers)}") + + if not isinstance(kernel_weights, torch.Tensor): + raise TypeError(f"Input kernel_weights is not torch.Tensor. Got {type(kernel_weights)}") + + if not isinstance(affine_weights, torch.Tensor): + raise TypeError(f"Input affine_weights is not torch.Tensor. Got {type(affine_weights)}") + + if not len(image.shape) == 4: + raise ValueError(f"Invalid shape for image, expected BxCxHxW. Got {image.shape}") + + if not len(kernel_centers.shape) == 3: + raise ValueError(f"Invalid shape for kernel_centers, expected BxNx2. Got {kernel_centers.shape}") + + if not len(kernel_weights.shape) == 3: + raise ValueError(f"Invalid shape for kernel_weights, expected BxNx2. Got {kernel_weights.shape}") + + if not len(affine_weights.shape) == 3: + raise ValueError(f"Invalid shape for affine_weights, expected BxNx2. Got {affine_weights.shape}") + + batch_size, _, h, w = image.shape + coords: torch.Tensor = create_meshgrid(h, w, device=image.device, dtype=image.dtype, normalized_coordinates=True) + coords = coords.reshape(-1, 2).expand(batch_size, -1, -1) + warped: torch.Tensor = warp_points_tps(coords, kernel_centers, kernel_weights, affine_weights) + warped = warped.view(-1, h, w, 2) + warped_image: torch.Tensor = nn.functional.grid_sample( + image, warped, padding_mode=padding_mode, align_corners=align_corners + ) + + return warped_image diff --git a/kornia/geometry/vector.py b/kornia/geometry/vector.py new file mode 100644 index 0000000..0deb9f1 --- /dev/null +++ b/kornia/geometry/vector.py @@ -0,0 +1,310 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Optional, Tuple, Union, cast + +import torch +import torch.nn.functional as F + +from kornia.core.check import KORNIA_CHECK +from kornia.core.tensor_wrapper import TensorWrapper, _wrap # type: ignore[attr-defined] +from kornia.geometry.linalg import batched_dot_product, batched_squared_norm + +__all__ = ["Scalar", "Vector2", "Vector3"] + + +# TODO: implement more functionality to validate +class Scalar(TensorWrapper): + """Wrap a tensor representing a scalar value.""" + + def __init__(self, data: torch.Tensor) -> None: + super().__init__(data) + + +class Vector3(TensorWrapper): + """Wrap a tensor representing a 3D vector.""" + + def __init__(self, vector: torch.Tensor) -> None: + super().__init__(vector) + KORNIA_CHECK(vector.shape[-1] == 3) + + def __repr__(self) -> str: + return f"x: {self.x}\ny: {self.y}\nz: {self.z}" + + def __getitem__(self, idx: Union[slice, int, torch.Tensor]) -> "Vector3": + return Vector3(self.data[idx, ...]) + + @property + def x(self) -> torch.Tensor: + """Return the x-coordinate stored in the last tensor dimension.""" + return self.data[..., 0] + + @property + def y(self) -> torch.Tensor: + """Return the y-coordinate stored in the last tensor dimension.""" + return self.data[..., 1] + + @property + def z(self) -> torch.Tensor: + """Return the z-coordinate stored in the last tensor dimension.""" + return self.data[..., 2] + + def normalized(self) -> "Vector3": + """Return a copy with unit Euclidean length. + + Returns: + New :class:`Vector3` with the same leading shape as this vector. + The last dimension is normalized with the L2 norm, so each + ``(x, y, z)`` vector has length one when the input norm is nonzero. + """ + return Vector3(F.normalize(self.data, p=2, dim=-1)) + + def dot(self, right: "Vector3") -> Scalar: + """Compute dot products with another 3D vector wrapper. + + Args: + right: Right-hand :class:`Vector3` operand with compatible leading + dimensions. + + Returns: + :class:`Scalar` containing :math:`x_1 x_2 + y_1 y_2 + z_1 z_2` for + each leading element. + """ + return Scalar(batched_dot_product(self.data, right.data)) + + def squared_norm(self) -> Scalar: + """Compute squared Euclidean lengths of the wrapped vectors. + + Returns: + :class:`Scalar` containing :math:`x^2 + y^2 + z^2` for each + vector. + """ + return Scalar(batched_squared_norm(self.data)) + + @classmethod + def random( + cls, + shape: Optional[Tuple[int, ...]] = None, + device: Optional[Union[str, torch.device]] = None, + dtype: Optional[torch.dtype] = None, + ) -> "Vector3": + """Create random 3D vectors with optional leading dimensions. + + Args: + shape: Optional leading dimensions before the final coordinate + dimension. For example, ``shape=(B, N)`` creates + :math:`(B, N, 3)` vectors. + device: Target device for the generated tensor. + dtype: Target dtype for the generated tensor. + + Returns: + :class:`Vector3` wrapping a random tensor with shape + ``(*shape, 3)``. + """ + if shape is None: + shape = () + return cls(torch.rand((*shape, 3), device=device, dtype=dtype)) + + # TODO: polish overload + # @overload + # @classmethod + # def from_coords( + # cls, x: Tensor, y: Tensor, z: Tensor, device=None, dtype=None + # ) -> "Vector3": + # KORNIA_CHECK(isinstance(x, Tensor)) + # KORNIA_CHECK(type(x) == type(y) == type(z)) + # return wrap(as_tensor((x, y, z), device=device, dtype=dtype), Vector3) + + # TODO: polish overload + # @overload + # @classmethod + # def from_coords( + # cls, x: float, y: float, z: float, device=None, dtype=None + # ) -> "Vector3": + # KORNIA_CHECK(isinstance(x, float)) + # KORNIA_CHECK(type(x) == type(y) == type(z)) + # return wrap(as_tensor((x, y, z), device=device, dtype=dtype), Vector3) + + @classmethod + def from_coords( + cls, + x: Union[float, torch.Tensor], + y: Union[float, torch.Tensor], + z: Union[float, torch.Tensor], + device: Optional[Union[str, torch.device]] = None, + dtype: Optional[torch.dtype] = None, + ) -> "Vector3": + """Construct a 3D vector wrapper from x, y, and z coordinates. + + All coordinate inputs must share the same Python type (all floats or + all tensors). For tensor inputs, coordinates are stacked along the last + dimension to produce ``(..., 3)`` output. + + Args: + x: X-coordinate value or tensor. + y: Y-coordinate value or tensor. + z: Z-coordinate value or tensor. + device: Device used when scalar inputs are converted to tensor. + dtype: Dtype used when scalar inputs are converted to tensor. + + Returns: + :class:`Vector3` containing the assembled coordinates. + """ + KORNIA_CHECK(type(x) is type(y) is type(z)) + KORNIA_CHECK(isinstance(x, torch.Tensor | float)) + if isinstance(x, float): + return _wrap(torch.as_tensor((x, y, z), device=device, dtype=dtype), Vector3) + # TODO: this is totally insane ... + tensors: Tuple[torch.Tensor, ...] = (x, cast(torch.Tensor, y), cast(torch.Tensor, z)) + return _wrap(torch.stack(tensors, -1), Vector3) + + +class Vector2(TensorWrapper): + """Wrap a tensor representing a 2D vector.""" + + def __init__(self, vector: torch.Tensor) -> None: + super().__init__(vector) + KORNIA_CHECK(vector.shape[-1] == 2) + + def __repr__(self) -> str: + return f"x: {self.x}\ny: {self.y}" + + def __getitem__(self, idx: Union[slice, int, torch.Tensor]) -> "Vector2": + return Vector2(self.data[idx, ...]) + + @property + def x(self) -> torch.Tensor: + """Return the x-coordinate stored in the last tensor dimension.""" + return self.data[..., 0] + + @property + def y(self) -> torch.Tensor: + """Return the y-coordinate stored in the last tensor dimension.""" + return self.data[..., 1] + + def normalized(self) -> "Vector2": + """Return a copy with unit Euclidean length. + + Returns: + New :class:`Vector2` with the same leading shape as this vector. + The last dimension is normalized so each ``(x, y)`` vector has + length one when the input norm is nonzero. + """ + return Vector2(F.normalize(self.data, p=2, dim=-1)) + + def dot(self, right: "Vector2") -> Scalar: + """Compute dot products with another 2D vector wrapper. + + Args: + right: Right-hand :class:`Vector2` operand with compatible leading + dimensions. + + Returns: + :class:`Scalar` containing :math:`x_1 x_2 + y_1 y_2` for each + leading element. + """ + return Scalar(batched_dot_product(self.data, right.data)) + + def squared_norm(self) -> Scalar: + """Compute squared Euclidean lengths of the wrapped vectors. + + Returns: + :class:`Scalar` containing :math:`x^2 + y^2` for each vector. + """ + return Scalar(batched_squared_norm(self.data)) + + @classmethod + def random( + cls, + shape: Optional[Tuple[int, ...]] = None, + device: Optional[Union[str, torch.device]] = None, + dtype: Optional[torch.dtype] = None, + ) -> "Vector2": + """Create random 2D vectors with optional leading dimensions. + + Args: + shape: Optional leading dimensions before the final coordinate + dimension. For example, ``shape=(B, N)`` creates + :math:`(B, N, 2)` vectors. + device: Target device for the generated tensor. + dtype: Target dtype for the generated tensor. + + Returns: + :class:`Vector2` wrapping a random tensor with shape + ``(*shape, 2)``. + """ + if shape is None: + shape = () + return cls(torch.rand((*shape, 2), device=device, dtype=dtype)) + + @classmethod + def from_coords( + cls, + x: Union[float, torch.Tensor], + y: Union[float, torch.Tensor], + device: Optional[Union[str, torch.device]] = None, + dtype: Optional[torch.dtype] = None, + ) -> "Vector2": + """Construct a 2D vector wrapper from x and y coordinates. + + Both coordinates must share the same Python type (both floats or both + tensors). For tensor inputs, coordinates are stacked along the last + dimension to produce ``(..., 2)`` output. + + Args: + x: X-coordinate value or tensor. + y: Y-coordinate value or tensor. + device: Device used when scalar inputs are converted to tensor. + dtype: Dtype used when scalar inputs are converted to tensor. + + Returns: + :class:`Vector2` containing the assembled coordinates. + """ + KORNIA_CHECK(type(x) is type(y)) + KORNIA_CHECK(isinstance(x, torch.Tensor | float)) + if isinstance(x, float): + return _wrap(torch.as_tensor((x, y), device=device, dtype=dtype), Vector2) + # TODO: this is totally insane ... + tensors: Tuple[torch.Tensor, ...] = (x, cast(torch.Tensor, y)) + return _wrap(torch.stack(tensors, -1), Vector2) + + +Vec3 = Vector3 +Vec2 = Vector2 + +# TODO: adapt to TensorWrapper + +# class UnitVector(Module): +# def __init__(self, vector: torch.Tensor) -> None: +# super().__init__() +# KORNIA_CHECK_SHAPE(vector, ["B", "N"]) +# self._vector = Parameter(vector) +# +# @property +# def vector(self) -> Tensor: +# return self._vector +# +# @classmethod +# def from_unit_vector(cls, v: Tensor) -> "UnitVector": +# # TODO: add checks https://github.com/strasdat/Sophus/blob/23.04-beta/cpp/sophus/geometry/ray.h#L59 +# return UnitVector(_VectorType(v)) +# +# @classmethod +# def from_vector(cls, v: Tensor) -> "UnitVector": +# """From a vector and normalize.""" +# return UnitVector(_VectorType(v).normalized()) +# diff --git a/kornia/image/__init__.py b/kornia/image/__init__.py new file mode 100644 index 0000000..002ba3b --- /dev/null +++ b/kornia/image/__init__.py @@ -0,0 +1,57 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Image submodule for Kornia. + +This package provides image data structures and utilities, including image layout, size, pixel format, +and channel order definitions. +""" + +from .base import ChannelsOrder, ImageLayout, ImageSize, PixelFormat +from .draw import draw_convex_polygon, draw_line, draw_point2d, draw_rectangle +from .image import Image +from .image_print import image_to_string, print_image +from .utils import ( + ImageToTensor, + image_list_to_tensor, + image_to_tensor, + make_grid, + perform_keep_shape_image, + perform_keep_shape_video, + tensor_to_image, +) + +__all__ = [ + "ChannelsOrder", + "Image", + "ImageLayout", + "ImageSize", + "ImageToTensor", + "PixelFormat", + "draw_convex_polygon", + "draw_line", + "draw_point2d", + "draw_rectangle", + "image_list_to_tensor", + "image_to_string", + "image_to_tensor", + "make_grid", + "perform_keep_shape_image", + "perform_keep_shape_video", + "print_image", + "tensor_to_image", +] diff --git a/kornia/image/base.py b/kornia/image/base.py new file mode 100644 index 0000000..413abfa --- /dev/null +++ b/kornia/image/base.py @@ -0,0 +1,141 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + +import torch + +from kornia.core.check import KORNIA_CHECK_SHAPE + + +@dataclass(frozen=True) +class ImageSize: + r"""Data class to represent image shape. + + Args: + height: image height. + width: image width. + + Example: + >>> size = ImageSize(3, 4) + >>> size.height + 3 + >>> size.width + 4 + + """ + + height: int | torch.Tensor + width: int | torch.Tensor + + +class ColorSpace(Enum): + r"""Enum that represents the color space of an image.""" + + UNKNOWN = 0 # for now, in case of multi band images + GRAY = 1 + RGB = 2 + BGR = 3 + + +@dataclass(frozen=True) +class PixelFormat: + r"""Data class to represent the pixel format of an image. + + Args: + color_space: color space. + bit_depth: the number of bits per channel. + + Example: + >>> pixel_format = PixelFormat(ColorSpace.RGB, 8) + >>> pixel_format.color_space + + >>> pixel_format.bit_depth + 8 + + """ + + color_space: ColorSpace + bit_depth: int + + +class ChannelsOrder(Enum): + r"""Enum that represents the channels order of an image.""" + + CHANNELS_FIRST = 0 + CHANNELS_LAST = 1 + + +@dataclass(frozen=True) +class ImageLayout: + """Data class to represent the layout of an image. + + Args: + image_size: image size. + channels: number of channels. + channels_order: channels order. + + Example: + >>> layout = ImageLayout(ImageSize(3, 4), 3, ChannelsOrder.CHANNELS_LAST) + >>> layout.image_size + ImageSize(height=3, width=4) + >>> layout.channels + 3 + >>> layout.channels_order + + + """ + + image_size: ImageSize + channels: int + channels_order: ChannelsOrder + + +def KORNIA_CHECK_IMAGE_LAYOUT( + x: torch.Tensor, + layout: ImageLayout, + msg: str | None = None, + raises: bool = True, +) -> bool: + """Check tensor shape matches the expected ImageLayout. + + Args: + x: tensor to validate. + layout: expected image layout. + msg: custom error message. + raises: if True, raise ShapeError on mismatch. + + Returns: + True if shape matches, False otherwise (when raises=False). + + """ + if layout.channels_order == ChannelsOrder.CHANNELS_FIRST: + shape = [str(layout.channels), str(layout.image_size.height), str(layout.image_size.width)] + elif layout.channels_order == ChannelsOrder.CHANNELS_LAST: + shape = [str(layout.image_size.height), str(layout.image_size.width), str(layout.channels)] + else: + if raises: + raise NotImplementedError(f"Layout {layout.channels_order} not implemented.") + return False + + return KORNIA_CHECK_SHAPE(x, shape, msg, raises) + + +# TODO: define CompressedImage diff --git a/kornia/image/draw.py b/kornia/image/draw.py new file mode 100644 index 0000000..e6269e3 --- /dev/null +++ b/kornia/image/draw.py @@ -0,0 +1,392 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Utilities for drawing on images.""" + +from typing import List, Optional, Tuple, Union + +import torch +from torch import Tensor + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SHAPE + +# TODO: implement width of the line + + +def draw_point2d(image: Tensor, points: Tensor, color: Tensor) -> Tensor: + r"""Set one or more coordinates in a Tensor to a color. + + Args: + image: the input image on which to draw the points with shape :math`(C,H,W)` or :math`(H,W)`. + points: the [x, y] points to be drawn on the image. + color: the color of the pixel with :math`(C)` where :math`C` is the number of channels of the image. + + Return: + The image with points set to the color. + + """ + KORNIA_CHECK( + (len(image.shape) == 2 and len(color.shape) == 1) or (image.shape[0] == color.shape[0]), + "Color dim must match the channel dims of the provided image", + ) + points = points.to(dtype=torch.int64, device=image.device) + x, y = zip(*points) + if len(color.shape) == 1: + color = torch.unsqueeze(color, dim=1) + color = color.to(dtype=image.dtype, device=image.device) + if len(image.shape) == 2: + image[y, x] = color + else: + image[:, y, x] = color + return image + + +def _draw_pixel(image: torch.Tensor, x: int, y: int, color: torch.Tensor) -> None: + r"""Draw a pixel into an image. + + Args: + image: the input image to where to draw the lines with shape :math`(C,H,W)`. + x: the x coordinate of the pixel. + y: the y coordinate of the pixel. + color: the color of the pixel with :math`(C)` where :math`C` is the number of channels of the image. + + Return: + Nothing is returned. + + """ + image[:, y, x] = color + + +def draw_line(image: torch.Tensor, p1: torch.Tensor, p2: torch.Tensor, color: torch.Tensor) -> torch.Tensor: + r"""Draw a single line into an image. + + Args: + image: the input image to where to draw the lines with shape :math`(C,H,W)`. + p1: the start point [x y] of the line with shape (2, ) or (B, 2). + p2: the end point [x y] of the line with shape (2, ) or (B, 2). + color: the color of the line with shape :math`(C)` where :math`C` is the number of channels of the image. + + Return: + the image with containing the line. + + Examples: + >>> image = torch.zeros(1, 8, 8) + >>> draw_line(image, torch.tensor([6, 4]), torch.tensor([1, 4]), torch.tensor([255])) + tensor([[[ 0., 0., 0., 0., 0., 0., 0., 0.], + [ 0., 0., 0., 0., 0., 0., 0., 0.], + [ 0., 0., 0., 0., 0., 0., 0., 0.], + [ 0., 0., 0., 0., 0., 0., 0., 0.], + [ 0., 255., 255., 255., 255., 255., 255., 0.], + [ 0., 0., 0., 0., 0., 0., 0., 0.], + [ 0., 0., 0., 0., 0., 0., 0., 0.], + [ 0., 0., 0., 0., 0., 0., 0., 0.]]]) + + """ + if (p1.shape[0] != p2.shape[0]) or (p1.shape[-1] != 2 or p2.shape[-1] != 2): + raise ValueError( + "Input points must be 2D points with shape (2, ) or (B, 2) and must have the same batch sizes." + ) + if ( + (p1[..., 0] < 0).any() + or (p1[..., 0] >= image.shape[-1]).any() + or (p1[..., 1] < 0).any() + or (p1[..., 1] >= image.shape[-2]).any() + ): + raise ValueError("p1 is out of bounds.") + if ( + (p2[..., 0] < 0).any() + or (p2[..., 0] >= image.shape[-1]).any() + or (p2[..., 1] < 0).any() + or (p2[..., 1] >= image.shape[-2]).any() + ): + raise ValueError("p2 is out of bounds.") + + if len(image.size()) != 3: + raise ValueError("image must have 3 dimensions (C,H,W).") + + if color.size(0) != image.size(0): + raise ValueError("color must have the same number of channels as the image.") + + # move p1 and p2 to the same device as the input image + # move color to the same device and dtype as the input image + p1 = p1.to(image.device).to(torch.int64) + p2 = p2.to(image.device).to(torch.int64) + color = color.to(image) + + x1, y1 = p1[..., 0], p1[..., 1] + x2, y2 = p2[..., 0], p2[..., 1] + dx = x2 - x1 + dy = y2 - y1 + dx_sign = torch.sign(dx) + dy_sign = torch.sign(dy) + dx, dy = torch.abs(dx), torch.abs(dy) + dx_zero_mask = dx == 0 + dy_zero_mask = dy == 0 + dx_gt_dy_mask = (dx > dy) & ~(dx_zero_mask | dy_zero_mask) + rest_mask = ~(dx_zero_mask | dy_zero_mask | dx_gt_dy_mask) + + dx_zero_x_coords, dx_zero_y_coords = [], [] + dy_zero_x_coords, dy_zero_y_coords = [], [] + dx_gt_dy_x_coords, dx_gt_dy_y_coords = [], [] + rest_x_coords, rest_y_coords = [], [] + + if dx_zero_mask.any(): + dx_zero_x_coords = [ + x for x_i, dy_i in zip(x1[dx_zero_mask], dy[dx_zero_mask]) for x in x_i.repeat(int(dy_i.item() + 1)) + ] + dx_zero_y_coords = [ + y + for y_i, s, dy_ in zip(y1[dx_zero_mask], dy_sign[dx_zero_mask], dy[dx_zero_mask]) + for y in (y_i + s * torch.arange(0, dy_ + 1, 1, device=image.device)) + ] + + if dy_zero_mask.any(): + dy_zero_x_coords = [ + x + for x_i, s, dx_i in zip(x1[dy_zero_mask], dx_sign[dy_zero_mask], dx[dy_zero_mask]) + for x in (x_i + s * torch.arange(0, dx_i + 1, 1, device=image.device)) + ] + dy_zero_y_coords = [ + y for y_i, dx_i in zip(y1[dy_zero_mask], dx[dy_zero_mask]) for y in y_i.repeat(int(dx_i.item() + 1)) + ] + + if dx_gt_dy_mask.any(): + dx_gt_dy_x_coords = [ + x + for x_i, s, dx_i in zip(x1[dx_gt_dy_mask], dx_sign[dx_gt_dy_mask], dx[dx_gt_dy_mask]) + for x in (x_i + s * torch.arange(0, dx_i + 1, 1, device=image.device)) + ] + dx_gt_dy_y_coords = [ + y + for y_i, s, dx_i, dy_i in zip( + y1[dx_gt_dy_mask], dy_sign[dx_gt_dy_mask], dx[dx_gt_dy_mask], dy[dx_gt_dy_mask] + ) + for y in ( + y_i + s * torch.arange(0, dy_i + 1, dy_i / dx_i, device=image.device)[: int(dx_i.item()) + 1].ceil() + ) + ] + if rest_mask.any(): + rest_x_coords = [ + x + for x_i, s, dx_i, dy_ in zip(x1[rest_mask], dx_sign[rest_mask], dx[rest_mask], dy[rest_mask]) + for x in ( + x_i + s * torch.arange(0, dx_i + 1, dx_i / dy_, device=image.device)[: int(dy_.item()) + 1].ceil() + ) + ] + rest_y_coords = [ + y + for y_i, s, dy_i in zip(y1[rest_mask], dy_sign[rest_mask], dy[rest_mask]) + for y in (y_i + s * torch.arange(0, dy_i + 1, 1, device=image.device)) + ] + x_coords = torch.clamp( + torch.tensor(dx_zero_x_coords + dy_zero_x_coords + dx_gt_dy_x_coords + rest_x_coords).long(), + min=0, + max=image.shape[-1] - 1, + ) + y_coords = torch.clamp( + torch.tensor(dx_zero_y_coords + dy_zero_y_coords + dx_gt_dy_y_coords + rest_y_coords).long(), + min=0, + max=image.shape[-2] - 1, + ) + image[:, y_coords, x_coords] = color.view(-1, 1) + return image + + +def draw_rectangle( + image: torch.Tensor, rectangle: torch.Tensor, color: Optional[torch.Tensor] = None, fill: Optional[bool] = None +) -> torch.Tensor: + r"""Draw N rectangles on a batch of image tensors. + + Args: + image: is tensor of BxCxHxW. + rectangle: represents number of rectangles to draw in BxNx4 + N is the number of boxes to draw per batch index[x1, y1, x2, y2] + 4 is in (top_left.x, top_left.y, bot_right.x, bot_right.y). + color: a size 1, size 3, BxNx1, or BxNx3 tensor. + If C is 3, and color is 1 channel it will be broadcasted. + fill: is a flag used to fill the boxes with color if True. + + Returns: + This operation modifies image inplace but also returns the drawn tensor for + convenience with same shape the of the input BxCxHxW. + + Example: + >>> img = torch.rand(2, 3, 10, 12) + >>> rect = torch.tensor([[[0, 0, 4, 4]], [[4, 4, 10, 10]]]) + >>> out = draw_rectangle(img, rect) + + """ + batch, c, h, w = image.shape + batch_rect, num_rectangle, num_points = rectangle.shape + if batch != batch_rect: + raise AssertionError("Image batch and rectangle batch must be equal") + if num_points != 4: + raise AssertionError("Number of points in rectangle must be 4") + + # clone rectangle, in case it's been expanded assignment from clipping causes problems + rectangle = rectangle.long().clone() + + # clip rectangle to hxw bounds + rectangle[:, :, 1::2] = torch.clamp(rectangle[:, :, 1::2], 0, h - 1) + rectangle[:, :, ::2] = torch.clamp(rectangle[:, :, ::2], 0, w - 1) + + if color is None: + color = torch.tensor([0.0] * c).expand(batch, num_rectangle, c) + + if fill is None: + fill = False + + if len(color.shape) == 1: + color = color.expand(batch, num_rectangle, c) + b, n, color_channels = color.shape + + if color_channels == 1 and c == 3: + color = color.expand(batch, num_rectangle, c) + + for b in range(batch): + for n in range(num_rectangle): + if fill: + image[ + b, + :, + int(rectangle[b, n, 1]) : int(rectangle[b, n, 3] + 1), + int(rectangle[b, n, 0]) : int(rectangle[b, n, 2] + 1), + ] = color[b, n, :, None, None] + else: + image[b, :, int(rectangle[b, n, 1]) : int(rectangle[b, n, 3] + 1), rectangle[b, n, 0]] = color[ + b, n, :, None + ] + image[b, :, int(rectangle[b, n, 1]) : int(rectangle[b, n, 3] + 1), rectangle[b, n, 2]] = color[ + b, n, :, None + ] + image[b, :, rectangle[b, n, 1], int(rectangle[b, n, 0]) : int(rectangle[b, n, 2] + 1)] = color[ + b, n, :, None + ] + image[b, :, rectangle[b, n, 3], int(rectangle[b, n, 0]) : int(rectangle[b, n, 2] + 1)] = color[ + b, n, :, None + ] + + return image + + +def _get_convex_edges(polygon: Tensor, h: int, w: int) -> Tuple[Tensor, Tensor]: + r"""Get the left and right edges of a polygon for each y-coordinate y \in [0, h). + + Args: + polygon: represents polygons to draw in BxNx2 + N is the number of points + 2 is (x, y). + h: bottom most coordinate (top coordinate is assumed to be 0) + w: right most coordinate (left coordinate is assumed to be 0) + + Returns: + The left and right edges of the polygon of shape (B,B). + + """ + dtype = polygon.dtype + + # Check if polygons are in loop closed format, if not -> make it so + if not torch.allclose(polygon[..., -1, :], polygon[..., 0, :]): + polygon = torch.cat((polygon, polygon[..., :1, :]), dim=-2) # (B, N+1, 2) + + # Partition points into edges + x_start, y_start = polygon[..., :-1, 0], polygon[..., :-1, 1] + x_end, y_end = polygon[..., 1:, 0], polygon[..., 1:, 1] + + # Create scanlines, edge dx/dy, and produce x values + ys = torch.arange(h, device=polygon.device, dtype=dtype) + dx = ((x_end - x_start) / (y_end - y_start + 1e-12)).clamp(-w, w) + xs = (ys[..., :, None] - y_start[..., None, :]) * dx[..., None, :] + x_start[..., None, :] + + # Only count edge in their active regions (i.e between the vertices) + valid_edges = (y_start[..., None, :] <= ys[..., :, None]).logical_and(ys[..., :, None] <= y_end[..., None, :]) + valid_edges |= (y_start[..., None, :] >= ys[..., :, None]).logical_and(ys[..., :, None] >= y_end[..., None, :]) + x_left_edges = xs.clone() + x_left_edges[~valid_edges] = w + x_right_edges = xs.clone() + x_right_edges[~valid_edges] = -1 + + # Find smallest and largest x values for the valid edges + x_left = x_left_edges.min(dim=-1).values + x_right = x_right_edges.max(dim=-1).values + return x_left, x_right + + +def _batch_polygons(polygons: List[Tensor]) -> Tensor: + r"""Convert a List of variable length polygons into a fixed size tensor. + + Works by repeating the last element in the tensor. + + Args: + polygons: List of variable length polygons of shape [N_1 x 2, N_2 x 2, ..., N_B x 2]. + B is the batch size, + N_i is the number of points, + 2 is (x, y). + + Returns: + A fixed size tensor of shape (B, N, 2) where N = max_i(N_i) + + """ + B, N = len(polygons), len(max(polygons, key=len)) + batched_polygons = torch.zeros(B, N, 2, dtype=polygons[0].dtype, device=polygons[0].device) + for b, p in enumerate(polygons): + batched_polygons[b] = torch.cat((p, p[-1:].expand(N - len(p), 2))) if len(p) < N else p + return batched_polygons + + +def draw_convex_polygon(images: Tensor, polygons: Union[Tensor, List[Tensor]], colors: Tensor) -> Tensor: + r"""Draws convex polygons on a batch of image tensors. + + Args: + images: is tensor of BxCxHxW. + polygons: represents polygons as points, either BxNx2 or List of variable length polygons. + N is the number of points. + 2 is (x, y). + colors: a B x 3 tensor or 3 tensor with color to fill in. + + Returns: + This operation modifies image inplace but also returns the drawn tensor for + convenience with same shape the of the input BxCxHxW. + + Note: + This function assumes a coordinate system (0, h - 1), (0, w - 1) in the image, with (0, 0) being the center + of the top-left pixel and (w - 1, h - 1) being the center of the bottom-right coordinate. + + Example: + >>> img = torch.rand(1, 3, 12, 16) + >>> poly = torch.tensor([[[4, 4], [12, 4], [12, 8], [4, 8]]]) + >>> color = torch.tensor([[0.5, 0.5, 0.5]]) + >>> out = draw_convex_polygon(img, poly, color) + + """ + # TODO: implement optional linetypes for smooth edges + KORNIA_CHECK_SHAPE(images, ["B", "C", "H", "W"]) + b_i, c_i, h_i, w_i, device = *images.shape, images.device + if isinstance(polygons, List): + polygons = _batch_polygons(polygons) + b_p, _, xy, device_p, dtype_p = *polygons.shape, polygons.device, polygons.dtype + if len(colors.shape) == 1: + colors = colors.expand(b_i, c_i) + b_c, _, device_c = *colors.shape, colors.device + KORNIA_CHECK(xy == 2, "Polygon vertices must be xy, i.e. 2-dimensional") + KORNIA_CHECK(b_i == b_p == b_c, "Image, polygon, and color must have same batch dimension") + KORNIA_CHECK(device == device_p == device_c, "Image, polygon, and color must have same device") + + x_left, x_right = _get_convex_edges(polygons, h_i, w_i) + ws = torch.arange(w_i, device=device, dtype=dtype_p)[None, None, :] + fill_region = (ws >= x_left[..., :, None]) & (ws <= x_right[..., :, None]) + images.mul_(~fill_region[:, None]).add_(fill_region[:, None] * colors[..., None, None]) + return images diff --git a/kornia/image/image.py b/kornia/image/image.py new file mode 100644 index 0000000..40b8ba6 --- /dev/null +++ b/kornia/image/image.py @@ -0,0 +1,400 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Union + +import torch +from torch.utils.dlpack import from_dlpack, to_dlpack + +import kornia.color +from kornia.core.check import KORNIA_CHECK +from kornia.image.base import ( + KORNIA_CHECK_IMAGE_LAYOUT, + ChannelsOrder, + ColorSpace, + ImageLayout, + ImageSize, + PixelFormat, +) +from kornia.image.image_print import image_to_string +from kornia.io.io import ImageLoadType, load_image, write_image + +# placeholder for numpy +np_ndarray = Any +DLPack = Any + + +class Image: + r"""Class that holds an Image torch.Tensor representation. + + .. note:: + + Disclaimer: This class provides the minimum functionality for image manipulation. However, as soon + as you start to experiment with advanced torch.Tensor manipulation, you might expect fancy + polymorphic behaviours. + + .. warning:: + + This API is experimental and might suffer changes in the future. + + Args: + data: a torch torch.Tensor containing the image data. + layout: a dataclass containing the image layout information. + + Examples: + >>> # from a torch.tensor + >>> data = torch.randint(0, 255, (3, 4, 5), dtype=torch.uint8) # CxHxW + >>> pixel_format = PixelFormat( + ... color_space=ColorSpace.RGB, + ... bit_depth=8, + ... ) + >>> layout = ImageLayout( + ... image_size=ImageSize(4, 5), + ... channels=3, + ... channels_order=ChannelsOrder.CHANNELS_FIRST, + ... ) + >>> img = Image(data, pixel_format, layout) + >>> assert img.channels == 3 + + >>> # from a numpy array (like opencv) + >>> data = np.ones((4, 5, 3), dtype=np.uint8) # HxWxC + >>> img = Image.from_numpy(data, color_space=ColorSpace.RGB) + >>> assert img.channels == 3 + >>> assert img.width == 5 + >>> assert img.height == 4 + + """ + + def __init__(self, data: torch.Tensor, pixel_format: PixelFormat, layout: ImageLayout) -> None: + """Image constructor. + + Args: + data: a torch torch.Tensor containing the image data. + pixel_format: the pixel format of the image. + layout: a dataclass containing the image layout information. + + """ + KORNIA_CHECK_IMAGE_LAYOUT(data, layout) + KORNIA_CHECK(data.element_size() == pixel_format.bit_depth // 8, "Invalid bit depth.") + + self._data = data + self._pixel_format = pixel_format + self._layout = layout + + def __repr__(self) -> str: + return f"Image data: {self.data}\nPixel Format: {self.pixel_format}\n Layout: {self.layout}" + + # TODO: explore use TensorWrapper + def to(self, device: Union[str, torch.device, None] = None, dtype: Union[torch.dtype, None] = None) -> Image: + """Move the image to the given device and dtype. + + Args: + device: the device to move the image to. + dtype: the data type to cast the image to. + + Returns: + Image: the image moved to the given device and dtype. + + """ + if device is not None and isinstance(device, torch.dtype): + dtype, device = device, None + # put the data to the device and dtype + self._data = self.data.to(device, dtype) + return self + + # TODO: explore use TensorWrapper + def clone(self) -> Image: + """Return a copy of the image.""" + return Image(self.data.clone(), self.pixel_format, self.layout) + + @property + def data(self) -> torch.Tensor: + """Return the underlying torch.Tensor data.""" + return self._data + + @property + def shape(self) -> tuple[int, ...]: + """Return the image shape.""" + return self.data.shape + + @property + def dtype(self) -> torch.dtype: + """Return the image data type.""" + return self.data.dtype + + @property + def device(self) -> torch.device: + """Return the image device.""" + return self.data.device + + @property + def pixel_format(self) -> PixelFormat: + """Return the pixel format.""" + return self._pixel_format + + @property + def layout(self) -> ImageLayout: + """Return the image layout.""" + return self._layout + + @property + def channels(self) -> int: + """Return the number channels of the image.""" + return self.layout.channels + + @property + def image_size(self) -> ImageSize: + """Return the image size.""" + return self.layout.image_size + + @property + def height(self) -> int: + """Return the image height (columns).""" + return int(self.layout.image_size.height) + + @property + def width(self) -> int: + """Return the image width (rows).""" + return int(self.layout.image_size.width) + + @property + def channels_order(self) -> ChannelsOrder: + """Return the channels order.""" + return self.layout.channels_order + + # TODO: figure out a better way map this function + def float(self) -> Image: + """Return the image as float.""" + self._data = self.data.float() + return self + + def to_gray(self) -> Image: + """Convert the image to grayscale.""" + src = self._pixel_format.color_space + data = self._data + + if src == ColorSpace.GRAY: + return self + + is_channels_last = self._layout.channels_order == ChannelsOrder.CHANNELS_LAST + if is_channels_last: + data = data.permute(0, 3, 1, 2) if data.ndim == 4 else data.permute(2, 0, 1) + + # Perform the color space conversion + if src == ColorSpace.RGB: + out = kornia.color.rgb_to_grayscale(data) + elif src == ColorSpace.BGR: + out = kornia.color.bgr_to_grayscale(data) + else: + raise ValueError(f"Unsupported source color space for to_gray(): {src}") + + if is_channels_last: + if out.ndim == 4: + out = out.permute(0, 2, 3, 1) + elif out.ndim == 3: + out = out.permute(1, 2, 0) + else: + raise ValueError(f"Unexpected shape after grayscale conversion: {out.shape}") + + new_pf = PixelFormat(color_space=ColorSpace.GRAY, bit_depth=self._pixel_format.bit_depth) + new_layout = ImageLayout(self._layout.image_size, channels=1, channels_order=self._layout.channels_order) + return Image(out, new_pf, new_layout) + + def to_rgb(self) -> Image: + """Convert the image to RGB.""" + src = self._pixel_format.color_space + data = self._data + + if src == ColorSpace.RGB: + return self + + is_channels_last = self._layout.channels_order == ChannelsOrder.CHANNELS_LAST + if is_channels_last: + data = data.permute(0, 3, 1, 2) if data.ndim == 4 else data.permute(2, 0, 1) + + if src == ColorSpace.GRAY: + out = kornia.color.grayscale_to_rgb(data) + elif src == ColorSpace.BGR: + out = data[:, [2, 1, 0], ...] if data.ndim == 4 else data[[2, 1, 0], ...] + else: + raise ValueError(f"Unsupported source color space for to_rgb(): {src}") + + if is_channels_last: + out = out.permute(0, 2, 3, 1) if out.ndim == 4 else out.permute(1, 2, 0) + + new_pf = PixelFormat(color_space=ColorSpace.RGB, bit_depth=self._pixel_format.bit_depth) + new_layout = ImageLayout(self._layout.image_size, channels=3, channels_order=self._layout.channels_order) + return Image(out, new_pf, new_layout) + + def to_bgr(self) -> Image: + """Convert the image to BGR.""" + src = self._pixel_format.color_space + data = self._data + + if src == ColorSpace.BGR: + return self + + is_channels_last = self._layout.channels_order == ChannelsOrder.CHANNELS_LAST + if is_channels_last: + data = data.permute(0, 3, 1, 2) if data.ndim == 4 else data.permute(2, 0, 1) + + if src == ColorSpace.GRAY: + rgb_data = kornia.color.grayscale_to_rgb(data) + out = rgb_data[:, [2, 1, 0], ...] if rgb_data.ndim == 4 else rgb_data[[2, 1, 0], ...] + elif src == ColorSpace.RGB: + out = data[:, [2, 1, 0], ...] if data.ndim == 4 else data[[2, 1, 0], ...] + else: + raise ValueError(f"Unsupported source color space for to_bgr(): {src}") + + if is_channels_last: + out = out.permute(0, 2, 3, 1) if out.ndim == 4 else out.permute(1, 2, 0) + + new_pf = PixelFormat(color_space=ColorSpace.BGR, bit_depth=self._pixel_format.bit_depth) + new_layout = ImageLayout(self._layout.image_size, channels=3, channels_order=self._layout.channels_order) + return Image(out, new_pf, new_layout) + + @classmethod + def from_numpy( + cls, + data: np_ndarray, + color_space: ColorSpace = ColorSpace.RGB, + channels_order: ChannelsOrder = ChannelsOrder.CHANNELS_LAST, + ) -> Image: + """Construct an image torch.Tensor from a numpy array. + + Args: + data: a numpy array containing the image data. + color_space: the color space of the image. + pixel_format: the pixel format of the image. + channels_order: what dimension the channels are in the image torch.Tensor. + + Example: + >>> data = np.ones((4, 5, 3), dtype=np.uint8) # HxWxC + >>> img = Image.from_numpy(data, color_space=ColorSpace.RGB) + >>> assert img.channels == 3 + >>> assert img.width == 5 + >>> assert img.height == 4 + + """ + if channels_order == ChannelsOrder.CHANNELS_LAST: + image_size = ImageSize(height=data.shape[0], width=data.shape[1]) + channels = data.shape[2] + elif channels_order == ChannelsOrder.CHANNELS_FIRST: + image_size = ImageSize(height=data.shape[1], width=data.shape[2]) + channels = data.shape[0] + else: + raise ValueError("channels_order must be either `CHANNELS_LAST` or `CHANNELS_FIRST`") + + # create the pixel format based on the input data + pixel_format = PixelFormat(color_space=color_space, bit_depth=data.itemsize * 8) + + # create the image layout based on the input data + layout = ImageLayout(image_size=image_size, channels=channels, channels_order=channels_order) + + # create the image torch.Tensor + return cls(torch.from_numpy(data), pixel_format, layout) + + def to_numpy(self) -> np_ndarray: + """Return a numpy array in cpu from the image torch.Tensor.""" + return self.data.cpu().detach().numpy() + + @classmethod + def from_dlpack(cls, data: DLPack) -> Image: + """Construct an image torch.Tensor from a DLPack capsule. + + Args: + data: a DLPack capsule from numpy, tvm or jax. + + Example: + >>> x = np.ones((4, 5, 3)) + >>> img = Image.from_dlpack(x.__dlpack__()) + + """ + _data: torch.Tensor = from_dlpack(data) + + pixel_format = PixelFormat(color_space=ColorSpace.RGB, bit_depth=_data.element_size() * 8) + + # create the image layout based on the input data + layout = ImageLayout( + image_size=ImageSize(height=_data.shape[1], width=_data.shape[2]), + channels=_data.shape[0], + channels_order=ChannelsOrder.CHANNELS_FIRST, + ) + + return cls(_data, pixel_format, layout) + + def to_dlpack(self) -> DLPack: + """Return a DLPack capsule from the image torch.Tensor.""" + return to_dlpack(self.data) + + @classmethod + def from_file(cls, file_path: str | Path) -> Image: + """Construct an image torch.Tensor from a file. + + Args: + file_path: the path to the file to read the image from. + + """ + # TODO: allow user to specify the desired type and device + data: torch.Tensor = load_image(file_path, desired_type=ImageLoadType.RGB8, device="cpu") + + pixel_format = PixelFormat(color_space=ColorSpace.RGB, bit_depth=data.element_size() * 8) + + layout = ImageLayout( + image_size=ImageSize(height=data.shape[1], width=data.shape[2]), + channels=data.shape[0], + channels_order=ChannelsOrder.CHANNELS_FIRST, + ) + return cls(data, pixel_format, layout) + + def write(self, file_path: str | Path) -> None: + """Write the image to a file. + + For now, only support writing to JPEG format. + + Args: + file_path: the path to the file to write the image to. + + Example: + >>> data = np.ones((4, 5, 3), dtype=np.uint8) # HxWxC + >>> img = Image.from_numpy(data) + >>> img.write("test.jpg") + + """ + data = self.data + if self.channels_order == ChannelsOrder.CHANNELS_LAST: + data = data.permute(2, 0, 1) + write_image(file_path, data) + + def print(self, max_width: int = 256) -> None: + """Print the image torch.Tensor to the console. + + Args: + max_width: the maximum width of the image to print. + + .. code-block:: python + + img = Image.from_file("panda.png") + img.print() + + .. image:: https://github.com/kornia/data/blob/main/print_image.png?raw=true + + """ + print(image_to_string(self.data, max_width)) diff --git a/kornia/image/image_print.py b/kornia/image/image_print.py new file mode 100644 index 0000000..cd77a12 --- /dev/null +++ b/kornia/image/image_print.py @@ -0,0 +1,420 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +"""Convert an image torch.Tensor to an ANSI text string (xterm-256color). + +Nice long listing of all 256 colors and their codes. + +Taken from https://gist.github.com/klange/1687427 +""" + +import re +from typing import Tuple, Union + +import torch +from torch import float16, float32, float64 + +import kornia +from kornia.core.check import KORNIA_CHECK_IS_IMAGE, KORNIA_CHECK_SHAPE + +# color look-up table +# 8-bit, RGB hex +CLUT = [ + # Primary 3-bit (8 colors). Unique representation! + ("00", "000000"), + ("01", "800000"), + ("02", "008000"), + ("03", "808000"), + ("04", "000080"), + ("05", "800080"), + ("06", "008080"), + ("07", "c0c0c0"), + # Equivalent "bright" versions of original 8 colors. + ("08", "808080"), + ("09", "ff0000"), + ("10", "00ff00"), + ("11", "ffff00"), + ("12", "0000ff"), + ("13", "ff00ff"), + ("14", "00ffff"), + ("15", "ffffff"), + # Strictly ascending. + ("16", "000000"), + ("17", "00005f"), + ("18", "000087"), + ("19", "0000af"), + ("20", "0000d7"), + ("21", "0000ff"), + ("22", "005f00"), + ("23", "005f5f"), + ("24", "005f87"), + ("25", "005faf"), + ("26", "005fd7"), + ("27", "005fff"), + ("28", "008700"), + ("29", "00875f"), + ("30", "008787"), + ("31", "0087af"), + ("32", "0087d7"), + ("33", "0087ff"), + ("34", "00af00"), + ("35", "00af5f"), + ("36", "00af87"), + ("37", "00afaf"), + ("38", "00afd7"), + ("39", "00afff"), + ("40", "00d700"), + ("41", "00d75f"), + ("42", "00d787"), + ("43", "00d7af"), + ("44", "00d7d7"), + ("45", "00d7ff"), + ("46", "00ff00"), + ("47", "00ff5f"), + ("48", "00ff87"), + ("49", "00ffaf"), + ("50", "00ffd7"), + ("51", "00ffff"), + ("52", "5f0000"), + ("53", "5f005f"), + ("54", "5f0087"), + ("55", "5f00af"), + ("56", "5f00d7"), + ("57", "5f00ff"), + ("58", "5f5f00"), + ("59", "5f5f5f"), + ("60", "5f5f87"), + ("61", "5f5faf"), + ("62", "5f5fd7"), + ("63", "5f5fff"), + ("64", "5f8700"), + ("65", "5f875f"), + ("66", "5f8787"), + ("67", "5f87af"), + ("68", "5f87d7"), + ("69", "5f87ff"), + ("70", "5faf00"), + ("71", "5faf5f"), + ("72", "5faf87"), + ("73", "5fafaf"), + ("74", "5fafd7"), + ("75", "5fafff"), + ("76", "5fd700"), + ("77", "5fd75f"), + ("78", "5fd787"), + ("79", "5fd7af"), + ("80", "5fd7d7"), + ("81", "5fd7ff"), + ("82", "5fff00"), + ("83", "5fff5f"), + ("84", "5fff87"), + ("85", "5fffaf"), + ("86", "5fffd7"), + ("87", "5fffff"), + ("88", "870000"), + ("89", "87005f"), + ("90", "870087"), + ("91", "8700af"), + ("92", "8700d7"), + ("93", "8700ff"), + ("94", "875f00"), + ("95", "875f5f"), + ("96", "875f87"), + ("97", "875faf"), + ("98", "875fd7"), + ("99", "875fff"), + ("100", "878700"), + ("101", "87875f"), + ("102", "878787"), + ("103", "8787af"), + ("104", "8787d7"), + ("105", "8787ff"), + ("106", "87af00"), + ("107", "87af5f"), + ("108", "87af87"), + ("109", "87afaf"), + ("110", "87afd7"), + ("111", "87afff"), + ("112", "87d700"), + ("113", "87d75f"), + ("114", "87d787"), + ("115", "87d7af"), + ("116", "87d7d7"), + ("117", "87d7ff"), + ("118", "87ff00"), + ("119", "87ff5f"), + ("120", "87ff87"), + ("121", "87ffaf"), + ("122", "87ffd7"), + ("123", "87ffff"), + ("124", "af0000"), + ("125", "af005f"), + ("126", "af0087"), + ("127", "af00af"), + ("128", "af00d7"), + ("129", "af00ff"), + ("130", "af5f00"), + ("131", "af5f5f"), + ("132", "af5f87"), + ("133", "af5faf"), + ("134", "af5fd7"), + ("135", "af5fff"), + ("136", "af8700"), + ("137", "af875f"), + ("138", "af8787"), + ("139", "af87af"), + ("140", "af87d7"), + ("141", "af87ff"), + ("142", "afaf00"), + ("143", "afaf5f"), + ("144", "afaf87"), + ("145", "afafaf"), + ("146", "afafd7"), + ("147", "afafff"), + ("148", "afd700"), + ("149", "afd75f"), + ("150", "afd787"), + ("151", "afd7af"), + ("152", "afd7d7"), + ("153", "afd7ff"), + ("154", "afff00"), + ("155", "afff5f"), + ("156", "afff87"), + ("157", "afffaf"), + ("158", "afffd7"), + ("159", "afffff"), + ("160", "d70000"), + ("161", "d7005f"), + ("162", "d70087"), + ("163", "d700af"), + ("164", "d700d7"), + ("165", "d700ff"), + ("166", "d75f00"), + ("167", "d75f5f"), + ("168", "d75f87"), + ("169", "d75faf"), + ("170", "d75fd7"), + ("171", "d75fff"), + ("172", "d78700"), + ("173", "d7875f"), + ("174", "d78787"), + ("175", "d787af"), + ("176", "d787d7"), + ("177", "d787ff"), + ("178", "d7af00"), + ("179", "d7af5f"), + ("180", "d7af87"), + ("181", "d7afaf"), + ("182", "d7afd7"), + ("183", "d7afff"), + ("184", "d7d700"), + ("185", "d7d75f"), + ("186", "d7d787"), + ("187", "d7d7af"), + ("188", "d7d7d7"), + ("189", "d7d7ff"), + ("190", "d7ff00"), + ("191", "d7ff5f"), + ("192", "d7ff87"), + ("193", "d7ffaf"), + ("194", "d7ffd7"), + ("195", "d7ffff"), + ("196", "ff0000"), + ("197", "ff005f"), + ("198", "ff0087"), + ("199", "ff00af"), + ("200", "ff00d7"), + ("201", "ff00ff"), + ("202", "ff5f00"), + ("203", "ff5f5f"), + ("204", "ff5f87"), + ("205", "ff5faf"), + ("206", "ff5fd7"), + ("207", "ff5fff"), + ("208", "ff8700"), + ("209", "ff875f"), + ("210", "ff8787"), + ("211", "ff87af"), + ("212", "ff87d7"), + ("213", "ff87ff"), + ("214", "ffaf00"), + ("215", "ffaf5f"), + ("216", "ffaf87"), + ("217", "ffafaf"), + ("218", "ffafd7"), + ("219", "ffafff"), + ("220", "ffd700"), + ("221", "ffd75f"), + ("222", "ffd787"), + ("223", "ffd7af"), + ("224", "ffd7d7"), + ("225", "ffd7ff"), + ("226", "ffff00"), + ("227", "ffff5f"), + ("228", "ffff87"), + ("229", "ffffaf"), + ("230", "ffffd7"), + ("231", "ffffff"), + # Gray-scale range. + ("232", "080808"), + ("233", "121212"), + ("234", "1c1c1c"), + ("235", "262626"), + ("236", "303030"), + ("237", "3a3a3a"), + ("238", "444444"), + ("239", "4e4e4e"), + ("240", "585858"), + ("241", "626262"), + ("242", "6c6c6c"), + ("243", "767676"), + ("244", "808080"), + ("245", "8a8a8a"), + ("246", "949494"), + ("247", "9e9e9e"), + ("248", "a8a8a8"), + ("249", "b2b2b2"), + ("250", "bcbcbc"), + ("251", "c6c6c6"), + ("252", "d0d0d0"), + ("253", "dadada"), + ("254", "e4e4e4"), + ("255", "eeeeee"), +] +SHORT2RGB_DICT = dict(CLUT) +RGB2SHORT_DICT = {v: k for k, v in SHORT2RGB_DICT.items()} + + +def _str2hex(hexstr: str) -> int: + return int(hexstr, 16) + + +def _strip_hash(rgb: str) -> str: + return rgb.removeprefix("#") + + +def short2rgb(short: str) -> str: + """Convert short to RGB code.""" + return SHORT2RGB_DICT[short] + + +def rgb2short(rgb: str) -> Tuple[str, str]: + """Find the closest xterm-256 approximation to the given RGB value. + + Args: + rgb: Hex code representing an RGB value, eg, 'abcdef'. + + Returns: + String between 0 and 255, compatible with xterm. + + Example: + >>> rgb2short('123456') + ('23', '005f5f') + >>> rgb2short('ffffff') + ('231', 'ffffff') + >>> rgb2short('0DADD6') # vimeo logo + ('38', '00afd7') + + """ + rgb = _strip_hash(rgb) + incs = (0x00, 0x5F, 0x87, 0xAF, 0xD7, 0xFF) + # Break 6-char RGB code into 3 integer vals. + parts = [int(h, 16) for h in re.split(r"(..)(..)(..)", rgb)[1:4]] + res = [] + for part in parts: + i = 0 + while i < len(incs) - 1: + s, b = incs[i], incs[i + 1] # smaller, bigger + if s <= part <= b: + s1 = abs(s - part) + b1 = abs(b - part) + if s1 < b1: + closest = s + else: + closest = b + res.append(closest) + break + i += 1 + _res = "".join([f"{i:02x}" for i in res]) + equiv = RGB2SHORT_DICT[_res] + return equiv, _res + + +def image_to_string(image: torch.Tensor, max_width: int = 256) -> str: + """Obtain the closest xterm-256 approximation string from an image torch.Tensor. + + The torch.Tensor shall be either 0~1 float type or 0~255 long type. + + Args: + image: an RGB image with shape :math:`3HW`. + max_width: maximum width of the input image. + """ + KORNIA_CHECK_IS_IMAGE(image, None, raises=True) + KORNIA_CHECK_SHAPE(image, ["C", "H", "W"]) + + if image.dtype not in [torch.bfloat16, float16, float32, float64]: + image = image / 255.0 + + if image.shape[-1] > max_width: + new_h = image.size(-2) * max_width // image.size(-1) + image = kornia.geometry.resize(image, (new_h, max_width)) + + image = (image * 255).clamp(0, 255).long() + + H, W = image.shape[-2:] + flat = image.permute(1, 2, 0).reshape(-1, 3) + + rgb2short_fn = rgb2short + lines = [] + idx = 0 + for _ in range(H): + row_parts = [] + for _ in range(W): + r, g, b = flat[idx].tolist() + h = f"{r:02x}{g:02x}{b:02x}" + short, _ = rgb2short_fn(h) + row_parts.append(f"\033[48;5;{short}m ") + idx += 1 + row_parts.append("\033[0m\n") + lines.append("".join(row_parts)) + + return "".join(lines) + + +def print_image(image: Union[str, torch.Tensor], max_width: int = 96) -> None: + """Print an image to the terminal. + + .. image:: https://github.com/kornia/data/blob/main/print_image.png?raw=true + + Args: + image: path to a valid image file or a torch.Tensor. + max_width: maximum width to print to terminal. + + Note: + Need to use `print_image(...)`. + + """ + if isinstance(image, str): + from kornia.io import ImageLoadType # pylint: disable=C0415 + + img = kornia.io.load_image(image, ImageLoadType.RGB8) + elif isinstance(image, torch.Tensor): + img = image + else: + raise RuntimeError(f"Expect image type to be either torch.Tensor or str. Got {type(image)}.") + print(image_to_string(img, max_width)) diff --git a/kornia/image/utils.py b/kornia/image/utils.py new file mode 100644 index 0000000..fc38a88 --- /dev/null +++ b/kornia/image/utils.py @@ -0,0 +1,367 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from functools import wraps +from typing import Any, Callable, List, Optional + +import torch +from torch import nn +from torch.nn import functional as F + + +def image_to_tensor(image: Any, keepdim: bool = True) -> torch.Tensor: + """Convert a numpy image to a PyTorch 4d tensor image. + + Args: + image: image of the form :math:`(H, W, C)`, :math:`(H, W)` or + :math:`(B, H, W, C)`. + keepdim: If ``False`` unsqueeze the input image to match the shape + :math:`(B, H, W, C)`. + + Returns: + tensor of the form :math:`(B, C, H, W)` if keepdim is ``False``, + :math:`(C, H, W)` otherwise. + + Example: + >>> img = np.ones((3, 3)) + >>> image_to_tensor(img).shape + torch.Size([1, 3, 3]) + + >>> img = np.ones((4, 4, 1)) + >>> image_to_tensor(img).shape + torch.Size([1, 4, 4]) + + >>> img = np.ones((4, 4, 3)) + >>> image_to_tensor(img, keepdim=False).shape + torch.Size([1, 3, 4, 4]) + + """ + if len(image.shape) > 4 or len(image.shape) < 2: + raise ValueError("Input size must be a two, three or four dimensional array") + + input_shape = image.shape + tensor: torch.Tensor = torch.from_numpy(image) + + if len(input_shape) == 2: + # (H, W) -> (1, H, W) + tensor = tensor.unsqueeze(0) + elif len(input_shape) == 3: + # (H, W, C) -> (C, H, W) + tensor = tensor.permute(2, 0, 1) + elif len(input_shape) == 4: + # (B, H, W, C) -> (B, C, H, W) + tensor = tensor.permute(0, 3, 1, 2) + keepdim = True # no need to unsqueeze + else: + raise ValueError(f"Cannot process image with shape {input_shape}") + + return tensor.unsqueeze(0) if not keepdim else tensor + + +def image_list_to_tensor(images: List[Any]) -> torch.Tensor: + """Convert a list of numpy images to a PyTorch 4d tensor image. + + Args: + images: list of images, each of the form :math:`(H, W, C)`. + Image shapes must be consistent + + Returns: + tensor of the form :math:`(B, C, H, W)`. + + Example: + >>> imgs = [np.ones((4, 4, 1)), np.zeros((4, 4, 1))] + >>> image_list_to_tensor(imgs).shape + torch.Size([2, 1, 4, 4]) + + """ + if not images: + raise ValueError("Input list of images is empty") + + images_t = [] + for img in images: + if not torch.is_tensor(img): + img = torch.as_tensor(img) + images_t.append(img) + + shape = images_t[0].shape + if len(shape) != 3: + raise ValueError("Each image must have shape (H, W, C)") + if any(img.shape != shape for img in images_t): + raise ValueError("All images must have the same shape") + + # Stack into (N, H, W, C) then permute to (N, C, H, W) + return torch.stack(images_t, dim=0).permute(0, 3, 1, 2) + + +def _to_bchw(tensor: torch.Tensor) -> torch.Tensor: + """Convert a PyTorch tensor image to BCHW format. + + Args: + tensor (torch.Tensor): image of the form :math:`(*, H, W)`. + + Returns: + input tensor of the form :math:`(B, C, H, W)`. + + """ + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(tensor)}") + + if len(tensor.shape) < 2: + raise ValueError(f"Input size must be a two, three or four dimensional tensor. Got {tensor.shape}") + + if len(tensor.shape) == 2: + tensor = tensor.unsqueeze(0) + + if len(tensor.shape) == 3: + tensor = tensor.unsqueeze(0) + + if len(tensor.shape) > 4: + tensor = tensor.view(-1, tensor.shape[-3], tensor.shape[-2], tensor.shape[-1]) + + return tensor + + +def _to_bcdhw(tensor: torch.Tensor) -> torch.Tensor: + """Convert a PyTorch tensor image to BCDHW format. + + Args: + tensor (torch.Tensor): image of the form :math:`(*, D, H, W)`. + + Returns: + input tensor of the form :math:`(B, C, D, H, W)`. + + """ + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(tensor)}") + + if len(tensor.shape) < 3: + raise ValueError(f"Input size must be a three, four or five dimensional tensor. Got {tensor.shape}") + + if len(tensor.shape) == 3: + tensor = tensor.unsqueeze(0) + + if len(tensor.shape) == 4: + tensor = tensor.unsqueeze(0) + + if len(tensor.shape) > 5: + tensor = tensor.view(-1, tensor.shape[-4], tensor.shape[-3], tensor.shape[-2], tensor.shape[-1]) + + return tensor + + +def tensor_to_image(tensor: torch.Tensor, keepdim: bool = False, force_contiguous: bool = False) -> Any: + """Convert a PyTorch tensor image to a numpy image. + + In case the tensor is in the GPU, it will be copied back to CPU. + + Args: + tensor: image of the form :math:`(H, W)`, :math:`(C, H, W)` or + :math:`(B, C, H, W)`. + keepdim: If ``False`` squeeze the input image to match the shape + :math:`(H, W, C)` or :math:`(H, W)`. + force_contiguous: If ``True`` call `contiguous` to the tensor before + + Returns: + image of the form :math:`(H, W)`, :math:`(H, W, C)` or :math:`(B, H, W, C)`. + + Example: + >>> img = torch.ones(1, 3, 3) + >>> tensor_to_image(img).shape + (3, 3) + + >>> img = torch.ones(3, 4, 4) + >>> tensor_to_image(img).shape + (4, 4, 3) + + """ + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(tensor)}") + + if len(tensor.shape) > 4 or len(tensor.shape) < 2: + raise ValueError("Input size must be a two, three or four dimensional tensor") + + input_shape = tensor.shape + image = tensor.cpu().detach() + + if len(input_shape) == 2: + # (H, W) -> (H, W) + pass + elif len(input_shape) == 3: + # (C, H, W) -> (H, W, C) + if input_shape[0] == 1: + # Grayscale for proper plt.imshow needs to be (H,W) + image = image.squeeze() + else: + image = image.permute(1, 2, 0) + elif len(input_shape) == 4: + # (B, C, H, W) -> (B, H, W, C) + image = image.permute(0, 2, 3, 1) + if input_shape[0] == 1 and not keepdim: + image = image.squeeze(0) + if input_shape[1] == 1: + image = image.squeeze(-1) + else: + raise ValueError(f"Cannot process tensor with shape {input_shape}") + + # make sure the image is contiguous + if force_contiguous: + image = image.contiguous() + + return image.numpy() + + +class ImageToTensor(nn.Module): + """Converts a numpy image to a PyTorch 4d tensor image. + + Args: + keepdim: If ``False`` unsqueeze the input image to match the shape :math:`(B, H, W, C)`. + + """ + + def __init__(self, keepdim: bool = False) -> None: + super().__init__() + self.keepdim = keepdim + + def forward(self, x: Any) -> torch.Tensor: + """Convert a NumPy-style image object into a PyTorch image tensor. + + Args: + x: Input image array accepted by :func:`image_to_tensor`. Common + layouts are ``(H, W)``, ``(H, W, C)``, or ``(B, H, W, C)``, + where :math:`B` is batch size, :math:`H` is height, + :math:`W` is width, and :math:`C` is channel count. + + Returns: + Tensor returned by :func:`image_to_tensor`. When ``self.keepdim`` is + ``False``, a batch dimension is added so image data follows + PyTorch's channel-first convention. + """ + return image_to_tensor(x, keepdim=self.keepdim) + + +def make_grid(tensor: torch.Tensor, n_row: Optional[int] = None, padding: int = 2) -> torch.Tensor: + """Convert a batched tensor to one image with padding in between. + + Args: + tensor: A batched tensor of shape (B, C, H, W). + n_row: Number of images displayed in each row of the grid. + padding: The amount of padding to add between images. + + Returns: + torch.Tensor: The combined image grid. + + """ + if not isinstance(tensor, torch.Tensor): + raise TypeError("Input tensor must be a PyTorch tensor.") + + B, C, H, W = tensor.shape + if n_row is None: + n_row = int(torch.sqrt(torch.tensor(B, dtype=torch.float32)).ceil().item()) + n_col = (B + n_row - 1) // n_row + + padded_H = H + padding + padded_W = W + padding + + # pad each image on right and bottom with `padding` zeros + tensor_padded = F.pad(tensor, (0, padding, 0, padding)) + + total = n_row * n_col + if total > B: + pad_tiles = torch.zeros((total - B, C, padded_H, padded_W), dtype=tensor.dtype, device=tensor.device) + tensor_padded = torch.cat((tensor_padded, pad_tiles), dim=0) + + # ensure contiguous memory layout before reshaping / permuting + tensor_padded = tensor_padded.contiguous() + + # reshape into (n_row, n_col, C, padded_H, padded_W) + grid = tensor_padded.view(n_row, n_col, C, padded_H, padded_W) + + # permute to (C, n_row, padded_H, n_col, padded_W) then collapse + grid = grid.permute(2, 0, 3, 1, 4).contiguous() + combined = grid.view(C, n_row * padded_H, n_col * padded_W) + + # crop trailing right/bottom padding to match original + combined_H = n_row * padded_H - padding + combined_W = n_col * padded_W - padding + combined = combined[:, :combined_H, :combined_W] + + return combined + + +def perform_keep_shape_image(f: Callable[..., torch.Tensor]) -> Callable[..., torch.Tensor]: + """Apply `f` to an image of arbitrary leading dimensions `(*, C, H, W)`. + + It works by first viewing the image as `(B, C, H, W)`, applying the function and re-viewing the image as original + shape. + """ + + @wraps(f) + def _wrapper(input: torch.Tensor, *args: Any, **kwargs: Any) -> torch.Tensor: + if not isinstance(input, torch.Tensor): + raise TypeError(f"Input input type is not a torch.Tensor. Got {type(input)}") + + if input.shape.numel() == 0: + raise ValueError("Invalid input tensor, it is empty.") + + input_shape = input.shape + input = _to_bchw(input) # view input as (B, C, H, W) + output = f(input, *args, **kwargs) + if len(input_shape) == 3: + output = output[0] + + if len(input_shape) == 2: + output = output[0, 0] + + if len(input_shape) > 4: + output = output.view(*(input_shape[:-3] + output.shape[-3:])) + + return output + + return _wrapper + + +def perform_keep_shape_video(f: Callable[..., torch.Tensor]) -> Callable[..., torch.Tensor]: + """Apply `f` to an image of arbitrary leading dimensions `(*, C, D, H, W)`. + + It works by first viewing the image as `(B, C, D, H, W)`, applying the function and re-viewing the image as original + shape. + """ + + @wraps(f) + def _wrapper(input: torch.Tensor, *args: Any, **kwargs: Any) -> torch.Tensor: + if not isinstance(input, torch.Tensor): + raise TypeError(f"Input input type is not a torch.Tensor. Got {type(input)}") + + if input.numel() == 0: + raise ValueError("Invalid input tensor, it is empty.") + + input_shape = input.shape + input = _to_bcdhw(input) # view input as (B, C, D, H, W) + output = f(input, *args, **kwargs) + if len(input_shape) == 4: + output = output[0] + + if len(input_shape) == 3: + output = output[0, 0] + + if len(input_shape) > 5: + output = output.view(*(input_shape[:-4] + output.shape[-4:])) + + return output + + return _wrapper diff --git a/kornia/io/__init__.py b/kornia/io/__init__.py new file mode 100644 index 0000000..32ceece --- /dev/null +++ b/kornia/io/__init__.py @@ -0,0 +1,25 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""IO submodule for Kornia. + +This package provides image input/output utilities, including image loading and writing functions. +""" + +from .io import ImageLoadType, load_image, write_image + +__all__ = ["ImageLoadType", "load_image", "write_image"] diff --git a/kornia/io/io.py b/kornia/io/io.py new file mode 100644 index 0000000..4766a0f --- /dev/null +++ b/kornia/io/io.py @@ -0,0 +1,250 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from enum import Enum +from pathlib import Path +from typing import Any, Union + +import kornia_rs +import torch + +import kornia +from kornia.core.check import KORNIA_CHECK +from kornia.image.utils import image_to_tensor, tensor_to_image + + +class ImageLoadType(Enum): + r"""Enum to specify the desired image type.""" + + UNCHANGED = 0 + GRAY8 = 1 + RGB8 = 2 + RGBA8 = 3 + GRAY32 = 4 + RGB32 = 5 + + +_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" + + +def _read_png_color_type(path_file: Path) -> int | None: + """Read the color type byte from a PNG file header. + + Returns None if the file is truncated or has an invalid PNG signature. + PNG color types: 0=Grayscale, 2=RGB, 3=Indexed, 4=Grayscale+Alpha, 6=RGBA. + """ + with open(path_file, "rb") as f: + # Need 26 bytes: 8 (signature) + 8 (IHDR chunk header) + 8 (width/height) + 1 (bit depth) + 1 (color type) + header = f.read(26) + if len(header) < 26 or header[:8] != _PNG_SIGNATURE: + return None + return header[25] + + +# Map PNG color type byte to kornia_rs read mode. +# Types not listed here fall through to read_image_any (which handles RGB). +# Note: color type 4 (Grayscale+Alpha) is intentionally omitted — kornia_rs +# does not support it, so it falls through to read_image_any as RGB. +_PNG_COLOR_TYPE_TO_MODE: dict[int, str] = { + 0: "mono", # Grayscale + 3: "mono", # Indexed (palette) — decoded as single channel by kornia_rs + 6: "rgba", # RGBA +} + + +def _load_image_to_tensor(path_file: Path, device: Union[str, torch.device, None]) -> torch.Tensor: + """Read an image file and decode using the Kornia Rust backend. + + The decoded image is returned as numpy array with shape HxWxC. + + Args: + path_file: Path to a valid image file. + device: the device where you want to get your image placed. + + Return: + Image torch.Tensor with shape :math:`(3,H,W)`. + + """ + # read image and return as `np.ndarray` with shape HxWxC + if path_file.suffix.lower() in [".jpg", ".jpeg"]: + img = kornia_rs.read_image_jpegturbo(str(path_file)) + elif path_file.suffix.lower() == ".png": + color_type = _read_png_color_type(path_file) + # None (truncated/invalid header) intentionally falls through to read_image_any + mode = _PNG_COLOR_TYPE_TO_MODE.get(color_type) + if mode is None or mode == "rgb": + # RGB is the default for read_image_any; use it for unknown types too + img = kornia_rs.read_image_any(str(path_file)) + else: + img = kornia_rs.read_image_png_u8(str(path_file), mode) + else: + img = kornia_rs.read_image_any(str(path_file)) + + # convert the image to torch.Tensor with shape CxHxW + img_t = image_to_tensor(img, keepdim=True) + + # move the torch.Tensor to the desired device, + dev = device if isinstance(device, torch.device) or device is None else torch.device(device) + + return img_t.to(device=dev) + + +def _to_float32(image: torch.Tensor) -> torch.Tensor: + """Convert an image torch.Tensor to float32.""" + KORNIA_CHECK(image.dtype == torch.uint8) + return image.float() / 255.0 + + +def _to_uint8(image: torch.Tensor) -> torch.Tensor: + """Convert an image torch.Tensor to uint8.""" + KORNIA_CHECK(image.dtype == torch.float32) + return image.mul(255.0).byte() + + +def _convert_image_type(image: torch.Tensor, desired_type: ImageLoadType) -> torch.Tensor: + """Convert a raw CxHxW uint8 image tensor to the desired type.""" + channels = image.shape[0] + + if desired_type == ImageLoadType.UNCHANGED: + return image + + # Normalize RGBA to RGB for types that don't need alpha + if channels == 4 and desired_type != ImageLoadType.RGBA8: + image = _to_uint8(kornia.color.rgba_to_rgb(_to_float32(image))) + channels = 3 + + match (desired_type, channels): + case (ImageLoadType.GRAY8, 1): + return image + case (ImageLoadType.GRAY8, 3): + return kornia.color.rgb_to_grayscale(image) + case (ImageLoadType.RGB8, 3): + return image + case (ImageLoadType.RGB8, 1): + return kornia.color.grayscale_to_rgb(image) + case (ImageLoadType.RGBA8, 4): + return image + case (ImageLoadType.RGBA8, 3): + return _to_uint8(kornia.color.rgb_to_rgba(_to_float32(image), 0.0)) + case (ImageLoadType.RGBA8, 1): + return _to_uint8(kornia.color.rgb_to_rgba(kornia.color.grayscale_to_rgb(_to_float32(image)), 0.0)) + case (ImageLoadType.GRAY32, 1): + return _to_float32(image) + case (ImageLoadType.GRAY32, 3): + return kornia.color.rgb_to_grayscale(_to_float32(image)) + case (ImageLoadType.RGB32, 3): + return _to_float32(image) + case (ImageLoadType.RGB32, 1): + return kornia.color.grayscale_to_rgb(_to_float32(image)) + case _: + raise NotImplementedError(f"Unsupported conversion: {channels} channels to {desired_type}") + + +def load_image( + path_file: str | Path, + desired_type: ImageLoadType = ImageLoadType.RGB32, + device: Union[str, torch.device, None] = "cpu", +) -> torch.Tensor: + """Read an image file and decode using the Kornia Rust backend. + + Args: + path_file: Path to a valid image file. + desired_type: the desired image type, defined by color space and dtype. + device: the device where you want to get your image placed. + + Return: + Image torch.Tensor with shape :math:`(3,H,W)`. + + """ + if not isinstance(path_file, Path): + path_file = Path(path_file) + + # read the image using the kornia_rs package + image: torch.Tensor = _load_image_to_tensor(path_file, device) # CxHxW + + return _convert_image_type(image, desired_type) + + +def _write_uint8_image(path_file: Path, img_np: Any, quality: int) -> None: + """Write uint8 image to file.""" + if path_file.suffix.lower() in [".jpg", ".jpeg"]: + mode = "mono" if img_np.ndim == 2 or (img_np.ndim == 3 and img_np.shape[-1] == 1) else "rgb" + kornia_rs.write_image_jpeg(str(path_file), img_np, mode=mode, quality=quality) + elif path_file.suffix.lower() == ".png": + mode = "mono" if img_np.ndim == 2 or (img_np.ndim == 3 and img_np.shape[-1] == 1) else "rgb" + kornia_rs.write_image_png_u8(str(path_file), img_np, mode=mode) + elif path_file.suffix.lower() == ".tiff": + mode = "mono" if img_np.ndim == 2 or (img_np.ndim == 3 and img_np.shape[-1] == 1) else "rgb" + kornia_rs.write_image_tiff_u8(str(path_file), img_np, mode=mode) + else: + raise NotImplementedError(f"Unsupported file extension: {path_file.suffix} for uint8 image") + + +def _write_uint16_image(path_file: Path, img_np: Any) -> None: + """Write uint16 image to file.""" + mode = "mono" if img_np.ndim == 2 or (img_np.ndim == 3 and img_np.shape[-1] == 1) else "rgb" + if path_file.suffix.lower() == ".png": + kornia_rs.write_image_png_u16(str(path_file), img_np, mode=mode) + elif path_file.suffix.lower() == ".tiff": + kornia_rs.write_image_tiff_u16(str(path_file), img_np, mode=mode) + else: + raise NotImplementedError(f"Unsupported file extension: {path_file.suffix} for uint16 image") + + +def _write_float32_image(path_file: Path, img_np: Any) -> None: + """Write float32 image to file.""" + mode = "mono" if img_np.ndim == 2 or (img_np.ndim == 3 and img_np.shape[-1] == 1) else "rgb" + if path_file.suffix.lower() == ".tiff": + kornia_rs.write_image_tiff_f32(str(path_file), img_np, mode=mode) + else: + raise NotImplementedError(f"Unsupported file extension: {path_file.suffix} for float32 image") + + +def write_image(path_file: str | Path, image: torch.Tensor, quality: int = 80) -> None: + """Save an image file using the Kornia Rust backend. + + Args: + path_file: Path to a valid image file. + image: Image torch.Tensor with shape :math:`(3,H,W)`, `(1,H,W)` and `(H,W)`. + quality: The quality of the JPEG encoding. If the file extension is .png or .tiff, the quality is ignored. + + Return: + None. + """ + if not isinstance(path_file, Path): + path_file = Path(path_file) + + KORNIA_CHECK( + path_file.suffix in [".jpg", ".jpeg", ".png", ".tiff"], + f"Invalid file extension: {path_file}, only .jpg, .jpeg, .png and .tiff are supported.", + ) + + KORNIA_CHECK(image.dim() >= 2, f"Invalid image shape: {image.shape}. Must be at least 2D.") + + img_np = tensor_to_image(image, keepdim=True, force_contiguous=True) # HxWxC + if img_np.ndim == 2: + img_np = img_np[..., None] # ensures channel dimension + if image.dtype == torch.uint8: + _write_uint8_image(path_file, img_np, quality) + elif image.dtype == torch.uint16: + _write_uint16_image(path_file, img_np) + elif image.dtype == torch.float32: + _write_float32_image(path_file, img_np) + else: + raise NotImplementedError(f"Unsupported image dtype: {image.dtype}") diff --git a/kornia/losses/__init__.py b/kornia/losses/__init__.py new file mode 100644 index 0000000..1f2666b --- /dev/null +++ b/kornia/losses/__init__.py @@ -0,0 +1,110 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Losses submodule for Kornia. + +This package provides a collection of loss functions for computer vision tasks, including SSIM, +PSNR, Dice, Focal, Lovasz, and more. +""" + +from __future__ import annotations + +from .cauchy import CauchyLoss, cauchy_loss +from .charbonnier import CharbonnierLoss, charbonnier_loss +from .depth_smooth import InverseDepthSmoothnessLoss, inverse_depth_smoothness_loss +from .dice import DiceLoss, dice_loss +from .divergence import js_div_loss_2d, kl_div_loss_2d +from .focal import BinaryFocalLossWithLogits, FocalLoss, binary_focal_loss_with_logits, focal_loss +from .geman_mcclure import GemanMcclureLoss, geman_mcclure_loss +from .hausdorff import HausdorffERLoss, HausdorffERLoss3D +from .lovasz_hinge import LovaszHingeLoss, lovasz_hinge_loss +from .lovasz_softmax import LovaszSoftmaxLoss, lovasz_softmax_loss +from .ms_ssim import MS_SSIMLoss +from .mutual_information import ( + MIKernel, + MILossFromRef, + MILossFromRef2D, + MILossFromRef3D, + NMILossFromRef, + NMILossFromRef2D, + NMILossFromRef3D, + mutual_information_loss, + mutual_information_loss_2d, + mutual_information_loss_3d, + normalized_mutual_information_loss, + normalized_mutual_information_loss_2d, + normalized_mutual_information_loss_3d, +) +from .one_hot import one_hot +from .psnr import PSNRLoss, psnr_loss +from .ssim import SSIMLoss, ssim_loss +from .ssim3d import SSIM3DLoss, ssim3d_loss +from .total_variation import TotalVariation, total_variation +from .tversky import TverskyLoss, tversky_loss +from .welsch import WelschLoss, welsch_loss + +__all__ = [ + "BinaryFocalLossWithLogits", + "CauchyLoss", + "CharbonnierLoss", + "DiceLoss", + "FocalLoss", + "GemanMcclureLoss", + "HausdorffERLoss", + "HausdorffERLoss3D", + "InverseDepthSmoothnessLoss", + "LovaszHingeLoss", + "LovaszSoftmaxLoss", + "MIKernel", + "MILossFromRef", + "MILossFromRef2D", + "MILossFromRef3D", + "MS_SSIMLoss", + "NMILossFromRef", + "NMILossFromRef2D", + "NMILossFromRef3D", + "PSNRLoss", + "SSIM3DLoss", + "SSIMLoss", + "TotalVariation", + "TverskyLoss", + "WelschLoss", + "binary_focal_loss_with_logits", + "cauchy_loss", + "charbonnier_loss", + "dice_loss", + "focal_loss", + "geman_mcclure_loss", + "inverse_depth_smoothness_loss", + "js_div_loss_2d", + "kl_div_loss_2d", + "lovasz_hinge_loss", + "lovasz_softmax_loss", + "mutual_information_loss", + "mutual_information_loss_2d", + "mutual_information_loss_3d", + "normalized_mutual_information_loss", + "normalized_mutual_information_loss_2d", + "normalized_mutual_information_loss_3d", + "one_hot", + "psnr_loss", + "ssim3d_loss", + "ssim_loss", + "total_variation", + "tversky_loss", + "welsch_loss", +] diff --git a/kornia/losses/_utils.py b/kornia/losses/_utils.py new file mode 100644 index 0000000..c26a8bb --- /dev/null +++ b/kornia/losses/_utils.py @@ -0,0 +1,58 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional + +import torch + + +def mask_ignore_pixels( + target: torch.Tensor, ignore_index: Optional[int] +) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + """Replace ignored target labels with a valid class and return their mask. + + Loss functions often need class indices to stay inside the valid class + range before one-hot encoding or gathering. This helper keeps valid labels + unchanged, maps ignored positions to class ``0`` temporarily, and returns a + mask so callers can remove those positions from the final loss. + + Args: + target: Target label tensor with arbitrary shape. + ignore_index: Label value that should be ignored. If ``None``, no + masking is applied. + + Returns: + Tuple ``(target, target_mask)``. ``target`` is either the original + tensor or a copy where ignored labels were replaced by zero. + ``target_mask`` is ``None`` when no ignored pixels are present; + otherwise it is a boolean tensor with ``True`` at valid positions. + """ + if ignore_index is None: + return target, None + + target_mask = target != ignore_index + + if target_mask.all(): + return target, None + + # map invalid pixels to a valid class (0) + # they need to be manually excluded from the loss computation after + target = target.where(target_mask, target.new_zeros(1)) + + return target, target_mask diff --git a/kornia/losses/cauchy.py b/kornia/losses/cauchy.py new file mode 100644 index 0000000..88df954 --- /dev/null +++ b/kornia/losses/cauchy.py @@ -0,0 +1,143 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SAME_DEVICE, KORNIA_CHECK_SAME_SHAPE + + +def cauchy_loss(img1: torch.Tensor, img2: torch.Tensor, reduction: str = "none") -> torch.Tensor: + r"""Criterion that computes the Cauchy [2] (aka. Lorentzian) loss. + + According to [1], we compute the Cauchy loss as follows: + + .. math:: + + \text{WL}(x, y) = log(\frac{1}{2} (x - y)^{2} + 1) + + Where: + - :math:`x` is the prediction. + - :math:`y` is the target to be regressed to. + + Reference: + [1] https://arxiv.org/pdf/1701.03077.pdf + [2] https://files.is.tue.mpg.de/black/papers/cviu.63.1.1996.pdf + + Args: + img1: the predicted torch.Tensor with shape :math:`(*)`. + img2: the target torch.Tensor with the same shape as img1. + reduction: Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction + will be applied (default), ``'mean'``: the sum of the output will be divided + by the number of elements in the output, ``'sum'``: the output will be + summed. + + Return: + a scalar with the computed loss. + + Example: + >>> img1 = torch.randn(2, 3, 32, 32, requires_grad=True) + >>> img2 = torch.randn(2, 3, 32, 32) + >>> output = cauchy_loss(img1, img2, reduction="mean") + >>> output.backward() + + """ + KORNIA_CHECK_IS_TENSOR(img1) + + KORNIA_CHECK_IS_TENSOR(img2) + + KORNIA_CHECK_SAME_SHAPE(img1, img2) + + KORNIA_CHECK_SAME_DEVICE(img1, img2) + + KORNIA_CHECK( + reduction in ("mean", "sum", "none", None), f"Given type of reduction is not supported. Got: {reduction}" + ) + + # compute loss + loss = torch.log1p(0.5 * torch.square(img1 - img2)) + + # perform reduction + if reduction == "mean": + loss = loss.mean() + elif reduction == "sum": + loss = loss.sum() + elif reduction == "none" or reduction is None: + pass + else: + raise NotImplementedError("Invalid reduction option.") + + return loss + + +class CauchyLoss(nn.Module): + r"""Criterion that computes the Cauchy [2] (aka. Lorentzian) loss. + + According to [1], we compute the Cauchy loss as follows: + + .. math:: + + \text{WL}(x, y) = log(\frac{1}{2} (x - y)^{2} + 1) + + Where: + - :math:`x` is the prediction. + - :math:`y` is the target to be regressed to. + + Reference: + [1] https://arxiv.org/pdf/1701.03077.pdf + [2] https://files.is.tue.mpg.de/black/papers/cviu.63.1.1996.pdf + + Args: + reduction: Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction + will be applied (default), ``'mean'``: the sum of the output will be divided + by the number of elements in the output, ``'sum'``: the output will be + summed. + + Shape: + - img1: the predicted torch.Tensor with shape :math:`(*)`. + - img2: the target torch.Tensor with the same shape as img1. + + Example: + >>> criterion = CauchyLoss(reduction="mean") + >>> img1 = torch.randn(2, 3, 32, 2107, requires_grad=True) + >>> img2 = torch.randn(2, 3, 32, 2107) + >>> output = criterion(img1, img2) + >>> output.backward() + + """ + + def __init__(self, reduction: str = "none") -> None: + super().__init__() + self.reduction = reduction + + def forward(self, img1: torch.Tensor, img2: torch.Tensor) -> torch.Tensor: + """Compute the Cauchy robust regression loss. + + Args: + img1: Predicted tensor with arbitrary shape. + img2: Target tensor with the same shape as ``img1``. + + Returns: + Loss tensor reduced according to ``self.reduction``. The Cauchy + penalty grows more slowly than squared error for large residuals, + making it less sensitive to outliers. + """ + return cauchy_loss(img1=img1, img2=img2, reduction=self.reduction) diff --git a/kornia/losses/charbonnier.py b/kornia/losses/charbonnier.py new file mode 100644 index 0000000..ff9090b --- /dev/null +++ b/kornia/losses/charbonnier.py @@ -0,0 +1,156 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +from torch import nn + +# from torch import Tensor (use torch.Tensor instead) +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SAME_DEVICE, KORNIA_CHECK_SAME_SHAPE + + +def charbonnier_loss(img1: torch.Tensor, img2: torch.Tensor, reduction: str = "none") -> torch.Tensor: + r"""Criterion that computes the Charbonnier [2] (aka. L1-L2 [3]) loss. + + According to [1], we compute the Charbonnier loss as follows: + + .. math:: + + \text{WL}(x, y) = \sqrt{(x - y)^{2} + 1} - 1 + + Where: + - :math:`x` is the prediction. + - :math:`y` is the target to be regressed to. + + Reference: + [1] https://arxiv.org/pdf/1701.03077.pdf + [2] https://ieeexplore.ieee.org/document/413553 + [3] https://hal.inria.fr/inria-00074015/document + [4] https://arxiv.org/pdf/1712.05927.pdf + + .. note:: + This implementation follows the formulation by Barron [1]. Other works utilize + a slightly different implementation (see [4]). + + Args: + img1: the predicted torch.Tensor with shape :math:`(*)`. + img2: the target torch.Tensor with the same shape as img1. + reduction: Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction + will be applied (default), ``'mean'``: the sum of the output will be divided + by the number of elements in the output, ``'sum'``: the output will be + summed. + + Return: + a scalar with the computed loss. + + Example: + >>> img1 = torch.randn(2, 3, 32, 32, requires_grad=True) + >>> img2 = torch.randn(2, 3, 32, 32) + >>> output = charbonnier_loss(img1, img2, reduction="sum") + >>> output.backward() + + """ + KORNIA_CHECK_IS_TENSOR(img1) + + KORNIA_CHECK_IS_TENSOR(img2) + + KORNIA_CHECK_SAME_SHAPE(img1, img2) + + KORNIA_CHECK_SAME_DEVICE(img1, img2) + + KORNIA_CHECK( + reduction in ("mean", "sum", "none", None), f"Given type of reduction is not supported. Got: {reduction}" + ) + + # compute loss + loss = ((img1 - img2) ** 2 + 1.0).sqrt() - 1.0 + + # perform reduction + if reduction == "mean": + loss = loss.mean() + elif reduction == "sum": + loss = loss.sum() + elif reduction == "none" or reduction is None: + pass + else: + raise NotImplementedError("Invalid reduction option.") + + return loss + + +class CharbonnierLoss(nn.Module): + r"""Criterion that computes the Charbonnier [2] (aka. L1-L2 [3]) loss. + + According to [1], we compute the Charbonnier loss as follows: + + .. math:: + + \text{WL}(x, y) = \sqrt{(x - y)^{2} + 1} - 1 + + Where: + - :math:`x` is the prediction. + - :math:`y` is the target to be regressed to. + + Reference: + [1] https://arxiv.org/pdf/1701.03077.pdf + [2] https://ieeexplore.ieee.org/document/413553 + [3] https://hal.inria.fr/inria-00074015/document + [4] https://arxiv.org/pdf/1712.05927.pdf + + .. note:: + This implementation follows the formulation by Barron [1]. Other works utilize + a slightly different implementation (see [4]). + + Args: + reduction: Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction + will be applied (default), ``'mean'``: the sum of the output will be divided + by the number of elements in the output, ``'sum'``: the output will be + summed. + + Shape: + - img1: the predicted torch.Tensor with shape :math:`(*)`. + - img2: the target torch.Tensor with the same shape as img1. + + Example: + >>> criterion = CharbonnierLoss(reduction="mean") + >>> img1 = torch.randn(2, 3, 32, 2107, requires_grad=True) + >>> img2 = torch.randn(2, 3, 32, 2107) + >>> output = criterion(img1, img2) + >>> output.backward() + + """ + + def __init__(self, reduction: str = "none") -> None: + super().__init__() + self.reduction = reduction + + def forward(self, img1: torch.Tensor, img2: torch.Tensor) -> torch.Tensor: + """Compute the Charbonnier robust regression loss. + + Args: + img1: Predicted tensor with arbitrary shape. + img2: Target tensor with the same shape as ``img1``. + + Returns: + Loss tensor reduced according to ``self.reduction``. The + Charbonnier penalty is a smooth approximation of absolute error and + remains differentiable near zero residual. + """ + return charbonnier_loss(img1=img1, img2=img2, reduction=self.reduction) diff --git a/kornia/losses/depth_smooth.py b/kornia/losses/depth_smooth.py new file mode 100644 index 0000000..671b5e6 --- /dev/null +++ b/kornia/losses/depth_smooth.py @@ -0,0 +1,134 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +from torch import nn + +# Based on +# https://github.com/tensorflow/models/blob/master/research/struct2depth/model.py#L625-L641 + + +def _gradient_x(img: torch.Tensor) -> torch.Tensor: + if len(img.shape) != 4: + raise AssertionError(img.shape) + return img[:, :, :, :-1] - img[:, :, :, 1:] + + +def _gradient_y(img: torch.Tensor) -> torch.Tensor: + if len(img.shape) != 4: + raise AssertionError(img.shape) + return img[:, :, :-1, :] - img[:, :, 1:, :] + + +def inverse_depth_smoothness_loss(idepth: torch.Tensor, image: torch.Tensor) -> torch.Tensor: + r"""Criterion that computes image-aware inverse depth smoothness loss. + + .. math:: + + \text{loss} = \left | \partial_x d_{ij} \right | e^{-\left \| + \partial_x I_{ij} \right \|} + \left | + \partial_y d_{ij} \right | e^{-\left \| \partial_y I_{ij} \right \|} + + Args: + idepth: tensor with the inverse depth with shape :math:`(N, 1, H, W)`. + image: tensor with the input image with shape :math:`(N, 3, H, W)`. + + Return: + a scalar with the computed loss. + + Examples: + >>> idepth = torch.rand(1, 1, 4, 5) + >>> image = torch.rand(1, 3, 4, 5) + >>> loss = inverse_depth_smoothness_loss(idepth, image) + + """ + if not isinstance(idepth, torch.Tensor): + raise TypeError(f"Input idepth type is not a torch.Tensor. Got {type(idepth)}") + + if not isinstance(image, torch.Tensor): + raise TypeError(f"Input image type is not a torch.Tensor. Got {type(image)}") + + if not len(idepth.shape) == 4: + raise ValueError(f"Invalid idepth shape, we expect BxCxHxW. Got: {idepth.shape}") + + if not len(image.shape) == 4: + raise ValueError(f"Invalid image shape, we expect BxCxHxW. Got: {image.shape}") + + if not idepth.shape[-2:] == image.shape[-2:]: + raise ValueError(f"idepth and image shapes must be the same. Got: {idepth.shape} and {image.shape}") + + if not idepth.device == image.device: + raise ValueError(f"idepth and image must be in the same device. Got: {idepth.device} and {image.device}") + + if not idepth.dtype == image.dtype: + raise ValueError(f"idepth and image must be in the same dtype. Got: {idepth.dtype} and {image.dtype}") + + # compute the gradients + idepth_dx: torch.Tensor = _gradient_x(idepth) + idepth_dy: torch.Tensor = _gradient_y(idepth) + image_dx: torch.Tensor = _gradient_x(image) + image_dy: torch.Tensor = _gradient_y(image) + + # compute image weights + weights_x: torch.Tensor = torch.exp(-torch.mean(torch.abs(image_dx), dim=1, keepdim=True)) + weights_y: torch.Tensor = torch.exp(-torch.mean(torch.abs(image_dy), dim=1, keepdim=True)) + + # apply image weights to depth + smoothness_x: torch.Tensor = torch.abs(idepth_dx * weights_x) + smoothness_y: torch.Tensor = torch.abs(idepth_dy * weights_y) + + return torch.mean(smoothness_x) + torch.mean(smoothness_y) + + +class InverseDepthSmoothnessLoss(nn.Module): + r"""Criterion that computes image-aware inverse depth smoothness loss. + + .. math:: + + \text{loss} = \left | \partial_x d_{ij} \right | e^{-\left \| + \partial_x I_{ij} \right \|} + \left | + \partial_y d_{ij} \right | e^{-\left \| \partial_y I_{ij} \right \|} + + Shape: + - Inverse Depth: :math:`(N, 1, H, W)` + - Image: :math:`(N, 3, H, W)` + - Output: scalar + + Examples: + >>> idepth = torch.rand(1, 1, 4, 5) + >>> image = torch.rand(1, 3, 4, 5) + >>> smooth = InverseDepthSmoothnessLoss() + >>> loss = smooth(idepth, image) + + """ + + def forward(self, idepth: torch.Tensor, image: torch.Tensor) -> torch.Tensor: + """Compute image-aware smoothness regularization for inverse depth. + + Args: + idepth: Inverse-depth tensor with shape :math:`(B, 1, H, W)`. + image: RGB image tensor with shape :math:`(B, 3, H, W)` used to + reduce the smoothness penalty across visible image edges. + + Returns: + Scalar tensor containing the edge-aware smoothness loss. Smoothness + is encouraged in flat image regions and relaxed near strong image + gradients. + """ + return inverse_depth_smoothness_loss(idepth, image) diff --git a/kornia/losses/dice.py b/kornia/losses/dice.py new file mode 100644 index 0000000..87f40ec --- /dev/null +++ b/kornia/losses/dice.py @@ -0,0 +1,210 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Module containing the Sørensen-Dice Coefficient loss.""" + +from __future__ import annotations + +from typing import Optional + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR +from kornia.losses._utils import mask_ignore_pixels +from kornia.losses.one_hot import one_hot + +# based on: +# https://github.com/kevinzakka/pytorch-goodies/blob/master/losses.py +# https://github.com/Lightning-AI/metrics/blob/v0.11.3/src/torchmetrics/functional/classification/dice.py#L66-L207 + + +def dice_loss( + pred: torch.Tensor, + target: torch.Tensor, + average: str = "micro", + eps: float = 1e-8, + weight: Optional[torch.Tensor] = None, + ignore_index: Optional[int] = -100, +) -> torch.Tensor: + r"""Criterion that computes Sørensen-Dice Coefficient loss. + + According to [1], we compute the Sørensen-Dice Coefficient as follows: + + .. math:: + + \text{Dice}(x, class) = \frac{2 |X \cap Y|}{|X| + |Y|} + + Where: + - :math:`X` expects to be the scores of each class. + - :math:`Y` expects to be the one-hot torch.Tensor with the class labels. + + the loss, is finally computed as: + + .. math:: + + \text{loss}(x, class) = 1 - \text{Dice}(x, class) + + Reference: + [1] https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient + + Args: + pred: logits torch.Tensor with shape :math:`(N, C, H, W)` where C = number of classes. + target: labels torch.Tensor with shape :math:`(N, H, W)` where each value + is in range :math:`0 ≤ targets[i] ≤ C-1`. + average: + Reduction applied in multi-class scenario: + - ``'micro'`` [default]: Calculate the loss across all classes. + - ``'macro'``: Calculate the loss for each class separately and average the metrics across classes. + eps: Scalar to enforce numerical stabiliy. + weight: weights for classes with shape :math:`(num\_of\_classes,)`. + ignore_index: labels with this value are ignored in the loss computation. + + Return: + One-element torch.Tensor of the computed loss. + + Example: + >>> N = 5 # num_classes + >>> pred = torch.randn(1, N, 3, 5, requires_grad=True) + >>> target = torch.empty(1, 3, 5, dtype=torch.long).random_(N) + >>> output = dice_loss(pred, target) + >>> output.backward() + + """ + KORNIA_CHECK_IS_TENSOR(pred) + + if not len(pred.shape) == 4: + raise ValueError(f"Invalid pred shape, we expect BxNxHxW. Got: {pred.shape}") + + if not pred.shape[-2:] == target.shape[-2:]: + raise ValueError(f"pred and target shapes must be the same. Got: {pred.shape} and {target.shape}") + + if not pred.device == target.device: + raise ValueError(f"pred and target must be in the same device. Got: {pred.device} and {target.device}") + num_of_classes = pred.shape[1] + possible_average = {"micro", "macro"} + KORNIA_CHECK(average in possible_average, f"The `average` has to be one of {possible_average}. Got: {average}") + + # compute F.softmax over the classes axis + pred_soft: torch.Tensor = F.softmax(pred, dim=1) + + target, target_mask = mask_ignore_pixels(target, ignore_index) + + # create the labels one hot torch.Tensor + target_one_hot: torch.Tensor = one_hot(target, num_classes=pred.shape[1], device=pred.device, dtype=pred.dtype) + + # mask ignore pixels + if target_mask is not None: + target_mask.unsqueeze_(1) + target_one_hot = target_one_hot * target_mask + pred_soft = pred_soft * target_mask + + # compute the actual dice score + if weight is not None: + KORNIA_CHECK_IS_TENSOR(weight, "weight must be torch.Tensor or None.") + KORNIA_CHECK( + (weight.shape[0] == num_of_classes and weight.numel() == num_of_classes), + f"weight shape must be (num_of_classes,): ({num_of_classes},), got {weight.shape}", + ) + KORNIA_CHECK( + weight.device == pred.device, + f"weight and pred must be in the same device. Got: {weight.device} and {pred.device}", + ) + else: + weight = pred.new_ones(pred.shape[1]) + + # set dimensions for the appropriate averaging + dims: tuple[int, ...] = (2, 3) + + if average == "micro": + dims = (1, *dims) + + weight = weight.view(-1, 1, 1) + pred_soft = pred_soft * weight + target_one_hot = target_one_hot * weight + + intersection = torch.sum(pred_soft * target_one_hot, dims) + cardinality = torch.sum(pred_soft + target_one_hot, dims) + + dice_score = 2.0 * intersection / (cardinality + eps) + dice_loss = -dice_score + 1.0 + + # reduce the loss across samples (and classes in case of `macro` averaging) + if average == "macro": + dice_loss = (dice_loss * weight).sum(-1) / weight.sum() + + dice_loss = torch.mean(dice_loss) + + return dice_loss + + +class DiceLoss(nn.Module): + r"""Criterion that computes Sørensen-Dice Coefficient loss. + + Dice-based objectives are common in medical and semantic segmentation because pixel classes + are often highly imbalanced. This loss directly optimizes region-level agreement. + + Args: + average: Reduction strategy for multi-class computation. Use "micro" to aggregate + classes globally, or "macro" to average class-wise Dice scores. + eps: Small constant added to the denominator for numerical stability. + weight: Optional class-weight tensor of shape :math:`(C,)`. + ignore_index: Label value to exclude from loss computation. + + Shapes: + - pred: :math:`(N, C, H, W)` where C is the number of classes. + - target: :math:`(N, H, W)` where each value is in the range :math:`[0, C-1]`. + - Output: scalar by default. + """ + + def __init__( + self, + average: str = "micro", + eps: float = 1e-8, + weight: Optional[torch.Tensor] = None, + ignore_index: Optional[int] = -100, + ) -> None: + """Initialize the Dice loss module. + + Args: + average: Reduction strategy for multi-class computation. + eps: Small constant added for numerical stability. + weight: Optional class-weight tensor of shape :math:`(C,)`. + ignore_index: Label value to exclude from loss computation. + """ + super().__init__() + self.average = average + self.eps = eps + self.weight = weight + self.ignore_index = ignore_index + + def forward(self, pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + """Compute Sørensen-Dice loss for segmentation logits and labels. + + Args: + pred: Logit tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is batch size, :math:`C` is number of classes, + :math:`H` is height, and :math:`W` is width. + target: Integer target labels with shape :math:`(B, H, W)`. + + Returns: + Scalar tensor containing ``1 - Dice`` after applying the configured + averaging strategy, class weights, numerical epsilon, and optional + ignored label handling. + """ + return dice_loss(pred, target, self.average, self.eps, self.weight, self.ignore_index) diff --git a/kornia/losses/divergence.py b/kornia/losses/divergence.py new file mode 100644 index 0000000..d367d1e --- /dev/null +++ b/kornia/losses/divergence.py @@ -0,0 +1,90 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +r"""Losses based on the divergence between probability distributions.""" + +from __future__ import annotations + +import torch +import torch.nn.functional as F + + +def _kl_div_2d(p: torch.Tensor, q: torch.Tensor) -> torch.Tensor: + # D_KL(P || Q) + batch, chans, height, width = p.shape + unsummed_kl = F.kl_div( + q.reshape(batch * chans, height * width).log(), p.reshape(batch * chans, height * width), reduction="none" + ) + kl_values = unsummed_kl.sum(-1).view(batch, chans) + return kl_values + + +def _js_div_2d(p: torch.Tensor, q: torch.Tensor) -> torch.Tensor: + # JSD(P || Q) + m = 0.5 * (p + q) + return 0.5 * _kl_div_2d(p, m) + 0.5 * _kl_div_2d(q, m) + + +# TODO: add this to the main module +def _reduce_loss(losses: torch.Tensor, reduction: str) -> torch.Tensor: + if reduction == "none": + return losses + return torch.mean(losses) if reduction == "mean" else torch.sum(losses) + + +def js_div_loss_2d(pred: torch.Tensor, target: torch.Tensor, reduction: str = "mean") -> torch.Tensor: + r"""Calculate the Jensen-Shannon divergence loss between heatmaps. + + Args: + pred: the input torch.Tensor with shape :math:`(B, N, H, W)`. + target: the target torch.Tensor with shape :math:`(B, N, H, W)`. + reduction: Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction + will be applied, ``'mean'``: the sum of the output will be divided by + the number of elements in the output, ``'sum'``: the output will be + summed. + + Examples: + >>> pred = torch.full((1, 1, 2, 4), 0.125) + >>> loss = js_div_loss_2d(pred, pred) + >>> loss.item() + 0.0 + + """ + return _reduce_loss(_js_div_2d(target, pred), reduction) + + +def kl_div_loss_2d(pred: torch.Tensor, target: torch.Tensor, reduction: str = "mean") -> torch.Tensor: + r"""Calculate the Kullback-Leibler divergence loss between heatmaps. + + Args: + pred: the input torch.Tensor with shape :math:`(B, N, H, W)`. + target: the target torch.Tensor with shape :math:`(B, N, H, W)`. + reduction: Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction + will be applied, ``'mean'``: the sum of the output will be divided by + the number of elements in the output, ``'sum'``: the output will be + summed. + + Examples: + >>> pred = torch.full((1, 1, 2, 4), 0.125) + >>> loss = kl_div_loss_2d(pred, pred) + >>> loss.item() + 0.0 + + """ + return _reduce_loss(_kl_div_2d(target, pred), reduction) diff --git a/kornia/losses/focal.py b/kornia/losses/focal.py new file mode 100644 index 0000000..7302046 --- /dev/null +++ b/kornia/losses/focal.py @@ -0,0 +1,397 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE +from kornia.losses._utils import mask_ignore_pixels +from kornia.losses.one_hot import one_hot + +# based on: +# https://github.com/zhezh/focalloss/blob/master/focalloss.py + + +def focal_loss( + pred: torch.Tensor, + target: torch.Tensor, + alpha: Optional[float], + gamma: float = 2.0, + reduction: str = "none", + weight: Optional[torch.Tensor] = None, + ignore_index: Optional[int] = -100, +) -> torch.Tensor: + r"""Criterion that computes Focal loss. + + According to :cite:`lin2018focal`, the Focal loss is computed as follows: + + .. math:: + + \text{FL}(p_t) = -\alpha_t (1 - p_t)^{\gamma} \, \text{log}(p_t) + + Where: + - :math:`p_t` is the model's estimated probability for each class. + + Args: + pred: logits torch.Tensor with shape :math:`(N, C, *)` where C = number of classes. + target: labels torch.Tensor with shape :math:`(N, *)` where each value is an integer + representing correct classification :math:`target[i] \in [0, C)`. + alpha: Weighting factor :math:`\alpha \in [0, 1]`. + gamma: Focusing parameter :math:`\gamma >= 0`. + reduction: Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction + will be applied, ``'mean'``: the sum of the output will be divided by + the number of elements in the output, ``'sum'``: the output will be + summed. + weight: weights for classes with shape :math:`(num\_of\_classes,)`. + ignore_index: labels with this value are ignored in the loss computation. + + Return: + the computed loss. + + Example: + >>> C = 5 # num_classes + >>> pred = torch.randn(1, C, 3, 5, requires_grad=True) + >>> target = torch.randint(C, (1, 3, 5)) + >>> kwargs = {"alpha": 0.5, "gamma": 2.0, "reduction": 'mean'} + >>> output = focal_loss(pred, target, **kwargs) + >>> output.backward() + + """ + KORNIA_CHECK_SHAPE(pred, ["B", "C", "*"]) + out_size = (pred.shape[0],) + pred.shape[2:] + KORNIA_CHECK( + (pred.shape[0] == target.shape[0] and target.shape[1:] == pred.shape[2:]), + f"Expected target size {out_size}, got {target.shape}", + ) + KORNIA_CHECK( + pred.device == target.device, + f"pred and target must be in the same device. Got: {pred.device} and {target.device}", + ) + + target, target_mask = mask_ignore_pixels(target, ignore_index) + + # create the labels one hot torch.Tensor + target_one_hot: torch.Tensor = one_hot(target, num_classes=pred.shape[1], device=pred.device, dtype=pred.dtype) + + # mask ignore pixels + if target_mask is not None: + target_mask.unsqueeze_(1) + target_one_hot = target_one_hot * target_mask + + # compute F.softmax over the classes axis + log_pred_soft: torch.Tensor = pred.log_softmax(1) + + # compute the actual focal loss + loss_tmp: torch.Tensor = -torch.pow(1.0 - log_pred_soft.exp(), gamma) * log_pred_soft * target_one_hot + + num_of_classes = pred.shape[1] + broadcast_dims = [-1] + [1] * len(pred.shape[2:]) + if alpha is not None: + alpha_fac = torch.tensor( + [1 - alpha] + [alpha] * (num_of_classes - 1), dtype=loss_tmp.dtype, device=loss_tmp.device + ) + alpha_fac = alpha_fac.view(broadcast_dims) + loss_tmp = alpha_fac * loss_tmp + + if weight is not None: + KORNIA_CHECK_IS_TENSOR(weight, "weight must be torch.Tensor or None.") + KORNIA_CHECK( + (weight.shape[0] == num_of_classes and weight.numel() == num_of_classes), + f"weight shape must be (num_of_classes,): ({num_of_classes},), got {weight.shape}", + ) + KORNIA_CHECK( + weight.device == pred.device, + f"weight and pred must be in the same device. Got: {weight.device} and {pred.device}", + ) + + weight = weight.view(broadcast_dims) + loss_tmp = weight * loss_tmp + + if reduction == "none": + loss = loss_tmp + elif reduction == "mean": + loss = torch.mean(loss_tmp) + elif reduction == "sum": + loss = torch.sum(loss_tmp) + else: + raise NotImplementedError(f"Invalid reduction mode: {reduction}") + return loss + + +class FocalLoss(nn.Module): + r"""Criterion that computes Focal loss. + + According to :cite:`lin2018focal`, the Focal loss is computed as follows: + + .. math:: + + \text{FL}(p_t) = -\alpha_t (1 - p_t)^{\gamma} \, \text{log}(p_t) + + Where: + - :math:`p_t` is the model's estimated probability for each class. + + Args: + alpha: Weighting factor :math:`\alpha \in [0, 1]`. + gamma: Focusing parameter :math:`\gamma >= 0`. + reduction: Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction + will be applied, ``'mean'``: the sum of the output will be divided by + the number of elements in the output, ``'sum'``: the output will be + summed. + weight: weights for classes with shape :math:`(num\_of\_classes,)`. + ignore_index: labels with this value are ignored in the loss computation. + + Shape: + - Pred: :math:`(N, C, *)` where C = number of classes. + - Target: :math:`(N, *)` where each value is an integer + representing correct classification :math:`target[i] \in [0, C)`. + + Example: + >>> C = 5 # num_classes + >>> pred = torch.randn(1, C, 3, 5, requires_grad=True) + >>> target = torch.randint(C, (1, 3, 5)) + >>> kwargs = {"alpha": 0.5, "gamma": 2.0, "reduction": 'mean'} + >>> criterion = FocalLoss(**kwargs) + >>> output = criterion(pred, target) + >>> output.backward() + + """ + + def __init__( + self, + alpha: Optional[float], + gamma: float = 2.0, + reduction: str = "none", + weight: Optional[torch.Tensor] = None, + ignore_index: Optional[int] = -100, + ) -> None: + super().__init__() + self.alpha: Optional[float] = alpha + self.gamma: float = gamma + self.reduction: str = reduction + self.weight: Optional[torch.Tensor] = weight + self.ignore_index: Optional[int] = ignore_index + + def forward(self, pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + """Compute multi-class focal loss from class logits. + + Args: + pred: Logit tensor with shape :math:`(B, C, *)`, where :math:`B` + is batch size, :math:`C` is number of classes, and ``*`` is + any number of spatial dimensions. + target: Integer class-label tensor with shape :math:`(B, *)`. + + Returns: + Focal-loss tensor reduced according to ``self.reduction``. The + focusing factor ``self.gamma`` down-weights easy examples, while + ``self.alpha`` and ``self.weight`` provide optional class weighting. + """ + return focal_loss(pred, target, self.alpha, self.gamma, self.reduction, self.weight, self.ignore_index) + + +def binary_focal_loss_with_logits( + pred: torch.Tensor, + target: torch.Tensor, + alpha: Optional[float] = 0.25, + gamma: float = 2.0, + reduction: str = "none", + pos_weight: Optional[torch.Tensor] = None, + weight: Optional[torch.Tensor] = None, + ignore_index: Optional[int] = -100, +) -> torch.Tensor: + r"""Criterion that computes Binary Focal loss. + + According to :cite:`lin2018focal`, the Focal loss is computed as follows: + + .. math:: + + \text{FL}(p_t) = -\alpha_t (1 - p_t)^{\gamma} \, \text{log}(p_t) + + Where: + - :math:`p_t` is the model's estimated probability for each class. + + Args: + pred: logits torch.Tensor with shape :math:`(N, C, *)` where C = number of classes. + target: labels torch.Tensor with the same shape as pred :math:`(N, C, *)` + where each value is between 0 and 1. + alpha: Weighting factor :math:`\alpha \in [0, 1]`. + gamma: Focusing parameter :math:`\gamma >= 0`. + reduction: Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction + will be applied, ``'mean'``: the sum of the output will be divided by + the number of elements in the output, ``'sum'``: the output will be + summed. + pos_weight: a weight of positive examples with shape :math:`(num\_of\_classes,)`. + It is possible to trade off recall and precision by adding weights to positive examples. + weight: weights for classes with shape :math:`(num\_of\_classes,)`. + ignore_index: labels with this value are ignored in the loss computation. + + Returns: + the computed loss. + + Examples: + >>> C = 3 # num_classes + >>> pred = torch.randn(1, C, 5, requires_grad=True) + >>> target = torch.randint(2, (1, C, 5)) + >>> kwargs = {"alpha": 0.25, "gamma": 2.0, "reduction": 'mean'} + >>> output = binary_focal_loss_with_logits(pred, target, **kwargs) + >>> output.backward() + + """ + KORNIA_CHECK_SHAPE(pred, ["B", "C", "*"]) + KORNIA_CHECK(pred.shape == target.shape, f"Expected target size {pred.shape}, got {target.shape}") + KORNIA_CHECK( + pred.device == target.device, + f"pred and target must be in the same device. Got: {pred.device} and {target.device}", + ) + + log_probs_pos: torch.Tensor = nn.functional.logsigmoid(pred) + log_probs_neg: torch.Tensor = nn.functional.logsigmoid(-pred) + + target, target_mask = mask_ignore_pixels(target, ignore_index) + + if target_mask is not None: + # mask ignore pixels + log_probs_neg = log_probs_neg * target_mask + log_probs_pos = log_probs_pos * target_mask + + pos_term: torch.Tensor = -log_probs_neg.exp().pow(gamma) * target * log_probs_pos + neg_term: torch.Tensor = -log_probs_pos.exp().pow(gamma) * (1.0 - target) * log_probs_neg + if alpha is not None: + pos_term = alpha * pos_term + neg_term = (1.0 - alpha) * neg_term + + num_of_classes = pred.shape[1] + broadcast_dims = [-1] + [1] * len(pred.shape[2:]) + if pos_weight is not None: + KORNIA_CHECK_IS_TENSOR(pos_weight, "pos_weight must be torch.Tensor or None.") + KORNIA_CHECK( + (pos_weight.shape[0] == num_of_classes and pos_weight.numel() == num_of_classes), + f"pos_weight shape must be (num_of_classes,): ({num_of_classes},), got {pos_weight.shape}", + ) + KORNIA_CHECK( + pos_weight.device == pred.device, + f"pos_weight and pred must be in the same device. Got: {pos_weight.device} and {pred.device}", + ) + + pos_weight = pos_weight.view(broadcast_dims) + pos_term = pos_weight * pos_term + + loss_tmp: torch.Tensor = pos_term + neg_term + if weight is not None: + KORNIA_CHECK_IS_TENSOR(weight, "weight must be torch.Tensor or None.") + KORNIA_CHECK( + (weight.shape[0] == num_of_classes and weight.numel() == num_of_classes), + f"weight shape must be (num_of_classes,): ({num_of_classes},), got {weight.shape}", + ) + KORNIA_CHECK( + weight.device == pred.device, + f"weight and pred must be in the same device. Got: {weight.device} and {pred.device}", + ) + + weight = weight.view(broadcast_dims) + loss_tmp = weight * loss_tmp + + if reduction == "none": + loss = loss_tmp + elif reduction == "mean": + loss = torch.mean(loss_tmp) + elif reduction == "sum": + loss = torch.sum(loss_tmp) + else: + raise NotImplementedError(f"Invalid reduction mode: {reduction}") + return loss + + +class BinaryFocalLossWithLogits(nn.Module): + r"""Criterion that computes Focal loss. + + According to :cite:`lin2018focal`, the Focal loss is computed as follows: + + .. math:: + + \text{FL}(p_t) = -\alpha_t (1 - p_t)^{\gamma} \, \text{log}(p_t) + + torch.where: + - :math:`p_t` is the model's estimated probability for each class. + + Args: + alpha: Weighting factor :math:`\alpha \in [0, 1]`. + gamma: Focusing parameter :math:`\gamma >= 0`. + reduction: Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction + will be applied, ``'mean'``: the sum of the output will be divided by + the number of elements in the output, ``'sum'``: the output will be + summed. + pos_weight: a weight of positive examples with shape :math:`(num\_of\_classes,)`. + It is possible to trade off recall and precision by adding weights to positive examples. + weight: weights for classes with shape :math:`(num\_of\_classes,)`. + ignore_index: labels with this value are ignored in the loss computation. + + Shape: + - Pred: :math:`(N, C, *)` where C = number of classes. + - Target: the same shape as Pred :math:`(N, C, *)` + where each value is between 0 and 1. + + Examples: + >>> C = 3 # num_classes + >>> pred = torch.randn(1, C, 5, requires_grad=True) + >>> target = torch.randint(2, (1, C, 5)) + >>> kwargs = {"alpha": 0.25, "gamma": 2.0, "reduction": 'mean'} + >>> criterion = BinaryFocalLossWithLogits(**kwargs) + >>> output = criterion(pred, target) + >>> output.backward() + + """ + + def __init__( + self, + alpha: Optional[float], + gamma: float = 2.0, + reduction: str = "none", + pos_weight: Optional[torch.Tensor] = None, + weight: Optional[torch.Tensor] = None, + ignore_index: Optional[int] = -100, + ) -> None: + super().__init__() + self.alpha: Optional[float] = alpha + self.gamma: float = gamma + self.reduction: str = reduction + self.pos_weight: Optional[torch.Tensor] = pos_weight + self.weight: Optional[torch.Tensor] = weight + self.ignore_index: Optional[int] = ignore_index + + def forward(self, pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + """Compute binary focal loss from logits. + + Args: + pred: Logit tensor for binary predictions with arbitrary shape. + target: Binary target tensor with the same shape as ``pred``. + + Returns: + Binary focal-loss tensor reduced according to ``self.reduction``. + Positive examples may be reweighted by ``self.pos_weight`` and + examples matching ``self.ignore_index`` are ignored when configured. + """ + return binary_focal_loss_with_logits( + pred, target, self.alpha, self.gamma, self.reduction, self.pos_weight, self.weight, self.ignore_index + ) diff --git a/kornia/losses/geman_mcclure.py b/kornia/losses/geman_mcclure.py new file mode 100644 index 0000000..7d5c149 --- /dev/null +++ b/kornia/losses/geman_mcclure.py @@ -0,0 +1,145 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SAME_DEVICE, KORNIA_CHECK_SAME_SHAPE + + +def geman_mcclure_loss(img1: torch.Tensor, img2: torch.Tensor, reduction: str = "none") -> torch.Tensor: + r"""Criterion that computes the Geman-McClure loss [2]. + + According to [1], we compute the Geman-McClure loss as follows: + + .. math:: + + \text{WL}(x, y) = \frac{2 (x - y)^{2}}{(x - y)^{2} + 4} + + Where: + - :math:`x` is the prediction. + - :math:`y` is the target to be regressed to. + + Reference: + [1] https://arxiv.org/pdf/1701.03077.pdf + [2] Bayesian image analysis: An application to single photon emission tomography, Geman and McClure, 1985 + + Args: + img1: the predicted torch.Tensor with shape :math:`(*)`. + img2: the target torch.Tensor with the same shape as img1. + reduction: Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction + will be applied (default), ``'mean'``: the sum of the output will be divided + by the number of elements in the output, ``'sum'``: the output will be + summed. + + Return: + a scalar with the computed loss. + + Example: + >>> img1 = torch.randn(2, 3, 32, 32, requires_grad=True) + >>> img2 = torch.randn(2, 3, 32, 32) + >>> output = geman_mcclure_loss(img1, img2, reduction="mean") + >>> output.backward() + + """ + KORNIA_CHECK_IS_TENSOR(img1) + + KORNIA_CHECK_IS_TENSOR(img2) + + KORNIA_CHECK_SAME_SHAPE(img1, img2) + + KORNIA_CHECK_SAME_DEVICE(img1, img2) + + KORNIA_CHECK( + reduction in ("mean", "sum", "none", None), f"Given type of reduction is not supported. Got: {reduction}" + ) + + # compute loss + diff = img1 - img2 + diff2 = torch.square(diff) + loss = 2.0 * diff2 / (diff2 + 4.0) + + # perform reduction + if reduction == "mean": + loss = loss.mean() + elif reduction == "sum": + loss = loss.sum() + elif reduction == "none" or reduction is None: + pass + else: + raise NotImplementedError("Invalid reduction option.") + + return loss + + +class GemanMcclureLoss(nn.Module): + r"""Criterion that computes the Geman-McClure loss [2]. + + According to [1], we compute the Geman-McClure loss as follows: + + .. math:: + + \text{WL}(x, y) = \frac{2 (x - y)^{2}}{(x - y)^{2} + 4} + + Where: + - :math:`x` is the prediction. + - :math:`y` is the target to be regressed to. + + Reference: + [1] https://arxiv.org/pdf/1701.03077.pdf + [2] Bayesian image analysis: An application to single photon emission tomography, Geman and McClure, 1985 + + Args: + reduction: Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction + will be applied (default), ``'mean'``: the sum of the output will be divided + by the number of elements in the output, ``'sum'``: the output will be + summed. + + Shape: + - img1: the predicted torch.Tensor with shape :math:`(*)`. + - img2: the target torch.Tensor with the same shape as img1. + + Example: + >>> criterion = GemanMcclureLoss(reduction="mean") + >>> img1 = torch.randn(2, 3, 32, 2107, requires_grad=True) + >>> img2 = torch.randn(2, 3, 32, 2107) + >>> output = criterion(img1, img2) + >>> output.backward() + + """ + + def __init__(self, reduction: str = "none") -> None: + super().__init__() + self.reduction = reduction + + def forward(self, img1: torch.Tensor, img2: torch.Tensor) -> torch.Tensor: + """Compute the Geman-McClure robust regression loss. + + Args: + img1: Predicted tensor with arbitrary shape. + img2: Target tensor with the same shape as ``img1``. + + Returns: + Loss tensor reduced according to ``self.reduction``. The robust + penalty limits the contribution of large residuals compared with a + plain squared-error loss. + """ + return geman_mcclure_loss(img1=img1, img2=img2, reduction=self.reduction) diff --git a/kornia/losses/hausdorff.py b/kornia/losses/hausdorff.py new file mode 100644 index 0000000..1b8bf38 --- /dev/null +++ b/kornia/losses/hausdorff.py @@ -0,0 +1,275 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Callable + +import torch +from torch import nn + + +class _HausdorffERLossBase(nn.Module): + """Base class for binary Hausdorff loss based on morphological erosion. + + This is an Hausdorff Distance (HD) Loss that based on morphological erosion,which provided + a differentiable approximation of Hausdorff distance as stated in :cite:`karimi2019reducing`. + The code is refactored on top of `here `__. + + Args: + alpha: controls the erosion rate in each iteration. + k: the number of iterations of erosion. + reduction: Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. + 'none': no reduction will be applied, 'mean': the weighted mean of the output is taken, + 'sum': the output will be summed. + + Returns: + Estimated Hausdorff Loss. + + """ + + conv: Callable[..., torch.Tensor] + max_pool: Callable[..., torch.Tensor] + + def __init__(self, alpha: float = 2.0, k: int = 10, reduction: str = "mean") -> None: + super().__init__() + self.alpha = alpha + self.k = k + self.reduction = reduction + self.register_buffer("kernel", self.get_kernel()) + + def get_kernel(self) -> torch.Tensor: + """Get kernel for image morphology convolution.""" + raise NotImplementedError + + def perform_erosion(self, pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + bound = (pred - target) ** 2 + + kernel = torch.as_tensor(self.kernel, device=pred.device, dtype=pred.dtype) + eroded = torch.zeros_like(bound, device=pred.device, dtype=pred.dtype) + mask = torch.ones_like(bound, device=pred.device, dtype=torch.bool) + + # Same padding, assuming kernel is odd and square (cube) shaped. + padding = (kernel.size(-1) - 1) // 2 + for k in range(self.k): + # compute convolution with kernel + dilation = self.conv(bound, weight=kernel, padding=padding, groups=1) + # apply soft thresholding at 0.5 and F.normalize + erosion = dilation - 0.5 + erosion[erosion < 0] = 0 + + # image-wise differences for 2D images + erosion_max = self.max_pool(erosion) + erosion_min = -self.max_pool(-erosion) + # No normalization needed if `max - min = 0` + _to_norm = (erosion_max - erosion_min) != 0 + to_norm = _to_norm.squeeze() + if to_norm.any(): + # NOTE: avoid in-place ops like below, which will not pass gradcheck: + # erosion[to_norm] = (erosion[to_norm] - erosion_min[to_norm]) / ( + # erosion_max[to_norm] - erosion_min[to_norm]) + _erosion_to_fill = (erosion - erosion_min) / (erosion_max - erosion_min) + erosion = torch.where(mask * _to_norm, _erosion_to_fill, erosion) + + # save erosion and add to loss + eroded = eroded + erosion * (k + 1) ** self.alpha + bound = erosion + + return eroded + + def forward(self, pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + """Compute Hausdorff loss. + + Args: + pred: predicted torch.Tensor with a shape of :math:`(B, C, H, W)` or :math:`(B, C, D, H, W)`. + Each channel is as binary as: 1 -> fg, 0 -> bg. + target: target torch.Tensor with a shape of :math:`(B, 1, H, W)` or :math:`(B, C, D, H, W)`. + + Returns: + Estimated Hausdorff Loss. + + """ + if not (pred.shape[2:] == target.shape[2:] and pred.size(0) == target.size(0) and target.size(1) == 1): + raise ValueError( + "Prediction and target need to be of same size, and target should not be one-hot." + f"Got {pred.shape} and {target.shape}." + ) + + if pred.size(1) < target.max().item(): + raise ValueError("Invalid target value.") + + out = torch.stack( + [ + self.perform_erosion( + pred[:, i : i + 1], + torch.where( + target == i, + torch.tensor(1, device=target.device, dtype=target.dtype), + torch.tensor(0, device=target.device, dtype=target.dtype), + ), + ) + for i in range(pred.size(1)) + ] + ) + + if self.reduction == "mean": + out = out.mean() + elif self.reduction == "sum": + out = out.sum() + elif self.reduction == "none": + pass + else: + raise NotImplementedError(f"reduction `{self.reduction}` has not been implemented yet.") + + return out + + +class HausdorffERLoss(_HausdorffERLossBase): + r"""Binary Hausdorff loss based on morphological erosion. + + Hausdorff Distance loss measures the maximum distance of a predicted segmentation boundary to + the nearest ground-truth edge pixel. For two segmentation point sets X and Y , + the one-sided HD from X to Y is defined as: + + .. math:: + + hd(X,Y) = \max_{x \in X} \min_{y \in Y}||x - y||_2 + + Furthermore, the bidirectional HD is: + + .. math:: + + HD(X,Y) = max(hd(X, Y), hd(Y, X)) + + This is an Hausdorff Distance (HD) Loss that based on morphological erosion, which provided + a differentiable approximation of Hausdorff distance as stated in :cite:`karimi2019reducing`. + The code is refactored on top of `here `__. + + Args: + alpha: controls the erosion rate in each iteration. + k: the number of iterations of erosion. + reduction: Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. + 'none': no reduction will be applied, 'mean': the weighted mean of the output is taken, + 'sum': the output will be summed. + + Examples: + >>> hdloss = HausdorffERLoss() + >>> input = torch.randn(5, 3, 20, 20) + >>> target = (torch.rand(5, 1, 20, 20) * 2).long() + >>> res = hdloss(input, target) + + """ + + conv = torch.conv2d + max_pool = nn.AdaptiveMaxPool2d(1) + + def get_kernel(self) -> torch.Tensor: + """Get kernel for image morphology convolution.""" + cross = torch.tensor([[[0, 1, 0], [1, 1, 1], [0, 1, 0]]]) + kernel = cross * 0.2 + return kernel[None] + + def forward(self, pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + """Compute Hausdorff loss. + + Args: + pred: predicted torch.Tensor with a shape of :math:`(B, C, H, W)`. + Each channel is as binary as: 1 -> fg, 0 -> bg. + target: target torch.Tensor with a shape of :math:`(B, 1, H, W)`. + + Returns: + Estimated Hausdorff Loss. + + """ + if pred.dim() != 4: + raise ValueError(f"Only 2D images supported. Got {pred.dim()}.") + + if not (target.max() < pred.size(1) and target.min() >= 0 and target.dtype == torch.long): + raise ValueError( + f"Expect long type target value in range (0, {pred.size(1)}). ({target.min()}, {target.max()})" + ) + return super().forward(pred, target) + + +class HausdorffERLoss3D(_HausdorffERLossBase): + r"""Binary 3D Hausdorff loss based on morphological erosion. + + Hausdorff Distance loss measures the maximum distance of a predicted segmentation boundary to + the nearest ground-truth edge pixel. For two segmentation point sets X and Y , + the one-sided HD from X to Y is defined as: + + .. math:: + + hd(X,Y) = \max_{x \in X} \min_{y \in Y}||x - y||_2 + + Furthermore, the bidirectional HD is: + + .. math:: + + HD(X,Y) = max(hd(X, Y), hd(Y, X)) + + This is a 3D Hausdorff Distance (HD) Loss that based on morphological erosion, which provided + a differentiable approximation of Hausdorff distance as stated in :cite:`karimi2019reducing`. + The code is refactored on top of `here `__. + + Args: + alpha: controls the erosion rate in each iteration. + k: the number of iterations of erosion. + reduction: Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. + 'none': no reduction will be applied, 'mean': the weighted mean of the output is taken, + 'sum': the output will be summed. + + Examples: + >>> hdloss = HausdorffERLoss3D() + >>> input = torch.randn(5, 3, 20, 20, 20) + >>> target = (torch.rand(5, 1, 20, 20, 20) * 2).long() + >>> res = hdloss(input, target) + + """ + + conv = torch.conv3d + max_pool = nn.AdaptiveMaxPool3d(1) + + def get_kernel(self) -> torch.Tensor: + """Get kernel for image morphology convolution.""" + cross = torch.tensor([[[0, 1, 0], [1, 1, 1], [0, 1, 0]]]) + bound = torch.tensor([[[0, 0, 0], [0, 1, 0], [0, 0, 0]]]) + # NOTE: The original repo claimed it shaped as (3, 1, 3, 3) + # which Jian suspect it is wrongly implemented. + # https://github.com/PatRyg99/HausdorffLoss/blob/9f580acd421af648e74b45d46555ccb7a876c27c/hausdorff_loss.py#L94 + kernel = torch.stack([bound, cross, bound], 1) * (1 / 7) + return kernel[None] + + def forward(self, pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + """Compute 3D Hausdorff loss. + + Args: + pred: predicted torch.Tensor with a shape of :math:`(B, C, D, H, W)`. + Each channel is as binary as: 1 -> fg, 0 -> bg. + target: target torch.Tensor with a shape of :math:`(B, 1, D, H, W)`. + + Returns: + Estimated Hausdorff Loss. + + """ + if pred.dim() != 5: + raise ValueError(f"Only 3D images supported. Got {pred.dim()}.") + + return super().forward(pred, target) diff --git a/kornia/losses/lovasz_hinge.py b/kornia/losses/lovasz_hinge.py new file mode 100644 index 0000000..6b88c80 --- /dev/null +++ b/kornia/losses/lovasz_hinge.py @@ -0,0 +1,167 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +from torch import Tensor, nn + +from kornia.core.check import KORNIA_CHECK_SHAPE + +# based on: +# https://github.com/bermanmaxim/LovaszSoftmax + + +def lovasz_hinge_loss(pred: Tensor, target: Tensor) -> Tensor: + r"""Criterion that computes a surrogate binary intersection-over-union (IoU) loss. + + According to [2], we compute the IoU as follows: + + .. math:: + + \text{IoU}(x, class) = \frac{|X \cap Y|}{|X \cup Y|} + + [1] approximates this fomular with a surrogate, which is fully differentable. + + Where: + - :math:`X` expects to be the scores of each class. + - :math:`Y` expects to be the binary tensor with the class labels. + + the loss, is finally computed as: + + .. math:: + + \text{loss}(x, class) = 1 - \text{IoU}(x, class) + + Reference: + [1] http://proceedings.mlr.press/v37/yub15.pdf + [2] https://arxiv.org/pdf/1705.08790.pdf + + .. note:: + This loss function only supports binary labels. For multi-class labels please + use the Lovasz-Softmax loss. + + Args: + pred: logits tensor with shape :math:`(N, 1, H, W)`. + target: labels tensor with shape :math:`(N, H, W)` with binary values. + + Return: + a scalar with the computed loss. + + Example: + >>> N = 1 # num_classes + >>> pred = torch.randn(1, N, 3, 5, requires_grad=True) + >>> target = torch.empty(1, 3, 5, dtype=torch.long).random_(N) + >>> output = lovasz_hinge_loss(pred, target) + >>> output.backward() + + """ + KORNIA_CHECK_SHAPE(pred, ["B", "1", "H", "W"]) + + KORNIA_CHECK_SHAPE(target, ["B", "H", "W"]) + + if not pred.shape[-2:] == target.shape[-2:]: + raise ValueError(f"pred and target shapes must be the same. Got: {pred.shape} and {target.shape}") + + if not pred.device == target.device: + raise ValueError(f"pred and target must be in the same device. Got: {pred.device} and {target.device}") + + # flatten pred and target [B, -1] and to float + pred_flatten: Tensor = pred.reshape(pred.shape[0], -1) + target_flatten: Tensor = target.reshape(target.shape[0], -1) + + # get shapes + B, N = pred_flatten.shape + + # compute actual loss + signs = 2.0 * target_flatten - 1.0 + errors = 1.0 - pred_flatten * signs + errors_sorted, permutation = errors.sort(dim=1, descending=True) + batch_index: Tensor = torch.arange(B, device=pred.device).reshape(-1, 1).repeat(1, N).reshape(-1) + target_sorted: Tensor = target_flatten[batch_index, permutation.view(-1)] + target_sorted = target_sorted.view(B, N) + target_sorted_sum: Tensor = target_sorted.sum(1, keepdim=True) + intersection: Tensor = target_sorted_sum - target_sorted.cumsum(1) + union: Tensor = target_sorted_sum + (1.0 - target_sorted).cumsum(1) + gradient: Tensor = 1.0 - intersection / union + if N > 1: + gradient[..., 1:] = gradient[..., 1:] - gradient[..., :-1] + loss: Tensor = (errors_sorted.relu() * gradient).sum(1).mean() + return loss + + +class LovaszHingeLoss(nn.Module): + r"""Criterion that computes a surrogate binary intersection-over-union (IoU) loss. + + According to [2], we compute the IoU as follows: + + .. math:: + + \text{IoU}(x, class) = \frac{|X \cap Y|}{|X \cup Y|} + + [1] approximates this fomular with a surrogate, which is fully differentable. + + Where: + - :math:`X` expects to be the scores of each class. + - :math:`Y` expects to be the binary tensor with the class labels. + + the loss, is finally computed as: + + .. math:: + + \text{loss}(x, class) = 1 - \text{IoU}(x, class) + + Reference: + [1] http://proceedings.mlr.press/v37/yub15.pdf + [2] https://arxiv.org/pdf/1705.08790.pdf + + .. note:: + This loss function only supports binary labels. For multi-class labels please + use the Lovasz-Softmax loss. + + Args: + pred: logits tensor with shape :math:`(N, 1, H, W)`. + labels: labels tensor with shape :math:`(N, H, W)` with binary values. + + Return: + a scalar with the computed loss. + + Example: + >>> N = 1 # num_classes + >>> criterion = LovaszHingeLoss() + >>> pred = torch.randn(1, N, 3, 5, requires_grad=True) + >>> target = torch.empty(1, 3, 5, dtype=torch.long).random_(N) + >>> output = criterion(pred, target) + >>> output.backward() + + """ + + def __init__(self) -> None: + super().__init__() + + def forward(self, pred: Tensor, target: Tensor) -> Tensor: + """Compute binary Lovasz hinge loss for segmentation logits. + + Args: + pred: Binary logit tensor with shape :math:`(B, 1, H, W)`. + target: Binary label tensor with shape :math:`(B, H, W)`. + + Returns: + Scalar tensor containing the Lovasz hinge surrogate for the + intersection-over-union objective. + """ + return lovasz_hinge_loss(pred=pred, target=target) diff --git a/kornia/losses/lovasz_softmax.py b/kornia/losses/lovasz_softmax.py new file mode 100644 index 0000000..c7e8d9d --- /dev/null +++ b/kornia/losses/lovasz_softmax.py @@ -0,0 +1,198 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional + +import torch +from torch import Tensor, nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE + +# based on: +# https://github.com/bermanmaxim/LovaszSoftmax + + +def lovasz_softmax_loss(pred: Tensor, target: Tensor, weight: Optional[Tensor] = None) -> Tensor: + r"""Criterion that computes a surrogate multi-class intersection-over-union (IoU) loss. + + According to [1], we compute the IoU as follows: + + .. math:: + + \text{IoU}(x, class) = \frac{|X \cap Y|}{|X \cup Y|} + + [1] approximates this fomular with a surrogate, which is fully differentable. + + Where: + - :math:`X` expects to be the scores of each class. + - :math:`Y` expects to be the long tensor with the class labels. + + the loss, is finally computed as: + + .. math:: + + \text{loss}(x, class) = 1 - \text{IoU}(x, class) + + Reference: + [1] https://arxiv.org/pdf/1705.08790.pdf + + .. note:: + This loss function only supports multi-class (C > 1) labels. For binary + labels please use the Lovasz-Hinge loss. + + Args: + pred: logits tensor with shape :math:`(N, C, H, W)` where C = number of classes > 1. + target: labels tensor with shape :math:`(N, H, W)` where each value + is in range :math:`0 ≤ targets[i] ≤ C-1`. + weight: weights for classes with shape :math:`(num\_of\_classes,)`. + + Return: + a scalar with the computed loss. + + Example: + >>> N = 5 # num_classes + >>> pred = torch.randn(1, N, 3, 5, requires_grad=True) + >>> target = torch.empty(1, 3, 5, dtype=torch.long).random_(N) + >>> output = lovasz_softmax_loss(pred, target) + >>> output.backward() + + """ + KORNIA_CHECK_SHAPE(pred, ["B", "N", "H", "W"]) + + KORNIA_CHECK_SHAPE(target, ["B", "H", "W"]) + + if not pred.shape[1] > 1: + raise ValueError(f"Invalid pred shape, we expect BxNxHxW, with N > 1. Got: {pred.shape}") + + if not pred.shape[-2:] == target.shape[-2:]: + raise ValueError(f"pred and target shapes must be the same. Got: {pred.shape} and {target.shape}") + + if not pred.device == target.device: + raise ValueError(f"pred and target must be in the same device. Got: {pred.device} and {target.device}") + + num_of_classes = pred.shape[1] + # compute the actual dice score + if weight is not None: + KORNIA_CHECK_IS_TENSOR(weight, "weight must be Tensor or None.") + KORNIA_CHECK( + (weight.shape[0] == num_of_classes and weight.numel() == num_of_classes), + f"weight shape must be (num_of_classes,): ({num_of_classes},), got {weight.shape}", + ) + KORNIA_CHECK( + weight.device == pred.device, + f"weight and pred must be in the same device. Got: {weight.device} and {pred.device}", + ) + + # flatten pred [B, C, -1] and target [B, -1] and to float + pred_flatten: Tensor = pred.reshape(pred.shape[0], pred.shape[1], -1) + target_flatten: Tensor = target.reshape(target.shape[0], -1) + + # get shapes + B, C, N = pred_flatten.shape + + # compute softmax over the classes axis + pred_soft: Tensor = pred_flatten.softmax(1) + + # compute actual loss + foreground: Tensor = ( + torch.nn.functional.one_hot(target_flatten.to(torch.int64), num_classes=C).permute(0, 2, 1).to(pred.dtype) + ) + errors: Tensor = (pred_soft - foreground).abs() + errors_sorted, permutations = torch.sort(errors, dim=2, descending=True) + batch_index = torch.arange(B, device=pred.device).unsqueeze(1).unsqueeze(2).expand(B, C, N) + target_sorted = target_flatten[batch_index, permutations] + target_sorted_sum = target_sorted.sum(2, keepdim=True) + intersection = target_sorted_sum - target_sorted.cumsum(2) + union = target_sorted_sum + (1.0 - target_sorted).cumsum(2) + gradient = 1.0 - intersection / union + if N > 1: + gradient[..., 1:] = gradient[..., 1:] - gradient[..., :-1] + weighted_errors = errors_sorted * gradient + loss_per_class = weighted_errors.sum(2).mean(0) + if weight is not None: + loss_per_class *= weight + final_loss: Tensor = loss_per_class.mean() + return final_loss + + +class LovaszSoftmaxLoss(nn.Module): + r"""Criterion that computes a surrogate multi-class intersection-over-union (IoU) loss. + + According to [1], we compute the IoU as follows: + + .. math:: + + \text{IoU}(x, class) = \frac{|X \cap Y|}{|X \cup Y|} + + [1] approximates this fomular with a surrogate, which is fully differentable. + + Where: + - :math:`X` expects to be the scores of each class. + - :math:`Y` expects to be the binary tensor with the class labels. + + the loss, is finally computed as: + + .. math:: + + \text{loss}(x, class) = 1 - \text{IoU}(x, class) + + Reference: + [1] https://arxiv.org/pdf/1705.08790.pdf + + .. note:: + This loss function only supports multi-class (C > 1) labels. For binary + labels please use the Lovasz-Hinge loss. + + Args: + pred: logits tensor with shape :math:`(N, C, H, W)` where C = number of classes > 1. + labels: labels tensor with shape :math:`(N, H, W)` where each value + is in range :math:`0 ≤ targets[i] ≤ C-1`. + weight: weights for classes with shape :math:`(num\_of\_classes,)`. + + Return: + a scalar with the computed loss. + + Example: + >>> N = 5 # num_classes + >>> criterion = LovaszSoftmaxLoss() + >>> pred = torch.randn(1, N, 3, 5, requires_grad=True) + >>> target = torch.empty(1, 3, 5, dtype=torch.long).random_(N) + >>> output = criterion(pred, target) + >>> output.backward() + + """ + + def __init__(self, weight: Optional[Tensor] = None) -> None: + super().__init__() + self.weight = weight + + def forward(self, pred: Tensor, target: Tensor) -> Tensor: + """Compute multi-class Lovasz-Softmax loss. + + Args: + pred: Class logit tensor with shape :math:`(B, C, H, W)`, where + :math:`C` is the number of classes. + target: Integer class-label tensor with shape :math:`(B, H, W)`. + + Returns: + Scalar tensor containing the Lovasz-Softmax surrogate for the + mean intersection-over-union objective, using ``self.weight`` when + class weights are configured. + """ + return lovasz_softmax_loss(pred=pred, target=target, weight=self.weight) diff --git a/kornia/losses/ms_ssim.py b/kornia/losses/ms_ssim.py new file mode 100644 index 0000000..c4455f1 --- /dev/null +++ b/kornia/losses/ms_ssim.py @@ -0,0 +1,204 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional, Sequence + +import torch +import torch.nn.functional as F +from torch import nn + +# Based on: +# https://github.com/psyrocloud/MS-SSIM_L1_LOSS + + +class MS_SSIMLoss(nn.Module): + r"""Creates a criterion that computes MSSIM + L1 loss. + + According to [1], we compute the MS_SSIM + L1 loss as follows: + + .. math:: + \text{loss}(x, y) = \alpha \cdot \mathcal{L_{MSSIM}}(x,y)+(1 - \alpha) \cdot G_\alpha \cdot \mathcal{L_1}(x,y) + + Where: + - :math:`\alpha` is the weight parameter. + - :math:`x` and :math:`y` are the reconstructed and true reference images. + - :math:`\mathcal{L_{MSSIM}}` is the MS-SSIM loss. + - :math:`G_\alpha` is the sigma values for computing multi-scale SSIM. + - :math:`\mathcal{L_1}` is the L1 loss. + + Reference: + [1]: https://research.nvidia.com/sites/default/files/pubs/2017-03_Loss-Functions-for/NN_ImgProc.pdf#page11 + + Args: + sigmas: gaussian sigma values. + data_range: the range of the images. + K: k values. + alpha : specifies the alpha value + compensation: specifies the scaling coefficient. + reduction : Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the sum of the output will be divided by the number of elements + in the output, ``'sum'``: the output will be summed. + + Returns: + The computed loss. + + Shape: + - Input1: :math:`(N, C, H, W)`. + - Input2: :math:`(N, C, H, W)`. + - Output: :math:`(N, H, W)` or scalar if reduction is set to ``'mean'`` or ``'sum'``. + + Examples: + >>> input1 = torch.rand(1, 3, 5, 5) + >>> input2 = torch.rand(1, 3, 5, 5) + >>> criterion = kornia.losses.MS_SSIMLoss() + >>> loss = criterion(input1, input2) + + """ + + def __init__( + self, + sigmas: Sequence[float] = (0.5, 1.0, 2.0, 4.0, 8.0), + data_range: float = 1.0, + K: tuple[float, float] = (0.01, 0.03), + alpha: float = 0.025, + compensation: float = 200.0, + reduction: str = "mean", + ) -> None: + super().__init__() + self.DR: float = data_range + self.C1: float = (K[0] * data_range) ** 2 + self.C2: float = (K[1] * data_range) ** 2 + self.pad = int(2 * sigmas[-1]) + self.alpha: float = alpha + self.compensation: float = compensation + self.reduction: str = reduction + + # Set filter size + filter_size = int(4 * sigmas[-1] + 1) + g_masks = torch.zeros((3 * len(sigmas), 1, filter_size, filter_size)) + + # Compute mask at different scales + for idx, sigma in enumerate(sigmas): + g_masks[3 * idx + 0, 0, :, :] = self._fspecial_gauss_2d(filter_size, sigma) + g_masks[3 * idx + 1, 0, :, :] = self._fspecial_gauss_2d(filter_size, sigma) + g_masks[3 * idx + 2, 0, :, :] = self._fspecial_gauss_2d(filter_size, sigma) + + self.register_buffer("_g_masks", g_masks) + + def _fspecial_gauss_1d( + self, size: int, sigma: float, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None + ) -> torch.Tensor: + """Create 1-D gauss kernel. + + Args: + size: the size of gauss kernel. + sigma: sigma of normal distribution. + device: device to store the result on. + dtype: dtype of the result. + + Returns: + 1D kernel (size). + + """ + coords = torch.arange(size, device=device, dtype=dtype) + coords -= size // 2 + g = torch.exp(-(coords**2) / (2 * sigma**2)) + g /= g.sum() + return g.reshape(-1) + + def _fspecial_gauss_2d( + self, size: int, sigma: float, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None + ) -> torch.Tensor: + """Create 2-D gauss kernel. + + Args: + size: the size of gauss kernel. + sigma: sigma of normal distribution. + device: device to store the result on. + dtype: dtype of the result. + + Returns: + 2D kernel (size x size). + + """ + gaussian_vec = self._fspecial_gauss_1d(size, sigma, device, dtype) + return torch.outer(gaussian_vec, gaussian_vec) + + def forward(self, img1: torch.Tensor, img2: torch.Tensor) -> torch.Tensor: + """Compute MS_SSIM loss. + + Args: + img1: the predicted image with shape :math:`(B, C, H, W)`. + img2: the target image with a shape of :math:`(B, C, H, W)`. + + Returns: + Estimated MS-SSIM_L1 loss. + + """ + if not isinstance(img1, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(img1)}") + + if not isinstance(img2, torch.Tensor): + raise TypeError(f"Output type is not a torch.Tensor. Got {type(img2)}") + + if not len(img1.shape) == len(img2.shape): + raise ValueError(f"Input shapes should be same. Got {type(img1)} and {type(img2)}.") + + g_masks: torch.Tensor = torch.jit.annotate(torch.Tensor, self._g_masks) + + CH: int = img1.shape[-3] + mux = F.conv2d(img1, g_masks, groups=CH, padding=self.pad) + muy = F.conv2d(img2, g_masks, groups=CH, padding=self.pad) + mux2 = mux * mux + muy2 = muy * muy + muxy = mux * muy + + sigmax2 = F.conv2d(img1 * img1, g_masks, groups=CH, padding=self.pad) - mux2 + sigmay2 = F.conv2d(img2 * img2, g_masks, groups=CH, padding=self.pad) - muy2 + sigmaxy = F.conv2d(img1 * img2, g_masks, groups=CH, padding=self.pad) - muxy + + lc = (2 * muxy + self.C1) / (mux2 + muy2 + self.C1) + cs = (2 * sigmaxy + self.C2) / (sigmax2 + sigmay2 + self.C2) + lM = lc[:, -1, :, :] * lc[:, -2, :, :] * lc[:, -3, :, :] + PIcs = cs.prod(dim=1) + + # Compute MS-SSIM loss + loss_ms_ssim = 1 - lM * PIcs + + # TODO: pass pointer to function e.g. to make more custom with mse, cosine, etc. + # Compute L1 loss + loss_l1 = F.l1_loss(img1, img2, reduction="none") + + # Compute average l1 loss in 3 channels + gaussian_l1 = F.conv2d(loss_l1, g_masks[-CH:], groups=CH, padding=self.pad).mean(1) + + # Compute MS-SSIM + L1 loss + loss = self.alpha * loss_ms_ssim + (1 - self.alpha) * gaussian_l1 / self.DR + loss = self.compensation * loss + + if self.reduction == "mean": + loss = torch.mean(loss) + elif self.reduction == "sum": + loss = torch.sum(loss) + elif self.reduction == "none": + pass + else: + raise NotImplementedError(f"Invalid reduction mode: {self.reduction}") + return loss diff --git a/kornia/losses/mutual_information.py b/kornia/losses/mutual_information.py new file mode 100644 index 0000000..e2fdc95 --- /dev/null +++ b/kornia/losses/mutual_information.py @@ -0,0 +1,895 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +from enum import Enum, member +from functools import partial + +import torch + + +def xu_kernel(x: torch.Tensor, window_radius: float = 1.0) -> torch.Tensor: + """Implementation of a 2nd-order polynomial kernel for Kernel density estimate (Xu et al., 2008). + + Support: [-window_radius, window_radius]. Returns 0 outside this range. + Ref: "Parzen-Window Based Normalized Mutual Information for Medical Image Registration", Eq. 22. + + Args: + x (torch.Tensor): signal, any shape + window_radius (float): radius of window for the kernel + + Returns: + torch.Tensor: transformed signal + """ + x_abs = x.abs().mul(1.0 / window_radius) + + poly1 = x_abs * (-1.8 * x_abs - 0.1) + 1.0 + poly2 = x_abs * (1.8 * x_abs - 3.7) + 1.9 + + return torch.where( + x_abs < 0.5, poly1, torch.where(x_abs <= 1.0, poly2, torch.tensor(0.0, device=x.device, dtype=x.dtype)) + ) + + +def rectangular_kernel(x: torch.Tensor, window_radius: float = 1.0) -> torch.Tensor: + """Implementation of a rectangular kernel. + + Support: [-window_radius, window_radius]. Returns 1.0 inside this range, 0.0 otherwise. + + Args: + x (torch.Tensor): signal, any shape + window_radius (float): radius of window for the kernel + + Returns: + torch.Tensor: transformed signal + """ + x = torch.abs(x) + return torch.where(x <= window_radius, 1.0, 0.0) + + +def truncated_gaussian_kernel(x: torch.Tensor, window_radius: float = 1.0) -> torch.Tensor: + """Implementation of a truncated Gaussian kernel. + + Support: [-window_radius, window_radius]. Returns Gaussian value inside this range, 0.0 otherwise. + Sigma is set to window_radius. + + Args: + x (torch.Tensor): signal, any shape + window_radius (float): radius of window for the kernel (used as sigma) + + Returns: + torch.Tensor: transformed signal + """ + sigma = window_radius + mask = torch.abs(x) <= window_radius + + gaussian_val = torch.exp(-0.5 * (x / sigma) ** 2) / (sigma * (2 * torch.pi) ** 0.5) + + return torch.where(mask, gaussian_val, 0.0) + + +class MIKernel(Enum): + """Available kernels for mutual-information density estimation. + + The enum values are callable kernels used to softly assign signal samples + to histogram bins. They are used by the entropy-based mutual-information + losses to build differentiable joint histograms. + """ + + xu = member(xu_kernel) + rectangular = member(rectangular_kernel) + truncated_gaussian = member(truncated_gaussian_kernel) + + +def _flatten_mask(mask: torch.Tensor | None) -> torch.Tensor: + if mask is None: + return torch.tensor([True]) + else: + return mask.view(-1) + + +def _normalize_signal(data: torch.Tensor, num_bins: int, eps: float = 1e-8) -> torch.Tensor: + min_val, _ = data.min(dim=-1) + max_val, _ = data.max(dim=-1) + diff = (max_val - min_val).unsqueeze(-1) + # signal is considered trivial if too low variation + return torch.where(diff > eps, (data - min_val.unsqueeze(-1)) / diff * num_bins, 0) + + +def _joint_histogram_to_entropies(joint_histogram: torch.Tensor, eps: float = 1e-8) -> torch.Tensor: + P_xy = joint_histogram + # clamp for numerical stability + P_xy = P_xy.clamp(eps) + # divide by sum to get a density + P_xy /= P_xy.sum(dim=(-1, -2), keepdim=True) + + P_x = P_xy.sum(dim=-2) + P_y = P_xy.sum(dim=-1) + H_xy = torch.sum(-P_xy * torch.log(P_xy), dim=(-1, -2)) + H_x = torch.sum(-P_x * torch.log(P_x), dim=-1) + H_y = torch.sum(-P_y * torch.log(P_y), dim=-1) + + return H_x, H_y, H_xy + + +class EntropyBasedLossBase(torch.nn.Module): + """A base class for entropy-based loss functions using kernel density estimation. + + This class provides the foundation for computing entropy-based losses between signals + by estimating probability distributions using kernel density estimation (KDE). It + computes joint histograms and derives entropy measures that can be used to quantify + the similarity or dissimilarity between signals. + + The class pre-processes a reference signal and provides methods to compute joint + histograms with other signals, from which various entropy measures (joint, marginal, + conditional, mutual information) can be derived in subclasses. + """ + + def __init__( + self, + reference_signal: torch.Tensor, + mask: torch.Tensor | None = None, + kernel_function: MIKernel = MIKernel.xu, + num_bins: int = 64, + window_radius: float = 1.0, + ) -> None: + """Initialize the entropy-based loss base module. + + Args: + reference_signal (torch.Tensor): reference signal to which + other signals will be compared by the forward method + mask (torch.Tensor | None): mask of roi in reference_signal, by default None + singleton shape, equal to reference_signal.shape[:-1]. It is a common mask for all the samples + in reference_signal. + kernel_function (MIKernel): Used kernel function for kernel + density estimate, by default MIKernel.xu + num_bins (int): number of signal value bins in kernel + density estimate, by default 64 + window_radius (float): radius of the kernel's support + interval, by default 1.0 + + Raises: + ValueError: If kernel_function is not a valid MIKernel member. + """ + super().__init__() + mask = self.fix_mask(mask, reference_signal) + self.eps = torch.finfo(reference_signal.dtype).eps + self.initial_shape = reference_signal.shape + signal = reference_signal[..., mask] + self.register_buffer("signal", _normalize_signal(signal, num_bins, self.eps)) + self.register_buffer("mask", mask) + self.num_bins = num_bins + if kernel_function not in MIKernel: + raise ValueError( + f"The passed_kernel_function is not an accepted MIKernel, the available options are {list(MIKernel)}." + ) + self.kernel_function = partial(kernel_function.value, window_radius=window_radius) + self.window_radius = window_radius + self.bin_centers = torch.arange(self.num_bins, device=self.signal.device) + + @staticmethod + def fix_mask(mask: torch.Tensor, masked_guy: torch.Tensor) -> torch.Tensor: + """Convert mask to correct full mask if it is None. + + Args: + mask (torch.Tensor | None): input mask + masked_guy (torch.Tensor): the tensor to be masked by mask + + Returns: + torch.Tensor: normalized mask + """ + if mask is None: + mask = torch.tensor(1, dtype=torch.bool) + if mask.ndim > 1: + raise ValueError("the mask has to be a common mask for all elements of the batch, not a batch of masks") + mask = mask.broadcast_to(masked_guy.shape[-1]) + return mask.to(masked_guy.device) + + # TODO: optimize method below, maybe with ihdex coordinates conversion + def trace_in_ref_mask(self, other_signal, other_mask): + """Align another masked signal with the reference mask positions. + + The reference signal can be stored with a mask, while the compared + signal may use a different mask over the same flattened coordinate + space. This helper restores the compared signal to the original flat + shape when needed, then selects the elements that correspond to the + stored reference mask. + + Args: + other_signal: Flattened signal values after applying ``other_mask``. + other_mask: Boolean mask describing which flattened positions are + present in ``other_signal``. + + Returns: + Signal values traced onto the reference mask used by this loss + instance. + """ + if other_mask.all(): + return other_signal[..., self.mask] + + intermediate = torch.zeros(self.initial_shape).to(other_signal) + intermediate[..., other_mask] = other_signal + return intermediate[..., self.mask] + + def _compute_joint_histogram( + self, other_signal: torch.Tensor, eps: float, other_mask: torch.Tensor | None = None + ) -> torch.Tensor: + """Compute the joint histogram between the reference signal and another signal. + + Uses kernel density estimation to estimate the joint probability distribution + between the reference signal and the provided other signal. The histogram is + computed by evaluating kernel functions at discretized signal values. + + Args: + other_signal (torch.Tensor): Signal to compare with the reference signal. + Must have the same shape as the reference signal. + eps (float): Epsilon value for numerical stability in computations. + other_mask (torch.Tensor): common mask of roi for all the samples in other_signal, defaults to None + + Returns: + torch.Tensor: Joint histogram tensor with shape [..., num_bins, num_bins] + representing the estimated joint probability distribution of the passed signals in the intersection + of rois. + + Raises: + ValueError: If other_signal has incompatible shape with reference signal. + """ + if other_signal.shape != self.initial_shape: + raise ValueError( + f"The two signals have incompatible shapes: {other_signal.shape} and {self.initial_shape}." + ) + # normalize in restriction to mask and recast in self.signal coords + other_mask = self.fix_mask(other_mask, other_signal) + other_signal = other_signal[..., other_mask] + other_signal = _normalize_signal(other_signal, num_bins=self.num_bins, eps=eps) + other_signal = self.trace_in_ref_mask(other_signal, other_mask) + common_mask = other_mask[self.mask] + + diff_1 = self.bin_centers.unsqueeze(-1) - self.signal[..., common_mask].unsqueeze(-2) + diff_2 = self.bin_centers.unsqueeze(-1) - other_signal[..., common_mask].unsqueeze(-2) + + vals_1 = self.kernel_function(diff_1) + vals_2 = self.kernel_function(diff_2) + + joint_histogram = torch.einsum("...in,...jn->...ij", vals_1, vals_2) + + return joint_histogram + + def entropies( + self, other_signal: torch.Tensor, other_mask: torch.Tensor | None = None + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Compute entropy measures between the reference signal and another signal. + + Calculates joint entropy and marginal entropies based on the joint histogram of the two signals. + + Args: + other_signal (torch.Tensor): Signal to compare with the reference signal. + Must have the same shape as the reference_signal passed at instantiation. + other_mask (torch.Tensor): common mask of roi for all the samples in other_signal, defaults to None + + Returns: + tuple[torch.Tensor, torch.Tensor, torch.Tensor]: A tuple containing: + - Marginal entropy H(X) of reference signal + - Marginal entropy H(Y) of other signal + - Joint entropy H(X,Y) + + All tensors have the same batch dimensions as the input signals. + + Note: + Subclasses should implement specific loss functions based on these entropy + measures (e.g., mutual information). + """ + joint_histogram = self._compute_joint_histogram(other_signal, self.eps, other_mask) + return _joint_histogram_to_entropies(joint_histogram, eps=self.eps) + + +class MILossFromRef(EntropyBasedLossBase): + """Mutual-information loss against a stored reference signal. + + The reference signal is normalized and stored during initialization by the + base class. Calling the module with another signal estimates both marginal + entropies and the joint entropy, then returns negative mutual information + so it can be minimized as a loss. + """ + + def forward(self, other_signal: torch.Tensor, other_mask: torch.Tensor | None = None) -> torch.Tensor: + """Compute differentiable mutual information for self.signal and other_signal. + + mi = (H(X) + H(Y) - H(X,Y)) + To have a loss function, the opposite is returned. + Can also handle two batches of flat tensors, then a batch of loss values is returned. + + Args: + other_signal: Batch of flat tensors shape (B,N) where B is + the tuple of batch dimensions for self.signal, possibly empty. + other_mask (torch.Tensor): common mask of roi for all the samples in other_signal, defaults to None + + Returns: + torch.Tensor: tensor of losses, shape B as above + """ + H_x, H_y, H_xy = self.entropies(other_signal, other_mask) + mi = H_x + H_y - H_xy + + return -mi + + +class NMILossFromRef(EntropyBasedLossBase): + """Normalized mutual-information loss against a stored reference signal. + + This variant divides the sum of marginal entropies by joint entropy before + negating the value. It is commonly useful when the amount of overlap or + intensity distribution scale differs between compared signals. + """ + + def forward(self, other_signal: torch.Tensor, other_mask: torch.Tensor | None = None) -> torch.Tensor: + """Compute differentiable normalized mutual information for self.signal and other_signal. + + nmi = (H(X) + H(Y)) / H(X,Y) + To have a loss function, the opposite is returned. + Can also handle two batches of flat tensors, then a batch of loss values is returned. + + Args: + other_signal: Batch of flat tensors shape (B,N) where B is + the tuple of batch dimensions for self.signal, possibly empty. + other_mask (torch.Tensor): common mask of roi for all the samples in other_signal, defaults to None + + Returns: + torch.Tensor: tensor of losses, shape B as above + """ + H_x, H_y, H_xy = self.entropies(other_signal, other_mask) + nmi = (H_x + H_y) / H_xy + + return -nmi + + +class MILossFromRef2D(MILossFromRef): + """Mutual Information loss module specifically designed for 2D image data. + + This class extends MILossFromRef to handle 2D image inputs by automatically + reshaping batched 2D images into the format expected by the base entropy + computation methods. It computes mutual information between reference 2D images + and other images using kernel density estimation. + """ + + def __init__( + self, + reference_signal: torch.Tensor, + mask: torch.Tensor | None = None, + kernel_function: MIKernel = MIKernel.xu, + num_bins: int = 64, + window_radius: float = 1, + ) -> None: + """Initialize the 2D Mutual Information loss module. + + Args: + reference_signal (torch.Tensor): reference signal to which + other signals will be compared by the forward method. + batch of 2D images, shape (B,H,W) where B are the batch + dimensions. + mask (torch.Tensor | None): common mask of roi for all the samples in reference_signal, by default None + kernel_function (MIKernel): Used kernel function for kernel + density estimate, by default MIKernel.xu + num_bins (int): number of signal value bins in kernel + density estimate, by default 64 + window_radius (float): radius of the kernel's support + interval, by default 1.0 + """ + super().__init__( + self.arrange_shape(reference_signal), _flatten_mask(mask), kernel_function, num_bins, window_radius + ) + + @staticmethod + def arrange_shape(tensor: torch.Tensor) -> torch.Tensor: + """Flatten the spatial dimensions of a 2D signal batch. + + Args: + tensor: Signal tensor with shape :math:`(*, H, W)`, where ``*`` is + any leading batch shape. + + Returns: + Tensor with shape :math:`(*, H * W)` suitable for histogram-based + mutual-information computation. + """ + return tensor.reshape(tensor.shape[:-2] + (-1,)) + + def forward( + self, + other_signal: torch.Tensor, + other_mask: torch.Tensor | None = None, + ) -> torch.Tensor: + """Compute differentiable mutual information for reference_signal and other_signal (both supposed 2D). + + mi = (H(X) + H(Y) - H(X,Y)) + To have a loss function, the opposite is returned. + Can also handle two batches of flat tensors, then a batch of loss values is returned. + + Args: + other_signal: Batch of flat tensors same shape (B,H,W) as + reference_signal passed for instantiation + other_mask (torch.Tensor): common mask of roi for all the samples in other_signal, defaults to None + + Returns: + torch.Tensor: tensor of losses, shape B as above + """ + return super().forward(self.arrange_shape(other_signal), _flatten_mask(other_mask)) + + +class MILossFromRef3D(MILossFromRef): + """Mutual Information loss module specifically designed for 3D image data. + + This class extends MILossFromRef to handle 3D image inputs by automatically + reshaping batched 3D images into the format expected by the base entropy + computation methods. It computes normalized mutual information between reference 3D images + and other images using kernel density estimation. + """ + + def __init__( + self, + reference_signal: torch.Tensor, + mask: torch.Tensor | None = None, + kernel_function: MIKernel = MIKernel.xu, + num_bins: int = 64, + window_radius: float = 1, + ) -> None: + """Initialize the 3D Mutual Information loss module. + + Args: + reference_signal (torch.Tensor): reference signal to which + other signals will be compared by the forward method. + batch of 3D images, shape (B,D,H,W) where B are the + batch dimensions. + mask (torch.Tensor | None): common mask of roi for all the samples in reference_signal, by default None + kernel_function (MIKernel): Used kernel function for kernel + density estimate, by default MIKernel.xu + num_bins (int): number of signal value bins in kernel + density estimate, by default 64 + window_radius (float): radius of the kernel's support + interval, by default 1.0 + """ + super().__init__( + self.arrange_shape(reference_signal), _flatten_mask(mask), kernel_function, num_bins, window_radius + ) + + @staticmethod + def arrange_shape(tensor: torch.Tensor) -> torch.Tensor: + """Flatten the spatial dimensions of a 3D signal batch. + + Args: + tensor: Signal tensor with shape :math:`(*, D, H, W)`, where + :math:`D` is depth, :math:`H` is height, and :math:`W` is + width. + + Returns: + Tensor with shape :math:`(*, D * H * W)` for entropy estimation. + """ + return tensor.reshape(tensor.shape[:-3] + (-1,)) + + def forward( + self, + other_signal: torch.Tensor, + other_mask: torch.Tensor | None = None, + ) -> torch.Tensor: + """Compute differentiable mutual information for reference_signal and other_signal (both supposed 3D). + + mi = (H(X) + H(Y) - H(X,Y)) + To have a loss function, the opposite is returned. + Can also handle two batches of flat tensors, then a batch of loss values is returned. + + Args: + other_signal: Batch of flat tensors same shape (B,D,H,W) as + reference_signal passed for instantiation + other_mask (torch.Tensor): common mask of roi for all the samples in other_signal, defaults to None + + Returns: + torch.Tensor: tensor of losses, shape B as above + """ + return super().forward(self.arrange_shape(other_signal), _flatten_mask(other_mask)) + + +class NMILossFromRef2D(NMILossFromRef): + """Normalized Mutual Information loss module specifically designed for 2D image data. + + This class extends NMILossFromRef to handle 2D image inputs by automatically + reshaping batched 2D images into the format expected by the base entropy + computation methods. It computes normalized mutual information between reference 2D images + and other images using kernel density estimation. + """ + + def __init__( + self, + reference_signal: torch.Tensor, + mask: torch.Tensor | None = None, + kernel_function: MIKernel = MIKernel.xu, + num_bins: int = 64, + window_radius: float = 1, + ) -> None: + """Initialize the 2D Normalized Mutual Information loss module. + + Args: + reference_signal (torch.Tensor): reference signal to which + other signals will be compared by the forward method. + batch of 2D images, shape (B,H,W) where B are the batch + dimensions. + mask (torch.Tensor | None): common mask of roi for all the samples in reference_signal, by default None + kernel_function (MIKernel): Used kernel function for kernel + density estimate, by default MIKernel.xu + num_bins (int): number of signal value bins in kernel + density estimate, by default 64 + window_radius (float): radius of the kernel's support + interval, by default 1.0 + """ + super().__init__( + self.arrange_shape(reference_signal), _flatten_mask(mask), kernel_function, num_bins, window_radius + ) + + @staticmethod + def arrange_shape(tensor: torch.Tensor) -> torch.Tensor: + """Flatten the spatial dimensions of a 2D signal batch. + + Args: + tensor: Signal tensor with shape :math:`(*, H, W)`. + + Returns: + Tensor with shape :math:`(*, H * W)` used by the normalized + mutual-information base implementation. + """ + return tensor.reshape(tensor.shape[:-2] + (-1,)) + + def forward( + self, + other_signal: torch.Tensor, + other_mask: torch.Tensor | None = None, + ) -> torch.Tensor: + """Compute differentiable mutual information for reference_signal and other_signal (both supposed 2D). + + nmi = (H(X) + H(Y)) / H(X,Y) + To have a loss function, the opposite is returned. + Can also handle two batches of tensors, then a batch of loss values is returned. + + Args: + other_signal: Batch of tensors same shape (B,H,W) as + reference_signal passed for instantiation + other_mask (torch.Tensor): common mask of roi for all the samples in other_signal, defaults to None + + Returns: + torch.Tensor: tensor of losses, shape B as above + """ + return super().forward(self.arrange_shape(other_signal), _flatten_mask(other_mask)) + + +class NMILossFromRef3D(NMILossFromRef): + """Normalized Mutual Information loss module specifically designed for 3D image data. + + This class extends NMILossFromRef to handle 3D image inputs by automatically + reshaping batched 3D images into the format expected by the base entropy + computation methods. It computes mutual information between reference 3D images + and other images using kernel density estimation. + """ + + def __init__( + self, + reference_signal: torch.Tensor, + mask: torch.Tensor | None = None, + kernel_function: MIKernel = MIKernel.xu, + num_bins: int = 64, + window_radius: float = 1, + ) -> None: + """Initialize the 3D Normalized Mutual Information loss module. + + Args: + reference_signal (torch.Tensor): reference signal to which + other signals will be compared by the forward method. + batch of 3D images, shape (B,D,H,W) where B are the + batch dimensions. + mask (torch.Tensor | None): common mask of roi for all the samples in reference_signal, by default None + kernel_function (MIKernel): Used kernel function for kernel + density estimate, by default MIKernel.xu + num_bins (int): number of signal value bins in kernel + density estimate, by default 64 + window_radius (float): radius of the kernel's support + interval, by default 1.0 + """ + super().__init__( + self.arrange_shape(reference_signal), _flatten_mask(mask), kernel_function, num_bins, window_radius + ) + + @staticmethod + def arrange_shape(tensor: torch.Tensor) -> torch.Tensor: + """Flatten the spatial dimensions of a 3D signal batch. + + Args: + tensor: Signal tensor with shape :math:`(*, D, H, W)`. + + Returns: + Tensor with shape :math:`(*, D * H * W)` used by the normalized + mutual-information base implementation. + """ + return tensor.reshape(tensor.shape[:-3] + (-1,)) + + def forward( + self, + other_signal: torch.Tensor, + other_mask: torch.Tensor | None = None, + ) -> torch.Tensor: + """Compute differentiable mutual information for reference_signal and other_signal (both supposed 3D). + + nmi = (H(X) + H(Y)) / H(X,Y) + To have a loss function, the opposite is returned. + Can also handle two batches of flat tensors, then a batch of loss values is returned. + + Args: + other_signal: Batch of tensors, same shape (B,D,H,W) as + reference_signal passed for instantiation + other_mask (torch.Tensor): common mask of roi for all the samples in other_signal, defaults to None + + Returns: + torch.Tensor: tensor of losses, shape B as above + """ + return super().forward(self.arrange_shape(other_signal), _flatten_mask(other_mask)) + + +def mutual_information_loss( + input: torch.Tensor, + target: torch.Tensor, + input_mask: torch.Tensor | None = None, + target_mask: torch.Tensor | None = None, + kernel_function: MIKernel = MIKernel.xu, + num_bins: int = 64, + window_radius: float = 1.0, +) -> torch.Tensor: + """Compute differentiable mutual information for flat tensors. + + mi = (H(X) + H(Y) - H(X,Y)) + To have a loss function, the opposite is returned. + Can also handle two batches of flat tensors, then a batch of loss values is returned. + + Args: + input (torch.Tensor): Batch of flat tensors shape (B,N) where B + is any batch dimensions tuple, possibly empty. + target (torch.Tensor): Batch of flat tensors, same shape as + input. + input_mask (torch.Tensor): mask of roi in input, defaults to None. + target_mask (torch.Tensor): mask of roi in target, defaults to None. + + kernel_function (MIKernel): Used kernel function for kernel + density estimate, by default MIKernel.xu + num_bins (int): The number of bins used for KDE, defaults to 64. + window_radius (float): The smoothing window radius in KDE, in + terms of bin width units, defaults to 1. + + Returns: + torch.Tensor: tensor of losses, shape B (common batch dims tuple + of input and target) + """ + module = MILossFromRef( + reference_signal=target, + mask=target_mask, + kernel_function=kernel_function, + num_bins=num_bins, + window_radius=window_radius, + ) + return module.forward(input, other_mask=input_mask) + + +def mutual_information_loss_2d( + input: torch.Tensor, + target: torch.Tensor, + input_mask: torch.Tensor | None = None, + target_mask: torch.Tensor | None = None, + kernel_function: MIKernel = MIKernel.xu, + num_bins: int = 64, + window_radius: float = 1.0, +) -> torch.Tensor: + """Compute differentiable mutual information for 2d tensors. + + mi = (H(X) + H(Y) - H(X,Y)) + To have a loss function, the opposite is returned. + Can also handle two batches of 2d tensors, then a batch of loss values is returned. + + Args: + input (torch.Tensor): Batch of 2d tensors shape (B,H,W) where B + is any batch dimensions tuple, possibly empty. + target (torch.Tensor): Batch of 2d tensors, same shape as input. + input_mask (torch.Tensor): mask of roi in input, defaults to None. + target_mask (torch.Tensor): mask of roi in target, defaults to None. + kernel_function (MIKernel): Used kernel function for kernel + density estimate, by default MIKernel.xu + num_bins (int): The number of bins used for KDE, defaults to 64. + window_radius (float): The smoothing window radius in KDE, in + terms of bin width units, defaults to 1. + + Returns: + torch.Tensor: tensor of losses, shape B (common batch dims tuple + of input and target) + """ + module = MILossFromRef2D( + reference_signal=target, + mask=target_mask, + kernel_function=kernel_function, + num_bins=num_bins, + window_radius=window_radius, + ) + return module.forward(input, other_mask=input_mask) + + +def mutual_information_loss_3d( + input: torch.Tensor, + target: torch.Tensor, + input_mask: torch.Tensor | None = None, + target_mask: torch.Tensor | None = None, + kernel_function: MIKernel = MIKernel.xu, + num_bins: int = 64, + window_radius: float = 1.0, +) -> torch.Tensor: + """Compute differentiable mutual information for 3d tensors. + + mi = (H(X) + H(Y) - H(X,Y)) + To have a loss function, the opposite is returned. + Can also handle two batches of 3d tensors, then a batch of loss values is returned. + + Args: + input (torch.Tensor): Batch of 3d tensors shape (B,D,H,W) where + B is any batch dimensions tuple, possibly empty. + target (torch.Tensor): Batch of 3d tensors, same shape as input. + input_mask (torch.Tensor): mask of roi in input, defaults to None. + target_mask (torch.Tensor): mask of roi in target, defaults to None. + + kernel_function (MIKernel): Used kernel function for kernel + density estimate, by default MIKernel.xu + num_bins (int): The number of bins used for KDE, defaults to 64. + window_radius (float): The smoothing window radius in KDE, in + terms of bin width units, defaults to 1. + + Returns: + torch.Tensor: tensor of losses, shape B (common batch dims tuple + of input and target) + """ + module = MILossFromRef3D( + reference_signal=target, + mask=target_mask, + kernel_function=kernel_function, + num_bins=num_bins, + window_radius=window_radius, + ) + return module.forward(input, other_mask=input_mask) + + +def normalized_mutual_information_loss( + input: torch.Tensor, + target: torch.Tensor, + input_mask: torch.Tensor | None = None, + target_mask: torch.Tensor | None = None, + kernel_function: MIKernel = MIKernel.xu, + num_bins: int = 64, + window_radius: float = 1.0, +) -> torch.Tensor: + """Compute differentiable normalized mutual information for flat tensors. + + nmi = (H(X) + H(Y)) / H(X,Y) + To have a loss function, the opposite is returned. + Can also handle two batches of flat tensors, then a batch of loss values is returned. + + Args: + input (torch.Tensor): Batch of flat tensors shape (B,N) where B + is any batch dimensions tuple, possibly empty. + target (torch.Tensor): Batch of flat tensors, same shape as + input. + input_mask (torch.Tensor): mask of roi in input, defaults to None. + target_mask (torch.Tensor): mask of roi in target, defaults to None. + + kernel_function (MIKernel): Used kernel function for kernel + density estimate, by default MIKernel.xu + num_bins (int): The number of bins used for KDE, defaults to 64. + window_radius (float): The smoothing window radius in KDE, in + terms of bin width units, defaults to 1. + + Returns: + torch.Tensor: tensor of losses, shape B (common batch dims tuple + of input and target) + """ + module = NMILossFromRef( + reference_signal=target, + kernel_function=kernel_function, + mask=target_mask, + num_bins=num_bins, + window_radius=window_radius, + ) + return module.forward(input, other_mask=input_mask) + + +def normalized_mutual_information_loss_2d( + input: torch.Tensor, + target: torch.Tensor, + input_mask: torch.Tensor | None = None, + target_mask: torch.Tensor | None = None, + kernel_function: MIKernel = MIKernel.xu, + num_bins: int = 64, + window_radius: float = 1.0, +) -> torch.Tensor: + """Compute differentiable normalized mutual information for 2d tensors. + + nmi = (H(X) + H(Y)) / H(X,Y) + To have a loss function, the opposite is returned. + Can also handle two batches of 2d tensors, then a batch of loss values is returned. + + Args: + input (torch.Tensor): Batch of 2d tensors shape (B,H,W) where B + is any batch dimensions tuple, possibly empty. + target (torch.Tensor): Batch of 2d tensors, same shape as input. + input_mask (torch.Tensor): mask of roi in input, defaults to None. + target_mask (torch.Tensor): mask of roi in target, defaults to None. + + kernel_function (MIKernel): Used kernel function for kernel + density estimate, by default MIKernel.xu + num_bins (int): The number of bins used for KDE, defaults to 64. + window_radius (float): The smoothing window radius in KDE, in + terms of bin width units, defaults to 1. + + Returns: + torch.Tensor: tensor of losses, shape B (common batch dims tuple + of input and target) + """ + module = NMILossFromRef2D( + reference_signal=target, + mask=target_mask, + kernel_function=kernel_function, + num_bins=num_bins, + window_radius=window_radius, + ) + return module.forward(input, other_mask=input_mask) + + +def normalized_mutual_information_loss_3d( + input: torch.Tensor, + target: torch.Tensor, + input_mask: torch.Tensor | None = None, + target_mask: torch.Tensor | None = None, + kernel_function: MIKernel = MIKernel.xu, + num_bins: int = 64, + window_radius: float = 1.0, +) -> torch.Tensor: + """Compute differentiable normalized mutual information for 3d tensors. + + nmi = (H(X) + H(Y)) / H(X,Y) + To have a loss function, the opposite is returned. + Can also handle two batches of 3d tensors, then a batch of loss values is returned. + + Args: + input (torch.Tensor): Batch of 3d tensors shape (B,D,H,W) where + B is any batch dimensions tuple, possibly empty. + target (torch.Tensor): Batch of 3d tensors, same shape as input. + input_mask (torch.Tensor): mask of roi in input, defaults to None. + target_mask (torch.Tensor): mask of roi in target, defaults to None. + + kernel_function (MIKernel): Used kernel function for kernel + density estimate, by default MIKernel.xu + num_bins (int): The number of bins used for KDE, defaults to 64. + window_radius (float): The smoothing window radius in KDE, in + terms of bin width units, defaults to 1. + + Returns: + torch.Tensor: tensor of losses, shape B (common batch dims tuple + of input and target) + """ + module = NMILossFromRef3D( + reference_signal=target, + mask=target_mask, + kernel_function=kernel_function, + num_bins=num_bins, + window_radius=window_radius, + ) + return module.forward(input, other_mask=input_mask) diff --git a/kornia/losses/one_hot.py b/kornia/losses/one_hot.py new file mode 100644 index 0000000..b82ce27 --- /dev/null +++ b/kornia/losses/one_hot.py @@ -0,0 +1,72 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch +import torch.nn.functional as F + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR + + +def one_hot( + labels: torch.Tensor, num_classes: int, device: torch.device, dtype: torch.dtype, eps: float = 1e-6 +) -> torch.Tensor: + r"""Convert an integer label x-D torch.Tensor to a one-hot (x+1)-D torch.Tensor. + + Args: + labels: torch.Tensor with labels of shape :math:`(N, *)`, where N is batch size. + Each value is an integer representing correct classification. + num_classes: number of classes in labels. + device: the desired device of returned torch.Tensor. + dtype: the desired data type of returned torch.Tensor. + eps: epsilon for numerical stability. + + Returns: + the labels in one hot torch.Tensor of shape :math:`(N, C, *)`, + + Examples: + >>> labels = torch.LongTensor([[[0, 1], [2, 0]]]) + >>> one_hot(labels, num_classes=3, device=torch.device('cpu'), dtype=torch.float32) + tensor([[[[1.0000e+00, 1.0000e-06], + [1.0000e-06, 1.0000e+00]], + + [[1.0000e-06, 1.0000e+00], + [1.0000e-06, 1.0000e-06]], + + [[1.0000e-06, 1.0000e-06], + [1.0000e+00, 1.0000e-06]]]]) + + """ + KORNIA_CHECK_IS_TENSOR(labels, "Input labels must be a torch.Tensor") + KORNIA_CHECK(labels.dtype == torch.int64, f"labels must be of dtype torch.int64. Got: {labels.dtype}") + KORNIA_CHECK(num_classes >= 1, f"The number of classes must be >= 1. Got: {num_classes}") + + # Use PyTorch's built-in one_hot function + one_hot_tensor = F.one_hot(labels, num_classes=num_classes) + + # PyTorch's one_hot adds the class dimension at the end: (*, num_classes) + # We need it at position 1: (N, C, *) + # Permute: move the last dimension (num_classes) to position 1 + ndim = labels.ndim + permute_dims = [0] + [ndim] + list(range(1, ndim)) + one_hot_tensor = one_hot_tensor.permute(*permute_dims) + + # Convert to desired dtype and device, then apply eps for numerical stability + one_hot_tensor = one_hot_tensor.to(dtype=dtype, device=device) + # Apply eps: multiply by (1-eps) and add eps to all elements + one_hot_tensor = one_hot_tensor * (1.0 - eps) + eps + + return one_hot_tensor diff --git a/kornia/losses/psnr.py b/kornia/losses/psnr.py new file mode 100644 index 0000000..91c4b7c --- /dev/null +++ b/kornia/losses/psnr.py @@ -0,0 +1,97 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +from torch import nn + +from kornia import metrics + + +def psnr_loss(image: torch.Tensor, target: torch.Tensor, max_val: float) -> torch.Tensor: + r"""Compute the PSNR loss. + + The loss is computed as follows: + + .. math:: + + \text{loss} = -\text{psnr(x, y)} + + See :meth:`~kornia.losses.psnr` for details abut PSNR. + + Args: + image: the input image with shape :math:`(*)`. + target : the labels image with shape :math:`(*)`. + max_val: The maximum value in the image tensor. + + Return: + the computed loss as a scalar. + + Examples: + >>> ones = torch.ones(1) + >>> psnr_loss(ones, 1.2 * ones, 2.) # 10 * log(4/((1.2-1)**2)) / log(10) + tensor(-20.0000) + + """ + return -1.0 * metrics.psnr(image, target, max_val) + + +class PSNRLoss(nn.Module): + r"""Create a criterion that calculates the PSNR loss. + + The loss is computed as follows: + + .. math:: + + \text{loss} = -\text{psnr(x, y)} + + See :meth:`~kornia.losses.psnr` for details abut PSNR. + + Args: + max_val: The maximum value in the image tensor. + + Shape: + - Image: arbitrary dimensional tensor :math:`(*)`. + - Target: arbitrary dimensional tensor :math:`(*)` same shape as image. + - Output: a scalar. + + Examples: + >>> ones = torch.ones(1) + >>> criterion = PSNRLoss(2.) + >>> criterion(ones, 1.2 * ones) # 10 * log(4/((1.2-1)**2)) / log(10) + tensor(-20.0000) + + """ + + def __init__(self, max_val: float) -> None: + super().__init__() + self.max_val: float = max_val + + def forward(self, image: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + """Compute the negative PSNR loss between two tensors. + + Args: + image: Predicted image or signal tensor with arbitrary shape. + target: Reference tensor with the same shape as ``image``. + + Returns: + Scalar tensor containing the PSNR-based loss computed with + ``self.max_val``. Lower values correspond to higher peak signal to + noise ratio. + """ + return psnr_loss(image, target, self.max_val) diff --git a/kornia/losses/ssim.py b/kornia/losses/ssim.py new file mode 100644 index 0000000..7cc56af --- /dev/null +++ b/kornia/losses/ssim.py @@ -0,0 +1,142 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +from torch import nn + +from kornia import metrics + + +def ssim_loss( + img1: torch.Tensor, + img2: torch.Tensor, + window_size: int, + max_val: float = 1.0, + eps: float = 1e-12, + reduction: str = "mean", + padding: str = "same", +) -> torch.Tensor: + r"""Compute a loss based on the SSIM measurement. + + The loss, or the Structural dissimilarity (DSSIM) is described as: + + .. math:: + + \text{loss}(x, y) = \frac{1 - \text{SSIM}(x, y)}{2} + + See :meth:`~kornia.losses.ssim` for details about SSIM. + + Args: + img1: the first input image with shape :math:`(B, C, H, W)`. + img2: the second input image with shape :math:`(B, C, H, W)`. + window_size: the size of the gaussian kernel to smooth the images. + max_val: the dynamic range of the images. + eps: Small value for numerically stability when dividing. + reduction : Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the sum of the output will be divided by the number of elements + in the output, ``'sum'``: the output will be summed. + padding: ``'same'`` | ``'valid'``. Whether to only use the "valid" convolution + area to compute SSIM to match the MATLAB implementation of original SSIM paper. + + Returns: + The loss based on the ssim index. + + Examples: + >>> input1 = torch.rand(1, 4, 5, 5) + >>> input2 = torch.rand(1, 4, 5, 5) + >>> loss = ssim_loss(input1, input2, 5) + + """ + # compute the ssim map + ssim_map: torch.Tensor = metrics.ssim(img1, img2, window_size, max_val, eps, padding) + + # compute and reduce the loss + loss = torch.clamp((1.0 - ssim_map) / 2, min=0, max=1) + + if reduction == "mean": + loss = torch.mean(loss) + elif reduction == "sum": + loss = torch.sum(loss) + elif reduction == "none": + pass + else: + raise NotImplementedError("Invalid reduction option.") + + return loss + + +class SSIMLoss(nn.Module): + r"""Create a criterion that computes a loss based on the SSIM measurement. + + The loss, or the Structural dissimilarity (DSSIM) is described as: + + .. math:: + + \text{loss}(x, y) = \frac{1 - \text{SSIM}(x, y)}{2} + + See :meth:`~kornia.losses.ssim_loss` for details about SSIM. + + Args: + window_size: the size of the gaussian kernel to smooth the images. + max_val: the dynamic range of the images. + eps: Small value for numerically stability when dividing. + reduction : Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the sum of the output will be divided by the number of elements + in the output, ``'sum'``: the output will be summed. + padding: ``'same'`` | ``'valid'``. Whether to only use the "valid" convolution + area to compute SSIM to match the MATLAB implementation of original SSIM paper. + + Returns: + The loss based on the ssim index. + + Examples: + >>> input1 = torch.rand(1, 4, 5, 5) + >>> input2 = torch.rand(1, 4, 5, 5) + >>> criterion = SSIMLoss(5) + >>> loss = criterion(input1, input2) + + """ + + def __init__( + self, window_size: int, max_val: float = 1.0, eps: float = 1e-12, reduction: str = "mean", padding: str = "same" + ) -> None: + super().__init__() + self.window_size: int = window_size + self.max_val: float = max_val + self.eps: float = eps + self.reduction: str = reduction + self.padding: str = padding + + def forward(self, img1: torch.Tensor, img2: torch.Tensor) -> torch.Tensor: + """Compute SSIM loss between two 2D image tensors. + + Args: + img1: First image batch with shape :math:`(B, C, H, W)`, where + :math:`B` is batch size, :math:`C` is channel count, + :math:`H` is height, and :math:`W` is width. + img2: Second image batch with the same shape as ``img1``. + + Returns: + SSIM loss tensor reduced according to ``self.reduction``. The loss + is derived from structural similarity, so lower values indicate + more similar local luminance, contrast, and structure. + """ + return ssim_loss(img1, img2, self.window_size, self.max_val, self.eps, self.reduction, self.padding) diff --git a/kornia/losses/ssim3d.py b/kornia/losses/ssim3d.py new file mode 100644 index 0000000..90db71e --- /dev/null +++ b/kornia/losses/ssim3d.py @@ -0,0 +1,143 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +from torch import nn + +from kornia import metrics + + +def ssim3d_loss( + img1: torch.Tensor, + img2: torch.Tensor, + window_size: int, + max_val: float = 1.0, + eps: float = 1e-12, + reduction: str = "mean", + padding: str = "same", +) -> torch.Tensor: + r"""Compute a loss based on the SSIM measurement. + + The loss, or the Structural dissimilarity (DSSIM) is described as: + + .. math:: + + \text{loss}(x, y) = \frac{1 - \text{SSIM}(x, y)}{2} + + See :meth:`~kornia.losses.ssim` for details about SSIM. + + Args: + img1: the first input image with shape :math:`(B, C, D, H, W)`. + img2: the second input image with shape :math:`(B, C, D, H, W)`. + window_size: the size of the gaussian kernel to smooth the images. + max_val: the dynamic range of the images. + eps: Small value for numerically stability when dividing. + reduction : Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the sum of the output will be divided by the number of elements + in the output, ``'sum'``: the output will be summed. + padding: ``'same'`` | ``'valid'``. Whether to only use the "valid" convolution + area to compute SSIM to match the MATLAB implementation of original SSIM paper. + + Returns: + The loss based on the ssim index. + + Examples: + >>> input1 = torch.rand(1, 4, 5, 5, 5) + >>> input2 = torch.rand(1, 4, 5, 5, 5) + >>> loss = ssim3d_loss(input1, input2, 5) + + """ + # compute the ssim map + ssim_map: torch.Tensor = metrics.ssim3d(img1, img2, window_size, max_val, eps, padding) + + # compute and reduce the loss + loss = 1.0 - ssim_map + + if reduction == "mean": + loss = loss.mean() + elif reduction == "sum": + loss = loss.sum() + elif reduction == "none": + pass + else: + raise NotImplementedError("Invalid reduction option.") + + return loss + + +class SSIM3DLoss(nn.Module): + r"""Create a criterion that computes a loss based on the SSIM measurement. + + The loss, or the Structural dissimilarity (DSSIM) is described as: + + .. math:: + + \text{loss}(x, y) = \frac{1 - \text{SSIM}(x, y)}{2} + + See :meth:`~kornia.losses.ssim_loss` for details about SSIM. + + Args: + window_size: the size of the gaussian kernel to smooth the images. + max_val: the dynamic range of the images. + eps: Small value for numerically stability when dividing. + reduction : Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the sum of the output will be divided by the number of elements + in the output, ``'sum'``: the output will be summed. + padding: ``'same'`` | ``'valid'``. Whether to only use the "valid" convolution + area to compute SSIM to match the MATLAB implementation of original SSIM paper. + + Returns: + The loss based on the ssim index. + + Examples: + >>> input1 = torch.rand(1, 4, 5, 5, 5) + >>> input2 = torch.rand(1, 4, 5, 5, 5) + >>> criterion = SSIM3DLoss(5) + >>> loss = criterion(input1, input2) + + """ + + def __init__( + self, window_size: int, max_val: float = 1.0, eps: float = 1e-12, reduction: str = "mean", padding: str = "same" + ) -> None: + super().__init__() + self.window_size: int = window_size + self.max_val: float = max_val + self.eps: float = eps + self.reduction: str = reduction + self.padding: str = padding + + def forward(self, img1: torch.Tensor, img2: torch.Tensor) -> torch.Tensor: + """Compute SSIM loss between two 3D image or volume tensors. + + Args: + img1: First volume batch with shape :math:`(B, C, D, H, W)`, where + :math:`B` is batch size, :math:`C` is channel count, + :math:`D` is depth, :math:`H` is height, and :math:`W` is + width. + img2: Second volume batch with the same shape as ``img1``. + + Returns: + SSIM3D loss tensor reduced according to ``self.reduction``. Lower + values indicate stronger local structural agreement between the two + volumes. + """ + return ssim3d_loss(img1, img2, self.window_size, self.max_val, self.eps, self.reduction, self.padding) diff --git a/kornia/losses/total_variation.py b/kornia/losses/total_variation.py new file mode 100644 index 0000000..babec87 --- /dev/null +++ b/kornia/losses/total_variation.py @@ -0,0 +1,116 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SHAPE + + +def total_variation(img: torch.Tensor, reduction: str = "sum") -> torch.Tensor: + r"""Compute Total Variation according to [1]. + + Args: + img: the input image with shape :math:`(*, H, W)`. + reduction : Specifies the reduction to apply to the output: ``'mean'`` | ``'sum'``. + ``'mean'``: the sum of the output will be divided by the number of elements + in the output, ``'sum'``: the output will be summed. + + Return: + a torch.Tensor with shape :math:`(*,)`. + + Examples: + >>> total_variation(torch.ones(4, 4)) + tensor(0.) + >>> total_variation(torch.ones(2, 5, 3, 4, 4)).shape + torch.Size([2, 5, 3]) + + .. note:: + See a working example `here `__. + Total Variation is formulated with summation, however this is not resolution invariant. + Thus, `reduction='mean'` was added as an optional reduction method. + + Reference: + [1] https://en.wikipedia.org/wiki/Total_variation + + """ + # TODO: here torchscript doesn't like KORNIA_CHECK_TYPE + if not isinstance(img, torch.Tensor): + raise TypeError(f"Not a torch.Tensor type. Got: {type(img)}") + + KORNIA_CHECK_SHAPE(img, ["*", "H", "W"]) + KORNIA_CHECK(reduction in ("mean", "sum"), f"Expected reduction to be one of 'mean'/'sum', but got '{reduction}'.") + + pixel_dif1 = img[..., 1:, :] - img[..., :-1, :] + pixel_dif2 = img[..., :, 1:] - img[..., :, :-1] + + res1 = pixel_dif1.abs() + res2 = pixel_dif2.abs() + + reduce_axes = (-2, -1) + if reduction == "mean": + if img.is_floating_point(): + res1 = res1.to(img).mean(dim=reduce_axes) + res2 = res2.to(img).mean(dim=reduce_axes) + else: + res1 = res1.float().mean(dim=reduce_axes) + res2 = res2.float().mean(dim=reduce_axes) + elif reduction == "sum": + res1 = res1.sum(dim=reduce_axes) + res2 = res2.sum(dim=reduce_axes) + else: + raise NotImplementedError("Invalid reduction option.") + + return res1 + res2 + + +class TotalVariation(nn.Module): + r"""Compute the Total Variation according to [1]. + + Shape: + - Input: :math:`(*, H, W)`. + - Output: :math:`(*,)`. + + Examples: + >>> tv = TotalVariation() + >>> output = tv(torch.ones((2, 3, 4, 4), requires_grad=True)) + >>> output.data + tensor([[0., 0., 0.], + [0., 0., 0.]]) + >>> output.sum().backward() # grad can be implicitly created only for scalar outputs + + Reference: + [1] https://en.wikipedia.org/wiki/Total_variation + + """ + + def forward(self, img: torch.Tensor) -> torch.Tensor: + """Compute total variation for an image or image batch. + + Args: + img: Tensor with shape :math:`(*, H, W)`, where ``*`` represents + optional leading dimensions such as batch and channels, + :math:`H` is height, and :math:`W` is width. + + Returns: + Tensor containing the sum of absolute horizontal and vertical + finite differences for each leading element. Lower values indicate + spatially smoother images. + """ + return total_variation(img) diff --git a/kornia/losses/tversky.py b/kornia/losses/tversky.py new file mode 100644 index 0000000..0e8a7f8 --- /dev/null +++ b/kornia/losses/tversky.py @@ -0,0 +1,185 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.losses._utils import mask_ignore_pixels + +# based on: +# https://github.com/kevinzakka/pytorch-goodies/blob/master/losses.py + + +def tversky_loss( + pred: torch.Tensor, + target: torch.Tensor, + alpha: float, + beta: float, + eps: float = 1e-8, + ignore_index: Optional[int] = -100, +) -> torch.Tensor: + r"""Criterion that computes Tversky Coefficient loss. + + According to :cite:`salehi2017tversky`, we compute the Tversky Coefficient as follows: + + .. math:: + + \text{S}(P, G, \alpha; \beta) = + \frac{|PG|}{|PG| + \alpha |P \setminus G| + \beta |G \setminus P|} + + Where: + - :math:`P` and :math:`G` are the predicted and ground truth binary + labels. + - :math:`\alpha` and :math:`\beta` control the magnitude of the + penalties for FPs and FNs, respectively. + + Note: + - :math:`\alpha = \beta = 0.5` => dice coeff + - :math:`\alpha = \beta = 1` => tanimoto coeff + - :math:`\alpha + \beta = 1` => F beta coeff + + Args: + pred: logits tensor with shape :math:`(N, C, H, W)` where C = number of classes. + target: labels tensor with shape :math:`(N, H, W)` where each value + is in range :math:`0 ≤ targets[i] ≤ C-1`. + alpha: the first coefficient in the denominator. + beta: the second coefficient in the denominator. + eps: scalar for numerical stability. + ignore_index: labels with this value are ignored in the loss computation. + + Return: + the computed loss. + + Example: + >>> N = 5 # num_classes + >>> pred = torch.randn(1, N, 3, 5, requires_grad=True) + >>> target = torch.empty(1, 3, 5, dtype=torch.long).random_(N) + >>> output = tversky_loss(pred, target, alpha=0.5, beta=0.5) + >>> output.backward() + + """ + if not isinstance(pred, torch.Tensor): + raise TypeError(f"pred type is not a torch.Tensor. Got {type(pred)}") + + if not len(pred.shape) == 4: + raise ValueError(f"Invalid pred shape, we expect BxNxHxW. Got: {pred.shape}") + + if not pred.shape[-2:] == target.shape[-2:]: + raise ValueError(f"pred and target shapes must be the same. Got: {pred.shape} and {target.shape}") + + if not pred.device == target.device: + raise ValueError(f"pred and target must be in the same device. Got: {pred.device} and {target.device}") + + # compute softmax over the classes axis + pred_soft = F.softmax(pred, dim=1) + target, target_mask = mask_ignore_pixels(target, ignore_index) + + p_true = pred_soft.gather(1, target.unsqueeze(1)) # (B,1,H,W) + + if target_mask is not None: + m = target_mask.unsqueeze(1).to(dtype=pred.dtype) + p_true = p_true * m + total = m.sum((1, 2, 3)) + else: + B, _, H, W = pred.shape + total = torch.full((B,), H * W, dtype=pred.dtype, device=pred.device) + + intersection = p_true.sum((1, 2, 3)) + # denominator = intersection + (alpha + beta) * (total - intersection) + eps + # instead of multiple ops, do it in one fused step: + denominator = torch.addcmul( + intersection, # base + total - intersection, # tensor1 + torch.full_like(total, alpha + beta), # tensor2 (scalar as tensor) + value=1.0, # (intersection) + 1 * (tensor1*tensor2) + ).add_(eps) # in-place add eps + score = intersection.div(denominator) + + return 1.0 - score.mean() + + +class TverskyLoss(nn.Module): + r"""Criterion that computes Tversky Coefficient loss. + + According to :cite:`salehi2017tversky`, we compute the Tversky Coefficient as follows: + + .. math:: + + \text{S}(P, G, \alpha; \beta) = + \frac{|PG|}{|PG| + \alpha |P \setminus G| + \beta |G \setminus P|} + + Where: + - :math:`P` and :math:`G` are the predicted and ground truth binary + labels. + - :math:`\alpha` and :math:`\beta` control the magnitude of the + penalties for FPs and FNs, respectively. + + Note: + - :math:`\alpha = \beta = 0.5` => dice coeff + - :math:`\alpha = \beta = 1` => tanimoto coeff + - :math:`\alpha + \beta = 1` => F beta coeff + + Args: + alpha: the first coefficient in the denominator. + beta: the second coefficient in the denominator. + eps: scalar for numerical stability. + ignore_index: labels with this value are ignored in the loss computation. + + Shape: + - Pred: :math:`(N, C, H, W)` where C = number of classes. + - Target: :math:`(N, H, W)` where each value is + :math:`0 ≤ targets[i] ≤ C-1`. + + Examples: + >>> N = 5 # num_classes + >>> criterion = TverskyLoss(alpha=0.5, beta=0.5) + >>> pred = torch.randn(1, N, 3, 5, requires_grad=True) + >>> target = torch.empty(1, 3, 5, dtype=torch.long).random_(N) + >>> output = criterion(pred, target) + >>> output.backward() + + """ + + def __init__(self, alpha: float, beta: float, eps: float = 1e-8, ignore_index: Optional[int] = -100) -> None: + super().__init__() + self.alpha: float = alpha + self.beta: float = beta + self.eps: float = eps + self.ignore_index: Optional[int] = ignore_index + + def forward(self, pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + """Compute the Tversky segmentation loss for class logits and labels. + + Args: + pred: Logit tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is batch size, :math:`C` is number of classes, + :math:`H` is height, and :math:`W` is width. + target: Integer class-label tensor with shape :math:`(B, H, W)`. + Labels matching ``self.ignore_index`` are excluded from the + loss when an ignore index is configured. + + Returns: + Scalar tensor containing the Tversky loss. ``self.alpha`` controls + the false-positive penalty and ``self.beta`` controls the + false-negative penalty. + """ + return tversky_loss(pred, target, self.alpha, self.beta, self.eps, self.ignore_index) diff --git a/kornia/losses/welsch.py b/kornia/losses/welsch.py new file mode 100644 index 0000000..d727d09 --- /dev/null +++ b/kornia/losses/welsch.py @@ -0,0 +1,144 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +from torch import nn + +# from torch import Tensor (use torch.Tensor instead) +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SAME_DEVICE, KORNIA_CHECK_SAME_SHAPE + + +def welsch_loss(img1: torch.Tensor, img2: torch.Tensor, reduction: str = "none") -> torch.Tensor: + r"""Criterion that computes the Welsch [2] (aka. Leclerc [3]) loss. + + According to [1], we compute the Welsch loss as follows: + + .. math:: + + \text{WL}(x, y) = 1 - exp(-\frac{1}{2} (x - y)^{2}) + + Where: + - :math:`x` is the prediction. + - :math:`y` is the target to be regressed to. + + Reference: + [1] https://arxiv.org/pdf/1701.03077.pdf + [2] https://www.tandfonline.com/doi/abs/10.1080/03610917808812083 + [3] https://link.springer.com/article/10.1007/BF00054839 + + Args: + img1: the predicted torch.Tensor with shape :math:`(*)`. + img2: the target torch.Tensor with the same shape as img1. + reduction: Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction + will be applied (default), ``'mean'``: the sum of the output will be divided + by the number of elements in the output, ``'sum'``: the output will be + summed. + + Return: + a scalar with the computed loss. + + Example: + >>> img1 = torch.randn(2, 3, 32, 32, requires_grad=True) + >>> img2 = torch.randn(2, 3, 32, 32) + >>> output = welsch_loss(img1, img2, reduction="mean") + >>> output.backward() + + """ + KORNIA_CHECK_IS_TENSOR(img1) + + KORNIA_CHECK_IS_TENSOR(img2) + + KORNIA_CHECK_SAME_SHAPE(img1, img2) + + KORNIA_CHECK_SAME_DEVICE(img1, img2) + + KORNIA_CHECK(reduction in ("mean", "sum", "none"), f"Given type of reduction is not supported. Got: {reduction}") + + # compute loss + loss = 1.0 - (-0.5 * (img1 - img2) ** 2).exp() + + # perform reduction + if reduction == "mean": + loss = loss.mean() + elif reduction == "sum": + loss = loss.sum() + elif reduction == "none": + pass + else: + raise NotImplementedError("Invalid reduction option.") + + return loss + + +class WelschLoss(nn.Module): + r"""Criterion that computes the Welsch [2] (aka. Leclerc [3]) loss. + + According to [1], we compute the Welsch loss as follows: + + .. math:: + + \text{WL}(x, y) = 1 - exp(-\frac{1}{2} (x - y)^{2}) + + Where: + - :math:`x` is the prediction. + - :math:`y` is the target to be regressed to. + + Reference: + [1] https://arxiv.org/pdf/1701.03077.pdf + [2] https://www.tandfonline.com/doi/abs/10.1080/03610917808812083 + [3] https://link.springer.com/article/10.1007/BF00054839 + + Args: + reduction: Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction + will be applied (default), ``'mean'``: the sum of the output will be divided + by the number of elements in the output, ``'sum'``: the output will be + summed. + + Shape: + - img1: the predicted torch.Tensor with shape :math:`(*)`. + - img2: the target torch.Tensor with the same shape as img1. + + Example: + >>> criterion = WelschLoss(reduction="mean") + >>> img1 = torch.randn(2, 3, 32, 1904, requires_grad=True) + >>> img2 = torch.randn(2, 3, 32, 1904) + >>> output = criterion(img1, img2) + >>> output.backward() + + """ + + def __init__(self, reduction: str = "none") -> None: + super().__init__() + self.reduction = reduction + + def forward(self, img1: torch.Tensor, img2: torch.Tensor) -> torch.Tensor: + """Compute the Welsch robust regression loss. + + Args: + img1: Predicted tensor with arbitrary shape. + img2: Target tensor with the same shape as ``img1``. + + Returns: + Loss tensor reduced according to ``self.reduction``. The Welsch + penalty saturates for large residuals, reducing the influence of + strong outliers. + """ + return welsch_loss(img1=img1, img2=img2, reduction=self.reduction) diff --git a/kornia/metrics/__init__.py b/kornia/metrics/__init__.py new file mode 100644 index 0000000..2127457 --- /dev/null +++ b/kornia/metrics/__init__.py @@ -0,0 +1,53 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Metrics submodule for Kornia. + +This package provides evaluation metrics for computer vision, such as accuracy, mean IoU, +PSNR, SSIM, endpoint error, and stereo disparity error. +""" + +from .accuracy import accuracy +from .average_meter import AverageMeter +from .confusion_matrix import confusion_matrix +from .disparity import mean_absolute_disparity_error, mean_bad_pixel_error, root_mean_squared_disparity_error +from .endpoint_error import AEPE, aepe, average_endpoint_error +from .mean_average_precision import mean_average_precision +from .mean_iou import mean_iou, mean_iou_bbox +from .psnr import psnr +from .ssim import SSIM, ssim +from .ssim3d import SSIM3D, ssim3d + +__all__ = [ + "AEPE", + "SSIM", + "SSIM3D", + "AverageMeter", + "accuracy", + "aepe", + "average_endpoint_error", + "confusion_matrix", + "mean_absolute_disparity_error", + "mean_average_precision", + "mean_bad_pixel_error", + "mean_iou", + "mean_iou_bbox", + "psnr", + "root_mean_squared_disparity_error", + "ssim", + "ssim3d", +] diff --git a/kornia/metrics/accuracy.py b/kornia/metrics/accuracy.py new file mode 100644 index 0000000..a729f73 --- /dev/null +++ b/kornia/metrics/accuracy.py @@ -0,0 +1,43 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import List, Tuple + +import torch + + +def accuracy(pred: torch.Tensor, target: torch.Tensor, topk: Tuple[int, ...] = (1,)) -> List[torch.Tensor]: + """Compute the accuracy over the k top predictions for the specified values of k. + + Args: + pred: the input torch.Tensor with the logits to evaluate. + target: the torch.Tensor containing the ground truth. + topk: the expected topk ranking. + + Example: + >>> logits = torch.tensor([[0, 1, 0]]) + >>> target = torch.tensor([[1]]) + >>> accuracy(logits, target) + [tensor(100.)] + + """ + maxk = min(max(topk), pred.size()[1]) + batch_size = target.size(0) + _, pred = pred.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.reshape(1, -1).expand_as(pred)) + return [correct[: min(k, maxk)].reshape(-1).float().sum(0) * 100.0 / batch_size for k in topk] diff --git a/kornia/metrics/average_meter.py b/kornia/metrics/average_meter.py new file mode 100644 index 0000000..ab6b714 --- /dev/null +++ b/kornia/metrics/average_meter.py @@ -0,0 +1,68 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Union + +import torch + + +class AverageMeter: + """Computes and stores the average and current value. + + Example: + >>> stats = AverageMeter() + >>> acc1 = torch.tensor(0.99) # coming from K.metrics.accuracy + >>> stats.update(acc1, n=1) # where n is batch size usually + >>> round(stats.avg, 2) + 0.99 + + """ + + val: Union[float, bool, torch.Tensor] + _avg: Union[float, torch.Tensor] + sum: Union[float, torch.Tensor] + count: int + + def __init__(self) -> None: + self.reset() + + def reset(self) -> None: + """Reset the current value, accumulated sum, count, and average.""" + self.val = 0 + self._avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val: Union[float, bool, torch.Tensor], n: int = 1) -> None: + """Add a new value to the running average. + + Args: + val: New scalar value or scalar tensor to accumulate. + n: Weight for ``val``. This is usually the batch size represented + by the value. + """ + self.val = val + self.sum += val * n + self.count += n + self._avg = self.sum / self.count + + @property + def avg(self) -> float: + """Return the accumulated average as a Python float.""" + if isinstance(self._avg, torch.Tensor): + return float(self._avg.item()) + return self._avg diff --git a/kornia/metrics/confusion_matrix.py b/kornia/metrics/confusion_matrix.py new file mode 100644 index 0000000..5f0eed2 --- /dev/null +++ b/kornia/metrics/confusion_matrix.py @@ -0,0 +1,86 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch + +# Inspired by: +# https://github.com/pytorch/tnt/blob/master/torchnet/meter/confusionmeter.py#L68-L73 + + +def confusion_matrix( + pred: torch.Tensor, target: torch.Tensor, num_classes: int, normalized: bool = False +) -> torch.Tensor: + r"""Compute confusion matrix to evaluate the accuracy of a classification. + + Args: + pred: tensor with estimated targets returned by a + classifier. The shape can be :math:`(B, *)` and must contain integer + values between 0 and K-1. + target: tensor with ground truth (correct) target + values. The shape can be :math:`(B, *)` and must contain integer + values between 0 and K-1, where targets are assumed to be provided as + one-hot vectors. + num_classes: total possible number of classes in target. + normalized: whether to return the confusion matrix normalized. + + Returns: + a tensor containing the confusion matrix with shape + :math:`(B, K, K)` where K is the number of classes. + + Example: + >>> logits = torch.tensor([[0, 1, 0]]) + >>> target = torch.tensor([[0, 1, 0]]) + >>> confusion_matrix(logits, target, num_classes=3) + tensor([[[2., 0., 0.], + [0., 1., 0.], + [0., 0., 0.]]]) + + """ + if not torch.is_tensor(pred) and pred.dtype is not torch.int64: + raise TypeError(f"Input pred type is not a torch.Tensor with torch.int64 dtype. Got {type(pred)}") + + if not torch.is_tensor(target) and target.dtype is not torch.int64: + raise TypeError(f"Input target type is not a torch.Tensor with torch.int64 dtype. Got {type(target)}") + if not pred.shape == target.shape: + raise ValueError(f"Inputs pred and target must have the same shape. Got: {pred.shape} and {target.shape}") + if not pred.device == target.device: + raise ValueError(f"Inputs must be in the same device. Got: {pred.device} - {target.device}") + + if not isinstance(num_classes, int) or num_classes < 2: + raise ValueError(f"The number of classes must be an integer bigger than two. Got: {num_classes}") + + batch_size: int = pred.shape[0] + + # hack for bitcounting 2 arrays together + # NOTE: torch.bincount does not implement batched version + pre_bincount: torch.Tensor = pred + target * num_classes + pre_bincount_vec: torch.Tensor = pre_bincount.view(batch_size, -1) + + confusion_list = [] + for iter_id in range(batch_size): + pb: torch.Tensor = pre_bincount_vec[iter_id] + bin_count: torch.Tensor = torch.bincount(pb, minlength=num_classes**2) + confusion_list.append(bin_count) + + confusion_vec: torch.Tensor = torch.stack(confusion_list) + confusion_mat: torch.Tensor = confusion_vec.view(batch_size, num_classes, num_classes).to(torch.float32) # BxKxK + + if normalized: + norm_val: torch.Tensor = torch.sum(confusion_mat, dim=1, keepdim=True) + confusion_mat = confusion_mat / (norm_val + 1e-6) + + return confusion_mat diff --git a/kornia/metrics/disparity.py b/kornia/metrics/disparity.py new file mode 100644 index 0000000..d3e20ed --- /dev/null +++ b/kornia/metrics/disparity.py @@ -0,0 +1,222 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Optional + +import torch + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SAME_SHAPE + + +def _is_broadcastable_to(shape: torch.Size, target_shape: torch.Size) -> bool: + """Check whether ``shape`` can be broadcast to ``target_shape``.""" + if len(shape) > len(target_shape): + return False + return all(s in (1, t) for s, t in zip(reversed(shape), reversed(target_shape), strict=False)) + + +def _check_disparity_inputs( + input: torch.Tensor, target: torch.Tensor, valid_mask: Optional[torch.Tensor] +) -> Optional[torch.Tensor]: + """Validate disparity metric inputs and return the valid mask broadcast to the input shape.""" + KORNIA_CHECK_IS_TENSOR(input) + KORNIA_CHECK_IS_TENSOR(target) + KORNIA_CHECK_SAME_SHAPE(input, target) + + if valid_mask is None: + return None + + KORNIA_CHECK_IS_TENSOR(valid_mask) + KORNIA_CHECK( + _is_broadcastable_to(valid_mask.shape, input.shape), + f"valid_mask shape must be broadcastable to the input shape. Got: {valid_mask.shape} and {input.shape}", + ) + + return valid_mask.to(torch.bool).broadcast_to(input.shape) + + +def _reduce_disparity_error(error: torch.Tensor, valid_mask: Optional[torch.Tensor], reduction: str) -> torch.Tensor: + """Reduce a per-pixel error map over the valid pixels according to ``reduction``.""" + if reduction == "mean": + error = error.mean() if valid_mask is None else error[valid_mask].mean() + elif reduction == "sum": + error = error.sum() if valid_mask is None else error[valid_mask].sum() + elif reduction == "none": + if valid_mask is not None: + error = torch.where(valid_mask, error, torch.zeros_like(error)) + else: + raise NotImplementedError("Invalid reduction option.") + + return error + + +def mean_absolute_disparity_error( + input: torch.Tensor, + target: torch.Tensor, + valid_mask: Optional[torch.Tensor] = None, + reduction: str = "mean", +) -> torch.Tensor: + r"""Compute the mean absolute error (MAE) between two disparity maps. + + Given predicted and ground truth disparity maps :math:`D` and :math:`D^{gt}` with + valid pixels :math:`\mathcal{V}`, the metric is: + + .. math:: + + \text{MAE}(D, D^{gt}) = \frac{1}{|\mathcal{V}|}\sum_{p \in \mathcal{V}} |D_{p} - D^{gt}_{p}| + + Args: + input: the predicted disparity map with arbitrary shape :math:`(*)`. + target: the ground truth disparity map with the same shape as ``input``. + valid_mask: optional mask broadcastable to the shape of ``input``, where nonzero + (``True``) values mark the pixels to evaluate. Non-boolean masks are converted + to boolean. If ``None``, all pixels are evaluated. + reduction: specifies the reduction to apply to the output: + ``'none'`` | ``'mean'`` | ``'sum'``. ``'mean'``: the error is averaged over the + valid pixels, ``'sum'``: the error is summed over the valid pixels, ``'none'``: no + reduction will be applied and the per-pixel error map is returned, with masked-out + positions set to zero. + + Return: + the computed metric as a scalar, or the per-pixel error map if ``reduction='none'``. + + Note: + If ``valid_mask`` selects no pixels, ``'mean'`` reduction returns ``nan``. + + Examples: + >>> input = torch.tensor([[0.0, 1.0], [2.0, 3.0]]) + >>> target = torch.tensor([[0.0, 1.0], [2.0, 4.0]]) + >>> mean_absolute_disparity_error(input, target) + tensor(0.2500) + >>> valid_mask = torch.tensor([[True, True], [True, False]]) + >>> mean_absolute_disparity_error(input, target, valid_mask) + tensor(0.) + + Reference: + D. Scharstein and R. Szeliski. A taxonomy and evaluation of dense two-frame stereo + correspondence algorithms. IJCV 2002. https://vision.middlebury.edu/stereo/taxonomy-IJCV.pdf + + """ + mask = _check_disparity_inputs(input, target, valid_mask) + error = (input - target).abs() + return _reduce_disparity_error(error, mask, reduction) + + +def root_mean_squared_disparity_error( + input: torch.Tensor, + target: torch.Tensor, + valid_mask: Optional[torch.Tensor] = None, + reduction: str = "mean", +) -> torch.Tensor: + r"""Compute the root mean squared error (RMSE) between two disparity maps. + + Given predicted and ground truth disparity maps :math:`D` and :math:`D^{gt}` with + valid pixels :math:`\mathcal{V}`, the metric is: + + .. math:: + + \text{RMSE}(D, D^{gt}) = + \sqrt{\frac{1}{|\mathcal{V}|}\sum_{p \in \mathcal{V}} (D_{p} - D^{gt}_{p})^{2}} + + Args: + input: the predicted disparity map with arbitrary shape :math:`(*)`. + target: the ground truth disparity map with the same shape as ``input``. + valid_mask: optional mask broadcastable to the shape of ``input``, where nonzero + (``True``) values mark the pixels to evaluate. Non-boolean masks are converted + to boolean. If ``None``, all pixels are evaluated. + reduction: specifies the reduction to apply to the squared error before the square + root: ``'none'`` | ``'mean'`` | ``'sum'``. ``'mean'``: the squared error is + averaged over the valid pixels, ``'sum'``: the squared error is summed over the + valid pixels, ``'none'``: no reduction will be applied and the per-pixel absolute + error map is returned, with masked-out positions set to zero. + + Return: + the computed metric as a scalar, or the per-pixel error map if ``reduction='none'``. + + Note: + If ``valid_mask`` selects no pixels, ``'mean'`` reduction returns ``nan``. + + Examples: + >>> input = torch.zeros(2, 2) + >>> target = torch.tensor([[0.0, 0.0], [0.0, 1.0]]) + >>> root_mean_squared_disparity_error(input, target) + tensor(0.5000) + + Reference: + D. Scharstein and R. Szeliski. A taxonomy and evaluation of dense two-frame stereo + correspondence algorithms. IJCV 2002. https://vision.middlebury.edu/stereo/taxonomy-IJCV.pdf + + """ + mask = _check_disparity_inputs(input, target, valid_mask) + error = (input - target) ** 2 + return _reduce_disparity_error(error, mask, reduction).sqrt() + + +def mean_bad_pixel_error( + input: torch.Tensor, + target: torch.Tensor, + threshold: float = 3.0, + valid_mask: Optional[torch.Tensor] = None, + reduction: str = "mean", +) -> torch.Tensor: + r"""Compute the bad pixel ratio between two disparity maps. + + A pixel is considered bad when its absolute disparity error is strictly greater than + ``threshold``. Given predicted and ground truth disparity maps :math:`D` and :math:`D^{gt}` + with valid pixels :math:`\mathcal{V}`, the metric is: + + .. math:: + + \text{Bad}_{\tau}(D, D^{gt}) = + \frac{1}{|\mathcal{V}|}\sum_{p \in \mathcal{V}} [|D_{p} - D^{gt}_{p}| > \tau] + + This corresponds to the bad-pixel percentage reported by the Middlebury and KITTI stereo + benchmarks, expressed as a fraction in :math:`[0, 1]` instead of a percentage. + + Args: + input: the predicted disparity map with arbitrary shape :math:`(*)`. + target: the ground truth disparity map with the same shape as ``input``. + threshold: the disparity error above which a pixel is considered bad. + valid_mask: optional mask broadcastable to the shape of ``input``, where nonzero + (``True``) values mark the pixels to evaluate. Non-boolean masks are converted + to boolean. If ``None``, all pixels are evaluated. + reduction: specifies the reduction to apply to the output: + ``'none'`` | ``'mean'`` | ``'sum'``. ``'mean'``: the fraction of bad pixels among + the valid pixels, ``'sum'``: the number of bad pixels among the valid pixels, + ``'none'``: no reduction will be applied and the per-pixel bad-pixel map is + returned, with masked-out positions set to zero. + + Return: + the computed metric as a scalar, or the per-pixel bad-pixel map if ``reduction='none'``. + + Note: + If ``valid_mask`` selects no pixels, ``'mean'`` reduction returns ``nan``. + + Examples: + >>> input = torch.zeros(2, 2) + >>> target = torch.tensor([[0.0, 1.0], [2.0, 4.0]]) + >>> mean_bad_pixel_error(input, target, threshold=1.5) + tensor(0.5000) + + Reference: + D. Scharstein and R. Szeliski. A taxonomy and evaluation of dense two-frame stereo + correspondence algorithms. IJCV 2002. https://vision.middlebury.edu/stereo/taxonomy-IJCV.pdf + + """ + mask = _check_disparity_inputs(input, target, valid_mask) + bad = ((input - target).abs() > threshold).to(input.dtype) + return _reduce_disparity_error(bad, mask, reduction) diff --git a/kornia/metrics/endpoint_error.py b/kornia/metrics/endpoint_error.py new file mode 100644 index 0000000..9e91594 --- /dev/null +++ b/kornia/metrics/endpoint_error.py @@ -0,0 +1,124 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch +from torch import Tensor, nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE + + +def aepe(input: torch.Tensor, target: torch.Tensor, reduction: str = "mean") -> torch.Tensor: + r"""Create a function that calculates the average endpoint error (AEPE) between 2 flow maps. + + AEPE is the endpoint error between two 2D vectors (e.g., optical flow). + Given a h x w x 2 optical flow map, the AEPE is: + + .. math:: + + \text{AEPE}=\frac{1}{hw}\sum_{i=1, j=1}^{h, w}\sqrt{(I_{i,j,1}-T_{i,j,1})^{2}+(I_{i,j,2}-T_{i,j,2})^{2}} + + Args: + input: the input flow map with shape :math:`(*, 2)`. + target: the target flow map with shape :math:`(*, 2)`. + reduction : Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the sum of the output will be divided by the number of elements + in the output, ``'sum'``: the output will be summed. + + Return: + the computed AEPE as a scalar. + + Examples: + >>> ones = torch.ones(4, 4, 2) + >>> aepe(ones, 1.2 * ones) + tensor(0.2828) + + Reference: + https://link.springer.com/content/pdf/10.1007/s11263-010-0390-2.pdf + + """ + KORNIA_CHECK_IS_TENSOR(input) + KORNIA_CHECK_IS_TENSOR(target) + KORNIA_CHECK_SHAPE(input, ["*", "2"]) + KORNIA_CHECK_SHAPE(target, ["*", "2"]) + KORNIA_CHECK( + input.shape == target.shape, f"input and target shapes must be the same. Got: {input.shape} and {target.shape}" + ) + + epe: Tensor = ((input[..., 0] - target[..., 0]) ** 2 + (input[..., 1] - target[..., 1]) ** 2).sqrt() + + if reduction == "mean": + epe = epe.mean() + elif reduction == "sum": + epe = epe.sum() + elif reduction == "none": + pass + else: + raise NotImplementedError("Invalid reduction option.") + + return epe + + +class AEPE(nn.Module): + r"""Computes the average endpoint error (AEPE) between 2 flow maps. + + EPE is the endpoint error between two 2D vectors (e.g., optical flow). + Given a h x w x 2 optical flow map, the AEPE is: + + .. math:: + + \text{AEPE}=\frac{1}{hw}\sum_{i=1, j=1}^{h, w}\sqrt{(I_{i,j,1}-T_{i,j,1})^{2}+(I_{i,j,2}-T_{i,j,2})^{2}} + + Args: + reduction : Specifies the reduction to apply to the + output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the sum of the output will be divided by the number of elements + in the output, ``'sum'``: the output will be summed. + + Shape: + - input: :math:`(*, 2)`. + - target :math:`(*, 2)`. + - output: :math:`(1)`. + + Examples: + >>> input1 = torch.rand(1, 4, 5, 2) + >>> input2 = torch.rand(1, 4, 5, 2) + >>> epe = AEPE(reduction="mean") + >>> epe = epe(input1, input2) + + """ + + def __init__(self, reduction: str = "mean") -> None: + super().__init__() + self.reduction: str = reduction + + def forward(self, input: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + """Compute average endpoint error between vector fields. + + Args: + input: Predicted vector field with shape :math:`(B, H, W, 2)`. + target: Target vector field with the same shape as ``input``. + + Returns: + Endpoint-error tensor reduced according to ``self.reduction``. + The endpoint error is the Euclidean distance between predicted and + target vectors at each spatial location. + """ + return aepe(input, target, self.reduction) + + +average_endpoint_error = aepe diff --git a/kornia/metrics/mean_average_precision.py b/kornia/metrics/mean_average_precision.py new file mode 100644 index 0000000..d75a8c9 --- /dev/null +++ b/kornia/metrics/mean_average_precision.py @@ -0,0 +1,181 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, List, Tuple + +import torch + +from .mean_iou import mean_iou_bbox + + +def mean_average_precision( + pred_boxes: List[torch.Tensor], + pred_labels: List[torch.Tensor], + pred_scores: List[torch.Tensor], + gt_boxes: List[torch.Tensor], + gt_labels: List[torch.Tensor], + n_classes: int, + threshold: float = 0.5, +) -> Tuple[torch.Tensor, Dict[int, float]]: + """Calculate the Mean Average Precision (mAP) of detected objects. + + Code altered from https://github.com/sgrvinod/a-PyTorch-Tutorial-to-Object-Detection/blob/master/utils.py#L271. + Background class (0 index) is excluded. + + Args: + pred_boxes: a torch.Tensor list of predicted bounding boxes. + pred_labels: a torch.Tensor list of predicted labels. + pred_scores: a torch.Tensor list of predicted labels' scores. + gt_boxes: a torch.Tensor list of ground truth bounding boxes. + gt_labels: a torch.Tensor list of ground truth labels. + n_classes: the number of classes. + threshold: count as a positive if the overlap is greater than the threshold. + + Returns: + mean average precision (mAP), list of average precisions for each class. + + Examples: + >>> boxes, labels, scores = torch.tensor([[100, 50, 150, 100.]]), torch.tensor([1]), torch.tensor([.7]) + >>> gt_boxes, gt_labels = torch.tensor([[100, 50, 150, 100.]]), torch.tensor([1]) + >>> mean_average_precision([boxes], [labels], [scores], [gt_boxes], [gt_labels], 2) + (tensor(1.), {1: 1.0}) + + """ + # these are all lists of tensors of the same length, i.e. number of images + if not len(pred_boxes) == len(pred_labels) == len(pred_scores) == len(gt_boxes) == len(gt_labels): + raise AssertionError + + # Store all (true) objects in a single continuous torch.Tensor while keeping track of the image it is from + gt_images = [] + for i, labels in enumerate(gt_labels): + gt_images.extend([i] * labels.size(0)) + # (n_objects), n_objects is the total no. of objects across all images + _gt_boxes = torch.cat(gt_boxes, 0) # (n_objects, 4) + _gt_labels = torch.cat(gt_labels, 0) # (n_objects) + _gt_images = torch.tensor(gt_images, device=_gt_boxes.device, dtype=torch.long) + + if not _gt_images.size(0) == _gt_boxes.size(0) == _gt_labels.size(0): + raise AssertionError + + # Store all detections in a single continuous torch.Tensor while keeping track of the image it is from + pred_images = [] + for i, labels in enumerate(pred_labels): + pred_images.extend([i] * labels.size(0)) + _pred_boxes = torch.cat(pred_boxes, 0) # (n_detections, 4) + _pred_labels = torch.cat(pred_labels, 0) # (n_detections) + _pred_scores = torch.cat(pred_scores, 0) # (n_detections) + _pred_images = torch.tensor(pred_images, device=_pred_boxes.device, dtype=torch.long) # (n_detections) + + if not _pred_images.size(0) == _pred_boxes.size(0) == _pred_labels.size(0) == _pred_scores.size(0): + raise AssertionError + + # Calculate APs for each class (except background) + average_precisions = torch.zeros( + (n_classes - 1), device=_pred_boxes.device, dtype=_pred_boxes.dtype + ) # (n_classes - 1) + for c in range(1, n_classes): + # Extract only objects with this class + gt_class_images = _gt_images[_gt_labels == c] # (n_class_objects) + gt_class_boxes = _gt_boxes[_gt_labels == c] # (n_class_objects, 4) + + # Keep track of which true objects with this class have already been 'detected' + # (n_class_objects) + gt_class_boxes_detected = torch.zeros( + (gt_class_images.size(0)), dtype=torch.uint8, device=gt_class_images.device + ) + + # Extract only detections with this class + pred_class_images = _pred_images[_pred_labels == c] # (n_class_detections) + pred_class_boxes = _pred_boxes[_pred_labels == c] # (n_class_detections, 4) + pred_class_scores = _pred_scores[_pred_labels == c] # (n_class_detections) + n_class_detections = pred_class_boxes.size(0) + if n_class_detections == 0: + continue + + # Sort detections in decreasing order of confidence/scores + pred_class_scores, sort_ind = torch.sort(pred_class_scores, dim=0, descending=True) # (n_class_detections) + pred_class_images = pred_class_images[sort_ind] # (n_class_detections) + pred_class_boxes = pred_class_boxes[sort_ind] # (n_class_detections, 4) + + # In the order of decreasing scores, check if true or false positive + gt_positives = torch.zeros( + (n_class_detections,), dtype=pred_class_boxes.dtype, device=pred_class_boxes.device + ) # (n_class_detections) + false_positives = torch.zeros( + (n_class_detections,), dtype=pred_class_boxes.dtype, device=pred_class_boxes.device + ) # (n_class_detections) + for d in range(n_class_detections): + this_detection_box = pred_class_boxes[d].unsqueeze(0) # (1, 4) + this_image = pred_class_images[d] # (), scalar + + # Find objects in the image with this class, their difficulties, and whether they have been detected before + object_boxes = gt_class_boxes[gt_class_images == this_image] # (n_class_objects_in_img) + # If no such object in this image, then the detection is a false positive + if object_boxes.size(0) == 0: + false_positives[d] = 1 + continue + + # Find maximum overlap of this detection with objects in this image of this class + overlaps = mean_iou_bbox(this_detection_box, object_boxes) # (1, n_class_objects_in_img) + max_overlap, ind = torch.max(overlaps.squeeze(0), dim=0) # (), () - scalars + + # 'ind' is the index of the object in these image-level tensors 'object_boxes', 'object_difficulties' + # In the original class-level tensors 'gt_class_boxes', etc., 'ind' corresponds to object with index... + original_ind = torch.tensor( + range(gt_class_boxes.size(0)), device=gt_class_boxes_detected.device, dtype=torch.long + )[gt_class_images == this_image][ind] + # We need 'original_ind' to update 'gt_class_boxes_detected' + + # If the maximum overlap is greater than the threshold of 0.5, it's a match + if max_overlap.item() > threshold: + # If this object has already not been detected, it's a true positive + if gt_class_boxes_detected[original_ind] == 0: + gt_positives[d] = 1 + gt_class_boxes_detected[original_ind] = 1 # this object has now been detected/accounted for + # Otherwise, it's a false positive (since this object is already accounted for) + else: + false_positives[d] = 1 + # Otherwise, the detection occurs in a different location than the actual object, and is a false positive + else: + false_positives[d] = 1 + + # Compute cumulative precision and recall at each detection in the order of decreasing scores + cumul_gt_positives = torch.cumsum(gt_positives, dim=0) # (n_class_detections) + cumul_false_positives = torch.cumsum(false_positives, dim=0) # (n_class_detections) + cumul_precision = cumul_gt_positives / ( + cumul_gt_positives + cumul_false_positives + 1e-10 + ) # (n_class_detections) + cumul_recall = cumul_gt_positives / _gt_boxes.size(0) # (n_class_detections) + + # Find the mean of the maximum of the precisions corresponding to recalls above the threshold 't' + recall_thresholds = torch.arange(start=0, end=1.1, step=0.1).tolist() # (11) + precisions = torch.zeros((len(recall_thresholds)), device=_gt_boxes.device, dtype=_gt_boxes.dtype) # (11) + for i, t in enumerate(recall_thresholds): + recalls_above_t = cumul_recall >= t + if recalls_above_t.any(): + precisions[i] = cumul_precision[recalls_above_t].max() + else: + precisions[i] = 0.0 + average_precisions[c - 1] = precisions.mean() # c is in [1, n_classes - 1] + + # Calculate Mean Average Precision (mAP) + mean_ap = average_precisions.mean() + + # Keep class-wise average precisions in a dictionary + ap_dict = {c + 1: float(v) for c, v in enumerate(average_precisions.tolist())} + + return mean_ap, ap_dict diff --git a/kornia/metrics/mean_iou.py b/kornia/metrics/mean_iou.py new file mode 100644 index 0000000..f865726 --- /dev/null +++ b/kornia/metrics/mean_iou.py @@ -0,0 +1,172 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch + +from .confusion_matrix import confusion_matrix + + +def mean_iou(pred: torch.Tensor, target: torch.Tensor, num_classes: int, eps: float = 1e-6) -> torch.Tensor: + r"""Calculate mean Intersection-Over-Union (mIOU). + + The function internally computes the confusion matrix. + + Args: + pred : tensor with estimated targets returned by a + classifier. The shape can be :math:`(B, *)` and must contain integer + values between 0 and K-1. + target: tensor with ground truth (correct) target + values. The shape can be :math:`(B, *)` and must contain integer + values between 0 and K-1, where targets are assumed to be provided as + one-hot vectors. + num_classes: total possible number of classes in target. + eps: epsilon for numerical stability. + + Returns: + a tensor representing the mean intersection-over union + with shape :math:`(B, K)` where K is the number of classes. + + Example: + >>> logits = torch.tensor([[0, 1, 0]]) + >>> target = torch.tensor([[0, 1, 0]]) + >>> mean_iou(logits, target, num_classes=3) + tensor([[1., 1., 1.]]) + + """ + if not torch.is_tensor(pred) and pred.dtype is not torch.int64: + raise TypeError(f"Input pred type is not a torch.Tensor with torch.int64 dtype. Got {type(pred)}") + + if not torch.is_tensor(target) and target.dtype is not torch.int64: + raise TypeError(f"Input target type is not a torch.Tensor with torch.int64 dtype. Got {type(target)}") + if not pred.shape == target.shape: + raise ValueError(f"Inputs pred and target must have the same shape. Got: {pred.shape} and {target.shape}") + if not pred.device == target.device: + raise ValueError(f"Inputs must be in the same device. Got: {pred.device} - {target.device}") + + if not isinstance(num_classes, int) or num_classes < 2: + raise ValueError(f"The number of classes must be an integer bigger than two. Got: {num_classes}") + + # we first compute the confusion matrix + conf_mat: torch.Tensor = confusion_matrix(pred, target, num_classes) + + # compute the actual intersection over union + sum_over_row = torch.sum(conf_mat, dim=1) + sum_over_col = torch.sum(conf_mat, dim=2) + conf_mat_diag = torch.diagonal(conf_mat, dim1=-2, dim2=-1) + denominator = sum_over_row + sum_over_col - conf_mat_diag + + # NOTE: we add epsilon so that samples that are neither in the + # prediction or ground truth are taken into account. + ious = (conf_mat_diag + eps) / (denominator + eps) + return ious + + +def _convert_boxes_to_xyxy(boxes: torch.Tensor, box_format: str) -> torch.Tensor: + """Convert bounding boxes from various formats to xyxy format. + + Args: + boxes: tensor of bounding boxes in shape (N, 4). + box_format: box format - one of 'xyxy', 'xywh', or 'cxcywh'. + + Returns: + boxes in xyxy format (x1, y1, x2, y2). + """ + if box_format == "xyxy": + return boxes + elif box_format == "xywh": + # (x, y, w, h) -> (x1, y1, x2, y2) + x, y, w, h = boxes[:, 0:1], boxes[:, 1:2], boxes[:, 2:3], boxes[:, 3:4] + x2 = x + w + y2 = y + h + return torch.cat([x, y, x2, y2], dim=1) + elif box_format == "cxcywh": + # (cx, cy, w, h) -> (x1, y1, x2, y2) + cx, cy, w, h = boxes[:, 0:1], boxes[:, 1:2], boxes[:, 2:3], boxes[:, 3:4] + x1 = cx - w / 2 + y1 = cy - h / 2 + x2 = cx + w / 2 + y2 = cy + h / 2 + return torch.cat([x1, y1, x2, y2], dim=1) + else: + raise ValueError(f"Unsupported box format: {box_format}. Must be one of 'xyxy', 'xywh', or 'cxcywh'.") + + +def mean_iou_bbox(boxes_1: torch.Tensor, boxes_2: torch.Tensor, box_format: str = "xyxy") -> torch.Tensor: + """Compute the IoU of the cartesian product of two sets of boxes. + + Args: + boxes_1: a tensor of bounding boxes in :math:`(B1, 4)`. + boxes_2: a tensor of bounding boxes in :math:`(B2, 4)`. + box_format: the bounding box format. Supported formats are: + - 'xyxy': (x1, y1, x2, y2) where (x1, y1) is top-left and (x2, y2) is bottom-right + - 'xywh': (x, y, w, h) where (x, y) is top-left, w is width, h is height + - 'cxcywh': (cx, cy, w, h) where (cx, cy) is center, w is width, h is height + Default: 'xyxy'. + + Returns: + a tensor in dimensions :math:`(B1, B2)`, representing the + intersection of each of the boxes in set 1 with respect to each of the boxes in set 2. + + Example: + >>> # XYXY format + >>> boxes_1 = torch.tensor([[40, 40, 60, 60], [30, 40, 50, 60]]) + >>> boxes_2 = torch.tensor([[40, 50, 60, 70], [30, 40, 40, 50]]) + >>> mean_iou_bbox(boxes_1, boxes_2) + tensor([[0.3333, 0.0000], + [0.1429, 0.2500]]) + >>> # XYWH format + >>> boxes_1_xywh = torch.tensor([[40, 40, 20, 20], [30, 40, 20, 20]]) + >>> boxes_2_xywh = torch.tensor([[40, 50, 20, 20], [30, 40, 10, 10]]) + >>> mean_iou_bbox(boxes_1_xywh, boxes_2_xywh, box_format='xywh') + tensor([[0.3333, 0.0000], + [0.1429, 0.2500]]) + >>> # CXCYWH format + >>> boxes_1_cxcywh = torch.tensor([[50, 50, 20, 20], [40, 50, 20, 20]]) + >>> boxes_2_cxcywh = torch.tensor([[50, 60, 20, 20], [35, 45, 10, 10]]) + >>> mean_iou_bbox(boxes_1_cxcywh, boxes_2_cxcywh, box_format='cxcywh') + tensor([[0.3333, 0.0000], + [0.1429, 0.2500]]) + + """ + # Convert boxes to xyxy format + boxes_1_xyxy = _convert_boxes_to_xyxy(boxes_1, box_format) + boxes_2_xyxy = _convert_boxes_to_xyxy(boxes_2, box_format) + + # Validate boxes are in proper xyxy format + if not ( + ((boxes_1_xyxy[:, 2] - boxes_1_xyxy[:, 0]) > 0).all() and ((boxes_1_xyxy[:, 3] - boxes_1_xyxy[:, 1]) > 0).all() + ): + raise AssertionError("Boxes_1 contains invalid boxes after conversion.") + if not ( + ((boxes_2_xyxy[:, 2] - boxes_2_xyxy[:, 0]) > 0).all() and ((boxes_2_xyxy[:, 3] - boxes_2_xyxy[:, 1]) > 0).all() + ): + raise AssertionError("Boxes_2 contains invalid boxes after conversion.") + + # Find intersection + lower_bounds = torch.max(boxes_1_xyxy[:, :2].unsqueeze(1), boxes_2_xyxy[:, :2].unsqueeze(0)) # (n1, n2, 2) + upper_bounds = torch.min(boxes_1_xyxy[:, 2:].unsqueeze(1), boxes_2_xyxy[:, 2:].unsqueeze(0)) # (n1, n2, 2) + intersection_dims = torch.clamp(upper_bounds - lower_bounds, min=0) # (n1, n2, 2) + intersection = intersection_dims[:, :, 0] * intersection_dims[:, :, 1] # (n1, n2) + + # Find areas of each box in both sets + areas_set_1 = (boxes_1_xyxy[:, 2] - boxes_1_xyxy[:, 0]) * (boxes_1_xyxy[:, 3] - boxes_1_xyxy[:, 1]) # (n1) + areas_set_2 = (boxes_2_xyxy[:, 2] - boxes_2_xyxy[:, 0]) * (boxes_2_xyxy[:, 3] - boxes_2_xyxy[:, 1]) # (n2) + + # Find the union + union = areas_set_1.unsqueeze(1) + areas_set_2.unsqueeze(0) - intersection # (n1, n2) + + return intersection / union # (n1, n2) diff --git a/kornia/metrics/psnr.py b/kornia/metrics/psnr.py new file mode 100644 index 0000000..9eaf88e --- /dev/null +++ b/kornia/metrics/psnr.py @@ -0,0 +1,67 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch +from torch.nn.functional import mse_loss as mse + + +def psnr(image: torch.Tensor, target: torch.Tensor, max_val: float) -> torch.Tensor: + r"""Create a function that calculates the PSNR between 2 images. + + PSNR is Peek Signal to Noise Ratio, which is similar to mean squared error. + Given an m x n image, the PSNR is: + + .. math:: + + \text{PSNR} = 10 \log_{10} \bigg(\frac{\text{MAX}_I^2}{MSE(I,T)}\bigg) + + where + + .. math:: + + \text{MSE}(I,T) = \frac{1}{mn}\sum_{i=0}^{m-1}\sum_{j=0}^{n-1} [I(i,j) - T(i,j)]^2 + + and :math:`\text{MAX}_I` is the maximum possible input value + (e.g for floating point images :math:`\text{MAX}_I=1`). + + Args: + image: the input image with arbitrary shape :math:`(*)`. + target: the labels image with arbitrary shape :math:`(*)`. + max_val: The maximum value in the input tensor. + + Return: + the computed loss as a scalar. + + Examples: + >>> ones = torch.ones(1) + >>> psnr(ones, 1.2 * ones, 2.) # 10 * log(4/((1.2-1)**2)) / log(10) + tensor(20.0000) + + Reference: + https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio#Definition + + """ + if not isinstance(image, torch.Tensor): + raise TypeError(f"Expected torch.Tensor but got {type(image)}.") + + if not isinstance(target, torch.Tensor): + raise TypeError(f"Expected torch.Tensor but got {type(target)}.") + + if image.shape != target.shape: + raise TypeError(f"Expected tensors of equal shapes, but got {image.shape} and {target.shape}") + + return 10.0 * torch.log10(max_val**2 / mse(image, target, reduction="mean")) diff --git a/kornia/metrics/ssim.py b/kornia/metrics/ssim.py new file mode 100644 index 0000000..5474a8c --- /dev/null +++ b/kornia/metrics/ssim.py @@ -0,0 +1,198 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import List + +import torch +from torch import nn + +from kornia.filters import filter2d_separable, get_gaussian_kernel1d +from kornia.filters.filter import _compute_padding + + +def _crop(img: torch.Tensor, cropping_shape: List[int]) -> torch.Tensor: + """Crop out the part of "valid" convolution area.""" + return torch.nn.functional.pad( + img, (-cropping_shape[2], -cropping_shape[3], -cropping_shape[0], -cropping_shape[1]) + ) + + +def ssim( + img1: torch.Tensor, + img2: torch.Tensor, + window_size: int, + max_val: float = 1.0, + eps: float = 1e-12, + padding: str = "same", +) -> torch.Tensor: + r"""Compute the Structural Similarity (SSIM) index map between two images. + + Measures the (SSIM) index between each element in the input `x` and target `y`. + + The index can be described as: + + .. math:: + + \text{SSIM}(x, y) = \frac{(2\mu_x\mu_y+c_1)(2\sigma_{xy}+c_2)} + {(\mu_x^2+\mu_y^2+c_1)(\sigma_x^2+\sigma_y^2+c_2)} + + where: + - :math:`c_1=(k_1 L)^2` and :math:`c_2=(k_2 L)^2` are two variables to + stabilize the division with weak denominator. + - :math:`L` is the dynamic range of the pixel-values (typically this is + :math:`2^{\#\text{bits per pixel}}-1`). + + Args: + img1: the first input image with shape :math:`(B, C, H, W)`. + img2: the second input image with shape :math:`(B, C, H, W)`. + window_size: the size of the gaussian kernel to smooth the images. + max_val: the dynamic range of the images. + eps: Small value for numerically stability when dividing. + padding: ``'same'`` | ``'valid'``. Whether to only use the "valid" convolution + area to compute SSIM to match the MATLAB implementation of original SSIM paper. + + Returns: + The ssim index map with shape :math:`(B, C, H, W)`. + + Examples: + >>> input1 = torch.rand(1, 4, 5, 5) + >>> input2 = torch.rand(1, 4, 5, 5) + >>> ssim_map = ssim(input1, input2, 5) # 1x4x5x5 + + """ + if not isinstance(img1, torch.Tensor): + raise TypeError(f"Input img1 type is not a torch.Tensor. Got {type(img1)}") + + if not isinstance(img2, torch.Tensor): + raise TypeError(f"Input img2 type is not a torch.Tensor. Got {type(img2)}") + + if not isinstance(max_val, float): + raise TypeError(f"Input max_val type is not a float. Got {type(max_val)}") + + if not len(img1.shape) == 4: + raise ValueError(f"Invalid img1 shape, we expect BxCxHxW. Got: {img1.shape}") + + if not len(img2.shape) == 4: + raise ValueError(f"Invalid img2 shape, we expect BxCxHxW. Got: {img2.shape}") + + if not img1.shape == img2.shape: + raise ValueError(f"img1 and img2 shapes must be the same. Got: {img1.shape} and {img2.shape}") + + # prepare kernel + kernel: torch.Tensor = get_gaussian_kernel1d(window_size, 1.5, device=img1.device, dtype=img1.dtype) + + # compute coefficients + C1: float = (0.01 * max_val) ** 2 + C2: float = (0.03 * max_val) ** 2 + + # compute local mean per channel + mu1: torch.Tensor = filter2d_separable(img1, kernel, kernel) + mu2: torch.Tensor = filter2d_separable(img2, kernel, kernel) + + cropping_shape: List[int] = [] + if padding == "valid": + height = width = kernel.shape[-1] + cropping_shape = _compute_padding([height, width]) + mu1 = _crop(mu1, cropping_shape) + mu2 = _crop(mu2, cropping_shape) + elif padding == "same": + pass + + mu1_sq = mu1**2 + mu2_sq = mu2**2 + mu1_mu2 = mu1 * mu2 + + mu_img1_sq = filter2d_separable(img1**2, kernel, kernel) + mu_img2_sq = filter2d_separable(img2**2, kernel, kernel) + mu_img1_img2 = filter2d_separable(img1 * img2, kernel, kernel) + + if padding == "valid": + mu_img1_sq = _crop(mu_img1_sq, cropping_shape) + mu_img2_sq = _crop(mu_img2_sq, cropping_shape) + mu_img1_img2 = _crop(mu_img1_img2, cropping_shape) + elif padding == "same": + pass + + # compute local sigma per channel + sigma1_sq = mu_img1_sq - mu1_sq + sigma2_sq = mu_img2_sq - mu2_sq + sigma12 = mu_img1_img2 - mu1_mu2 + + # compute the similarity index map + num: torch.Tensor = (2.0 * mu1_mu2 + C1) * (2.0 * sigma12 + C2) + den: torch.Tensor = (mu1_sq + mu2_sq + C1) * (sigma1_sq + sigma2_sq + C2) + + return num / (den + eps) + + +class SSIM(nn.Module): + r"""Create a module that computes the Structural Similarity (SSIM) index between two images. + + Measures the (SSIM) index between each element in the input `x` and target `y`. + + The index can be described as: + + .. math:: + + \text{SSIM}(x, y) = \frac{(2\mu_x\mu_y+c_1)(2\sigma_{xy}+c_2)} + {(\mu_x^2+\mu_y^2+c_1)(\sigma_x^2+\sigma_y^2+c_2)} + + where: + - :math:`c_1=(k_1 L)^2` and :math:`c_2=(k_2 L)^2` are two variables to + stabilize the division with weak denominator. + - :math:`L` is the dynamic range of the pixel-values (typically this is + :math:`2^{\#\text{bits per pixel}}-1`). + + Args: + window_size: the size of the gaussian kernel to smooth the images. + max_val: the dynamic range of the images. + eps: Small value for numerically stability when dividing. + padding: ``'same'`` | ``'valid'``. Whether to only use the "valid" convolution + area to compute SSIM to match the MATLAB implementation of original SSIM paper. + + Shape: + - Input: :math:`(B, C, H, W)`. + - Target :math:`(B, C, H, W)`. + - Output: :math:`(B, C, H, W)`. + + Examples: + >>> input1 = torch.rand(1, 4, 5, 5) + >>> input2 = torch.rand(1, 4, 5, 5) + >>> ssim = SSIM(5) + >>> ssim_map = ssim(input1, input2) # 1x4x5x5 + + """ + + def __init__(self, window_size: int, max_val: float = 1.0, eps: float = 1e-12, padding: str = "same") -> None: + super().__init__() + self.window_size: int = window_size + self.max_val: float = max_val + self.eps = eps + self.padding = padding + + def forward(self, img1: torch.Tensor, img2: torch.Tensor) -> torch.Tensor: + """Compute a 2D SSIM map between two image batches. + + Args: + img1: First image tensor with shape :math:`(B, C, H, W)`. + img2: Second image tensor with the same shape as ``img1``. + + Returns: + Tensor with shape :math:`(B, C, H, W)` containing local structural + similarity values for each channel and spatial location. + """ + return ssim(img1, img2, self.window_size, self.max_val, self.eps, self.padding) diff --git a/kornia/metrics/ssim3d.py b/kornia/metrics/ssim3d.py new file mode 100644 index 0000000..6f8a4cd --- /dev/null +++ b/kornia/metrics/ssim3d.py @@ -0,0 +1,199 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import List + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE +from kornia.filters import filter3d, get_gaussian_kernel3d +from kornia.filters.filter import _compute_padding + + +def _crop(img: torch.Tensor, cropping_shape: List[int]) -> torch.Tensor: + """Crop out the part of "valid" convolution area.""" + return F.pad( + img, + ( + -cropping_shape[4], + -cropping_shape[5], + -cropping_shape[2], + -cropping_shape[3], + -cropping_shape[0], + -cropping_shape[1], + ), + ) + + +def ssim3d( + img1: torch.Tensor, + img2: torch.Tensor, + window_size: int, + max_val: float = 1.0, + eps: float = 1e-12, + padding: str = "same", +) -> torch.Tensor: + r"""Compute the Structural Similarity (SSIM) index map between two images. + + Measures the (SSIM) index between each element in the input `x` and target `y`. + + The index can be described as: + + .. math:: + + \text{SSIM}(x, y) = \frac{(2\mu_x\mu_y+c_1)(2\sigma_{xy}+c_2)} + {(\mu_x^2+\mu_y^2+c_1)(\sigma_x^2+\sigma_y^2+c_2)} + + torch.where: + - :math:`c_1=(k_1 L)^2` and :math:`c_2=(k_2 L)^2` are two variables to + stabilize the division with weak denominator. + - :math:`L` is the dynamic range of the pixel-values (typically this is + :math:`2^{\#\text{bits per pixel}}-1`). + + Args: + img1: the first input image with shape :math:`(B, C, D, H, W)`. + img2: the second input image with shape :math:`(B, C, D, H, W)`. + window_size: the size of the gaussian kernel to smooth the images. + max_val: the dynamic range of the images. + eps: Small value for numerically stability when dividing. + padding: ``'same'`` | ``'valid'``. Whether to only use the "valid" convolution + area to compute SSIM to match the MATLAB implementation of original SSIM paper. + + Returns: + The ssim index map with shape :math:`(B, C, D, H, W)`. + + Examples: + >>> input1 = torch.rand(1, 4, 5, 5, 5) + >>> input2 = torch.rand(1, 4, 5, 5, 5) + >>> ssim_map = ssim3d(input1, input2, 5) # 1x4x5x5x5 + + """ + KORNIA_CHECK_IS_TENSOR(img1) + KORNIA_CHECK_IS_TENSOR(img2) + KORNIA_CHECK_SHAPE(img1, ["B", "C", "D", "H", "W"]) + KORNIA_CHECK_SHAPE(img2, ["B", "C", "D", "H", "W"]) + KORNIA_CHECK(img1.shape == img2.shape, f"img1 and img2 shapes must be the same. Got: {img1.shape} and {img2.shape}") + + if not isinstance(max_val, float): + raise TypeError(f"Input max_val type is not a float. Got {type(max_val)}") + + # prepare kernel + kernel: torch.Tensor = get_gaussian_kernel3d((window_size, window_size, window_size), (1.5, 1.5, 1.5)) + + # compute coefficients + C1: float = (0.01 * max_val) ** 2 + C2: float = (0.03 * max_val) ** 2 + + # compute local mean per channel + mu1: torch.Tensor = filter3d(img1, kernel) + mu2: torch.Tensor = filter3d(img2, kernel) + + cropping_shape: List[int] = [] + if padding == "valid": + depth, height, width = kernel.shape[-3:] + cropping_shape = _compute_padding([depth, height, width]) + mu1 = _crop(mu1, cropping_shape) + mu2 = _crop(mu2, cropping_shape) + elif padding == "same": + pass + + mu1_sq = mu1**2 + mu2_sq = mu2**2 + mu1_mu2 = mu1 * mu2 + + mu_img1_sq = filter3d(img1**2, kernel) + mu_img2_sq = filter3d(img2**2, kernel) + mu_img1_img2 = filter3d(img1 * img2, kernel) + + if padding == "valid": + mu_img1_sq = _crop(mu_img1_sq, cropping_shape) + mu_img2_sq = _crop(mu_img2_sq, cropping_shape) + mu_img1_img2 = _crop(mu_img1_img2, cropping_shape) + elif padding == "same": + pass + + # compute local sigma per channel + sigma1_sq = mu_img1_sq - mu1_sq + sigma2_sq = mu_img2_sq - mu2_sq + sigma12 = mu_img1_img2 - mu1_mu2 + + # compute the similarity index map + num: torch.Tensor = (2.0 * mu1_mu2 + C1) * (2.0 * sigma12 + C2) + den: torch.Tensor = (mu1_sq + mu2_sq + C1) * (sigma1_sq + sigma2_sq + C2) + + return num / (den + eps) + + +class SSIM3D(nn.Module): + r"""Create a module that computes the Structural Similarity (SSIM) index between two 3D images. + + Measures the (SSIM) index between each element in the input `x` and target `y`. + + The index can be described as: + + .. math:: + + \text{SSIM}(x, y) = \frac{(2\mu_x\mu_y+c_1)(2\sigma_{xy}+c_2)} + {(\mu_x^2+\mu_y^2+c_1)(\sigma_x^2+\sigma_y^2+c_2)} + + torch.where: + - :math:`c_1=(k_1 L)^2` and :math:`c_2=(k_2 L)^2` are two variables to + stabilize the division with weak denominator. + - :math:`L` is the dynamic range of the pixel-values (typically this is + :math:`2^{\#\text{bits per pixel}}-1`). + + Args: + window_size: the size of the gaussian kernel to smooth the images. + max_val: the dynamic range of the images. + eps: Small value for numerically stability when dividing. + padding: ``'same'`` | ``'valid'``. Whether to only use the "valid" convolution + area to compute SSIM to match the MATLAB implementation of original SSIM paper. + + Shape: + - Input: :math:`(B, C, D, H, W)`. + - Target :math:`(B, C, D, H, W)`. + - Output: :math:`(B, C, D, H, W)`. + + Examples: + >>> input1 = torch.rand(1, 4, 5, 5, 5) + >>> input2 = torch.rand(1, 4, 5, 5, 5) + >>> ssim = SSIM3D(5) + >>> ssim_map = ssim(input1, input2) # 1x4x5x5x5 + + """ + + def __init__(self, window_size: int, max_val: float = 1.0, eps: float = 1e-12, padding: str = "same") -> None: + super().__init__() + self.window_size: int = window_size + self.max_val: float = max_val + self.eps = eps + self.padding = padding + + def forward(self, img1: torch.Tensor, img2: torch.Tensor) -> torch.Tensor: + """Compute a 3D SSIM map between two volume batches. + + Args: + img1: First volume tensor with shape :math:`(B, C, D, H, W)`. + img2: Second volume tensor with the same shape as ``img1``. + + Returns: + Tensor with shape :math:`(B, C, D, H, W)` containing local + structural similarity values across depth, height, and width. + """ + return ssim3d(img1, img2, self.window_size, self.max_val, self.eps, self.padding) diff --git a/kornia/models/__init__.py b/kornia/models/__init__.py new file mode 100644 index 0000000..76d8e06 --- /dev/null +++ b/kornia/models/__init__.py @@ -0,0 +1,22 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Models submodule for Kornia. + +This package provides model architectures and utilities for state-of-the-art models for Visual Language +Models and Vision Language Action Models. +""" diff --git a/kornia/models/_hf_models/__init__.py b/kornia/models/_hf_models/__init__.py new file mode 100644 index 0000000..ed9124d --- /dev/null +++ b/kornia/models/_hf_models/__init__.py @@ -0,0 +1,18 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .hf_onnx_community import HFONNXComunnityModel, HFONNXComunnityModelLoader diff --git a/kornia/models/_hf_models/hf_onnx_community.py b/kornia/models/_hf_models/hf_onnx_community.py new file mode 100644 index 0000000..d429e54 --- /dev/null +++ b/kornia/models/_hf_models/hf_onnx_community.py @@ -0,0 +1,214 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import os +from typing import Any, Optional + +import torch + +from kornia.config import kornia_config +from kornia.core import ImageSequential +from kornia.core.external import onnx +from kornia.geometry.transform import resize +from kornia.models.base import ModelBaseMixin +from kornia.onnx import ONNXSequential +from kornia.onnx.utils import ONNXLoader, io_name_conversion + +from .preprocessor import PreprocessingLoader + + +class HFONNXComunnityModelLoader: + """Initializes the ONNXComunnityModelLoader for onnx-community repo of Hugging Face. + + Args: + model_name: The name of the model to load. + model_type: The type of the model to load. + cache_dir: The directory where models are cached locally. + Defaults to None, which will use a default `kornia.config.hub_onnx_dir` directory. + with_data: Whether to download the model weights such as `model.onnx_data`. + + """ + + def __init__( + self, model_name: str, model_type: str = "model", cache_dir: Optional[str] = None, with_data: bool = False + ) -> None: + self.model_name = model_name + self.model_type = model_type + self.with_data = with_data + + if cache_dir is None: + cache_dir = kornia_config.hub_onnx_dir + self.cache_dir = os.path.join(cache_dir, self.model_name) + self.model_url = ( + f"https://huggingface.co/onnx-community/{self.model_name}/resolve/main/onnx/{self.model_type}.onnx" + ) + self.config_url = ( + f"https://huggingface.co/onnx-community/{self.model_name}/resolve/main/preprocessor_config.json" + ) + + def load_model( + self, download: bool = True, io_name_mapping: Optional[dict[str, str]] = None, **kwargs: Any + ) -> onnx.ModelProto: # type:ignore + """Load the ONNX model file from the configured onnx-community repository. + + Args: + download: Whether to download the remote ONNX file when it is not already + available in the local cache directory. + io_name_mapping: Optional mapping used to rename ONNX graph input and + output names after loading. This is useful when composing the model + with Kornia pre-processing or post-processing graphs that expect + specific tensor names. + kwargs: Additional keyword arguments forwarded to :class:`ONNXLoader`. + + Returns: + The loaded ONNX model graph. If ``io_name_mapping`` is provided, the graph + is returned after its input and output names have been converted. + """ + onnx_model = ONNXLoader.load_model( + self.model_url, download=download, with_data=self.with_data, cache_dir=self.cache_dir, **kwargs + ) + + if io_name_mapping is not None: + onnx_model = io_name_conversion(onnx_model, io_name_mapping) + + return onnx_model + + def load_preprocessing( + self, + ) -> ImageSequential: + """Load the preprocessing pipeline described by the remote processor config. + + The Hugging Face onnx-community repositories store image preprocessing + settings in ``preprocessor_config.json``. This method reads that JSON file and + converts the supported operations, such as resize, rescale, and normalize, into + a Kornia :class:`~kornia.core.ImageSequential` module. + + Returns: + Sequential Kornia image preprocessing module built from the repository + configuration. + """ + json_req = ONNXLoader.load_config(self.config_url) + return PreprocessingLoader.from_json(json_req) + + def _add_metadata( + self, + model: onnx.ModelProto, # type:ignore + additional_metadata: Optional[dict[str, Any]] = None, + ) -> onnx.ModelProto: # type:ignore + if additional_metadata is None: + additional_metadata = {} + for key, value in additional_metadata.items(): + metadata_props = model.metadata_props.add() + metadata_props.key = key + metadata_props.value = str(value) + return model + + +class HFONNXComunnityModel(ONNXSequential, ModelBaseMixin): + """ONNX model wrapper for Kornia pipelines loaded from onnx-community repositories. + + The wrapper combines an optional preprocessing graph, the core ONNX model, and an + optional postprocessing graph into a single :class:`ONNXSequential` module. It also + keeps references to the individual components so callers can export either the + complete pipeline or only the model graph. + + Args: + model: Main ONNX model graph. + pre_processor: Optional ONNX graph executed before ``model``. + post_processor: Optional ONNX graph executed after ``model``. + name: Optional human-readable model name used for export filenames. + auto_ir_version_conversion: Whether to allow automatic ONNX IR version + conversion when composing the sequence. + io_maps: Optional list of input/output name mappings used when connecting the + ONNX graphs together. + """ + + name: str = "onnx_community_model" + + def __init__( + self, + model: onnx.ModelProto, # type: ignore + pre_processor: Optional[onnx.ModelProto] = None, # type: ignore + post_processor: Optional[onnx.ModelProto] = None, # type: ignore + name: Optional[str] = None, + auto_ir_version_conversion: bool = True, + io_maps: Optional[list[tuple[str, str]]] = None, + ) -> None: + model_list = [] if pre_processor is None else [pre_processor] + model_list.append(model) + if post_processor is not None: + model_list.append(post_processor) + super().__init__(*model_list, auto_ir_version_conversion=auto_ir_version_conversion, io_maps=io_maps) + if name is not None: + self.name = name + self.model = model + self.pre_processor = pre_processor + self.post_processor = post_processor + + def resize_back(self, images: torch.Tensor, target_images: torch.Tensor) -> torch.Tensor: + """Resize the input images back to the original size of target images. + + Args: + images: The input images to be resized. + target_images: The target images whose size is used as the reference for resizing. + + Returns: + The resized images. + + """ + if isinstance(target_images, torch.Tensor): + return resize(images, target_images.shape[-2:]) + raise RuntimeError + + def to_onnx( + self, + onnx_name: Optional[str] = None, + include_pre_and_post_processor: bool = True, + save: bool = True, + additional_metadata: Optional[list[tuple[str, str]]] = None, + **kwargs: Any, + ) -> onnx.ModelProto: # type:ignore + """Export a depth estimation model to ONNX format. + + Args: + onnx_name: + The name of the output ONNX file. If not provided, a default name in the + format "Kornia-.onnx" will be used. + include_pre_and_post_processor: + Whether to include the pre-processor and post-processor in the exported model. + save: + If to save the model or load it. + additional_metadata: + Additional metadata to add to the ONNX model. + kwargs: Additional arguments to convert to onnx. + + """ + if onnx_name is None: + onnx_name = f"kornia_{self.name}.onnx" + + if not include_pre_and_post_processor: + self._add_metadata(self.model, additional_metadata) + if save: + self._export(self.model, onnx_name, **kwargs) + return self.model + + self._add_metadata(self._combined_op, additional_metadata) + if save: + self._export(self._combined_op, onnx_name, **kwargs) + return self._combined_op diff --git a/kornia/models/_hf_models/preprocessor.py b/kornia/models/_hf_models/preprocessor.py new file mode 100644 index 0000000..64dafe3 --- /dev/null +++ b/kornia/models/_hf_models/preprocessor.py @@ -0,0 +1,149 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Any + +import torch +from torch import nn + +from kornia.core import ImageSequential +from kornia.enhance.normalize import Normalize +from kornia.enhance.rescale import Rescale +from kornia.geometry.transform import Resize + + +class PreprocessingLoader: + """Factory for converting image-processor settings into Kornia modules. + + Hugging Face image processor configuration files describe preprocessing as JSON + fields, for example whether to resize an image, multiply it by a rescale factor, + or normalize it with channel statistics. This helper maps the supported fields to + Kornia modules so the preprocessing can be composed with ONNX model pipelines. + """ + + @staticmethod + def normalize(mean: torch.Tensor, std: torch.Tensor) -> Normalize: + """Create a channel-wise normalization module. + + Args: + mean: Mean value used for each channel. The tensor is passed directly to + :class:`~kornia.enhance.normalize.Normalize`. + std: Standard deviation value used for each channel. + + Returns: + Kornia normalization module that subtracts ``mean`` and divides by ``std`` + for every image in the batch. + """ + return Normalize(mean=mean, std=std) + + @staticmethod + def rescale(rescale_factor: float) -> Rescale: + """Create an intensity rescaling module. + + Args: + rescale_factor: Multiplicative factor applied to image values. In the DPT + pipeline this is typically derived from the processor JSON so the + tensor range matches what the exported model expects. + + Returns: + Kornia rescaling module that multiplies the input image tensor by + ``rescale_factor``. + """ + return Rescale(factor=rescale_factor) + + @staticmethod + def resize(width: int, height: int) -> Resize: + """Create an image resize module from width and height values. + + Args: + width: Target image width, corresponding to the last spatial dimension + ``W`` of an image tensor. + height: Target image height, corresponding to the second-to-last spatial + dimension ``H`` of an image tensor. + + Returns: + Kornia resize module configured with output size ``(height, width)``. + """ + return Resize((height, width)) + + @staticmethod + def from_json(req: dict[str, Any]) -> ImageSequential: + """Build a preprocessing pipeline from a processor configuration dictionary. + + Args: + req: Parsed processor configuration. The dictionary must contain + ``image_processor_type`` so the loader can choose the matching + processor-specific parser. + + Returns: + Sequential Kornia module containing the supported preprocessing steps. + + Raises: + RuntimeError: If the requested image processor type is not supported by + this loader. + """ + if req["image_processor_type"] == "DPTImageProcessor": + return DPTImageProcessor.from_json(req) + raise RuntimeError(f"Unsupported image processor type: {req['image_processor_type']}") + + +class DPTImageProcessor(PreprocessingLoader): + """Parser for DPT image processor configurations. + + DPT models use a fixed image preprocessing recipe before inference. This parser + reads the supported JSON fields and constructs the equivalent Kornia pipeline in + the same order used by the processor configuration: resize, rescale, and normalize. + Padding is intentionally not handled here because the current implementation does + not include the extra parameters needed to reproduce that branch safely. + """ + + @staticmethod + def from_json(json_data: dict[str, Any]) -> ImageSequential: + """Convert a DPT processor JSON dictionary into a Kornia preprocessing pipeline. + + Args: + json_data: Parsed DPT processor configuration. Supported fields include + ``do_resize``, ``size``, ``do_rescale``, ``rescale_factor``, + ``do_normalize``, ``image_mean``, and ``image_std``. + + Returns: + :class:`~kornia.core.ImageSequential` containing the enabled preprocessing + modules in execution order. + + Raises: + NotImplementedError: If the configuration requests padding, which is not + implemented by this loader. + """ + preproc_list: list[nn.Module] = [] + if json_data["do_pad"]: + raise NotImplementedError + if json_data["do_resize"]: + # Missing some parameters such as `ensure_multiple_of`, `keep_aspect_ratio` + preproc_list.append( + PreprocessingLoader.resize(width=json_data["size"]["width"], height=json_data["size"]["height"]) + ) + if json_data["do_rescale"]: + preproc_list.append(PreprocessingLoader.rescale(rescale_factor=json_data["rescale_factor"] * 255)) + if json_data["do_normalize"]: + preproc_list.append( + PreprocessingLoader.normalize( + mean=torch.tensor([json_data["image_mean"]]), std=torch.tensor([json_data["image_std"]]) + ) + ) + return ImageSequential(*preproc_list) diff --git a/kornia/models/base.py b/kornia/models/base.py new file mode 100644 index 0000000..21a9852 --- /dev/null +++ b/kornia/models/base.py @@ -0,0 +1,172 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import datetime +import logging +import os +from abc import ABC, abstractmethod +from typing import Any, Generic, List, Optional, TypeVar, Union, cast + +import torch +from torch import nn + +from kornia.core.external import PILImage as Image +from kornia.image.utils import tensor_to_image +from kornia.io import write_image + +logger = logging.getLogger(__name__) + +ModelConfig = TypeVar("ModelConfig") + + +class ModelBaseMixin: + """Provide common properties and utilities for Kornia model classes.""" + + name: str = "model" + + def _tensor_to_type( + self, output: Union[torch.Tensor, List[torch.Tensor]], output_type: str, is_batch: bool = False + ) -> Union[torch.Tensor, List[torch.Tensor], List[Image.Image]]: # type: ignore + """Convert the output tensor to the desired type. + + Args: + output: The output tensor or list of tensors. + output_type: The desired output type. Accepted values are "torch" and "pil". + is_batch: If True, the output is expected to be a batch of tensors. + + Returns: + The converted output tensor or list of tensors. + + Raises: + RuntimeError: If the output type is not supported. + + """ + if output_type == "torch": + return output + elif output_type == "pil": + if isinstance(output, list): + return [tensor_to_image(t) for t in output] + else: + return tensor_to_image(output) + else: + raise RuntimeError(f"Output type {output_type} is not supported. Accepted values are 'torch' and 'pil'.") + + def save(self, output: Union[torch.Tensor, List[torch.Tensor]], directory: str, is_batch: bool = False) -> None: + """Save the output tensor to a directory. + + Args: + output: The output tensor or list of tensors. + directory: The directory to save the output. + is_batch: If True, the output is expected to be a batch of tensors. + + """ + os.makedirs(directory, exist_ok=True) + timestamp = datetime.datetime.now(tz=datetime.UTC).strftime("%Y%m%d_%H%M%S") + if isinstance(output, list): + for i, out in enumerate(output): + write_image(os.path.join(directory, f"{self.name}_{timestamp}_{i}.png"), out) + else: + write_image(os.path.join(directory, f"{self.name}_{timestamp}.png"), output) + logger.info(f"Outputs are saved in {directory}") + + def _save_outputs( + self, output: Union[torch.Tensor, List[torch.Tensor]], directory: Optional[str] = None, suffix: str = "" + ) -> None: + """Save the output tensor to a directory with an optional suffix. + + Args: + output: The output tensor or list of tensors. + directory: The directory to save the output. If None, a default directory is used. + suffix: Optional suffix to add to the filename. + + """ + if directory is None: + name = f"{self.name}{suffix}_{datetime.datetime.now(tz=datetime.UTC).strftime('%Y%m%d%H%M%S')!s}" + directory = os.path.join("kornia_outputs", name) + + os.makedirs(directory, exist_ok=True) + timestamp = datetime.datetime.now(tz=datetime.UTC).strftime("%Y%m%d_%H%M%S") + if isinstance(output, list): + for i, out in enumerate(output): + write_image(os.path.join(directory, f"{self.name}{suffix}_{timestamp}_{i}.png"), out) + else: + write_image(os.path.join(directory, f"{self.name}{suffix}_{timestamp}.png"), output) + logger.info(f"Outputs are saved in {directory}") + + +class ModelBase(ABC, nn.Module, ModelBaseMixin, Generic[ModelConfig]): + """Abstract model class with some utilities function.""" + + def load_checkpoint(self, checkpoint: str, device: Optional[torch.device] = None) -> None: + """Load checkpoint from a given url or file. + + Args: + checkpoint: The url or filepath for the respective checkpoint + device: The desired device to load the weights and move the model + + """ + if os.path.isfile(checkpoint): + with open(checkpoint, "rb") as f: + state_dict = torch.load(f, map_location=device) + else: + state_dict = torch.hub.load_state_dict_from_url(checkpoint, map_location=device) + + self.load_state_dict(state_dict) + + @staticmethod + @abstractmethod + def from_config(config: ModelConfig) -> ModelBase[ModelConfig]: + """Build/load the model. + + Args: + config: The specifications for the model be build/loaded + + """ + raise NotImplementedError + + def compile( + self, + *, + fullgraph: bool = False, + dynamic: bool = False, + backend: str = "inductor", + mode: Optional[str] = None, + options: Optional[dict[Any, Any]] = None, + disable: bool = False, + ) -> ModelBase[ModelConfig]: + """Compile this model with :func:`torch.compile`. + + Args: + fullgraph: Whether Dynamo should require a single full graph. + dynamic: Whether dynamic shape tracing is enabled. + backend: Compilation backend name passed to :func:`torch.compile`. + mode: Optional backend-specific compilation mode. + options: Optional backend-specific option dictionary. + disable: If ``True``, return an uncompiled model wrapper according + to PyTorch's compile semantics. + + Returns: + Compiled model object with the same high-level interface as this + instance. + """ + compiled = torch.compile( + self, fullgraph=fullgraph, dynamic=dynamic, backend=backend, mode=mode, options=options, disable=disable + ) + compiled = cast(ModelBase[ModelConfig], compiled) + return compiled diff --git a/kornia/models/common.py b/kornia/models/common.py new file mode 100644 index 0000000..3ebd166 --- /dev/null +++ b/kornia/models/common.py @@ -0,0 +1,222 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch +import torch.nn.functional as F +from torch import nn + + +class ConvNormAct(nn.Sequential): + """Apply a sequence of Convolution, Normalization, and Activation layers. + + Args: + in_channels: Number of input channels. + out_channels: Number of output channels. + kernel_size: Size of the convolution kernel. + stride: Stride of the convolution. Default: 1. + padding: Zero-padding added to both sides of the input. Default: 0. + groups: Number of blocked connections from input to output. Default: 1. + norm: Normalization layer type. Default: :class:`nn.BatchNorm2d`. + act: Activation layer type. Default: :class:`nn.ReLU`. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + stride: int = 1, + act: str = "relu", + groups: int = 1, + conv_naming: str = "conv", + norm_naming: str = "norm", + act_naming: str = "act", + ) -> None: + super().__init__() + if kernel_size % 2 == 0: + # even kernel_size -> asymmetric padding + # PPHGNetV2 (for RT-DETR) uses kernel 2 + # follow TensorFlow/PaddlePaddle: bottom/right side is padded 1 more than top/left + # NOTE: this does not account for stride=2 + p1 = (kernel_size - 1) // 2 + p2 = kernel_size - 1 - p1 + self.pad = nn.ZeroPad2d((p1, p2, p1, p2)) + padding = 0 + else: + padding = (kernel_size - 1) // 2 + conv = nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding, 1, groups, False) + norm = nn.BatchNorm2d(out_channels) + activation = {"relu": nn.ReLU, "silu": nn.SiLU, "none": nn.Identity}[act](inplace=True) + + self.__setattr__(conv_naming, conv) + self.__setattr__(norm_naming, norm) + self.__setattr__(act_naming, activation) + + +# Lightly adapted from +# https://github.com/facebookresearch/MaskFormer/blob/main/mask_former/modeling/transformer/transformer_predictor.py +class MLP(nn.Module): + """Implement a Multi-Layer Perceptron (MLP) for feature projection. + + Args: + input_dim: The number of input features. + hidden_dim: The number of features in hidden layers. + output_dim: The number of output features. + num_layers: The total number of layers. + sigmoid_output: Whether to apply a sigmoid to the final output. Default: False. + """ + + def __init__( + self, input_dim: int, hidden_dim: int, output_dim: int, num_layers: int, sigmoid_output: bool = False + ) -> None: + super().__init__() + self.num_layers = num_layers + h = [hidden_dim] * (num_layers - 1) + self.layers = nn.ModuleList(nn.Linear(n, k) for n, k in zip([input_dim, *h], [*h, output_dim])) + self.sigmoid_output = sigmoid_output + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply the configured MLP to the last tensor dimension. + + Args: + x: Input tensor whose last dimension equals ``input_dim``. + + Returns: + Tensor with the same leading dimensions as ``x`` and final + dimension ``output_dim``. If ``sigmoid_output`` is enabled, the + final values are passed through a sigmoid. + """ + for i, layer in enumerate(self.layers): + x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x) + if self.sigmoid_output: + x = F.sigmoid(x) + return x + + +# Adapted from timm +# https://github.com/huggingface/pytorch-image-models/blob/v0.9.2/timm/layers/drop.py#L137 +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" + + def __init__(self, drop_prob: float = 0.0, scale_by_keep: bool = True) -> None: + super().__init__() + self.drop_prob = drop_prob + self.scale_by_keep = scale_by_keep + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply stochastic depth to a residual branch. + + Args: + x: Residual-branch tensor with arbitrary shape. The first dimension + is treated as the batch dimension. + + Returns: + Tensor with the same shape as ``x``. During training, complete + samples may be dropped and optionally scaled by the keep + probability; during evaluation the input is returned unchanged. + """ + if self.drop_prob == 0.0 or not self.training: + return x + keep_prob = 1 - self.drop_prob + shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = x.new_empty(shape).bernoulli_(keep_prob) + if keep_prob > 0.0 and self.scale_by_keep: + random_tensor.div_(keep_prob) + return x * random_tensor + + +# From https://github.com/facebookresearch/detectron2/blob/main/detectron2/layers/batch_norm.py +# Itself from https://github.com/facebookresearch/ConvNeXt/blob/d1fa8f6fef0a165b27399986cc2bdacc92777e40/models/convnext.py#L119 # noqa +class LayerNorm2d(nn.Module): + """Apply Layer Normalization over a 4D input tensor (NCHW).""" + + def __init__(self, num_channels: int, eps: float = 1e-6) -> None: + super().__init__() + self.weight = nn.Parameter(torch.ones(num_channels)) + self.bias = nn.Parameter(torch.zeros(num_channels)) + self.eps = eps + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply channel-wise layer normalization to an NCHW tensor. + + Args: + x: Tensor with shape :math:`(B, C, H, W)`, where :math:`B` is batch + size, :math:`C` is channel count, :math:`H` is height, and + :math:`W` is width. + + Returns: + Tensor with the same shape as ``x`` after normalizing across the + channel dimension and applying learned affine parameters. + """ + u = x.mean(1, keepdim=True) + s = (x - u).pow(2).mean(1, keepdim=True) + x = (x - u) / (s + self.eps).sqrt() + x = self.weight[:, None, None] * x + self.bias[:, None, None] + return x + + +def window_partition(x: torch.Tensor, window_size: int) -> tuple[torch.Tensor, tuple[int, int]]: + """Partition into non-overlapping windows with padding if needed. + + Args: + x: input tokens with [B, H, W, C]. + window_size: window size. + + Returns: + windows: windows after partition with [B * num_windows, window_size, window_size, C]. + (Hp, Wp): padded height and width before partition + + """ + B, H, W, C = x.shape + + pad_h = (window_size - H % window_size) % window_size + pad_w = (window_size - W % window_size) % window_size + if pad_h > 0 or pad_w > 0: + x = F.pad(x, (0, 0, 0, pad_w, 0, pad_h)) + Hp, Wp = H + pad_h, W + pad_w + + x = x.view(B, Hp // window_size, window_size, Wp // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows, (Hp, Wp) + + +def window_unpartition( + windows: torch.Tensor, window_size: int, pad_hw: tuple[int, int], hw: tuple[int, int] +) -> torch.Tensor: + """Window unpartition into original sequences and removing padding. + + Args: + windows: input tokens with [B * num_windows, window_size, window_size, C]. + window_size: window size. + pad_hw: padded height and width (Hp, Wp). + hw: original height and width (H, W) before padding. + + Returns: + x: unpartitioned sequences with [B, H, W, C]. + + """ + Hp, Wp = pad_hw + H, W = hw + B = windows.shape[0] // (Hp * Wp // window_size // window_size) + x = windows.view(B, Hp // window_size, Wp // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, Hp, Wp, -1) + + if Hp > H or Wp > W: + x = x[:, :H, :W, :].contiguous() + return x diff --git a/kornia/models/depth_estimation/__init__.py b/kornia/models/depth_estimation/__init__.py new file mode 100644 index 0000000..db1217c --- /dev/null +++ b/kornia/models/depth_estimation/__init__.py @@ -0,0 +1,24 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Depth estimation models submodule for Kornia. + +This package provides model architectures and utilities for monocular depth estimation tasks. +""" + +from .base import * +from .depth_anything import * diff --git a/kornia/models/depth_estimation/base.py b/kornia/models/depth_estimation/base.py new file mode 100644 index 0000000..497d510 --- /dev/null +++ b/kornia/models/depth_estimation/base.py @@ -0,0 +1,126 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional, Union + +import torch + +from kornia.color.gray import grayscale_to_rgb +from kornia.core.external import PILImage as Image +from kornia.models._hf_models import HFONNXComunnityModel + +__all__ = ["DepthEstimation"] + + +class DepthEstimation(HFONNXComunnityModel): + """Base class for depth estimation models compatible with HuggingFace and ONNX.""" + + name: str = "depth_estimation" + + def __call__(self, images: Union[torch.Tensor, list[torch.Tensor]]) -> Union[torch.Tensor, list[torch.Tensor]]: # type: ignore[override] + """Detect objects in a given list of images. + + Args: + images: If list of RGB images. Each image is a torch.Tensor with shape :math:`(3, H, W)`. + If torch.Tensor, a torch.Tensor with shape :math:`(B, 3, H, W)`. + + Returns: + list of detections found in each image. For item in a batch, shape is :math:`(D, 6)`, where :math:`D` is the + number of detections in the given image, :math:`6` represents class id, score, and `xywh` bounding box. + + """ + if isinstance( + images, + ( + list, + tuple, + ), + ): + results = [super(DepthEstimation, self).__call__(image[None].cpu().numpy())[0] for image in images] + results = [ + self.resize_back(torch.tensor(result, device=image.device, dtype=image.dtype), image) + for result, image in zip(results, images) + ] + return results + + result = super().__call__(images.cpu().numpy())[0] + result = torch.tensor(result, device=images.device, dtype=images.dtype) + return self.resize_back(result, images) + + def visualize( + self, + images: torch.Tensor, + depth_maps: Optional[Union[torch.Tensor, list[torch.Tensor]]] = None, + output_type: str = "torch", + depth_type: str = "relative", + max_depth: int = 80, + ) -> Union[torch.Tensor, list[torch.Tensor], list[Image.Image]]: # type: ignore + """Draw the segmentation results. + + Args: + images: input tensor. + depth_maps: estimated depths. + output_type: type of the output. + depth_type: 'metric' or 'relative' depth. + max_depth: maximum depth value. Only valid for metric depth. + + Returns: + output tensor. + + """ + if depth_maps is None: + depth_maps = self(images) + output = [] + for depth_map in depth_maps: + if depth_type == "metric": + depth_map = depth_map / max_depth + elif depth_type == "relative": + depth_map = depth_map / depth_map.max() + else: + raise ValueError(f"Unsupported depth type `{depth_type}`.") + output.append(grayscale_to_rgb(depth_map)) + + return self._tensor_to_type(output, output_type, is_batch=isinstance(images, torch.Tensor)) + + def save( + self, + images: torch.Tensor, + depth_maps: Optional[Union[torch.Tensor, list[torch.Tensor]]] = None, + directory: Optional[str] = None, + output_type: str = "torch", + depth_type: str = "relative", + max_depth: int = 80, + ) -> None: + """Save the segmentation results. + + Args: + images: input tensor. + depth_maps: estimated depths. + output_type: type of the output. + depth_type: 'metric' or 'relative' depth. + max_depth: maximum depth value. Only valid for metric depth. + directory: where to store outputs. + + Returns: + output tensor. + + """ + outputs = self.visualize(images, depth_maps, output_type, depth_type=depth_type, max_depth=max_depth) + self._save_outputs(images, directory, suffix="_src") + self._save_outputs(outputs, directory, suffix="_depth") diff --git a/kornia/models/depth_estimation/depth_anything.py b/kornia/models/depth_estimation/depth_anything.py new file mode 100644 index 0000000..125530a --- /dev/null +++ b/kornia/models/depth_estimation/depth_anything.py @@ -0,0 +1,72 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Optional + +from kornia.models._hf_models import HFONNXComunnityModelLoader + +from .base import DepthEstimation + +__all__ = ["DepthAnythingONNXBuilder"] + + +class DepthAnythingONNXBuilder: + """Provide static methods to build DepthAnything models for ONNX inference.""" + + @staticmethod + def build( + model_name: str = "depth-anything-v2-small", model_type: str = "model", cache_dir: Optional[str] = None + ) -> DepthEstimation: + """Export a DepthAnything model to an ONNX model file. + + Args: + model_name: The name of the model to be loaded. Valid model names include: + - `depth-anything-v2-small` + - `depth-anything-v2-base` + - `depth-anything-v2-large` + model_type: + The type of the model to be loaded. Valid model types include: + - `model` + - `model_bnb4` + - `model_fp16` + - `model_int8` + - `model_q4` + - `model_quantized` + - `model_uint8` + cache_dir: + The directory where the model should be cached. + + Returns: + DepthEstimation: The depth estimation model. + + Note: + To use this model, load an image tensor (shape: ``(1, C, H, W)``) and call ``model.save(image)``. + + """ + if model_name not in [ + "depth-anything-v2-small", + "depth-anything-v2-base", + "depth-anything-v2-large", + ]: + raise ValueError(f"{model_name} is not a valid model name.") + loader = HFONNXComunnityModelLoader(model_name, model_type=model_type, cache_dir=cache_dir) + onnx_model = loader.load_model( + download=True, + io_name_mapping={"pixel_values": "input", "predicted_depth": "output"}, + ) + preproc = loader.load_preprocessing().to_onnx(save=False) + return DepthEstimation(onnx_model, pre_processor=preproc, name=f"{model_name}_{model_type}") diff --git a/kornia/models/dexined.py b/kornia/models/dexined.py new file mode 100644 index 0000000..d3116dd --- /dev/null +++ b/kornia/models/dexined.py @@ -0,0 +1,329 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# adapted from: https://github.com/xavysp/DexiNed/blob/d944b70eb6eaf40e22f8467c1e12919aa600d8e4/model.py + +from __future__ import annotations + +from collections import OrderedDict +from typing import ClassVar, Optional + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK +from kornia.core.mixin.onnx import ONNXExportMixin + +__all__ = ["DexiNed"] + +url: str = "http://cmp.felk.cvut.cz/~mishkdmy/models/DexiNed_BIPED_10.pth" + + +def weight_init(m: nn.Module) -> None: + """Initialize weights.""" + if isinstance(m, nn.Conv2d): + # torch.nn.init.xavier_uniform_(m.weight, gain=1.0) + torch.nn.init.xavier_normal_(m.weight, gain=1.0) + # torch.nn.init.normal_(m.weight, mean=0.0, std=0.01) + if m.weight.data.shape[1] == torch.Size([1]): + torch.nn.init.normal_(m.weight, mean=0.0) + + if m.bias is not None: + torch.nn.init.zeros_(m.bias) + + # for fusion layer + if isinstance(m, nn.ConvTranspose2d): + # torch.nn.init.xavier_uniform_(m.weight, gain=1.0) + torch.nn.init.xavier_normal_(m.weight, gain=1.0) + # torch.nn.init.normal_(m.weight, mean=0.0, std=0.01) + + if m.weight.data.shape[1] == torch.Size([1]): + torch.nn.init.normal_(m.weight, std=0.1) + if m.bias is not None: + torch.nn.init.zeros_(m.bias) + + +class CoFusion(nn.Module): + def __init__(self, in_ch: int, out_ch: int) -> None: + super().__init__() + self.conv1 = nn.Conv2d(in_ch, 64, kernel_size=3, stride=1, padding=1) + self.conv2 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1) + self.conv3 = nn.Conv2d(64, out_ch, kernel_size=3, stride=1, padding=1) + self.relu = nn.ReLU() + self.norm_layer1 = nn.GroupNorm(4, 64) + self.norm_layer2 = nn.GroupNorm(4, 64) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # fusecat = torch.cat(x, dim=1) + attn = self.relu(self.norm_layer1(self.conv1(x))) + attn = self.relu(self.norm_layer2(self.conv2(attn))) + attn = F.softmax(self.conv3(attn), dim=1) + + # return ((fusecat * attn).sum(1)).unsqueeze(1) + return ((x * attn).sum(1))[:, None, ...] + + +class _DenseLayer(nn.Sequential): + def __init__(self, input_features: int, out_features: int) -> None: + super().__init__( + OrderedDict( + [ + ("relu1", nn.ReLU(inplace=True)), + ("conv1", nn.Conv2d(input_features, out_features, kernel_size=3, stride=1, padding=2, bias=True)), + ("norm1", nn.BatchNorm2d(out_features)), + ("relu2", nn.ReLU(inplace=True)), + ("conv2", nn.Conv2d(out_features, out_features, kernel_size=3, stride=1, bias=True)), + ("norm2", nn.BatchNorm2d(out_features)), + ] + ) + ) + + def forward(self, x: list[torch.Tensor]) -> list[torch.Tensor]: + x1, x2 = x[0], x[1] + x3: torch.Tensor = x1 + for mod in self: + x3 = mod(x3) + return [0.5 * (x3 + x2), x2] + + +class _DenseBlock(nn.Sequential): + def __init__(self, num_layers: int, input_features: int, out_features: int) -> None: + super().__init__() + for i in range(num_layers): + layer = _DenseLayer(input_features, out_features) + self.add_module(f"denselayer{(i + 1)}", layer) + input_features = out_features + + def forward(self, x: list[torch.Tensor]) -> list[torch.Tensor]: + x_out = x + for mod in self: + x_out = mod(x_out) + return x_out + + +class UpConvBlock(nn.Module): + def __init__(self, in_features: int, up_scale: int) -> None: + super().__init__() + self.up_factor = 2 + self.constant_features = 16 + + layers = self.make_deconv_layers(in_features, up_scale) + KORNIA_CHECK(layers is not None, "layers cannot be none") + self.features = nn.Sequential(*layers) + + def make_deconv_layers(self, in_features: int, up_scale: int) -> nn.ModuleList: + layers = nn.ModuleList([]) + all_pads = [0, 0, 1, 3, 7] + for i in range(up_scale): + kernel_size = 2**up_scale + pad = all_pads[up_scale] # kernel_size-1 + out_features = self.compute_out_features(i, up_scale) + layers.append(nn.Conv2d(in_features, out_features, 1)) + layers.append(nn.ReLU(inplace=True)) + layers.append(nn.ConvTranspose2d(out_features, out_features, kernel_size, stride=2, padding=pad)) + in_features = out_features + return layers + + def compute_out_features(self, idx: int, up_scale: int) -> int: + return 1 if idx == up_scale - 1 else self.constant_features + + def forward(self, x: torch.Tensor, out_shape: list[int]) -> torch.Tensor: + out = self.features(x) + out = F.interpolate(out, out_shape, mode="bilinear") + return out + + +class SingleConvBlock(nn.Module): + def __init__(self, in_features: int, out_features: int, stride: int, use_bs: bool = True) -> None: + super().__init__() + self.use_bn = use_bs + self.conv = nn.Conv2d(in_features, out_features, 1, stride=stride, bias=True) + self.bn = nn.BatchNorm2d(out_features) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.conv(x) + if self.use_bn: + x = self.bn(x) + return x + + +class DoubleConvBlock(nn.Sequential): + def __init__( + self, + in_features: int, + mid_features: int, + out_features: Optional[int] = None, + stride: int = 1, + use_act: bool = True, + ) -> None: + super().__init__() + if out_features is None: + out_features = mid_features + self.add_module("conv1", nn.Conv2d(in_features, mid_features, 3, padding=1, stride=stride)) + self.add_module("bn1", nn.BatchNorm2d(mid_features)) + self.add_module("relu1", nn.ReLU(inplace=True)) + self.add_module("conv2", nn.Conv2d(mid_features, out_features, 3, padding=1)) + self.add_module("bn2", nn.BatchNorm2d(out_features)) + if use_act: + self.add_module("relu2", nn.ReLU(inplace=True)) + + +class DexiNed(ONNXExportMixin, nn.Module): + r"""Definition of the DXtrem network from :cite:`xsoria2020dexined`. + + Return: + A list of tensor with the intermediate features which the last element + is the edges map with shape :math:`(B,1,H,W)`. + + Example: + >>> img = torch.rand(1, 3, 320, 320) + >>> net = DexiNed(pretrained=False) + >>> out = net(img) + >>> out.shape + torch.Size([1, 1, 320, 320]) + + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, 1, -1, -1] + + def __init__(self, pretrained: bool) -> None: + super().__init__() + self.block_1 = DoubleConvBlock(3, 32, 64, stride=2) + self.block_2 = DoubleConvBlock(64, 128, use_act=False) + self.dblock_3 = _DenseBlock(2, 128, 256) # [128,256,100,100] + self.dblock_4 = _DenseBlock(3, 256, 512) + self.dblock_5 = _DenseBlock(3, 512, 512) + self.dblock_6 = _DenseBlock(3, 512, 256) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + # left skip connections, figure in Journal + self.side_1 = SingleConvBlock(64, 128, 2) + self.side_2 = SingleConvBlock(128, 256, 2) + self.side_3 = SingleConvBlock(256, 512, 2) + self.side_4 = SingleConvBlock(512, 512, 1) + self.side_5 = SingleConvBlock(512, 256, 1) + + # right skip connections, figure in Journal paper + self.pre_dense_2 = SingleConvBlock(128, 256, 2) + self.pre_dense_3 = SingleConvBlock(128, 256, 1) + self.pre_dense_4 = SingleConvBlock(256, 512, 1) + self.pre_dense_5 = SingleConvBlock(512, 512, 1) + self.pre_dense_6 = SingleConvBlock(512, 256, 1) + + # USNet + self.up_block_1 = UpConvBlock(64, 1) + self.up_block_2 = UpConvBlock(128, 1) + self.up_block_3 = UpConvBlock(256, 2) + self.up_block_4 = UpConvBlock(512, 3) + self.up_block_5 = UpConvBlock(512, 4) + self.up_block_6 = UpConvBlock(256, 4) + self.block_cat = SingleConvBlock(6, 1, stride=1, use_bs=False) # hed fusion method + # self.block_cat = CoFusion(6,6)# cats fusion method + + if pretrained: + self.load_from_file(url) + else: + self.apply(weight_init) + + def load_from_file(self, path_file: str) -> None: + """Load pretrained DexiNed weights and switch the module to evaluation mode. + + Args: + path_file: URL or local checkpoint path accepted by + :func:`torch.hub.load_state_dict_from_url`. + """ + # use torch.hub to load pretrained model + pretrained_dict = torch.hub.load_state_dict_from_url(path_file, map_location=torch.device("cpu")) + self.load_state_dict(pretrained_dict, strict=True) + self.eval() + + def get_features(self, x: torch.Tensor) -> list[torch.Tensor]: + """Compute the six multi-scale DexiNed side-output edge maps. + + Args: + x: Input RGB image tensor with shape :math:`(B, 3, H, W)`, where + :math:`B` is batch size, :math:`H` is height, and :math:`W` is + width. + + Returns: + List of six tensors aligned to the input spatial size. Each tensor + is a side-output edge response used by the final fusion block. + """ + # Block 1 + block_1 = self.block_1(x) + block_1_side = self.side_1(block_1) + + # Block 2 + block_2 = self.block_2(block_1) + block_2_down = self.maxpool(block_2) + block_2_add = block_2_down + block_1_side + block_2_side = self.side_2(block_2_add) + + # Block 3 + block_3_pre_dense = self.pre_dense_3(block_2_down) + block_3, _ = self.dblock_3([block_2_add, block_3_pre_dense]) + block_3_down = self.maxpool(block_3) # [128,256,50,50] + block_3_add = block_3_down + block_2_side + block_3_side = self.side_3(block_3_add) + + # Block 4 + block_2_resize_half = self.pre_dense_2(block_2_down) + block_4_pre_dense = self.pre_dense_4(block_3_down + block_2_resize_half) + block_4, _ = self.dblock_4([block_3_add, block_4_pre_dense]) + block_4_down = self.maxpool(block_4) + block_4_add = block_4_down + block_3_side + block_4_side = self.side_4(block_4_add) + + # Block 5 + block_5_pre_dense = self.pre_dense_5(block_4_down) # block_5_pre_dense_512 +block_4_down + block_5, _ = self.dblock_5([block_4_add, block_5_pre_dense]) + block_5_add = block_5 + block_4_side + + # Block 6 + block_6_pre_dense = self.pre_dense_6(block_5) + block_6, _ = self.dblock_6([block_5_add, block_6_pre_dense]) + + # upsampling blocks + out_shape = x.shape[-2:] + out_1 = self.up_block_1(block_1, out_shape) + out_2 = self.up_block_2(block_2, out_shape) + out_3 = self.up_block_3(block_3, out_shape) + out_4 = self.up_block_4(block_4, out_shape) + out_5 = self.up_block_5(block_5, out_shape) + out_6 = self.up_block_6(block_6, out_shape) + results = [out_1, out_2, out_3, out_4, out_5, out_6] + return results + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Predict a fused edge map from an RGB image batch. + + Args: + x: Input RGB image tensor with shape :math:`(B, 3, H, W)`. + + Returns: + Tensor with shape :math:`(B, 1, H, W)` containing the fused DexiNed + edge response. + """ + features = self.get_features(x) + + # concatenate multiscale outputs + block_cat = torch.cat(features, dim=1) # Bx6xHxW + block_cat = self.block_cat(block_cat) # Bx1xHxW + + return block_cat diff --git a/kornia/models/efficient_vit/__init__.py b/kornia/models/efficient_vit/__init__.py new file mode 100644 index 0000000..ed02d6e --- /dev/null +++ b/kornia/models/efficient_vit/__init__.py @@ -0,0 +1,25 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Contrib EfficientViT — Efficient Vision Transformer models for Kornia. + +This subpackage provides EfficientViT model classes and configurations. +""" + +from .model import EfficientViT, EfficientViTConfig + +__all__ = ["EfficientViT", "EfficientViTConfig"] diff --git a/kornia/models/efficient_vit/backbone.py b/kornia/models/efficient_vit/backbone.py new file mode 100644 index 0000000..04f5e6a --- /dev/null +++ b/kornia/models/efficient_vit/backbone.py @@ -0,0 +1,449 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# EfficientViT: Multi-Scale Linear Attention for High-Resolution Dense Prediction +# Han Cai, Junyan Li, Muyan Hu, Chuang Gan, Song Han +# International Conference on Computer Vision (ICCV), 2023 +from __future__ import annotations + +from typing import Any + +import torch +from torch import nn + +from kornia.models.efficient_vit.nn.ops import ( # type: ignore + ConvLayer, + DSConv, + EfficientViTBlock, + FusedMBConv, + IdentityLayer, + MBConv, + OpSequential, + ResBlock, + ResidualBlock, +) +from kornia.models.efficient_vit.utils import build_kwargs_from_config + + +class EfficientViTBackbone(nn.Module): + """Implement the EfficientViT backbone architecture. + + EfficientViT is a high-speed vision transformer designed for efficient + inference on mobile and edge devices by optimizing the attention mechanism + and structural blocks. + + Args: + width_list: List of widths for each stage. + depth_list: List of depths (number of blocks) for each stage. + in_channels: Number of input image channels. Default: 3. + dim: Dimension of the query, key, and value tensors in the attention mechanism. Default: 32. + expand_ratio: Expansion ratio for the MBConv blocks. Default: 4. + norm: Normalization layer type. Default: "bn2d". + act_func: Activation function type. Default: "hswish". + """ + + def __init__( + self, + width_list: list[int], + depth_list: list[int], + in_channels: int = 3, + dim: int = 32, + expand_ratio: float = 4, + norm: str = "bn2d", + act_func: str = "hswish", + ) -> None: + super().__init__() + + self.width_list = [] + # input stem + input_stem = [ + ConvLayer(in_channels=in_channels, out_channels=width_list[0], stride=2, norm=norm, act_func=act_func) + ] + for _ in range(depth_list[0]): + block = self.build_local_block( + in_channels=width_list[0], + out_channels=width_list[0], + stride=1, + expand_ratio=1, + norm=norm, + act_func=act_func, + ) + input_stem.append(ResidualBlock(block, IdentityLayer())) + in_channels = width_list[0] + self.input_stem = OpSequential(input_stem) + self.width_list.append(in_channels) + + # stages + stages = [] + for w, d in zip(width_list[1:3], depth_list[1:3]): + stage = [] + for i in range(d): + stride = 2 if i == 0 else 1 + block = self.build_local_block( + in_channels=in_channels, + out_channels=w, + stride=stride, + expand_ratio=expand_ratio, + norm=norm, + act_func=act_func, + ) + block = ResidualBlock(block, IdentityLayer() if stride == 1 else None) + stage.append(block) + in_channels = w + stages.append(OpSequential(stage)) + self.width_list.append(in_channels) + + for w, d in zip(width_list[3:], depth_list[3:]): + stage = [] + block = self.build_local_block( + in_channels=in_channels, + out_channels=w, + stride=2, + expand_ratio=expand_ratio, + norm=norm, + act_func=act_func, + fewer_norm=True, + ) + stage.append(ResidualBlock(block, None)) + in_channels = w + + for _ in range(d): + stage.append( + EfficientViTBlock( + in_channels=in_channels, dim=dim, expand_ratio=expand_ratio, norm=norm, act_func=act_func + ) + ) + stages.append(OpSequential(stage)) + self.width_list.append(in_channels) + self.stages = nn.ModuleList(stages) + + @staticmethod + def build_local_block( + in_channels: int, + out_channels: int, + stride: int, + expand_ratio: float, + norm: str, + act_func: str, + fewer_norm: bool = False, + ) -> nn.Module: + """Build the local convolution block used between EfficientViT stages. + + Args: + in_channels: Number of input feature channels. + out_channels: Number of output feature channels. + stride: Spatial stride for the block. + expand_ratio: Expansion ratio used by MBConv-style blocks. + norm: Normalization layer name. + act_func: Activation function name. + fewer_norm: If ``True``, omit selected normalization layers. + + Returns: + Depthwise-separable or inverted-bottleneck convolution block. + """ + if expand_ratio == 1: + block = DSConv( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + use_bias=(True, False) if fewer_norm else False, + norm=(None, norm) if fewer_norm else norm, + act_func=(act_func, None), + ) + else: + block = MBConv( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + expand_ratio=expand_ratio, + use_bias=(True, True, False) if fewer_norm else False, + norm=(None, None, norm) if fewer_norm else norm, + act_func=(act_func, act_func, None), + ) + return block + + def forward(self, x: torch.Tensor) -> dict[str, torch.Tensor]: + """Run the EfficientViT backbone and collect stage outputs. + + Args: + x: Image tensor with shape :math:`(B, C, H, W)`. + + Returns: + Dictionary containing the input, each stage output, and + ``"stage_final"`` for the final feature map. + """ + output_dict = {"input": x} + output_dict["stage0"] = x = self.input_stem(x) + for stage_id, stage in enumerate(self.stages, 1): + output_dict[f"stage{stage_id}"] = x = stage(x) + output_dict["stage_final"] = x + return output_dict + + +def efficientvit_backbone_b0(**kwargs: dict[str, Any]) -> EfficientViTBackbone: + """Create EfficientViT B0.""" + backbone = EfficientViTBackbone( + width_list=[8, 16, 32, 64, 128], + depth_list=[1, 2, 2, 2, 2], + dim=16, + **build_kwargs_from_config(kwargs, EfficientViTBackbone), + ) + return backbone + + +def efficientvit_backbone_b1(**kwargs: dict[str, Any]) -> EfficientViTBackbone: + """Create EfficientViT B1.""" + backbone = EfficientViTBackbone( + width_list=[16, 32, 64, 128, 256], + depth_list=[1, 2, 3, 3, 4], + dim=16, + **build_kwargs_from_config(kwargs, EfficientViTBackbone), + ) + return backbone + + +def efficientvit_backbone_b2(**kwargs: dict[str, Any]) -> EfficientViTBackbone: + """Create EfficientViT B2.""" + backbone = EfficientViTBackbone( + width_list=[24, 48, 96, 192, 384], + depth_list=[1, 3, 4, 4, 6], + dim=32, + **build_kwargs_from_config(kwargs, EfficientViTBackbone), + ) + return backbone + + +def efficientvit_backbone_b3(**kwargs: dict[str, Any]) -> EfficientViTBackbone: + """Create EfficientViT B3.""" + backbone = EfficientViTBackbone( + width_list=[32, 64, 128, 256, 512], + depth_list=[1, 4, 6, 6, 9], + dim=32, + **build_kwargs_from_config(kwargs, EfficientViTBackbone), + ) + return backbone + + +class EfficientViTLargeBackbone(nn.Module): + """Implement the large-scale variant of the EfficientViT backbone. + + This backbone is designed for high-resolution dense prediction tasks. It + utilizes multi-scale linear attention to achieve a global receptive field + while maintaining linear computational complexity relative to the input + resolution. + + Args: + width_list: List of channel widths for each stage of the backbone. + depth_list: List of number of blocks for each stage. + in_channels: Number of input image channels. Default: 3. + qkv_dim: The internal dimension for query, key, and value projections + in the attention layers. Default: 32. + norm: Normalization layer type to use (e.g., "bn2d", "ln"). + Default: "bn2d". + act_func: Activation function type to use (e.g., "gelu", "relu"). + Default: "gelu". + """ + + def __init__( + self, + width_list: list[int], + depth_list: list[int], + in_channels: int = 3, + qkv_dim: int = 32, + norm: str = "bn2d", + act_func: str = "gelu", + ) -> None: + super().__init__() + + self.width_list = [] + stages = [] + # stage 0 + stage0 = [ + ConvLayer(in_channels=in_channels, out_channels=width_list[0], stride=2, norm=norm, act_func=act_func) + ] + for _ in range(depth_list[0]): + block = self.build_local_block( + stage_id=0, + in_channels=width_list[0], + out_channels=width_list[0], + stride=1, + expand_ratio=1, + norm=norm, + act_func=act_func, + ) + stage0.append(ResidualBlock(block, IdentityLayer())) + in_channels = width_list[0] + stages.append(OpSequential(stage0)) + self.width_list.append(in_channels) + + for stage_id, (w, d) in enumerate(zip(width_list[1:4], depth_list[1:4]), start=1): + stage = [] + for i in range(d + 1): + stride = 2 if i == 0 else 1 + block = self.build_local_block( + stage_id=stage_id, + in_channels=in_channels, + out_channels=w, + stride=stride, + expand_ratio=4 if stride == 1 else 16, + norm=norm, + act_func=act_func, + fewer_norm=stage_id > 2, + ) + block = ResidualBlock(block, IdentityLayer() if stride == 1 else None) + stage.append(block) + in_channels = w + stages.append(OpSequential(stage)) + self.width_list.append(in_channels) + + for stage_id, (w, d) in enumerate(zip(width_list[4:], depth_list[4:]), start=4): + stage = [] + block = self.build_local_block( + stage_id=stage_id, + in_channels=in_channels, + out_channels=w, + stride=2, + expand_ratio=24, + norm=norm, + act_func=act_func, + fewer_norm=True, + ) + stage.append(ResidualBlock(block, None)) + in_channels = w + + for _ in range(d): + stage.append( + EfficientViTBlock( + in_channels=in_channels, dim=qkv_dim, expand_ratio=6, norm=norm, act_func=act_func + ) + ) + stages.append(OpSequential(stage)) + self.width_list.append(in_channels) + self.stages = nn.ModuleList(stages) + + @staticmethod + def build_local_block( + stage_id: int, + in_channels: int, + out_channels: int, + stride: int, + expand_ratio: float, + norm: str, + act_func: str, + fewer_norm: bool = False, + ) -> nn.Module: + """Build a local block for an EfficientViT large stage. + + Args: + stage_id: Index of the stage being constructed. + in_channels: Number of input feature channels. + out_channels: Number of output feature channels. + stride: Spatial stride for the block. + expand_ratio: Expansion ratio controlling intermediate channels. + norm: Normalization layer name. + act_func: Activation function name. + fewer_norm: If ``True``, use the reduced-normalization variant. + + Returns: + Residual, fused-MBConv, or MBConv block chosen for the stage. + """ + if expand_ratio == 1: + block = ResBlock( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + use_bias=(True, False) if fewer_norm else False, + norm=(None, norm) if fewer_norm else norm, + act_func=(act_func, None), + ) + elif stage_id <= 2: + block = FusedMBConv( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + expand_ratio=expand_ratio, + use_bias=(True, False) if fewer_norm else False, + norm=(None, norm) if fewer_norm else norm, + act_func=(act_func, None), + ) + else: + block = MBConv( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + expand_ratio=expand_ratio, + use_bias=(True, True, False) if fewer_norm else False, + norm=(None, None, norm) if fewer_norm else norm, + act_func=(act_func, act_func, None), + ) + return block + + def forward(self, x: torch.Tensor) -> dict[str, torch.Tensor]: + """Run the EfficientViT large backbone and collect stage outputs. + + Args: + x: Image tensor with shape :math:`(B, C, H, W)`. + + Returns: + Dictionary with entries for each stage and ``"stage_final"`` for + the final feature map. + """ + output_dict = {"input": x} + for stage_id, stage in enumerate(self.stages): + output_dict[f"stage{stage_id}"] = x = stage(x) + output_dict["stage_final"] = x + return output_dict + + +def efficientvit_backbone_l0(**kwargs: dict[str, Any]) -> EfficientViTLargeBackbone: + """Create EfficientViT L0.""" + backbone = EfficientViTLargeBackbone( + width_list=[32, 64, 128, 256, 512], + depth_list=[1, 1, 1, 4, 4], + **build_kwargs_from_config(kwargs, EfficientViTLargeBackbone), + ) + return backbone + + +def efficientvit_backbone_l1(**kwargs: dict[str, Any]) -> EfficientViTLargeBackbone: + """Create EfficientViT L.""" + backbone = EfficientViTLargeBackbone( + width_list=[32, 64, 128, 256, 512], + depth_list=[1, 1, 1, 6, 6], + **build_kwargs_from_config(kwargs, EfficientViTLargeBackbone), + ) + return backbone + + +def efficientvit_backbone_l2(**kwargs: dict[str, Any]) -> EfficientViTLargeBackbone: + """Create EfficientViT L2.""" + backbone = EfficientViTLargeBackbone( + width_list=[32, 64, 128, 256, 512], + depth_list=[1, 2, 2, 8, 8], + **build_kwargs_from_config(kwargs, EfficientViTLargeBackbone), + ) + return backbone + + +def efficientvit_backbone_l3(**kwargs: dict[str, Any]) -> EfficientViTLargeBackbone: + """Create EfficientViT L3.""" + backbone = EfficientViTLargeBackbone( + width_list=[64, 128, 256, 512, 1024], + depth_list=[1, 2, 2, 8, 8], + **build_kwargs_from_config(kwargs, EfficientViTLargeBackbone), + ) + return backbone diff --git a/kornia/models/efficient_vit/model.py b/kornia/models/efficient_vit/model.py new file mode 100644 index 0000000..d1c4cf0 --- /dev/null +++ b/kornia/models/efficient_vit/model.py @@ -0,0 +1,110 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Literal + +import torch + +from kornia.models.base import ModelBase +from kornia.models.efficient_vit import backbone as vit + + +def _get_base_url(model_type: Literal["b1", "b2", "b3"] = "b1", resolution: Literal[224, 256, 288] = 224) -> str: + """Return the base URL of the model weights.""" + return f"https://huggingface.co/kornia/efficientvit_imagenet_{model_type}_r{resolution}/resolve/main/{model_type}-r{resolution}.pt" + + +@dataclass +class EfficientViTConfig: + """Configuration to construct EfficientViT model. + + Model weights can be loaded from a checkpoint URL or local path. + The model weights are hosted on HuggingFace's model hub: https://huggingface.co/kornia. + + Args: + checkpoint: URL or local path of model weights. + + """ + + checkpoint: str = field(default_factory=_get_base_url) + + @classmethod + def from_pretrained( + cls, model_type: Literal["b1", "b2", "b3"], resolution: Literal[224, 256, 288] + ) -> EfficientViTConfig: + """Return a configuration object from a pre-trained model. + + Args: + model_type: model type, one of :obj:`"b1"`, :obj:`"b2"`, :obj:`"b3"`. + resolution: input resolution, one of :obj:`224`, :obj:`256`, :obj:`288`. + + """ + return cls(checkpoint=_get_base_url(model_type=model_type, resolution=resolution)) + + +class EfficientViT(ModelBase[EfficientViTConfig]): + """EfficientViT backbone model.""" + + def __init__(self, backbone: vit.EfficientViTBackbone | vit.EfficientViTLargeBackbone) -> None: + super().__init__() + self.backbone = backbone + + @staticmethod + def from_config(config: EfficientViTConfig) -> EfficientViT: + """Build the EfficientViT model from a configuration object. + + Args: + config: EfficientViT configuration object. See :class:`EfficientViTConfig`. + + Returns: + EfficientViT: the EfficientViT model. + + """ + # load the model from the checkpoint + try: + model_file = torch.hub.load_state_dict_from_url(config.checkpoint, map_location="cpu") + model_file = model_file["state_dict"] if "state_dict" in model_file else model_file + except RuntimeError: + raise RuntimeError(f"Unable to load the model from {config.checkpoint}.") from None + + file_name = config.checkpoint.split("/")[-1] + model_type = file_name.split("-")[0] + + if model_type not in ["b0", "b1", "b2", "b3", "l0", "l1", "l2", "l3"]: + raise ValueError(f"Unknown model type: {model_type}.") + + # create and load the model weights without strict until we polish the model files + model = getattr(vit, f"efficientvit_backbone_{model_type}")() + model.load_state_dict(model_file, strict=False) + + return EfficientViT(backbone=model) + + def forward(self, images: torch.Tensor) -> torch.Tensor: + """Extract features from the input images. + + Args: + images: input images tensor of shape :math:`(B, C, H, W)`. + + Returns: + Dict[str, torch.Tensor]: a dictionary containing the features. + + """ + feats = self.backbone(images) + return feats diff --git a/kornia/models/efficient_vit/nn/__init__.py b/kornia/models/efficient_vit/nn/__init__.py new file mode 100644 index 0000000..b453ad4 --- /dev/null +++ b/kornia/models/efficient_vit/nn/__init__.py @@ -0,0 +1,21 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Contrib EfficientViT NN — Neural network layers for EfficientViT models. + +This subpackage provides neural network building blocks for EfficientViT. +""" diff --git a/kornia/models/efficient_vit/nn/act.py b/kornia/models/efficient_vit/nn/act.py new file mode 100644 index 0000000..5cf39f7 --- /dev/null +++ b/kornia/models/efficient_vit/nn/act.py @@ -0,0 +1,47 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# EfficientViT: Multi-Scale Linear Attention for High-Resolution Dense Prediction +# Han Cai, Junyan Li, Muyan Hu, Chuang Gan, Song Han +# International Conference on Computer Vision (ICCV), 2023 +from __future__ import annotations + +from functools import partial +from typing import Any, Optional, Union + +from torch import nn + +from kornia.models.efficient_vit.utils import build_kwargs_from_config + +# register activation function here +REGISTERED_ACT_DICT: dict[str, type[nn.Module]] = { + "relu": nn.ReLU, + "relu6": nn.ReLU6, + "hswish": nn.Hardswish, + "silu": nn.SiLU, + "gelu": partial(nn.GELU, approximate="tanh"), # type: ignore +} + + +def build_act(name: Optional[str], **kwargs: dict[str, Any]) -> Union[nn.Module, None]: + """Return activation op.""" + if name in REGISTERED_ACT_DICT: + act_cls = REGISTERED_ACT_DICT[name] + args = build_kwargs_from_config(kwargs, act_cls) + return act_cls(**args) + + return None diff --git a/kornia/models/efficient_vit/nn/norm.py b/kornia/models/efficient_vit/nn/norm.py new file mode 100644 index 0000000..1cf00f6 --- /dev/null +++ b/kornia/models/efficient_vit/nn/norm.py @@ -0,0 +1,66 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# EfficientViT: Multi-Scale Linear Attention for High-Resolution Dense Prediction +# Han Cai, Junyan Li, Muyan Hu, Chuang Gan, Song Han +# International Conference on Computer Vision (ICCV), 2023 +from __future__ import annotations + +from typing import Any, Optional + +import torch +from torch import nn + +from kornia.models.efficient_vit.utils import build_kwargs_from_config + + +class LayerNorm2d(nn.LayerNorm): + """Apply Layer Normalization over a 4D input tensor (NCHW format).""" + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Normalize an NCHW feature map across channels. + + Args: + x: Tensor with shape :math:`(B, C, H, W)`. + + Returns: + Tensor with the same shape as ``x`` after channel-wise layer + normalization and optional affine scaling. + """ + out = x - torch.mean(x, dim=1, keepdim=True) + out = out / torch.sqrt(torch.square(out).mean(dim=1, keepdim=True) + self.eps) + if self.elementwise_affine: + out = out * self.weight.view(1, -1, 1, 1) + self.bias.view(1, -1, 1, 1) + return out + + +# register normalization function here +REGISTERED_NORM_DICT: dict[str, type] = {"bn2d": nn.BatchNorm2d, "ln": nn.LayerNorm, "ln2d": LayerNorm2d} + + +def build_norm(name: str = "bn2d", num_features: Optional[int] = None, **kwargs: Any) -> Optional[nn.Module]: + """Return norm op.""" + if name in ["ln", "ln2d"]: + kwargs["normalized_shape"] = num_features + else: + kwargs["num_features"] = num_features + if name in REGISTERED_NORM_DICT: + norm_cls = REGISTERED_NORM_DICT[name] + args = build_kwargs_from_config(kwargs, norm_cls) + return norm_cls(**args) + else: + return None diff --git a/kornia/models/efficient_vit/nn/ops.py b/kornia/models/efficient_vit/nn/ops.py new file mode 100644 index 0000000..188e24c --- /dev/null +++ b/kornia/models/efficient_vit/nn/ops.py @@ -0,0 +1,680 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# EfficientViT: Multi-Scale Linear Attention for High-Resolution Dense Prediction +# Han Cai, Junyan Li, Muyan Hu, Chuang Gan, Song Han +# International Conference on Computer Vision (ICCV), 2023 + +# type: ignore +from __future__ import annotations + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.models.efficient_vit.nn.act import build_act +from kornia.models.efficient_vit.nn.norm import build_norm +from kornia.models.efficient_vit.utils import get_same_padding, val2tuple + +__all__ = [ + "ConvLayer", + "DSConv", + "EfficientViTBlock", + "FusedMBConv", + "IdentityLayer", + "MBConv", + "OpSequential", + "ResBlock", + "ResidualBlock", +] + +################################################################################# +# Basic Layers # +################################################################################# + + +class ConvLayer(nn.Module): + """Implement a standard convolutional layer with Batch Normalization and Activation. + + Args: + in_channels: Number of input channels. + out_channels: Number of output channels. + kernel_size: Size of the convolving kernel. + stride: Stride of the convolution. Default: 1. + groups: Number of blocked connections from input to output. Default: 1. + use_bias: Whether to include a bias term in the convolution. Default: False. + dropout: Dropout rate to apply before convolution. Default: 0 (no dropout). + norm: Normalization layer type. Default: :class:`bn2d` (BatchNorm2d). + act_func: Activation layer type. Default: :class:`relu` (ReLU). + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size=3, + stride=1, + dilation=1, + groups=1, + use_bias=False, + dropout=0, + norm="bn2d", + act_func="relu", + ): + super().__init__() + + padding = get_same_padding(kernel_size) + padding *= dilation + + self.dropout = nn.Dropout2d(dropout, inplace=False) if dropout > 0 else None + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=(kernel_size, kernel_size), + stride=(stride, stride), + padding=padding, + dilation=(dilation, dilation), + groups=groups, + bias=use_bias, + ) + self.norm = build_norm(norm, num_features=out_channels) + self.act = build_act(act_func) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply dropout, convolution, optional normalization, and activation. + + Args: + x: Feature map tensor with shape :math:`(B, C, H, W)`. + + Returns: + Feature map after the configured convolutional layer stack. + """ + if self.dropout is not None: + x = self.dropout(x) + x = self.conv(x) + if self.norm: + x = self.norm(x) + if self.act: + x = self.act(x) + return x + + +class IdentityLayer(nn.Module): + """Implement a placeholder layer that returns the input as-is.""" + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Return the input tensor unchanged.""" + return x + + +################################################################################# +# Basic Blocks # +################################################################################# + + +class DSConv(nn.Module): + """Implement Depthwise Separable Convolution. + + This layer splits a standard convolution into a depthwise convolution + followed by a pointwise convolution to reduce parameters and FLOPs. + + Args: + in_channels: Number of input channels. + out_channels: Number of output channels. + kernel_size: Size of the convolving kernel. Default: 3. + stride: Stride of the convolution. Default: 1. + use_bias: Whether to use bias in the convolutional layers. Default: False. + norm: Normalization layers for the depthwise and pointwise stages. Default: ("bn2d", "bn2d"). + act_func: Activation functions for the depthwise and pointwise stages. Default: ("relu6", None). + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size=3, + stride=1, + use_bias=False, + norm=("bn2d", "bn2d"), + act_func=("relu6", None), + ): + super().__init__() + + use_bias = val2tuple(use_bias, 2) + norm = val2tuple(norm, 2) + act_func = val2tuple(act_func, 2) + + self.depth_conv = ConvLayer( + in_channels, + in_channels, + kernel_size, + stride, + groups=in_channels, + norm=norm[0], + act_func=act_func[0], + use_bias=use_bias[0], + ) + self.point_conv = ConvLayer( + in_channels, out_channels, 1, norm=norm[1], act_func=act_func[1], use_bias=use_bias[1] + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply depthwise then pointwise convolution. + + Args: + x: Feature map tensor with shape :math:`(B, C, H, W)`. + + Returns: + Feature map with ``out_channels`` produced by the pointwise + projection. + """ + x = self.depth_conv(x) + x = self.point_conv(x) + return x + + +class MBConv(nn.Module): + """Implement the Inverted Residual Block (Mobile Inverted Bottleneck). + + This block follows the MobileNetV2 design: a wide intermediate layer + (expansion) between two narrow layers, using depthwise convolutions + to maintain efficiency. + + Args: + in_channels: Number of input channels. + out_channels: Number of output channels. + kernel_size: Size of the depthwise convolution kernel. Default: 3. + stride: Stride of the depthwise convolution. Default: 1. + mid_channels: Number of intermediate expansion channels. If None, calculated + using expand_ratio. Default: None. + expand_ratio: Expansion factor for the intermediate channels relative to + in_channels. Default: 6. + use_bias: Whether to use bias in the convolutional layers. Default: False. + norm: Normalization layers for the expansion, depthwise, and pointwise stages. + Default: ("bn2d", "bn2d", "bn2d"). + act_func: Activation functions for the expansion, depthwise, and pointwise stages. + Default: ("relu6", "relu6", None). + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size=3, + stride=1, + mid_channels=None, + expand_ratio=6, + use_bias=False, + norm=("bn2d", "bn2d", "bn2d"), + act_func=("relu6", "relu6", None), + ): + super().__init__() + + use_bias = val2tuple(use_bias, 3) + norm = val2tuple(norm, 3) + act_func = val2tuple(act_func, 3) + mid_channels = mid_channels or round(in_channels * expand_ratio) + + self.inverted_conv = ConvLayer( + in_channels, mid_channels, 1, stride=1, norm=norm[0], act_func=act_func[0], use_bias=use_bias[0] + ) + self.depth_conv = ConvLayer( + mid_channels, + mid_channels, + kernel_size, + stride=stride, + groups=mid_channels, + norm=norm[1], + act_func=act_func[1], + use_bias=use_bias[1], + ) + self.point_conv = ConvLayer( + mid_channels, out_channels, 1, norm=norm[2], act_func=act_func[2], use_bias=use_bias[2] + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply an inverted bottleneck convolution block. + + Args: + x: Feature map tensor with shape :math:`(B, C, H, W)`. + + Returns: + Feature map after expansion, depthwise convolution, and projection. + """ + x = self.inverted_conv(x) + x = self.depth_conv(x) + x = self.point_conv(x) + return x + + +class FusedMBConv(nn.Module): + """Implement a fused version of the Inverted Residual Block for efficiency. + + This block improves throughput on certain hardware by replacing the + separate expansion and depthwise convolutions with a single regular + convolution, as seen in EfficientNet-V2. + + Args: + in_channels: Number of input channels. + out_channels: Number of output channels. + kernel_size: Size of the main convolution kernel. Default: 3. + stride: Stride of the convolution. Default: 1. + mid_channels: Number of intermediate expansion channels. If None, + calculated using expand_ratio. Default: None. + expand_ratio: Expansion factor for the intermediate channels relative + to in_channels. Default: 6. + groups: Number of groups for the main convolution. Default: 1. + use_bias: Whether to use bias in the convolutional layers. Default: False. + norm: Normalization layers for the fused and pointwise stages. + Default: ("bn2d", "bn2d"). + act_func: Activation functions for the fused and pointwise stages. + Default: ("relu6", None). + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size=3, + stride=1, + mid_channels=None, + expand_ratio=6, + groups=1, + use_bias=False, + norm=("bn2d", "bn2d"), + act_func=("relu6", None), + ): + super().__init__() + use_bias = val2tuple(use_bias, 2) + norm = val2tuple(norm, 2) + act_func = val2tuple(act_func, 2) + + mid_channels = mid_channels or round(in_channels * expand_ratio) + + self.spatial_conv = ConvLayer( + in_channels, + mid_channels, + kernel_size, + stride, + groups=groups, + use_bias=use_bias[0], + norm=norm[0], + act_func=act_func[0], + ) + self.point_conv = ConvLayer( + mid_channels, out_channels, 1, use_bias=use_bias[1], norm=norm[1], act_func=act_func[1] + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply a fused mobile inverted bottleneck convolution. + + Args: + x: Feature map tensor with shape :math:`(B, C, H, W)`. + + Returns: + Feature map after fused spatial convolution and pointwise + projection. + """ + x = self.spatial_conv(x) + x = self.point_conv(x) + return x + + +class ResBlock(nn.Module): + """Implement a standard residual block for EfficientViT. + + This block applies a series of convolutions and adds the original input + back to the output via a skip connection, helping to mitigate the + vanishing gradient problem in deep networks. + + Args: + in_channels: Number of input channels. + out_channels: Number of output channels. + kernel_size: Size of the convolving kernel. Default: 3. + stride: Stride of the convolution. Default: 1. + mid_channels: Number of intermediate channels. If None, calculated + using expand_ratio. Default: None. + expand_ratio: Expansion factor for the intermediate channels. Default: 1. + use_bias: Whether to use bias in the convolutional layers. Default: False. + norm: Normalization layers for the main and projection stages. + Default: ("bn2d", "bn2d"). + act_func: Activation functions for the main and projection stages. + Default: ("relu6", None). + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size=3, + stride=1, + mid_channels=None, + expand_ratio=1, + use_bias=False, + norm=("bn2d", "bn2d"), + act_func=("relu6", None), + ): + super().__init__() + use_bias = val2tuple(use_bias, 2) + norm = val2tuple(norm, 2) + act_func = val2tuple(act_func, 2) + + mid_channels = mid_channels or round(in_channels * expand_ratio) + + self.conv1 = ConvLayer( + in_channels, mid_channels, kernel_size, stride, use_bias=use_bias[0], norm=norm[0], act_func=act_func[0] + ) + self.conv2 = ConvLayer( + mid_channels, out_channels, kernel_size, 1, use_bias=use_bias[1], norm=norm[1], act_func=act_func[1] + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply the two-convolution residual-block main branch. + + Args: + x: Feature map tensor with shape :math:`(B, C, H, W)`. + + Returns: + Feature map after the block's convolutional transformations. + """ + x = self.conv1(x) + x = self.conv2(x) + return x + + +class LiteMLA(nn.Module): + r"""Lightweight multi-scale linear attention. + + This module implements a linear attention mechanism that avoids the quadratic + complexity of standard self-attention. It incorporates multi-scale depthwise + convolutions (scales) to aggregate local information effectively. + + Args: + in_channels: Number of input channels. + out_channels: Number of output channels. + heads: Number of attention heads. If None, calculated using heads_ratio. + Default: None. + heads_ratio: Ratio to determine the number of heads relative to channels. + Default: 1.0. + dim: Dimension of each attention head. Default: 8. + use_bias: Whether to use bias in the linear projections. Default: False. + norm: Normalization layers for the query/key/value and output projections. + Default: (None, "bn2d"). + act_func: Activation functions for the internal and output stages. + Default: (None, None). + kernel_func: The kernel function to use for linear attention (e.g., "relu"). + Default: "relu". + scales: Kernel sizes for the multi-scale depthwise convolutions. + Default: (5,). + eps: A small value to ensure numerical stability during division. + Default: 1.0e-15. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + heads: int or None = None, + heads_ratio: float = 1.0, + dim=8, + use_bias=False, + norm=(None, "bn2d"), + act_func=(None, None), + kernel_func="relu", + scales: tuple[int, ...] = (5,), + eps=1.0e-15, + ): + super().__init__() + self.eps = eps + heads = heads or int(in_channels // dim * heads_ratio) + + total_dim = heads * dim + + use_bias = val2tuple(use_bias, 2) + norm = val2tuple(norm, 2) + act_func = val2tuple(act_func, 2) + + self.dim = dim + self.qkv = ConvLayer(in_channels, 3 * total_dim, 1, use_bias=use_bias[0], norm=norm[0], act_func=act_func[0]) + self.aggreg = nn.ModuleList( + [ + nn.Sequential( + nn.Conv2d( + 3 * total_dim, + 3 * total_dim, + scale, + padding=get_same_padding(scale), + groups=3 * total_dim, + bias=use_bias[0], + ), + nn.Conv2d(3 * total_dim, 3 * total_dim, 1, groups=3 * heads, bias=use_bias[0]), + ) + for scale in scales + ] + ) + self.kernel_func = build_act(kernel_func, inplace=False) + + self.proj = ConvLayer( + total_dim * (1 + len(scales)), out_channels, 1, use_bias=use_bias[1], norm=norm[1], act_func=act_func[1] + ) + + @torch.amp.autocast(device_type="cuda", enabled=False) + def relu_linear_att(self, qkv: torch.Tensor) -> torch.Tensor: + B, _, H, W = list(qkv.size()) + + if qkv.dtype == torch.float16: + qkv = qkv.float() + + qkv = torch.reshape(qkv, (B, -1, 3 * self.dim, H * W)) + qkv = torch.transpose(qkv, -1, -2) + q, k, v = (qkv[..., 0 : self.dim], qkv[..., self.dim : 2 * self.dim], qkv[..., 2 * self.dim :]) + + # lightweight linear attention + q = self.kernel_func(q) + k = self.kernel_func(k) + + # linear matmul + trans_k = k.transpose(-1, -2) + + v = F.pad(v, (0, 1), mode="constant", value=1) + kv = torch.matmul(trans_k, v) + out = torch.matmul(q, kv) + out = out[..., :-1] / (out[..., -1:] + self.eps) + + out = torch.transpose(out, -1, -2) + out = torch.reshape(out, (B, -1, H, W)) + return out + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply lightweight multi-scale linear attention. + + Args: + x: Feature map tensor with shape :math:`(B, C, H, W)`. + + Returns: + Feature map after multi-scale query/key/value aggregation, linear + attention, and output projection. + """ + # generate multi-scale q, k, v + qkv = self.qkv(x) + multi_scale_qkv = [qkv] + for op in self.aggreg: + multi_scale_qkv.append(op(qkv)) + multi_scale_qkv = torch.cat(multi_scale_qkv, dim=1) + + out = self.relu_linear_att(multi_scale_qkv) + out = self.proj(out) + + return out + + +class EfficientViTBlock(nn.Module): + """Implement a single EfficientViT building block. + + This block consists of two main components: a Lightweight Multi-scale Linear + Attention (LiteMLA) layer and a Feed-Forward Network (FFN). It follows the + standard transformer-style residual architecture optimized for efficiency. + + Args: + in_channels: Number of input channels. + heads_ratio: Ratio to determine the number of attention heads in LiteMLA. + Default: 1.0. + dim: The dimension for the attention head and FFN layers. Default: 32. + expand_ratio: Expansion factor for the FFN (Feed-Forward Network). + Default: 4. + norm: Normalization layer type to use (e.g., "bn2d"). Default: "bn2d". + act_func: Activation function type to use (e.g., "hswish"). Default: "hswish". + """ + + def __init__( + self, + in_channels: int, + heads_ratio: float = 1.0, + dim=32, + expand_ratio: float = 4, + norm="bn2d", + act_func="hswish", + ): + super().__init__() + self.context_module = ResidualBlock( + LiteMLA( + in_channels=in_channels, out_channels=in_channels, heads_ratio=heads_ratio, dim=dim, norm=(None, norm) + ), + IdentityLayer(), + ) + local_module = MBConv( + in_channels=in_channels, + out_channels=in_channels, + expand_ratio=expand_ratio, + use_bias=(True, True, False), + norm=(None, None, norm), + act_func=(act_func, act_func, None), + ) + self.local_module = ResidualBlock(local_module, IdentityLayer()) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply an EfficientViT attention-plus-local block. + + Args: + x: Feature map tensor with shape :math:`(B, C, H, W)`. + + Returns: + Feature map with the same shape after the context module and local + MBConv module. + """ + x = self.context_module(x) + x = self.local_module(x) + return x + + +################################################################################# +# Functional Blocks # +################################################################################# + + +class ResidualBlock(nn.Module): + """Provide a flexible residual wrapper for a main branch and a shortcut. + + Args: + main: The primary neural network branch. + shortcut: The identity or projection shortcut branch. + post_act: Activation to apply after the summation. + pre_norm: Normalization to apply before the branches. + """ + + def __init__( + self, + main: nn.Module | None, + shortcut: nn.Module | None, + post_act: nn.Module | None = None, + pre_norm: nn.Module | None = None, + ) -> None: + super().__init__() + + self.pre_norm = pre_norm + self.main = main + self.shortcut = shortcut + self.post_act = build_act(post_act) + + def forward_main(self, x: torch.Tensor) -> torch.Tensor: + """Run the main branch, applying pre-normalization when configured. + + Args: + x: Input tensor for the residual block. + + Returns: + Output from ``self.main`` with optional ``self.pre_norm`` applied + first. + """ + if self.pre_norm is None: + return self.main(x) + else: + return self.main(self.pre_norm(x)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply the residual wrapper to an input tensor. + + Args: + x: Input tensor passed to the main and shortcut branches. + + Returns: + Tensor from the main branch, optionally added to the shortcut and + passed through ``self.post_act``. + """ + if self.main is None: + res = x + elif self.shortcut is None: + res = self.forward_main(x) + else: + res = self.forward_main(x) + self.shortcut(x) + if self.post_act: + res = self.post_act(res) + return res + + +class OpSequential(nn.Module): + """A container for sequential execution that handles optional or None modules. + + This class behaves similarly to :class:`nn.Sequential` but explicitly + allows for ``None`` elements within the operation list, which are + ignored during the forward pass. + + Args: + op_list: A list of modules to be executed sequentially. + Elements can be ``None``, in which case they act as an identity. + """ + + def __init__(self, op_list: list[nn.Module | None]): + super().__init__() + valid_op_list = [] + for op in op_list: + if op is not None: + valid_op_list.append(op) + self.op_list = nn.ModuleList(valid_op_list) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply each non-empty operation in sequence. + + Args: + x: Input tensor for the first operation. + + Returns: + Tensor after all stored operations have been applied. + """ + for op in self.op_list: + x = op(x) + return x diff --git a/kornia/models/efficient_vit/utils/__init__.py b/kornia/models/efficient_vit/utils/__init__.py new file mode 100644 index 0000000..a2d2b16 --- /dev/null +++ b/kornia/models/efficient_vit/utils/__init__.py @@ -0,0 +1,31 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Contrib EfficientViT Utils — Utility functions for EfficientViT models. + +This subpackage provides helper functions and utilities for EfficientViT implementations. +""" + +# EfficientViT: Multi-Scale Linear Attention for High-Resolution Dense Prediction +# Han Cai, Junyan Li, Muyan Hu, Chuang Gan, Song Han +# International Conference on Computer Vision (ICCV), 2023 + +# TODO: promote this to kornia +from .list import val2tuple +from .network import build_kwargs_from_config, get_same_padding + +__all__ = ["build_kwargs_from_config", "get_same_padding", "val2tuple"] diff --git a/kornia/models/efficient_vit/utils/list.py b/kornia/models/efficient_vit/utils/list.py new file mode 100644 index 0000000..e7b3135 --- /dev/null +++ b/kornia/models/efficient_vit/utils/list.py @@ -0,0 +1,44 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# EfficientViT: Multi-Scale Linear Attention for High-Resolution Dense Prediction +# Han Cai, Junyan Li, Muyan Hu, Chuang Gan, Song Han +# International Conference on Computer Vision (ICCV), 2023 +from __future__ import annotations + +from typing import Any, Union + + +def val2list(x: Union[list[Any], tuple[Any, ...], Any], repeat_time: int = 1) -> list[Any]: + """Convert value to list.""" + if isinstance(x, list): + return x + elif isinstance(x, tuple): + return list(x) + else: + return [x] * repeat_time + + +def val2tuple(x: Union[list[Any], tuple[Any, ...], Any], min_len: int = 1, idx_repeat: int = -1) -> tuple[Any, ...]: + """Convert value to tuple.""" + xlist = list(x) if isinstance(x, (list, tuple)) else [x] + cur_len = len(xlist) + if cur_len < min_len and cur_len > 0: + v = xlist[idx_repeat] + # Only append as many values as needed: + xlist[idx_repeat:idx_repeat] = [v] * (min_len - cur_len) + return tuple(xlist) diff --git a/kornia/models/efficient_vit/utils/network.py b/kornia/models/efficient_vit/utils/network.py new file mode 100644 index 0000000..6d8df4b --- /dev/null +++ b/kornia/models/efficient_vit/utils/network.py @@ -0,0 +1,43 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# EfficientViT: Multi-Scale Linear Attention for High-Resolution Dense Prediction +# Han Cai, Junyan Li, Muyan Hu, Chuang Gan, Song Han +# International Conference on Computer Vision (ICCV), 2023 +from __future__ import annotations + +from inspect import signature +from typing import Any, Union + + +def get_same_padding(kernel_size: Union[int, tuple[int, ...]]) -> Union[int, tuple[int, ...]]: + """Return padding values.""" + if isinstance(kernel_size, (tuple,)): + return tuple([get_same_padding(ks) for ks in kernel_size]) # type: ignore + + # assert kernel_size % 2 > 0, "kernel size should be odd number" + return kernel_size // 2 + + +def build_kwargs_from_config(config: dict[str, Any], target_func: Any) -> dict[str, Any]: + """Return kwargs from config object.""" + valid_keys = list(signature(target_func).parameters) + kwargs = {} + for key, value in config.items(): + if key in valid_keys: + kwargs[key] = value + return kwargs diff --git a/kornia/models/kimi_vl/__init__.py b/kornia/models/kimi_vl/__init__.py new file mode 100644 index 0000000..bdff9e6 --- /dev/null +++ b/kornia/models/kimi_vl/__init__.py @@ -0,0 +1,29 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .config import KimiVLConfig, KimiVLProjectorConfig, MoonViTConfig +from .model import KimiVLModel, KimiVLProjector +from .moonvit import MoonViT + +__all__ = [ + "KimiVLConfig", + "KimiVLModel", + "KimiVLProjector", + "KimiVLProjectorConfig", + "MoonViT", + "MoonViTConfig", +] diff --git a/kornia/models/kimi_vl/config.py b/kornia/models/kimi_vl/config.py new file mode 100644 index 0000000..ca58842 --- /dev/null +++ b/kornia/models/kimi_vl/config.py @@ -0,0 +1,91 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class MoonViTConfig: + """Configuration for MoonViT vision encoder. + + Args: + image_size: Default input image size used for positional embedding initialization. + The model supports variable resolution inputs through interpolation. + patch_size: Size of image patches. + num_channels: Number of input channels. + hidden_size: Hidden dimension size. + num_hidden_layers: Number of transformer layers. + num_attention_heads: Number of attention heads. + intermediate_size: Intermediate size in feed-forward network. + hidden_act: Activation function. + layer_norm_eps: Epsilon for layer normalization. + dropout_p: Dropout probability. + attention_dropout_p: Attention dropout probability. + rope_theta: Theta value for Rotary Positional Embeddings. + """ + + image_size: int = 384 + patch_size: int = 14 + num_channels: int = 3 + hidden_size: int = 1152 + num_hidden_layers: int = 27 + num_attention_heads: int = 16 + intermediate_size: int = 4304 + hidden_act: str = "gelu" + layer_norm_eps: float = 1e-6 + dropout_p: float = 0.0 + attention_dropout_p: float = 0.0 + rope_theta: float = 800000.0 + + +@dataclass +class KimiVLProjectorConfig: + """Configuration for KimiVL projector. + + Args: + input_dim: Input dimension (should match vision encoder's hidden_size). + hidden_dim: Hidden dimension of the MLP. + output_dim: Output dimension (embedding dim of the LLM). + dropout_p: Dropout probability. + """ + + input_dim: int = 1152 # Vision encoder output dimension + hidden_dim: int = 4608 # Hidden dim from official KimiVL weights + output_dim: int = 2048 # Output dim from official KimiVL weights + dropout_p: float = 0.0 + + +@dataclass +class KimiVLConfig: + """Configuration for KimiVL model. + + Args: + vision_config: Vision encoder configuration. + projector_config: Projector configuration. + """ + + vision_config: Optional[MoonViTConfig] = None + projector_config: Optional[KimiVLProjectorConfig] = None + + def __post_init__(self) -> None: + if self.vision_config is None: + self.vision_config = MoonViTConfig() + if self.projector_config is None: + self.projector_config = KimiVLProjectorConfig() diff --git a/kornia/models/kimi_vl/model.py b/kornia/models/kimi_vl/model.py new file mode 100644 index 0000000..fee3be1 --- /dev/null +++ b/kornia/models/kimi_vl/model.py @@ -0,0 +1,123 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional + +import torch +from torch import nn + +from .config import KimiVLConfig, KimiVLProjectorConfig +from .moonvit import MoonViT + +__all__ = ["KimiVLModel", "KimiVLProjector"] + + +class KimiVLProjector(nn.Module): + """KimiVL Projector with Pixel Unshuffle and MLP.""" + + def __init__(self, config: KimiVLProjectorConfig) -> None: + super().__init__() + self.downsample_ratio = 2 + + # Pre-norm (applied before pixel unshuffle, on the vision encoder output dimension) + self.pre_norm = nn.LayerNorm(config.input_dim) + + # After pixel unshuffle, the dimension becomes input_dim * (downsample_ratio ** 2) + mlp_input_dim = config.input_dim * (self.downsample_ratio**2) + + # MLP + self.mlp = nn.Sequential( + nn.Linear(mlp_input_dim, config.hidden_dim), + nn.GELU(), + nn.Linear(config.hidden_dim, config.output_dim), + nn.Dropout(config.dropout_p), + ) + + def forward(self, x: torch.Tensor, h: int, w: int) -> torch.Tensor: + """Forward pass. + + Args: + x: Input features (B, N, D) + h: Height in patches + w: Width in patches + + Returns: + Projected features (B, N/4, output_dim) + """ + B, _N, D = x.shape + + # Apply pre-norm + x = self.pre_norm(x) + + # Reshape to spatial (B, H, W, D) -> (B, D, H, W) + x = x.view(B, h, w, D).permute(0, 3, 1, 2) + + # Pixel unshuffle for spatial downsampling -> (B, D*4, H/2, W/2) + x = torch.nn.functional.pixel_unshuffle(x, self.downsample_ratio) + + # Flatten back -> (B, D*4, N/4) -> (B, N/4, D*4) + x = x.flatten(2).transpose(1, 2) + + # MLP + x = self.mlp(x) + + return x + + +class KimiVLModel(nn.Module): + """KimiVL Vision-Language Model (Vision Part). + + This model includes the Vision Encoder (MoonViT) and the Projector. + It does not include the LLM decoder, as Kornia focuses on the vision components. + + Args: + config: KimiVL configuration. + """ + + def __init__(self, config: KimiVLConfig) -> None: + super().__init__() + self.config = config + self.patch_size = config.vision_config.patch_size + + # Vision Encoder (MoonViT) + self.vision_encoder = MoonViT(config.vision_config) + + # Projector + self.projector = KimiVLProjector(config.projector_config) + + def forward(self, images: torch.Tensor, attention_mask: Optional[torch.Tensor] = None) -> torch.Tensor: + """Forward pass. + + Args: + images: (B, C, H, W) + attention_mask: (B, 1, N, N) or (B, N, N) optional mask for vision encoder. + + Returns: + Projected visual features (B, seq_len, output_dim) + """ + vision_features = self.vision_encoder(images, attention_mask=attention_mask) # (B, N, D) + + # Calculate patch grid size + H, W = images.shape[2], images.shape[3] + patch_size = self.patch_size + h_patches = H // patch_size + w_patches = W // patch_size + projected_features = self.projector(vision_features, h_patches, w_patches) + + return projected_features diff --git a/kornia/models/kimi_vl/moonvit.py b/kornia/models/kimi_vl/moonvit.py new file mode 100644 index 0000000..ade38af --- /dev/null +++ b/kornia/models/kimi_vl/moonvit.py @@ -0,0 +1,365 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional, Tuple + +import torch +import torch.nn.functional as F +from torch import nn + +from .config import MoonViTConfig + + +def apply_rotary_pos_emb(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + """Apply rotary positional embeddings. + + Args: + x: Input tensor of shape (batch, seq_len, head_dim). + cos: Cosine component of shape (seq_len, head_dim). + sin: Sine component of shape (seq_len, head_dim). + + Returns: + Tensor with rotary embeddings applied. + """ + # x: (batch, seq_len, head_dim) + # cos, sin: (seq_len, head_dim) or (1, seq_len, head_dim) + + # rotate_half + x1, x2 = x.chunk(2, dim=-1) + x_rotated = torch.cat((-x2, x1), dim=-1) + + return (x * cos) + (x_rotated * sin) + + +class MoonViTRotaryEmbedding(nn.Module): + """2D Rotary Positional Embedding.""" + + def __init__(self, dim: int, theta: float = 10000.0) -> None: + super().__init__() + self.dim = dim + self.theta = theta + + def forward(self, h: int, w: int, device: torch.device) -> Tuple[torch.Tensor, torch.Tensor]: + """Build cosine and sine tables for two-dimensional rotary position embeddings. + + The MoonViT encoder works on a flattened patch grid. This method receives the + grid height ``h`` and width ``w`` after patch embedding, creates one frequency + bank for the vertical axis and one for the horizontal axis, and then expands + them so each flattened patch token has a matching rotary embedding. The + returned tensors are later applied to the query and key vectors in attention. + + Args: + h: Number of patch rows in the image grid. + w: Number of patch columns in the image grid. + device: Device on which the cosine and sine tensors should be allocated. + + Returns: + A tuple containing the cosine and sine lookup tables, each with shape + :math:`(h * w, D)`, where ``D`` is the per-head rotary embedding dimension. + """ + # dim must be divisible by 2 for 2D RoPE (half for H, half for W) + # And each half must be divisible by 2 for complex rotation + dim_h = self.dim // 2 + dim_w = self.dim // 2 + + # Generate frequencies + inv_freq_h = 1.0 / (self.theta ** (torch.arange(0, dim_h, 2, device=device).float() / dim_h)) + inv_freq_w = 1.0 / (self.theta ** (torch.arange(0, dim_w, 2, device=device).float() / dim_w)) + + # Generate positions + seq_h = torch.arange(h, device=device, dtype=inv_freq_h.dtype) + seq_w = torch.arange(w, device=device, dtype=inv_freq_w.dtype) + + # Outer product to get (h, dim_h/2) and (w, dim_w/2) + freqs_h = torch.outer(seq_h, inv_freq_h) # (h, dim_h/2) + freqs_w = torch.outer(seq_w, inv_freq_w) # (w, dim_w/2) + + # Repeat h frequencies for each w + freqs_h = freqs_h.repeat_interleave(w, dim=0) # (h*w, dim_h/2) + + # Repeat w frequencies for each h + freqs_w = freqs_w.repeat(h, 1) # (h*w, dim_w/2) + + # Concatenate to get full embeddings + emb_h = torch.cat((freqs_h, freqs_h), dim=-1) # (seq_len, dim_h) + emb_w = torch.cat((freqs_w, freqs_w), dim=-1) # (seq_len, dim_w) + + emb = torch.cat((emb_h, emb_w), dim=-1) # (seq_len, dim) + + return emb.cos(), emb.sin() + + +class MoonViTAttention(nn.Module): + """Multi-head self-attention layer used by the MoonViT vision encoder. + + The layer projects input tokens into query, key, and value tensors, splits the + hidden dimension across attention heads, applies two-dimensional rotary positional + embeddings (RoPE) to the query and key tensors, and finally computes scaled + dot-product attention. The output is projected back to the model hidden size. + + Args: + config: MoonViT configuration containing the hidden size, number of attention + heads, and attention dropout probability. + """ + + def __init__(self, config: MoonViTConfig) -> None: + super().__init__() + self.num_heads = config.num_attention_heads + self.hidden_size = config.hidden_size + self.head_dim = config.hidden_size // config.num_attention_heads + + self.q_proj = nn.Linear(config.hidden_size, config.hidden_size) + self.k_proj = nn.Linear(config.hidden_size, config.hidden_size) + self.v_proj = nn.Linear(config.hidden_size, config.hidden_size) + self.out_proj = nn.Linear(config.hidden_size, config.hidden_size) + + self.dropout = nn.Dropout(config.attention_dropout_p) + + def forward( + self, + hidden_states: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Apply rotary self-attention to a sequence of visual patch tokens. + + Args: + hidden_states: Input token tensor with shape :math:`(B, N, D)`, where + ``B`` is the batch size, ``N`` is the number of flattened image + patches, and ``D`` is the hidden size. + cos: Cosine component of the rotary positional embedding with shape + :math:`(N, d)`, where ``d`` is the per-head dimension. + sin: Sine component of the rotary positional embedding with shape + :math:`(N, d)`. + attention_mask: Optional mask broadcastable to the scaled dot-product + attention scores. It can be used to prevent selected tokens from + attending to one another. + + Returns: + Tensor with shape :math:`(B, N, D)` containing the attended visual token + features after the output projection. + """ + batch_size, seq_len, _ = hidden_states.shape + + query = self.q_proj(hidden_states) + key = self.k_proj(hidden_states) + value = self.v_proj(hidden_states) + + # Reshape query, key, and value for multi-head attention + query = query.view(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + key = key.view(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + value = value.view(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + + # Apply RoPE + # cos, sin are (seq_len, head_dim) -> reshape to (1, 1, seq_len, head_dim) + cos = cos.view(1, 1, seq_len, self.head_dim) + sin = sin.view(1, 1, seq_len, self.head_dim) + + query = apply_rotary_pos_emb(query, cos, sin) + key = apply_rotary_pos_emb(key, cos, sin) + attn_output = F.scaled_dot_product_attention( + query, key, value, attn_mask=attention_mask, dropout_p=self.dropout.p if self.training else 0.0 + ) + + attn_output = attn_output.transpose(1, 2).contiguous().view(batch_size, seq_len, self.hidden_size) + return self.out_proj(attn_output) + + +class MoonViTMLP(nn.Module): + """Feed-forward MLP block used in MoonViT transformer layers. + + This module implements a two-layer projection with GELU activation and dropout, + following the standard transformer feed-forward network pattern. + + Args: + config: Model configuration containing ``hidden_size``, ``intermediate_size``, + and ``dropout_p`` used to construct the MLP. + """ + + def __init__(self, config: MoonViTConfig) -> None: + super().__init__() + self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size) + self.act = nn.GELU() + self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size) + self.dropout = nn.Dropout(config.dropout_p) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Project token features through the MoonViT feed-forward network. + + Args: + x: Input token features with shape :math:`(B, N, D)`, where ``B`` is the + batch size, ``N`` is the number of patch tokens, and ``D`` is the model + hidden size. + + Returns: + Tensor with shape :math:`(B, N, D)` after expansion to the intermediate + hidden size, GELU activation, projection back to ``D``, and dropout. + """ + x = self.fc1(x) + x = self.act(x) + x = self.fc2(x) + x = self.dropout(x) + return x + + +class MoonViTLayer(nn.Module): + """Single MoonViT transformer layer with pre-normalization, RoPE attention, and an MLP block. + + This layer applies layer normalization before both the self-attention and MLP submodules + (pre-norm transformer). The self-attention block uses rotary positional embeddings (RoPE) + via the provided cosine and sine tensors, and the output of each sub-block is added back + to the input (residual connections). + + Args: + config: Model configuration specifying hidden sizes, number of heads, dropout, and + normalization parameters. + """ + + def __init__(self, config: MoonViTConfig) -> None: + super().__init__() + self.norm1 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.attn = MoonViTAttention(config) + self.norm2 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.mlp = MoonViTMLP(config) + + def forward( + self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, attention_mask: Optional[torch.Tensor] = None + ) -> torch.Tensor: + """Run one pre-normalized MoonViT transformer block. + + The block first normalizes the input tokens before self-attention, adds the + attention result back through a residual connection, then repeats the same + pre-normalization and residual pattern for the MLP sub-block. + + Args: + x: Token tensor with shape :math:`(B, N, D)`, where ``B`` is the batch + size, ``N`` is the number of flattened image patches, and ``D`` is the + hidden size. + cos: Cosine rotary embedding table with shape :math:`(N, d)`, where ``d`` + is the per-head attention dimension. + sin: Sine rotary embedding table with shape :math:`(N, d)`. + attention_mask: Optional attention mask forwarded to the self-attention + layer. + + Returns: + Tensor with shape :math:`(B, N, D)` after attention, MLP processing, and + residual updates. + """ + x = x + self.attn(self.norm1(x), cos, sin, attention_mask) + x = x + self.mlp(self.norm2(x)) + return x + + +class MoonViTEncoder(nn.Module): + """Stack of MoonViT transformer layers. + + This encoder sequentially applies multiple :class:`MoonViTLayer` blocks to a + sequence of hidden states. Each layer consists of self-attention with rotary + positional embeddings followed by an MLP block, with residual connections and + layer normalization, producing an encoded representation of the input sequence. + """ + + def __init__(self, config: MoonViTConfig) -> None: + super().__init__() + self.layers = nn.ModuleList([MoonViTLayer(config) for _ in range(config.num_hidden_layers)]) + + def forward( + self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, attention_mask: Optional[torch.Tensor] = None + ) -> torch.Tensor: + """Encode visual patch tokens through the full MoonViT transformer stack. + + Args: + x: Input token tensor with shape :math:`(B, N, D)`, where ``B`` is batch + size, ``N`` is the flattened patch count, and ``D`` is hidden size. + cos: Cosine rotary embedding table shared by all encoder layers. + sin: Sine rotary embedding table shared by all encoder layers. + attention_mask: Optional attention mask passed unchanged to every + transformer layer. + + Returns: + Tensor with shape :math:`(B, N, D)` containing the encoded patch sequence + after all MoonViT layers have been applied. + """ + for layer in self.layers: + x = layer(x, cos, sin, attention_mask) + return x + + +class MoonViT(nn.Module): + """MoonViT Vision Encoder.""" + + def __init__(self, config: MoonViTConfig) -> None: + super().__init__() + self.config = config + + self.patch_embed = nn.Conv2d( + config.num_channels, config.hidden_size, kernel_size=config.patch_size, stride=config.patch_size + ) + + # Initialized for the default image size + num_patches = (config.image_size // config.patch_size) ** 2 + self.pos_embed = nn.Parameter(torch.randn(1, num_patches, config.hidden_size)) + + self.rope = MoonViTRotaryEmbedding(config.hidden_size // config.num_attention_heads, theta=config.rope_theta) + + self.encoder = MoonViTEncoder(config) + self.norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + def forward(self, pixel_values: torch.Tensor, attention_mask: Optional[torch.Tensor] = None) -> torch.Tensor: + """Forward pass. + + Args: + pixel_values: (B, C, H, W) + attention_mask: (B, 1, N, N) or (B, N, N) optional mask. + + Returns: + (B, seq_len, hidden_size) + """ + # Patch Embedding + x = self.patch_embed(pixel_values) # (B, D, H', W') + h_patches = x.shape[2] + w_patches = x.shape[3] + + x = x.flatten(2).transpose(1, 2) # (B, N, D) + + # Add Absolute Positional Embedding (with interpolation) + pos_embed = self.pos_embed + if x.shape[1] != pos_embed.shape[1]: + # Interpolate pos_embed to match current resolution + # pos_embed is (1, N_ref, D) -> (1, D, H_ref, W_ref) + h_ref = int(pos_embed.shape[1] ** 0.5) + if h_ref * h_ref != pos_embed.shape[1]: + raise ValueError("pos_embed shape is not a perfect square, cannot reshape to 2D grid.") + w_ref = h_ref + pos_embed = pos_embed.permute(0, 2, 1).view(1, -1, h_ref, w_ref) + + pos_embed = F.interpolate(pos_embed, size=(h_patches, w_patches), mode="bicubic", align_corners=False) + + # (1, D, H', W') -> (1, N, D) + pos_embed = pos_embed.flatten(2).transpose(1, 2) + + x = x + pos_embed + + # Generate RoPE + cos, sin = self.rope(h_patches, w_patches, x.device) # (N, head_dim) + x = self.encoder(x, cos, sin, attention_mask=attention_mask) + x = self.norm(x) + + return x diff --git a/kornia/models/paligemma/__init__.py b/kornia/models/paligemma/__init__.py new file mode 100644 index 0000000..18b4700 --- /dev/null +++ b/kornia/models/paligemma/__init__.py @@ -0,0 +1,21 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .configuration_paligemma import PaliGemmaConfig +from .modeling_paligemma import PaliGemma + +__all__ = ["PaliGemma", "PaliGemmaConfig"] diff --git a/kornia/models/paligemma/configuration_paligemma.py b/kornia/models/paligemma/configuration_paligemma.py new file mode 100644 index 0000000..ab5b7ca --- /dev/null +++ b/kornia/models/paligemma/configuration_paligemma.py @@ -0,0 +1,56 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Optional + +from kornia.models.siglip2.config import SigLip2VisionConfig + + +@dataclass +class PaliGemmaConfig: + """Configuration class for PaliGemma. + + Args: + vision_config: Configuration for the SigLip2 vision encoder. + vocab_size: Size of the vocabulary. + hidden_size: Dimension of the embeddings. + intermediate_size: Dimension of the intermediate (Feed Forward) layer. + num_hidden_layers: Number of Transformer layers in the text model. + num_attention_heads: Number of attention heads per transformer layer. + num_key_value_heads: Number of key/value heads (for multi-query attention). + head_dim: Dimension of each attention head. + max_position_embeddings: Maximum number of position embeddings. + rope_theta: The base period of the RoPE embeddings. + ignore_index: Index to ignore in the loss function. + image_token_index: Token index used for image placeholders. + """ + + vision_config: Optional[SigLip2VisionConfig] = field(default_factory=SigLip2VisionConfig) + vocab_size: int = 257152 + hidden_size: int = 2048 + intermediate_size: int = 16384 + num_hidden_layers: int = 18 + num_attention_heads: int = 8 + num_key_value_heads: int = 1 + head_dim: int = 256 + max_position_embeddings: int = 8192 + rope_theta: float = 10000.0 + ignore_index: int = -100 + image_token_index: int = 256000 diff --git a/kornia/models/paligemma/modeling_paligemma.py b/kornia/models/paligemma/modeling_paligemma.py new file mode 100644 index 0000000..0ddbf96 --- /dev/null +++ b/kornia/models/paligemma/modeling_paligemma.py @@ -0,0 +1,361 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional, Tuple + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.models.siglip2.vision_encoder import SigLip2VisionModel + +from .configuration_paligemma import PaliGemmaConfig + + +class GemmaRMSNorm(nn.Module): + """Root Mean Square Layer Normalization.""" + + def __init__(self, dim: int, eps: float = 1e-6) -> None: + super().__init__() + self.eps = eps + self.weight = nn.Parameter(torch.zeros(dim)) + + def _norm(self, x: torch.Tensor) -> torch.Tensor: + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply Gemma RMS normalization to hidden states. + + Args: + x: Hidden-state tensor with shape :math:`(B, N, D)`, where + :math:`B` is batch size, :math:`N` is sequence length, and + :math:`D` is hidden dimension. + + Returns: + Tensor with the same shape as ``x`` after root-mean-square + normalization and learned scaling. + """ + output = self._norm(x.float()).type_as(x) + return output * (1.0 + self.weight) + + +class GemmaRotaryEmbedding(nn.Module): + """Rotary Positional Embedding (RoPE).""" + + def __init__( + self, + dim: int, + max_position_embeddings: int = 2048, + base: int = 10000, + device: Optional[torch.device] = None, + ) -> None: + super().__init__() + + self.dim = dim + self.max_position_embeddings = max_position_embeddings + self.base = base + inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.int64).float().to(device) / dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + def forward(self, x: torch.Tensor, position_ids: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Create rotary sine and cosine embeddings for token positions. + + Args: + x: Tensor whose dtype is used for the returned embeddings. + position_ids: Position tensor with shape :math:`(B, N)`. + + Returns: + Tuple ``(cos, sin)`` containing rotary embedding factors with dtype + matching ``x``. + """ + inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1) + position_ids_expanded = position_ids[:, None, :].float() + + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() + sin = emb.sin() + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +def _rotate_half(x: torch.Tensor) -> torch.Tensor: + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb( + q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor +) -> Tuple[torch.Tensor, torch.Tensor]: + """Applies Rotary Positional Embedding to query and key states.""" + cos = cos.unsqueeze(1) + sin = sin.unsqueeze(1) + q_embed = (q * cos) + (_rotate_half(q) * sin) + k_embed = (k * cos) + (_rotate_half(k) * sin) + return q_embed, k_embed + + +class GemmaMLP(nn.Module): + """Multi-Layer Perceptron implementing the GeGLU pattern.""" + + def __init__(self, config: PaliGemmaConfig) -> None: + super().__init__() + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) + self.act_fn = nn.GELU() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply the gated Gemma feed-forward projection. + + Args: + x: Hidden-state tensor with shape :math:`(B, N, D)`. + + Returns: + Tensor with shape :math:`(B, N, D)` after gate projection, + activation, up projection, elementwise gating, and down projection. + """ + return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + + +class GemmaAttention(nn.Module): + """Multi-headed attention with RoPE and SDPA.""" + + def __init__(self, config: PaliGemmaConfig, layer_idx: Optional[int] = None) -> None: + super().__init__() + self.layer_idx = layer_idx + + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = config.head_dim + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + + self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) + self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) + self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + + self.rotary_emb = GemmaRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + base=int(self.rope_theta), + ) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + ) -> torch.Tensor: + """Apply grouped-query self-attention with rotary position embeddings. + + Args: + hidden_states: Input token tensor with shape :math:`(B, N, D)`. + attention_mask: Optional attention mask passed to scaled dot-product + attention. + position_ids: Token position ids with shape :math:`(B, N)`. + + Returns: + Tensor with shape :math:`(B, N, D)` after attention and output + projection. + """ + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + if position_ids is None: + raise ValueError("position_ids cannot be None for GemmaAttention") + + cos, sin = self.rotary_emb(value_states, position_ids=position_ids) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + key_states = torch.repeat_interleave(key_states, dim=1, repeats=self.num_key_value_groups) + value_states = torch.repeat_interleave(value_states, dim=1, repeats=self.num_key_value_groups) + + attn_output = F.scaled_dot_product_attention( + query_states, + key_states, + value_states, + attn_mask=attention_mask, + dropout_p=0.0, + is_causal=False, + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.view(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + return attn_output + + +class GemmaDecoderLayer(nn.Module): + """A single layer of the Gemma Decoder.""" + + def __init__(self, config: PaliGemmaConfig, layer_idx: int) -> None: + super().__init__() + self.hidden_size = config.hidden_size + + self.self_attn = GemmaAttention(config, layer_idx=layer_idx) + self.mlp = GemmaMLP(config) + self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=1e-6) + self.post_attention_layernorm = GemmaRMSNorm(config.hidden_size, eps=1e-6) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + ) -> torch.Tensor: + """Run one Gemma decoder layer. + + Args: + hidden_states: Input token tensor with shape :math:`(B, N, D)`. + attention_mask: Optional mask used by the attention block. + position_ids: Token position ids used to build rotary embeddings. + + Returns: + Tensor with shape :math:`(B, N, D)` after self-attention, MLP, and + residual connections. + """ + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + hidden_states = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + ) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + return hidden_states + + +class PaliGemma(nn.Module): + """PaliGemma Model for Vision-Language tasks. + + This model combines a SigLip2 Vision Encoder with a Gemma Language Decoder. + """ + + def __init__(self, config: PaliGemmaConfig) -> None: + super().__init__() + self.config = config + self.padding_idx = config.ignore_index + self.vocab_size = config.vocab_size + + if config.vision_config is None: + raise ValueError("vision_config cannot be None") + self.vision_tower = SigLip2VisionModel(config.vision_config) + + self.multi_modal_projector = nn.Linear(config.vision_config.hidden_size, config.hidden_size) + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=self.padding_idx) + + self.layers = nn.ModuleList( + [GemmaDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.norm = GemmaRMSNorm(config.hidden_size, eps=1e-6) + + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + self.apply(self._init_weights) + + def _init_weights(self, module: nn.Module) -> None: + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=0.02) + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=0.02) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + def forward( + self, + input_ids: torch.Tensor, + pixel_values: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Forward pass of the model. + + Args: + input_ids: Text tokens (batch, input_seq_len) + pixel_values: Images (batch, channels, height, width) + attention_mask: Optional attention mask. + position_ids: Optional position IDs. + + Returns: + logits: Prediction scores (batch, total_seq_len, vocab_size). + """ + vision_outputs = self.vision_tower(pixel_values) + + if isinstance(vision_outputs, (tuple, list)): + image_features = vision_outputs[1] + else: + image_features = vision_outputs + + if image_features.dim() != 3: + image_features = image_features.unsqueeze(1) + + image_features = self.multi_modal_projector(image_features) + + inputs_embeds = self.embed_tokens(input_ids) + + inputs_embeds = torch.cat([image_features, inputs_embeds], dim=1) + + if position_ids is None: + seq_length = inputs_embeds.shape[1] + position_ids = torch.arange(seq_length, dtype=torch.long, device=inputs_embeds.device) + position_ids = position_ids.unsqueeze(0).expand(inputs_embeds.shape[0], -1) + + hidden_states = inputs_embeds + + for layer in self.layers: + hidden_states = layer( + hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + ) + + hidden_states = self.norm(hidden_states) + logits = self.lm_head(hidden_states) + + return logits diff --git a/kornia/models/processors/__init__.py b/kornia/models/processors/__init__.py new file mode 100644 index 0000000..eabfb1e --- /dev/null +++ b/kornia/models/processors/__init__.py @@ -0,0 +1,181 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import List, Tuple, Union + +import torch +from torch import Tensor, nn + +from kornia.geometry.transform import resize + +from .naflex import NaFlex + +__all__ = [ + "NaFlex", + "OutputRangePostProcessor", + "ResizePostProcessor", + "ResizePreProcessor", +] + + +class ResizePreProcessor(nn.Module): + r"""Resize a list of image tensors to the given size. + + Additionally, also returns the original image sizes for further post-processing. + + Args: + height: Height of the resized image. + width: Width of the resized image. + interpolation_mode: Interpolation mode for image resizing. Supported values: + ``nearest``, ``bilinear``, ``bicubic``, ``area``, ``nearest-exact``. + Default: "bilinear". + + Example: + >>> import torch + >>> from kornia.models.processors import ResizePreProcessor + >>> processor = ResizePreProcessor(height=224, width=224) + >>> imgs = torch.randn(2, 3, 480, 640) + >>> resized, sizes = processor(imgs) + >>> print(resized.shape, sizes.shape) + torch.Size([2, 3, 224, 224]) torch.Size([2, 2]) + """ + + def __init__(self, height: int, width: int, interpolation_mode: str = "bilinear") -> None: + super().__init__() + self.size = (height, width) + self.interpolation_mode = interpolation_mode + + def forward(self, imgs: Union[Tensor, List[Tensor]]) -> Tuple[Tensor, Tensor]: + r"""Resize input images to the target size. + + Args: + imgs: Input images, either a tensor of shape :math:`(B, C, H, W)` or + a list of tensors of shape :math:`(C, H, W)`. + + Returns: + Tuple containing: + - resized_imgs: Resized images as a tensor of shape :math:`(B, C, H_{\text{new}}, W_{\text{new}})`. + - original_sizes: Original image sizes of shape :math:`(B, 2)` containing (height, width). + """ + resized_imgs: List[Tensor] = [] + + iters = len(imgs) if isinstance(imgs, list) else imgs.shape[0] + original_sizes = imgs[0].new_zeros((iters, 2)) + + for i in range(iters): + img = imgs[i] + original_sizes[i, 0] = img.shape[-2] # Height + original_sizes[i, 1] = img.shape[-1] # Width + resized_imgs.append(resize(img[None], size=self.size, interpolation=self.interpolation_mode)) + + return torch.cat(resized_imgs), original_sizes + + +class ResizePostProcessor(nn.Module): + r"""Rescale model outputs back to the original image dimensions. + + Args: + interpolation_mode: The algorithm used for upsampling. Supported values: + ``nearest``, ``bilinear``, ``bicubic``, ``area``, ``nearest-exact``. + Default: "bilinear". + + Example: + >>> import torch + >>> from kornia.models.processors import ResizePostProcessor + >>> processor = ResizePostProcessor() + >>> imgs = torch.randn(2, 3, 224, 224) + >>> original_sizes = torch.tensor([[480, 640], [360, 480]]) + >>> out = processor(imgs, original_sizes) + """ + + def __init__(self, interpolation_mode: str = "bilinear") -> None: + super().__init__() + self.interpolation_mode = interpolation_mode + + def forward(self, imgs: Union[Tensor, List[Tensor]], original_sizes: Tensor) -> Union[Tensor, List[Tensor]]: + r"""Resize model outputs back to original dimensions. + + Args: + imgs: Input images, either a tensor of shape :math:`(B, C, H, W)` or + a list of tensors of shape :math:`(C, H, W)`. + original_sizes: Original image sizes of shape :math:`(B, 2)` containing + (height, width) pairs. + + Returns: + Resized images in a batch with original dimensions. If ONNX export is active, + returns inputs unchanged with a warning. + + Warning: + ResizePostProcessor is not fully supported in ONNX export mode. + """ + import warnings + + resized_imgs: List[Tensor] = [] + + if torch.onnx.is_in_onnx_export(): + warnings.warn( + "ResizePostProcessor is not supported in ONNX export. " + "The output will not be resized back to the original size.", + stacklevel=2, + ) + return imgs + + iters = len(imgs) if isinstance(imgs, list) else imgs.shape[0] + + for i in range(iters): + img = imgs[i] + size = original_sizes[i] + resized_imgs.append( + resize(img[None], size=size.cpu().long().numpy().tolist(), interpolation=self.interpolation_mode) + ) + + return resized_imgs + + +class OutputRangePostProcessor(nn.Module): + r"""Clip and scale model outputs to a specific numerical range. + + Args: + min_val: Minimum value for clipping. Default: 0.0. + max_val: Maximum value for clipping. Default: 1.0. + + Example: + >>> import torch + >>> from kornia.models.processors import OutputRangePostProcessor + >>> processor = OutputRangePostProcessor(min_val=0.0, max_val=1.0) + >>> x = torch.randn(2, 3, 224, 224) + >>> out = processor(x) + >>> print(out.min(), out.max()) + tensor(0.) tensor(1.) + """ + + def __init__(self, min_val: float = 0.0, max_val: float = 1.0) -> None: + super().__init__() + self.min_val = min_val + self.max_val = max_val + + def forward(self, imgs: Union[Tensor, List[Tensor]]) -> Union[Tensor, List[Tensor]]: + r"""Clip output values to the specified range. + + Args: + imgs: Input images, either a tensor or list of tensors. + + Returns: + Clipped images with values in :math:`[\text{min\_val}, \text{max\_val}]`. + """ + if isinstance(imgs, torch.Tensor): + return torch.clamp(imgs, self.min_val, self.max_val) + return [img.clamp(self.min_val, self.max_val) for img in imgs] diff --git a/kornia/models/processors/naflex.py b/kornia/models/processors/naflex.py new file mode 100644 index 0000000..1bb350d --- /dev/null +++ b/kornia/models/processors/naflex.py @@ -0,0 +1,93 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +import math +from typing import Callable + +from torch import Tensor, nn +from torch.nn import functional as F + +__all__ = ["NaFlex"] + + +class NaFlex(nn.Module): + r"""NaFlex wrapper for vision embeddings that supports flexible image sizes. + + This module wraps a patch embedding function and a position embedding tensor. + It computes patch embeddings and dynamically interpolates the position embeddings + to match the input resolution using bilinear interpolation. + + Args: + patch_embedding_fcn: A callable (e.g., Conv2d) that takes images and returns + patch embeddings of shape :math:`(B, C, H, W)` or :math:`(B, N, C)`. + position_embedding: Position embedding tensor of shape :math:`(N, C)` to be + interpolated to match input resolution. + + Example: + >>> import torch + >>> from kornia.models.processors.naflex import NaFlex + >>> patch_fcn = torch.nn.Conv2d(3, 768, kernel_size=16, stride=16) + >>> pos_embed = torch.randn(196, 768) # 14x14 grid + >>> model = NaFlex(patch_fcn, pos_embed) + >>> x = torch.randn(1, 3, 224, 224) + >>> out = model(x) + >>> print(out.shape) + torch.Size([1, 196, 768]) + """ + + def __init__( + self, + patch_embedding_fcn: Callable[[Tensor], Tensor], + position_embedding: Tensor, + ) -> None: + super().__init__() + self.patch_embedding_fcn = patch_embedding_fcn + self.register_buffer("position_embedding", position_embedding) + + def forward(self, pixel_values: Tensor) -> Tensor: + r"""Forward pass through NaFlex wrapper. + + Args: + pixel_values: Input tensor of shape :math:`(B, C, H, W)`. + + Returns: + Embeddings with interpolated positional embeddings of shape :math:`(B, N, C)`. + + Raises: + ValueError: If original positional embedding does not form a square grid. + """ + embeddings = self.patch_embedding_fcn(pixel_values) + if embeddings.ndim == 4: + _, _, h_grid, w_grid = embeddings.shape + embeddings = embeddings.flatten(2).transpose(1, 2) + num_patches = h_grid * w_grid + else: + _, num_patches, _ = embeddings.shape + h_grid = int(math.sqrt(num_patches)) + w_grid = h_grid + if self.position_embedding.shape[0] == num_patches: + return embeddings + self.position_embedding.unsqueeze(0) + orig_num = int(self.position_embedding.shape[0]) + orig_grid = int(math.sqrt(orig_num)) + if orig_grid * orig_grid != orig_num: + raise ValueError(f"Original positional embedding is not a square grid (got {orig_num} embeddings)") + pos = self.position_embedding.view(orig_grid, orig_grid, -1).permute(2, 0, 1).unsqueeze(0) + pos_resized = F.interpolate(pos, size=(h_grid, w_grid), mode="bilinear", align_corners=False) + pos_resized = pos_resized.squeeze(0).permute(1, 2, 0).view(h_grid * w_grid, -1) + + return embeddings + pos_resized.unsqueeze(0) diff --git a/kornia/models/qwen25/__init__.py b/kornia/models/qwen25/__init__.py new file mode 100644 index 0000000..4372947 --- /dev/null +++ b/kornia/models/qwen25/__init__.py @@ -0,0 +1,20 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .qwen2_vl import Qwen2VLVisionTransformer + +__all__ = ["Qwen2VLVisionTransformer"] diff --git a/kornia/models/qwen25/qwen2_vl.py b/kornia/models/qwen25/qwen2_vl.py new file mode 100644 index 0000000..f39e949 --- /dev/null +++ b/kornia/models/qwen25/qwen2_vl.py @@ -0,0 +1,245 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional + +import torch +import torch.nn.functional as F +from torch import Tensor, nn +from torch.nn import Module + + +class Qwen2VLPatchMerger(Module): + """Patch merger block used in the Qwen2-VL vision encoder. + + Args: + dim: The output embedding dimension (e.g., 1280). + context_window: The context window size (unused in this skeleton but kept for API). + spatial_merge_size: The spatial merge size (unused in this skeleton). + """ + + def __init__(self, dim: int, context_window: int = 224, spatial_merge_size: int = 2) -> None: + super().__init__() + self.conv = nn.Conv2d(3, dim, kernel_size=14, stride=14) + self.ln_q = nn.LayerNorm(dim, eps=1e-6) + + def forward(self, x: Tensor) -> Tensor: + """Convert an image batch into Qwen2-VL patch tokens. + + Args: + x: Image tensor with shape :math:`(B, 3, H, W)`. + + Returns: + Token tensor with shape :math:`(B, N, D)`, where :math:`N` is the + number of extracted patches and :math:`D` is ``dim``. + """ + x = self.conv(x) + x = x.flatten(2) + x = x.transpose(1, 2) + x = self.ln_q(x) + return x + + +class Qwen2VLRotaryEmbedding(Module): + """Rotary positional embedding module used in Qwen2-VL vision-language layers. + + This module precomputes the inverse frequency spectrum required to build + rotary position embeddings (RoPE) for a given hidden dimension. The + frequencies are used to rotate query and key vectors in the attention mechanism, + encoding relative position information. + + Args: + dim: The feature dimension to be rotated. + theta: The base frequency scaling factor for the rotary embedding. + """ + + def __init__(self, dim: int, theta: float = 10000.0) -> None: + super().__init__() + self.dim = dim + self.theta = theta + + inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + def forward(self, x: Tensor, cu_seqlens: Optional[Tensor] = None) -> Tensor: + """Return rotary-position input unchanged for this lightweight module. + + Args: + x: Token tensor that would receive rotary position information. + cu_seqlens: Optional cumulative sequence lengths kept for API + compatibility with Qwen2-VL implementations. + + Returns: + The input tensor ``x`` unchanged. + """ + return x + + +class Qwen2VLVisionAttention(Module): + """Multi-head self-attention module used in the Qwen2-VL vision encoder. + + Args: + dim: Input feature dimension. + num_heads: Number of attention heads. + """ + + def __init__(self, dim: int, num_heads: int = 16) -> None: + super().__init__() + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.scale = self.head_dim**-0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=True) + self.proj = nn.Linear(dim, dim, bias=True) + + def forward(self, x: Tensor, cu_seqlens: Optional[Tensor] = None, rot_pos_emb: Optional[Tensor] = None) -> Tensor: + """Apply multi-head self-attention to vision tokens. + + Args: + x: Token tensor with shape :math:`(B, N, D)`. + cu_seqlens: Optional cumulative sequence lengths kept for API + compatibility. + rot_pos_emb: Optional rotary positional embedding tensor. + + Returns: + Tensor with shape :math:`(B, N, D)` after attention and output + projection. + """ + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] + + x = F.scaled_dot_product_attention(q, k, v, dropout_p=0.0) + + x = x.transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + return x + + +class Qwen2VLMLP(Module): + """FeedForward MLP used in the Qwen2-VL vision transformer blocks. + + Args: + dim: Input and output feature dimension. + hidden_dim: Dimension of the hidden layer. If None, defaults to 4 * dim. + """ + + def __init__(self, dim: int, hidden_dim: Optional[int] = None) -> None: + super().__init__() + if hidden_dim is None: + hidden_dim = 4 * dim + + self.fc1 = nn.Linear(dim, hidden_dim) + self.act = nn.GELU() + self.fc2 = nn.Linear(hidden_dim, dim) + + def forward(self, x: Tensor) -> Tensor: + """Apply the Qwen2-VL vision feed-forward network. + + Args: + x: Token tensor with shape :math:`(B, N, D)`. + + Returns: + Tensor with shape :math:`(B, N, D)` after expansion, GELU + activation, and projection. + """ + return self.fc2(self.act(self.fc1(x))) + + +class Qwen2VLVisionBlock(Module): + """Single transformer block for the Qwen2-VL vision encoder. + + Applies layer-normalized self-attention followed by an MLP, each with + residual connections. + + Args: + dim: Token embedding dimension. + num_heads: Number of attention heads. + mlp_ratio: MLP hidden dimension multiplier. + """ + + def __init__(self, dim: int, num_heads: int, mlp_ratio: float = 4.0) -> None: + super().__init__() + self.norm1 = nn.LayerNorm(dim, eps=1e-6) + self.attn = Qwen2VLVisionAttention(dim, num_heads=num_heads) + + self.norm2 = nn.LayerNorm(dim, eps=1e-6) + self.mlp = Qwen2VLMLP(dim, int(dim * mlp_ratio)) + + def forward(self, x: Tensor, rot_pos_emb: Optional[Tensor] = None) -> Tensor: + """Run one Qwen2-VL vision transformer block. + + Args: + x: Vision token tensor with shape :math:`(B, N, D)`. + rot_pos_emb: Optional rotary positional embedding passed to the + attention layer. + + Returns: + Tensor with shape :math:`(B, N, D)` after attention, MLP, and + residual connections. + """ + x = x + self.attn(self.norm1(x), rot_pos_emb=rot_pos_emb) + x = x + self.mlp(self.norm2(x)) + return x + + +class Qwen2VLVisionTransformer(Module): + """PyTorch implementation of the Qwen2-VL vision encoder. + + A ViT-style backbone composed of a patch merger followed by stacked + transformer blocks with rotary positional embeddings. + + Args: + embed_dim: Embedding dimension. + depth: Number of transformer blocks. + num_heads: Number of attention heads. + mlp_ratio: MLP expansion ratio. + patch_size: Patch size. + in_channels: Input channel count. + """ + + def __init__( + self, + embed_dim: int = 1280, + depth: int = 32, + num_heads: int = 16, + mlp_ratio: float = 4.0, + in_channels: int = 3, + ) -> None: + super().__init__() + self.patch_embed = Qwen2VLPatchMerger(embed_dim, context_window=224, spatial_merge_size=2) + self.blocks = nn.ModuleList([Qwen2VLVisionBlock(embed_dim, num_heads, mlp_ratio) for _ in range(depth)]) + self.rotary_pos_emb = Qwen2VLRotaryEmbedding(embed_dim // num_heads) + + def forward(self, x: Tensor) -> Tensor: + """Encode an image batch into Qwen2-VL vision tokens. + + Args: + x: Image tensor with shape :math:`(B, 3, H, W)`. + + Returns: + Token tensor with shape :math:`(B, N, D)` after patch embedding and + the configured stack of vision transformer blocks. + """ + x = self.patch_embed(x) + rot_pos_emb = self.rotary_pos_emb(x) + for block in self.blocks: + x = block(x, rot_pos_emb=rot_pos_emb) + + return x diff --git a/kornia/models/rt_detr/__init__.py b/kornia/models/rt_detr/__init__.py new file mode 100644 index 0000000..29b6e92 --- /dev/null +++ b/kornia/models/rt_detr/__init__.py @@ -0,0 +1,24 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Contrib RT-DETR — Real-time DEtection TRansformer models for Kornia. + +This subpackage provides RT-DETR model classes, configs, and post-processors. +""" + +from kornia.models.rt_detr.model import RTDETR, RTDETRConfig, RTDETRModelType +from kornia.models.rt_detr.post_processor import DETRPostProcessor diff --git a/kornia/models/rt_detr/architecture/__init__.py b/kornia/models/rt_detr/architecture/__init__.py new file mode 100644 index 0000000..028ddb3 --- /dev/null +++ b/kornia/models/rt_detr/architecture/__init__.py @@ -0,0 +1,21 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Contrib RT-DETR Architecture — Model architecture components for RT-DETR. + +This subpackage provides building blocks for RT-DETR model architectures. +""" diff --git a/kornia/models/rt_detr/architecture/hgnetv2.py b/kornia/models/rt_detr/architecture/hgnetv2.py new file mode 100644 index 0000000..2ec36a3 --- /dev/null +++ b/kornia/models/rt_detr/architecture/hgnetv2.py @@ -0,0 +1,216 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Based on code from +https://github.com/PaddlePaddle/PaddleDetection/blob/ec37e66685f3bc5a38cd13f60685acea175922e1/ +ppdet/modeling/backbones/hgnet_v2.py. +""" # noqa: D205 + +from __future__ import annotations + +from typing import NamedTuple + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK +from kornia.models.common import ConvNormAct + + +class StemBlock(nn.Module): + """Implement the entry-level Stem block for HGNetV2 backbones.""" + + def __init__(self, in_channels: int, mid_channels: int, out_channels: int) -> None: + super().__init__() + self.stem1 = ConvNormAct(in_channels, mid_channels, 3, 2) + self.stem2a = ConvNormAct(mid_channels, mid_channels // 2, 2) + self.stem2b = ConvNormAct(mid_channels // 2, mid_channels, 2) + self.stem3 = ConvNormAct(mid_channels * 2, mid_channels, 3, 2) + self.stem4 = ConvNormAct(mid_channels, out_channels, 1) + self.pool = nn.Sequential(nn.ZeroPad2d((0, 1, 0, 1)), nn.MaxPool2d(2, 1)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run the HGNetV2 stem and reduce the input spatial resolution. + + Args: + x: Image tensor with shape :math:`(B, C, H, W)`. + + Returns: + Stem feature map after strided convolutions, pooling, concatenation, + and channel projection. + """ + x = self.stem1(x) + x = torch.cat([self.pool(x), self.stem2b(self.stem2a(x))], 1) + x = self.stem4(self.stem3(x)) + return x + + +# Separable conv +class LightConvNormAct(nn.Sequential): + """Implement a composite layer containing Convolution, Normalization, and Activation. + + Args: + in_channels: Number of input channels. + out_channels: Number of output channels. + kernel_size: Size of the convolving kernel. + """ + + def __init__(self, in_channels: int, out_channels: int, kernel_size: int) -> None: + super().__init__() + self.conv1 = ConvNormAct(in_channels, out_channels, 1, act="none") # point-wise + self.conv2 = ConvNormAct(out_channels, out_channels, kernel_size, groups=out_channels) # depth-wise + + +class StageConfig(NamedTuple): + """Represent configuration parameters for a single HGNetV2 stage.""" + + in_channels: int + mid_channels: int + out_channels: int + num_blocks: int + downsample: bool + light_block: bool + kernel_size: int + layer_num: int + + +class HGBlock(nn.Module): + """Implement the High-Performance GPU block for HGNetV2. + + Args: + in_channels: The number of input channels. + config: The configuration object containing stage details. + identity: Whether to use a residual identity connection. + """ + + def __init__(self, in_channels: int, config: StageConfig, identity: bool) -> None: + super().__init__() + self.identity = identity + + layer_cls = LightConvNormAct if config.light_block else ConvNormAct + self.layers = nn.ModuleList() + for i in range(config.layer_num): + ch_in = in_channels if i == 0 else config.mid_channels + self.layers.append(layer_cls(ch_in, config.mid_channels, config.kernel_size)) + + total_channels = in_channels + config.mid_channels * config.layer_num + self.aggregation_squeeze_conv = ConvNormAct(total_channels, config.out_channels // 2, 1) + self.aggregation_excitation_conv = ConvNormAct(config.out_channels // 2, config.out_channels, 1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply one HGNetV2 aggregation block. + + Args: + x: Feature map tensor with shape :math:`(B, C, H, W)`. + + Returns: + Aggregated feature map. If ``self.identity`` is enabled, the input + is added as a residual connection. + """ + feats = [x] + for layer in self.layers: + feats.append(layer(feats[-1])) + out = torch.cat(feats, 1) + out = self.aggregation_squeeze_conv(out) + out = self.aggregation_excitation_conv(out) + return x + out if self.identity else out + + +class HGStage(nn.Sequential): + """Implement a single stage of the HGNetV2 architecture. + + Args: + config: The configuration object for the specific stage. + """ + + def __init__(self, config: StageConfig) -> None: + super().__init__() + ch_in = config.in_channels + self.downsample = ConvNormAct(ch_in, ch_in, 3, 2, "none", ch_in) if config.downsample else None + self.blocks = nn.Sequential( + HGBlock(ch_in, config, False), + *[HGBlock(config.out_channels, config, True) for _ in range(config.num_blocks - 1)], + ) + + +class PPHGNetV2(nn.Module): + """Implement the PPHGNetV2 backbone for real-time object detection. + + Args: + stem_channels: A list of three integers for the stem convolution channels. + stage_configs: A list of StageConfig objects for each of the four stages. + """ + + def __init__(self, stem_channels: list[int], stage_configs: list[StageConfig]) -> None: + KORNIA_CHECK(len(stem_channels) == 3) + KORNIA_CHECK(len(stage_configs) == 4) + super().__init__() + self.out_channels = [config.out_channels for config in stage_configs[-3:]] + self.stem = StemBlock(*stem_channels) + self.stages = nn.ModuleList() + for cfg in stage_configs: + self.stages.append(HGStage(cfg)) + + def forward(self, x: torch.Tensor) -> list[torch.Tensor]: + """Run the PPHGNetV2 backbone and return detection feature maps. + + Args: + x: Image tensor with shape :math:`(B, 3, H, W)`. + + Returns: + List ``[s3, s4, s5]`` containing the final three backbone stages + used by the detector neck. + """ + x = self.stem(x) + s2 = self.stages[0](x) + s3 = self.stages[1](s2) + s4 = self.stages[2](s3) + s5 = self.stages[3](s4) + return [s3, s4, s5] + + @staticmethod + def from_config(variant: str) -> PPHGNetV2: + """Create a PPHGNetV2 backbone from a named variant. + + Args: + variant: Backbone variant name. Currently ``"L"`` is supported. + + Returns: + Configured :class:`PPHGNetV2` instance. + """ + if variant == "L": + return PPHGNetV2( + stem_channels=[3, 32, 48], + stage_configs=[ + StageConfig(48, 48, 128, 1, False, False, 3, 6), + StageConfig(128, 96, 512, 1, True, False, 3, 6), + StageConfig(512, 192, 1024, 3, True, True, 5, 6), + StageConfig(1024, 384, 2048, 1, True, True, 5, 6), + ], + ) + elif variant == "X": + return PPHGNetV2( + stem_channels=[3, 32, 64], + stage_configs=[ + StageConfig(64, 64, 128, 1, False, False, 3, 6), + StageConfig(128, 128, 512, 2, True, False, 3, 6), + StageConfig(512, 256, 1024, 5, True, True, 5, 6), + StageConfig(1024, 512, 2048, 2, True, True, 5, 6), + ], + ) + else: + raise ValueError("Only variant L and X are supported") diff --git a/kornia/models/rt_detr/architecture/hybrid_encoder.py b/kornia/models/rt_detr/architecture/hybrid_encoder.py new file mode 100644 index 0000000..6bb2d45 --- /dev/null +++ b/kornia/models/rt_detr/architecture/hybrid_encoder.py @@ -0,0 +1,361 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Based on the code from +https://github.com/PaddlePaddle/PaddleDetection/blob/ec37e66685f3bc5a38cd13f60685acea175922e1/ +ppdet/modeling/transformers/hybrid_encoder.py. +""" # noqa: D205 + +from __future__ import annotations + +import copy +from typing import Optional + +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn.utils.fusion import fuse_conv_bn_weights + +from kornia.models.common import ConvNormAct + + +class RepVggBlock(nn.Module): + """Implement the re-parameterizable VGG-style block. + + Args: + in_channels: The number of input channels. + out_channels: The number of output channels. + """ + + def __init__(self, in_channels: int, out_channels: int) -> None: + super().__init__() + self.conv1 = ConvNormAct(in_channels, out_channels, 3, act="none") + self.conv2 = ConvNormAct(in_channels, out_channels, 1, act="none") + self.act = nn.SiLU(inplace=True) + self.conv: Optional[nn.Conv2d] = None + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply the RepVGG block in training or fused deployment form. + + Args: + x: Feature map tensor with shape :math:`(B, C, H, W)`. + + Returns: + Feature map after either the fused convolution or the summed + ``3x3`` and ``1x1`` branches followed by SiLU activation. + """ + if self.conv is not None: + out = self.act(self.conv(x)) + else: + out = self.act(self.conv1(x) + self.conv2(x)) + return out + + @torch.no_grad() + def optimize_for_deployment(self) -> None: + """Fuse convolution and batch-normalization branches for inference. + + The method replaces the parallel training-time branches with a single + equivalent ``3x3`` convolution stored in ``self.conv``. + """ + + def _fuse_conv_bn_weights(m: ConvNormAct) -> tuple[nn.Parameter, nn.Parameter]: + if m.norm.running_mean is None or m.norm.running_var is None: + raise ValueError + + return fuse_conv_bn_weights( + m.conv.weight, + m.conv.bias, + m.norm.running_mean, + m.norm.running_var, + m.norm.eps, + m.norm.weight, + m.norm.bias, + ) + + kernel3x3, bias3x3 = _fuse_conv_bn_weights(self.conv1) + kernel1x1, bias1x1 = _fuse_conv_bn_weights(self.conv2) + kernel3x3.add_(F.pad(kernel1x1, [1, 1, 1, 1])) + bias3x3.add_(bias1x1) + + self.conv = nn.Conv2d(kernel3x3.shape[1], kernel3x3.shape[0], 3, 1, 1) + self.conv.weight = kernel3x3 + self.conv.bias = bias3x3 + + +class CSPRepLayer(nn.Module): + """Implement the Cross-Stage Partial Rep-layer. + + Args: + in_channels: The number of input channels. + out_channels: The number of output channels. + num_blocks: The number of RepVggBlocks to include. + expansion: The expansion factor for the internal channels. Default: 1.0. + """ + + def __init__(self, in_channels: int, out_channels: int, num_blocks: int, expansion: float = 1.0) -> None: + super().__init__() + hidden_channels = int(out_channels * expansion) + self.conv1 = ConvNormAct(in_channels, hidden_channels, 1, act="silu") + self.conv2 = ConvNormAct(in_channels, hidden_channels, 1, act="silu") + self.bottlenecks = nn.Sequential(*[RepVggBlock(hidden_channels, hidden_channels) for _ in range(num_blocks)]) + self.conv3 = ( + ConvNormAct(hidden_channels, out_channels, 1, act="silu") + if hidden_channels != out_channels + else nn.Identity() + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Fuse cross-stage partial RepVGG features. + + Args: + x: Feature map tensor with shape :math:`(B, C, H, W)`. + + Returns: + Feature map after one branch passes through RepVGG bottlenecks and + is added to the shortcut branch, then projected to output channels. + """ + return self.conv3(self.bottlenecks(self.conv1(x)) + self.conv2(x)) + + +# almost identical to nn.TransformerEncoderLayer +# but add positional embeddings to q and k +class AIFI(nn.Module): + """Implement the All-scale Indoor/Outdoor Feature Interaction (AIFI) module. + + Args: + embed_dim: The dimension of the input embeddings. + num_heads: The number of attention heads. + dim_feedforward: The dimension of the feed-forward network. + dropout: The dropout probability. Default: 0.0. + """ + + def __init__(self, embed_dim: int, num_heads: int, dim_feedforward: int, dropout: float = 0.0) -> None: + super().__init__() + self.self_attn = nn.MultiheadAttention(embed_dim, num_heads, dropout) # NOTE: batch_first = False + + self.linear1 = nn.Linear(embed_dim, dim_feedforward) + self.dropout = nn.Dropout(dropout) + self.linear2 = nn.Linear(dim_feedforward, embed_dim) + + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + self.norm1 = nn.LayerNorm(embed_dim) + self.norm2 = nn.LayerNorm(embed_dim) + self.act = nn.GELU() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply all-scale feature interaction to one feature map. + + Args: + x: Feature map tensor with shape :math:`(B, C, H, W)`. + + Returns: + Tensor with shape :math:`(B, C, H, W)` after positional + self-attention and feed-forward refinement. + """ + # using post-norm + N, C, H, W = x.shape + x = x.permute(2, 3, 0, 1).flatten(0, 1) # (N, C, H, W) -> (H * W, N, C) + + # NOTE: cache pos_emb as buffer + pos_emb = self.build_2d_sincos_pos_emb(W, H, C, device=x.device, dtype=x.dtype) + q = k = x + pos_emb + + attn, _ = self.self_attn(q, k, x, need_weights=True) + x = self.norm1(x + self.dropout1(attn)) + x = self.norm2(x + self.dropout2(self.ffn(x))) + + x = x.view(H, W, N, C).permute(2, 3, 0, 1) # (H * W, N, C) -> (N, C, H, W) + return x + + def ffn(self, x: torch.Tensor) -> torch.Tensor: + """Apply the feed-forward sublayer used inside AIFI. + + Args: + x: Token tensor with last dimension equal to the embedding size. + + Returns: + Tensor with the same shape as ``x`` after linear expansion, + activation, dropout, and projection. + """ + return self.linear2(self.dropout(self.act(self.linear1(x)))) + + # TODO: make this into a reusable function + # https://github.com/facebookresearch/moco-v3/blob/main/vits.py#L53 + # https://github.com/PaddlePaddle/PaddleDetection/blob/79267419e1743157f376a7cb251e01caa3338ce0/ppdet/modeling/transformers/hybrid_encoder.py#L217 + @staticmethod + def build_2d_sincos_pos_emb( + w: int, + h: int, + embed_dim: int, + temp: float = 10_000.0, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ) -> torch.Tensor: + """Construct 2D sin-cos positional embeddings. + + Args: + w: width of the image or feature map + h: height of the image or feature map + embed_dim: embedding dimension + temp: temperature coefficient + device: device to place the positional embeddings + dtype: data type of the positional embeddings + + Returns: + positional embeddings, shape :math:`(H * W, 1, C)` + + """ + xs = torch.arange(w, device=device, dtype=dtype) + ys = torch.arange(h, device=device, dtype=dtype) + grid_x, grid_y = torch.meshgrid([xs, ys], indexing="ij") + + pos_dim = embed_dim // 4 + omega = torch.arange(pos_dim, device=device, dtype=dtype) / pos_dim + omega = 1.0 / (temp**omega) + + out_x = grid_x.reshape(-1, 1) * omega.view(1, -1) + out_y = grid_y.reshape(-1, 1) * omega.view(1, -1) + + pos_emb = torch.cat([out_x.sin(), out_x.cos(), out_y.sin(), out_y.cos()], 1) + return pos_emb.unsqueeze(1) # (H * W, 1, C) + + +class TransformerEncoder(nn.Module): + """Implement a transformer encoder comprising multiple encoder layers. + + Args: + encoder_layer: An instance of a transformer encoder layer. + num_layers: The total number of encoder layers to stack. + """ + + def __init__(self, encoder_layer: nn.Module, num_layers: int) -> None: + super().__init__() + self.layers = nn.ModuleList([copy.deepcopy(encoder_layer) for _ in range(num_layers)]) + self.num_layers = num_layers + + def forward( + self, src: torch.Tensor + ) -> torch.Tensor: # NOTE: Missing src_mask: torch.Tensor = None, pos_embed: torch.Tensor = None + """Apply the stacked transformer encoder layers. + + Args: + src: Source feature tensor passed through each encoder layer. + + Returns: + Tensor after all cloned encoder layers have been applied. + """ + output = src + for layer in self.layers: + output = layer(output) + + return output + + +class CCFM(nn.Module): + """Implement the Cross-Column Feature Mixing (CCFM) module. + + Args: + num_fmaps: The number of input feature maps. + hidden_dim: The hidden dimension for the fusion layers. + expansion: The expansion ratio for the internal layers. Default: 1.0. + """ + + def __init__(self, num_fmaps: int, hidden_dim: int, expansion: float = 1.0) -> None: + super().__init__() + self.lateral_convs = nn.ModuleList() + self.fpn_blocks = nn.ModuleList() + for _ in range(num_fmaps - 1): + self.lateral_convs.append(ConvNormAct(hidden_dim, hidden_dim, 1, 1, "silu")) + self.fpn_blocks.append(CSPRepLayer(hidden_dim * 2, hidden_dim, 3, expansion)) + + self.downsample_convs = nn.ModuleList() + self.pan_blocks = nn.ModuleList() + for _ in range(num_fmaps - 1): + self.downsample_convs.append(ConvNormAct(hidden_dim, hidden_dim, 3, 2, "silu")) + self.pan_blocks.append(CSPRepLayer(hidden_dim * 2, hidden_dim, 3, expansion)) + + def forward(self, fmaps: list[torch.Tensor]) -> list[torch.Tensor]: + """Fuse multi-scale feature maps with FPN and PAN paths. + + Args: + fmaps: Feature maps ordered from high resolution to low resolution. + + Returns: + List of fused feature maps ordered from high resolution to low + resolution after top-down and bottom-up mixing. + """ + # fmaps is ordered from hi-res to low-res + fmaps = list(fmaps) # shallow clone + + # new_fmaps is ordered from low-res to hi-res + new_fmaps = [fmaps.pop()] + while fmaps: + new_fmaps[-1] = self.lateral_convs[len(new_fmaps) - 1](new_fmaps[-1]) + up_lowres_fmap = F.interpolate(new_fmaps[-1], scale_factor=2.0, mode="nearest") + hires_fmap = fmaps.pop() + + concat_fmap = torch.cat([up_lowres_fmap, hires_fmap], 1) + new_fmaps.append(self.fpn_blocks[len(new_fmaps) - 1](concat_fmap)) + + fmaps = [new_fmaps.pop()] + while new_fmaps: + down_hires_fmap = self.downsample_convs[len(fmaps) - 1](fmaps[-1]) + lowres_fmap = new_fmaps.pop() + + concat_fmap = torch.cat([down_hires_fmap, lowres_fmap], 1) + fmaps.append(self.pan_blocks[len(fmaps) - 1](concat_fmap)) + + return fmaps + + +class HybridEncoder(nn.Module): + """Implement the Efficient Hybrid Encoder for RT-DETR. + + This module combines multi-scale feature fusion with Intra-scale + Feature Interaction using transformer layers. + """ + + def __init__(self, in_channels: list[int], hidden_dim: int, dim_feedforward: int, expansion: float = 1.0) -> None: + super().__init__() + self.input_proj = nn.ModuleList( + [ + ConvNormAct( # To align the naming strategy for the official weights + in_ch, hidden_dim, 1, act="none", conv_naming="0", norm_naming="1", act_naming="2" + ) + for in_ch in in_channels + ] + ) + encoder_layer = AIFI(hidden_dim, 8, dim_feedforward) + self.encoder = nn.Sequential(TransformerEncoder(encoder_layer, 1)) + self.ccfm = CCFM(len(in_channels), hidden_dim, expansion) + + def forward(self, fmaps: list[torch.Tensor]) -> list[torch.Tensor]: + """Project and fuse backbone feature maps for RT-DETR. + + Args: + fmaps: List of backbone feature maps with decreasing spatial + resolution. + + Returns: + List of encoded and cross-scale fused feature maps used by the + RT-DETR detection head. + """ + projected_maps = [proj(fmap) for proj, fmap in zip(self.input_proj, fmaps)] + projected_maps[-1] = self.encoder(projected_maps[-1]) + new_fmaps = self.ccfm(projected_maps) + return new_fmaps diff --git a/kornia/models/rt_detr/architecture/resnet_d.py b/kornia/models/rt_detr/architecture/resnet_d.py new file mode 100644 index 0000000..7bc8164 --- /dev/null +++ b/kornia/models/rt_detr/architecture/resnet_d.py @@ -0,0 +1,219 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""ResNet-D as described in https://arxiv.org/abs/1812.01187. + +Based on the code from https://github.com/pytorch/vision/blob/v0.15.1/torchvision/models/resnet.py +https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.6/ppdet/modeling/backbones/resnet.py +""" + +from __future__ import annotations + +from collections import OrderedDict + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK +from kornia.models.common import ConvNormAct + + +def _make_shortcut(in_channels: int, out_channels: int, stride: int) -> nn.Module: + return ( + nn.Sequential( + OrderedDict([("pool", nn.AvgPool2d(2, 2)), ("conv", ConvNormAct(in_channels, out_channels, 1, act="none"))]) + ) + if stride == 2 + else ConvNormAct(in_channels, out_channels, 1, act="none") + ) + + +class BasicBlockD(nn.Module): + """Implement the Basic Block for the ResNet-D variant.""" + + expansion = 1 + + def __init__(self, in_channels: int, out_channels: int, stride: int, shortcut: bool) -> None: + KORNIA_CHECK(stride in {1, 2}) + super().__init__() + self.convs = nn.Sequential( + OrderedDict( + [ + ("branch2a", ConvNormAct(in_channels, out_channels, 3, stride=stride)), + ("branch2b", ConvNormAct(out_channels, out_channels, 3, act="none")), + ] + ) + ) + self.short = nn.Identity() if shortcut else _make_shortcut(in_channels, out_channels, stride) + self.relu = nn.ReLU(inplace=True) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply a basic ResNet-D residual block. + + Args: + x: Feature map tensor with shape :math:`(B, C, H, W)`. + + Returns: + Tensor after two convolutions, shortcut projection if needed, and + ReLU activation. + """ + return self.relu(self.convs(x) + self.short(x)) + + +class BottleneckD(nn.Module): + """Implement the Bottleneck Block for the ResNet-D variant.""" + + expansion = 4 + + def __init__(self, in_channels: int, out_channels: int, stride: int, shortcut: bool) -> None: + KORNIA_CHECK(stride in {1, 2}) + super().__init__() + expanded_out_channels = out_channels * self.expansion + self.convs = nn.Sequential( + OrderedDict( + [ + ("branch2a", ConvNormAct(in_channels, out_channels, 1)), + ("branch2b", ConvNormAct(out_channels, out_channels, 3, stride=stride)), + ("branch2c", ConvNormAct(out_channels, expanded_out_channels, 1, act="none")), + ] + ) + ) + self.short = nn.Identity() if shortcut else _make_shortcut(in_channels, expanded_out_channels, stride) + self.relu = nn.ReLU(inplace=True) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply a bottleneck ResNet-D residual block. + + Args: + x: Feature map tensor with shape :math:`(B, C, H, W)`. + + Returns: + Tensor after ``1x1``, ``3x3``, and ``1x1`` convolutions, shortcut + addition, and ReLU activation. + """ + return self.relu(self.convs(x) + self.short(x)) + + +class Block(nn.Sequential): + """Implement a residual block for the ResNet-D architecture. + + Args: + blocks: A module containing the sequence of operations for the block. + """ + + def __init__(self, blocks: nn.Module) -> None: + super().__init__() + self.blocks = blocks + + +class ResNetD(nn.Module): + """Implement the ResNet-D architecture. + + Args: + n_blocks: A list of integers representing the number of blocks in each stage. + block: The block class type (BasicBlockD or BottleneckD). + """ + + def __init__(self, n_blocks: list[int], block: type[BasicBlockD | BottleneckD]) -> None: + KORNIA_CHECK(len(n_blocks) == 4) + super().__init__() + in_channels = 64 + self.conv1 = nn.Sequential( + OrderedDict( + [ + ("conv1_1", ConvNormAct(3, in_channels // 2, 3, stride=2)), + ("conv1_2", ConvNormAct(in_channels // 2, in_channels // 2, 3)), + ("conv1_3", ConvNormAct(in_channels // 2, in_channels, 3)), + ("pool", nn.MaxPool2d(3, stride=2, padding=1)), + ] + ) + ) + + res2, in_channels = self.make_stage(in_channels, 64, 1, n_blocks[0], block) + res3, in_channels = self.make_stage(in_channels, 128, 2, n_blocks[1], block) + res4, in_channels = self.make_stage(in_channels, 256, 2, n_blocks[2], block) + res5, in_channels = self.make_stage(in_channels, 512, 2, n_blocks[3], block) + + self.res_layers = nn.ModuleList([res2, res3, res4, res5]) + + self.out_channels = [ch * block.expansion for ch in [128, 256, 512]] + + @staticmethod + def make_stage( + in_channels: int, out_channels: int, stride: int, n_blocks: int, block: type[BasicBlockD | BottleneckD] + ) -> tuple[nn.Module, int]: + """Build one ResNet-D stage. + + Args: + in_channels: Number of input channels for the first block. + out_channels: Base output channels for the stage before expansion. + stride: Stride used by the first block. + n_blocks: Number of residual blocks in the stage. + block: Residual block class to instantiate. + + Returns: + Tuple ``(stage, channels)`` containing the stage module and its + expanded output channel count. + """ + stage = Block( + nn.Sequential( + block(in_channels, out_channels, stride, False), + *[block(out_channels * block.expansion, out_channels, 1, True) for _ in range(n_blocks - 1)], + ) + ) + return stage, out_channels * block.expansion + + def forward(self, x: torch.Tensor) -> list[torch.Tensor]: + """Run the ResNet-D backbone and return detector feature maps. + + Args: + x: Image tensor with shape :math:`(B, 3, H, W)`. + + Returns: + List ``[res3, res4, res5]`` containing the final three feature + stages used by RT-DETR. + """ + x = self.conv1(x) + res2 = self.res_layers[0](x) + res3 = self.res_layers[1](res2) + res4 = self.res_layers[2](res3) + res5 = self.res_layers[3](res4) + return [res3, res4, res5] + + @staticmethod + def from_config(variant: str | int) -> ResNetD: + """Create a ResNet-D backbone from a depth variant. + + Args: + variant: ResNet depth, such as ``18``, ``34``, ``50``, or ``101``. + + Returns: + Configured :class:`ResNetD` instance. + """ + variant = str(variant) + if variant == "18": + return ResNetD([2, 2, 2, 2], BasicBlockD) + elif variant == "34": + return ResNetD([3, 4, 6, 3], BasicBlockD) + elif variant == "50": + return ResNetD([3, 4, 6, 3], BottleneckD) + elif variant == "101": + return ResNetD([3, 4, 23, 3], BottleneckD) + elif variant == "152": + return ResNetD([3, 8, 36, 3], BottleneckD) + else: + raise ValueError("Only variant 18, 34, 50, 101, and 152 are supported") diff --git a/kornia/models/rt_detr/architecture/rtdetr_head.py b/kornia/models/rt_detr/architecture/rtdetr_head.py new file mode 100644 index 0000000..2d7f6a5 --- /dev/null +++ b/kornia/models/rt_detr/architecture/rtdetr_head.py @@ -0,0 +1,546 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Based on code from: +# https://github.com/lyuwenyu/RT-DETR/blob/8a1b85a91f527bed0c4e2424a7a6b4bcdd200fb1/rtdetr_pytorch/src/zoo/rtdetr/rtdetr_decoder.py + +from __future__ import annotations + +import copy +from typing import Optional + +import torch +from torch import nn + +from kornia.models.common import MLP, ConvNormAct + + +def _inverse_sigmoid(x: torch.Tensor, eps: float = 1e-5) -> torch.Tensor: + """Inverse sigmoid function. + + Args: + x: input tensor + eps: epsilon value for numerical stability + + Returns: + output tensor + + """ + out = x.clip(min=0.0, max=1.0) + return torch.log(out.clip(min=eps) / (1.0 - out).clip(min=eps)) + + +def _deformable_attention_kernel( + value: torch.Tensor, + value_spatial_shapes: list[tuple[int, int]], + sampling_locations: torch.Tensor, + attention_weights: torch.Tensor, +) -> torch.Tensor: + """Deformable Attention Kernel used in Deformable DETR. + + Described in https://arxiv.org/abs/2010.04159. + + Args: + value: shape (N, Lv, n_head * C) + value_spatial_shapes: [(H0, W0), (H1, W1), ...] + sampling_locations: shape (N, Lq, n_head, n_levels, n_points, 2) + attention_weights: shape (N, Lq, n_head, n_levels, n_points) + + Returns: + output, shape (N, Lq, n_head * C) + + """ + bs, _, n_head, c = value.shape + _, Len_q, _, n_levels, n_points, _ = sampling_locations.shape + + split_shape: list[int] = [h * w for h, w in value_spatial_shapes] + value_list = value.split(split_shape, dim=1) + sampling_grids = 2 * sampling_locations - 1 + + sampling_value_list: list[torch.Tensor] = [] + for level, (h, w) in enumerate(value_spatial_shapes): + # N_, H_*W_, M_, D_ -> N_, H_*W_, M_*D_ -> N_, M_*D_, H_*W_ -> N_*M_, D_, H_, W_ + value_l_ = value_list[level].flatten(2).permute(0, 2, 1).reshape(bs * n_head, c, h, w) + # N_, Lq_, M_, P_, 2 -> N_, M_, Lq_, P_, 2 -> N_*M_, Lq_, P_, 2 + sampling_grid_l_ = sampling_grids[:, :, :, level].permute(0, 2, 1, 3, 4).flatten(0, 1) + # N_*M_, D_, Lq_, P_ + sampling_value_l_ = torch.nn.functional.grid_sample( + value_l_, sampling_grid_l_, mode="bilinear", padding_mode="zeros", align_corners=False + ) + sampling_value_list.append(sampling_value_l_) + + # (N_, Lq_, M_, L_, P_) -> (N_, M_, Lq_, L_, P_) -> (N_*M_, 1, Lq_, L_*P_) + attention_weights = attention_weights.permute(0, 2, 1, 3, 4).reshape(bs * n_head, 1, Len_q, n_levels * n_points) + output = ( + (torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights) + .sum(-1) + .reshape(bs, n_head * c, Len_q) + ) + + return output.permute(0, 2, 1) + + +class MultiScaleDeformableAttention(nn.Module): + """Multi-scale Deformable Attention used in Deformable DETR. + + Described in https://arxiv.org/abs/2010.04159. + """ + + def __init__(self, embed_dim: int, num_heads: int, num_levels: int, num_points: int) -> None: + super().__init__() + self.num_heads = num_heads + self.num_levels = num_levels + self.num_points = num_points + self.total_points = num_heads * num_levels * num_points + + self.head_dim = embed_dim // num_heads + # KORNIA_CHECK not working with onnx + # assert self.head_dim * num_heads == embed_dim, "embed_dim must be divisible by num_heads" + + self.sampling_offsets = nn.Linear(embed_dim, self.total_points * 2) + self.attention_weights = nn.Linear(embed_dim, self.total_points) + self.value_proj = nn.Linear(embed_dim, embed_dim) + self.output_proj = nn.Linear(embed_dim, embed_dim) + + def forward( + self, + query: torch.Tensor, + reference_points: torch.Tensor, + value: torch.Tensor, + value_spatial_shapes: list[tuple[int, int]], + ) -> torch.Tensor: + """Run forward. + + Args: + query: shape (N, Lq, C) + reference_points: shape (N, Lq, n_levels, 4) + value: shape (N, Lv, C) + value_spatial_shapes: [(H0, W0), (H1, W1), ...] + + Returns: + output, shape (N, Lq, C) + + """ + N, Lenq, _ = query.shape + _, Len_v, _ = value.shape + + sampling_offsets = self.sampling_offsets(query).reshape( + N, Lenq, self.num_heads, self.num_levels, self.num_points, 2 + ) + attention_weights = self.attention_weights(query).reshape( + N, Lenq, self.num_heads, self.num_levels * self.num_points + ) + attention_weights = attention_weights.softmax(-1).reshape( + N, Lenq, self.num_heads, self.num_levels, self.num_points + ) + + # prepare the spatial sampling grid + reference_points_cxcy = reference_points[:, :, None, :, None, :2] + reference_points_wh = reference_points[:, :, None, :, None, 2:] + sampling_locations = reference_points_cxcy + sampling_offsets / self.num_points * reference_points_wh * 0.5 + + value_buf = self.value_proj(value).reshape(N, Len_v, self.num_heads, self.head_dim) + + out = _deformable_attention_kernel(value_buf, value_spatial_shapes, sampling_locations, attention_weights) + + # final projection + out = self.output_proj(out) + + return out + + +class TransformerDecoderLayer(nn.Module): + """Deformable Transformer Decoder layer used in Deformable DETR. + + Described in: https://arxiv.org/abs/2010.04159. + """ + + def __init__(self, embed_dim: int, num_heads: int, dropout: float, num_levels: int, num_points: int) -> None: + super().__init__() + # self attn + self.self_attn = nn.MultiheadAttention(embed_dim, num_heads, dropout, batch_first=True) + self.dropout1 = nn.Dropout(dropout) + self.norm1 = nn.LayerNorm(embed_dim) + + # cross attn + self.cross_attn = MultiScaleDeformableAttention(embed_dim, num_heads, num_levels, num_points) + self.dropout2 = nn.Dropout(dropout) + self.norm2 = nn.LayerNorm(embed_dim) + + # ffn + self.linear1 = nn.Linear(embed_dim, embed_dim * 4) + self.activation = nn.ReLU(inplace=True) + self.dropout3 = nn.Dropout(dropout) + self.linear2 = nn.Linear(embed_dim * 4, embed_dim) + self.dropout4 = nn.Dropout(dropout) + self.norm3 = nn.LayerNorm(embed_dim) + + def _ffn(self, x: torch.Tensor) -> torch.Tensor: + return self.linear2(self.dropout3(self.activation(self.linear1(x)))) + + def forward( + self, + tgt: torch.Tensor, + ref_points: torch.Tensor, + memory: torch.Tensor, + memory_spatial_shapes: list[tuple[int, int]], + memory_level_start_index: Optional[list[int]] = None, + attn_mask: Optional[torch.Tensor] = None, + memory_mask: Optional[torch.Tensor] = None, + query_pos_embed: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Run one RT-DETR transformer decoder layer. + + Args: + tgt: Query tensor with shape :math:`(B, N_q, D)`. + ref_points: Reference points for deformable cross-attention. + memory: Flattened encoder memory tensor. + memory_spatial_shapes: Spatial shape for each memory level. + memory_level_start_index: Optional start index for each level. + attn_mask: Optional self-attention mask. + memory_mask: Optional memory mask. + query_pos_embed: Optional positional embedding for object queries. + + Returns: + Updated query tensor with shape :math:`(B, N_q, D)`. + """ + # TODO: rename variables because is confusing + # self attention + q = k = tgt + query_pos_embed + out, _ = self.self_attn(q, k, value=tgt) + tgt = self.norm1(tgt + self.dropout1(out)) + + # cross attention + out = self.cross_attn(tgt + query_pos_embed, ref_points, memory, memory_spatial_shapes) + tgt = self.norm2(tgt + self.dropout2(out)) + + # ffn + out = self.norm3(tgt + self.dropout4(self._ffn(tgt))) + + return out + + +class TransformerDecoder(nn.Module): + """Implement the transformer decoder for generating object queries.""" + + def __init__(self, hidden_dim: int, decoder_layer: nn.Module, num_layers: int, eval_idx: int = -1) -> None: + super().__init__() + self.layers = nn.ModuleList([copy.deepcopy(decoder_layer) for _ in range(num_layers)]) + self.hidden_dim = hidden_dim + self.num_layers = num_layers + self.eval_idx = eval_idx if eval_idx >= 0 else num_layers + eval_idx + + def forward( + self, + tgt: torch.Tensor, + ref_points_unact: torch.Tensor, + memory: torch.Tensor, + memory_spatial_shapes: list[tuple[int, int]], + memory_level_start_index: list[int], + bbox_head: nn.ModuleList, + score_head: nn.ModuleList, + query_pos_head: nn.Module, + attn_mask: Optional[torch.Tensor] = None, + memory_mask: Optional[torch.Tensor] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Decode object queries into bounding boxes and class logits. + + Args: + tgt: Initial query tensor with shape :math:`(B, N_q, D)`. + ref_points_unact: Unactivated reference boxes for the queries. + memory: Flattened encoder feature memory. + memory_spatial_shapes: Spatial shape for each feature level. + memory_level_start_index: Start index for each level in ``memory``. + bbox_head: Per-layer box prediction heads. + score_head: Per-layer class prediction heads. + query_pos_head: Module that maps reference boxes to query position + embeddings. + attn_mask: Optional query attention mask. + memory_mask: Optional encoder memory mask. + + Returns: + Tuple ``(boxes, logits)`` stacked from the selected decoder layer. + """ + output: torch.Tensor = tgt + dec_out_bboxes: list[torch.Tensor] = [] + dec_out_logits: list[torch.Tensor] = [] + ref_points_detach = torch.sigmoid(ref_points_unact) + + for i, layer in enumerate(self.layers): + ref_points_input = ref_points_detach.unsqueeze(2) + query_pos_embed: torch.Tensor = query_pos_head(ref_points_detach) + + output = layer( + output, + ref_points_input, + memory, + memory_spatial_shapes, + memory_level_start_index, + attn_mask, + memory_mask, + query_pos_embed, + ) + + inter_ref_bbox = torch.sigmoid(bbox_head[i](output) + _inverse_sigmoid(ref_points_detach)) + + # TODO: will be supported later + # if self.training: + # dec_out_logits.append(score_head[i](output)) + # if i == 0: + # dec_out_bboxes.append(inter_ref_bbox) + # else: + # dec_out_bboxes.append(torch.sigmoid(bbox_head[i](output) + _inverse_sigmoid(ref_points))) + # elif i == self.eval_idx: + # dec_out_logits.append(score_head[i](output)) + # dec_out_bboxes.append(inter_ref_bbox) + # break + if i == self.eval_idx: + dec_out_logits.append(score_head[i](output)) + dec_out_bboxes.append(inter_ref_bbox) + break + + # ref_points_detach = inter_ref_bbox.detach( + # ) if self.training else inter_ref_bbox + ref_points_detach = inter_ref_bbox + + return torch.stack(dec_out_bboxes), torch.stack(dec_out_logits) + + +class RTDETRHead(nn.Module): + """Implement the RT-DETR prediction head for object detection.""" + + def __init__( + self, + num_classes: int, + hidden_dim: int, + num_queries: int, + in_channels: list[int], + num_decoder_layers: int, + num_heads: int = 8, + num_decoder_points: int = 4, + num_levels: int = 3, + dropout: float = 0.0, + num_denoising: int = 100, + ) -> None: + super().__init__() + self.num_classes = num_classes + self.num_queries = num_queries + # TODO: verify this is correct + if len(in_channels) > num_levels: + raise ValueError(f"`num_levels` cannot be greater than {len(in_channels)}. Got {num_levels}.") + self.num_levels = num_levels + + # build the input projection layers + self.input_proj = nn.ModuleList() + for ch_in in in_channels: + self.input_proj.append(ConvNormAct(ch_in, hidden_dim, 1, act="none")) + # NOTE: might be missing some layers here ? + # https://github.com/lyuwenyu/RT-DETR/blob/main/rtdetr_pytorch/src/zoo/rtdetr/rtdetr_decoder.py#L403-L410 + + # NOTE: need to be integrated with the TransformerDecoderLayer + decoder_layer = TransformerDecoderLayer( + embed_dim=hidden_dim, + num_heads=num_heads, + dropout=dropout, + num_levels=self.num_levels, + num_points=num_decoder_points, + ) + + self.decoder = TransformerDecoder( + hidden_dim=hidden_dim, decoder_layer=decoder_layer, num_layers=num_decoder_layers + ) + + # denoising part + if num_denoising > 0: + self.denoising_class_embed = nn.Embedding( + num_classes + 1, hidden_dim, padding_idx=num_classes + ) # not used in evaluation + + # decoder embedding + self.query_pos_head = MLP(4, 2 * hidden_dim, hidden_dim, num_layers=2) + + # encoder head + self.enc_output = nn.Sequential(nn.Linear(hidden_dim, hidden_dim), nn.LayerNorm(hidden_dim)) + self.enc_score_head = nn.Linear(hidden_dim, num_classes) + self.enc_bbox_head = MLP(hidden_dim, hidden_dim, 4, num_layers=3) + + # decoder head + self.dec_score_head = nn.ModuleList([nn.Linear(hidden_dim, num_classes) for _ in range(num_decoder_layers)]) + self.dec_bbox_head = nn.ModuleList( + [MLP(hidden_dim, hidden_dim, 4, num_layers=3) for _ in range(num_decoder_layers)] + ) + + def forward(self, feats: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Predict RT-DETR class logits and bounding boxes from neck features. + + Args: + feats: Multi-level feature maps from the encoder/neck. + + Returns: + Tuple ``(logits, boxes)`` for the final decoder output. ``logits`` + stores class scores and ``boxes`` stores normalized box + coordinates. + """ + # input projection and embedding + memory, spatial_shapes, level_start_index = self._get_encoder_input(feats) + + # prepare denoising training + denoising_class, denoising_bbox_unact, attn_mask = None, None, None + + target, init_ref_points_unact, _enc_topk_bboxes, _enc_topk_logits = self._get_decoder_input( + memory, spatial_shapes, denoising_class, denoising_bbox_unact + ) + + # decoder + out_bboxes, out_logits = self.decoder( + target, + init_ref_points_unact, + memory, + spatial_shapes, + level_start_index, + self.dec_bbox_head, + self.dec_score_head, + self.query_pos_head, + attn_mask=attn_mask, + ) + + return out_logits[-1], out_bboxes[-1] + + def _get_encoder_input(self, feats: torch.Tensor) -> tuple[torch.Tensor, list[tuple[int, int]], list[int]]: + # get projection features + proj_feats: list[torch.Tensor] = [self.input_proj[i](feat) for i, feat in enumerate(feats)] + + if self.num_levels > len(proj_feats): + len_srcs = len(proj_feats) + for i in range(len_srcs, self.num_levels): + if i == len_srcs: + proj_feats.append(self.input_proj[i](feats[-1])) + else: + proj_feats.append(self.input_proj[i](proj_feats[-1])) + + # get encoder inputs + feat_flatten_list: list[torch.Tensor] = [] + spatial_shapes: list[tuple[int, int]] = [] + level_start_index: list[int] = [0] + + for _, feat in enumerate(proj_feats): + _, _, h, w = feat.shape + # [b, c, h, w] -> [b, h*w, c] + feat_flatten_list.append(feat.flatten(2).permute(0, 2, 1)) + # [num_levels, 2] + spatial_shapes.append((h, w)) + # [l], start index of each level + level_start_index.append(h * w + level_start_index[-1]) + + # [b, l, c] + feat_flatten: torch.Tensor = torch.cat(feat_flatten_list, 1) + + level_start_index.pop() + return feat_flatten, spatial_shapes, level_start_index + + def _get_decoder_input( + self, + memory: torch.Tensor, + spatial_shapes: list[tuple[int, int]], + denoising_class: Optional[torch.Tensor] = None, + denoising_bbox_unact: Optional[torch.Tensor] = None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + # prepare input for decoder + # TODO: cache anchors and valid_mask as buffers + anchors, valid_mask = self._generate_anchors(spatial_shapes, device=memory.device, dtype=memory.dtype) + + # memory = torch.where(valid_mask, memory, 0) + memory = valid_mask.to(memory) * memory # TODO fix type error for onnx export + + output_memory = self.enc_output(memory) + + enc_outputs_class = self.enc_score_head(output_memory) + enc_outputs_coord_unact = self.enc_bbox_head(output_memory) + anchors + + _, topk_ind = torch.topk(enc_outputs_class.max(-1).values, self.num_queries, dim=1) + + reference_points_unact = enc_outputs_coord_unact.gather( + dim=1, index=topk_ind.unsqueeze(-1).repeat(1, 1, enc_outputs_coord_unact.shape[-1]) + ) + + enc_topk_bboxes = torch.sigmoid(reference_points_unact) + if denoising_bbox_unact is not None: + reference_points_unact = torch.concat([denoising_bbox_unact, reference_points_unact], 1) + + enc_topk_logits = enc_outputs_class.gather( + dim=1, index=topk_ind.unsqueeze(-1).repeat(1, 1, enc_outputs_class.shape[-1]) + ) + + # extract region features + target = output_memory.gather(dim=1, index=topk_ind.unsqueeze(-1).repeat(1, 1, output_memory.shape[-1])) + + if denoising_class is not None: + target = torch.concat([denoising_class, target], 1) + + return target.detach(), reference_points_unact.detach(), enc_topk_bboxes, enc_topk_logits + + @staticmethod + def _generate_anchors( + spatial_shapes: list[tuple[int, int]], + grid_size: float = 0.05, + eps: float = 0.01, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Generate anchors for RT-DETR. + + Args: + spatial_shapes: shape (width, height) of the feature maps + grid_size: size of the grid + eps: specify the minimum and maximum size of the anchors + device: device to place the anchors + dtype: data type for the anchors + + Returns: + logit of anchors and mask + + """ + # TODO: might make this (or some parts of it) into a separate reusable function + anchors_list: list[torch.Tensor] = [] + + for i, (h, w) in enumerate(spatial_shapes): + # TODO: fix later kornia.geometry.create_meshgrid() + grid_y, grid_x = torch.meshgrid( + [torch.arange(h, device=device, dtype=dtype), torch.arange(w, device=device, dtype=dtype)], + indexing="ij", + ) + grid_xy = torch.stack([grid_x, grid_y], -1) # HxWx2 + + # this satisfies onnx export + wh = torch.empty(2, device=device, dtype=dtype) + wh[0] = w + wh[1] = h + + grid_xy = (grid_xy + 0.5) / wh # normalize to [0, 1] + grid_wh = torch.ones_like(grid_xy) * grid_size * (2.0**i) + anchors_list.append(torch.cat([grid_xy, grid_wh], -1).reshape(-1, h * w, 4)) + + anchors = torch.cat(anchors_list, 1) + valid_mask = ((anchors > eps) * (anchors < 1 - eps)).all(-1, keepdim=True) + anchors = torch.log(anchors / (1 - anchors)) # anchors.logit() fails ONNX export + + inf_t = torch.empty(1, device=device, dtype=dtype) + inf_t[0] = float("inf") + + anchors = torch.where(valid_mask, anchors, inf_t) + + return anchors, valid_mask diff --git a/kornia/models/rt_detr/model.py b/kornia/models/rt_detr/model.py new file mode 100644 index 0000000..8d5244c --- /dev/null +++ b/kornia/models/rt_detr/model.py @@ -0,0 +1,344 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Based on code from PaddleDetection.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from enum import Enum +from typing import Any, Optional + +import torch + +from kornia.core.mixin.onnx import ONNXExportMixin +from kornia.models.base import ModelBase +from kornia.models.rt_detr.architecture.hgnetv2 import PPHGNetV2 +from kornia.models.rt_detr.architecture.hybrid_encoder import HybridEncoder +from kornia.models.rt_detr.architecture.resnet_d import ResNetD +from kornia.models.rt_detr.architecture.rtdetr_head import RTDETRHead + +URLs = { + "rtdetr_r18vd": "https://github.com/lyuwenyu/storage/releases/download/v0.1/rtdetr_r18vd_dec3_6x_coco_from_paddle.pth", + "rtdetr_r34vd": "https://github.com/lyuwenyu/storage/releases/download/v0.1/rtdetr_r34vd_dec4_6x_coco_from_paddle.pth", + "rtdetr_r50vd_m": "https://github.com/lyuwenyu/storage/releases/download/v0.1/rtdetr_r50vd_m_6x_coco_from_paddle.pth", + "rtdetr_r50vd": "https://github.com/lyuwenyu/storage/releases/download/v0.1/rtdetr_r50vd_6x_coco_from_paddle.pth", + "rtdetr_r101vd": "https://github.com/lyuwenyu/storage/releases/download/v0.1/rtdetr_r101vd_6x_coco_from_paddle.pth", +} + + +class RTDETRModelType(Enum): + """Enum class that maps RT-DETR model type.""" + + resnet18d = 0 + resnet34d = 1 + resnet50d = 2 + resnet101d = 3 + hgnetv2_l = 4 + hgnetv2_x = 5 + resnet50d_m = 6 + + +@dataclass +class RTDETRConfig: + """Configuration to construct RT-DETR model. + + Args: + model_type: model variant. Available models are + + - ResNetD-18: ``0``, ``'resnet18d'`` or :attr:`RTDETRModelType.resnet18d` + - ResNetD-34: ``1``, ``'resnet34d'`` or :attr:`RTDETRModelType.resnet34d` + - ResNetD-50: ``2``, ``'resnet50d'`` or :attr:`RTDETRModelType.resnet50d` + - ResNetD-101: ``3``, ``'resnet101d'`` or :attr:`RTDETRModelType.resnet101d` + - HGNetV2-L: ``4``, ``'hgnetv2_l'`` or :attr:`RTDETRModelType.hgnetv2_l` + - HGNetV2-X: ``5``, ``'hgnetv2_x'`` or :attr:`RTDETRModelType.hgnetv2_x` + + num_classes: number of classes. + checkpoint: URL or local path of model weights. + neck_hidden_dim: hidden dim for neck. + neck_dim_feedforward: feed-forward network dim for neck. + neck_expansion: expansion ratio for neck. + head_hidden_dim: hidden dim for head. + head_num_queries: number of queries for Deformable DETR transformer decoder. + head_num_decoder_layers: number of decoder layers for Deformable DETR transformer decoder. + + """ + + model_type: RTDETRModelType | str | int + num_classes: int + input_size: int = 640 + checkpoint: Optional[str] = None + + neck_hidden_dim: Optional[int] = None + neck_dim_feedforward: Optional[int] = None + neck_expansion: Optional[float] = None + head_hidden_dim: int = 256 + head_num_queries: int = 300 + head_num_decoder_layers: Optional[int] = None + confidence_threshold: float = 0.3 + + @staticmethod + def from_name(model_name: str, num_classes: int = 80) -> RTDETRConfig: + """Load model without pretrained weights. + + Args: + model_name: 'rtdetr_r18vd', 'rtdetr_r34vd', 'rtdetr_r50vd_m', 'rtdetr_r50vd', 'rtdetr_r101vd'. + num_classes: Number of classes to detect. + + """ + if model_name == "rtdetr_r18vd": + config = RTDETRConfig(RTDETRModelType.resnet18d, num_classes, input_size=640) + elif model_name == "rtdetr_r34vd": + config = RTDETRConfig(RTDETRModelType.resnet34d, num_classes, input_size=640) + elif model_name == "rtdetr_r50vd_m": + config = RTDETRConfig(RTDETRModelType.resnet50d_m, num_classes, input_size=640) + elif model_name == "rtdetr_r50vd": + config = RTDETRConfig(RTDETRModelType.resnet50d, num_classes, input_size=640) + elif model_name == "rtdetr_r101vd": + config = RTDETRConfig(RTDETRModelType.resnet101d, num_classes, input_size=640) + else: + raise ValueError + + return config + + +class RTDETR(ONNXExportMixin, ModelBase[RTDETRConfig]): + """RT-DETR Object Detection model, as described in https://arxiv.org/abs/2304.08069.""" + + def __init__(self, backbone: ResNetD | PPHGNetV2, encoder: HybridEncoder, decoder: RTDETRHead): + """Construct RT-DETR Object Detection model. + + Args: + backbone: backbone network for feature extraction. + encoder: neck network for feature fusion. + decoder: head network to decode features into detection results. + + """ + super().__init__() + self.backbone = backbone + self.encoder = encoder + self.decoder = decoder + + @staticmethod + def from_config(config: RTDETRConfig) -> RTDETR: + """Construct RT-DETR Object Detection model from a config object. + + Args: + config: configuration object for RT-DETR. + + .. note:: + For ``config.neck_hidden_dim``, ``config.neck_dim_feedforward``, ``config.neck_expansion``, and + ``config.head_num_decoder_layers``, if they are ``None``, their values will be replaced with the + default values depending on the ``config.model_type``. See the source code for the default values. + + """ + model_type = config.model_type + if isinstance(model_type, int): + model_type = RTDETRModelType(model_type) + elif isinstance(model_type, str): + model_type = getattr(RTDETRModelType, model_type) + + backbone: ResNetD | PPHGNetV2 + + if model_type == RTDETRModelType.resnet18d: + backbone = ResNetD.from_config(18) + neck_hidden_dim = config.neck_hidden_dim or 256 + neck_dim_feedforward = config.neck_dim_feedforward or 1024 + head_num_decoder_layers = config.head_num_decoder_layers or 3 + neck_expansion = config.neck_expansion or 0.5 + + elif model_type == RTDETRModelType.resnet34d: + backbone = ResNetD.from_config(34) + neck_hidden_dim = config.neck_hidden_dim or 256 + neck_dim_feedforward = config.neck_dim_feedforward or 1024 + head_num_decoder_layers = config.head_num_decoder_layers or 4 + neck_expansion = config.neck_expansion or 0.5 + + elif model_type == RTDETRModelType.resnet50d: + backbone = ResNetD.from_config(50) + neck_hidden_dim = config.neck_hidden_dim or 256 + neck_dim_feedforward = config.neck_dim_feedforward or 1024 + head_num_decoder_layers = config.head_num_decoder_layers or 6 + neck_expansion = config.neck_expansion or 1.0 + + elif model_type == RTDETRModelType.resnet50d_m: + backbone = ResNetD.from_config(50) + neck_hidden_dim = config.neck_hidden_dim or 256 + neck_dim_feedforward = config.neck_dim_feedforward or 1024 + head_num_decoder_layers = config.head_num_decoder_layers or 6 + neck_expansion = config.neck_expansion or 0.5 + + elif model_type == RTDETRModelType.resnet101d: + backbone = ResNetD.from_config(101) + neck_hidden_dim = config.neck_hidden_dim or 384 + neck_dim_feedforward = config.neck_dim_feedforward or 2048 + head_num_decoder_layers = config.head_num_decoder_layers or 6 + neck_expansion = config.neck_expansion or 1.0 + + elif model_type == RTDETRModelType.hgnetv2_l: + backbone = PPHGNetV2.from_config("L") + neck_hidden_dim = config.neck_hidden_dim or 256 + neck_dim_feedforward = config.neck_dim_feedforward or 1024 + head_num_decoder_layers = config.head_num_decoder_layers or 6 + neck_expansion = config.neck_expansion or 1.0 + + elif model_type == RTDETRModelType.hgnetv2_x: + backbone = PPHGNetV2.from_config("X") + neck_hidden_dim = config.neck_hidden_dim or 384 + neck_dim_feedforward = config.neck_dim_feedforward or 2048 + head_num_decoder_layers = config.head_num_decoder_layers or 6 + neck_expansion = config.neck_expansion or 1.0 + + model = RTDETR( + backbone, + HybridEncoder(backbone.out_channels, neck_hidden_dim, neck_dim_feedforward, neck_expansion), + RTDETRHead( + num_classes=config.num_classes, + hidden_dim=config.head_hidden_dim, + num_queries=config.head_num_queries, + in_channels=[neck_hidden_dim] * 3, + num_decoder_layers=head_num_decoder_layers, + ), + ) + + if config.checkpoint: + model.load_checkpoint(config.checkpoint) + return model + + @staticmethod + def from_pretrained(model_name: str) -> RTDETR: + """Load model from pretrained weights. + + Args: + model_name: 'rtdetr_r18vd', 'rtdetr_r34vd', 'rtdetr_r50vd_m', 'rtdetr_r50vd', 'rtdetr_r101vd'. + + """ + if model_name not in URLs: + raise ValueError(f"No pretrained model for '{model_name}'. Please select from {list(URLs.keys())}.") + + state_dict = torch.hub.load_state_dict_from_url( + URLs[model_name], map_location="cuda:0" if torch.cuda.is_available() else "cpu" + ) + + def map_name(old_name: str) -> str: + new_name = old_name + + # Encoder renaming + new_name = re.sub("encoder.pan_blocks", "encoder.ccfm.pan_blocks", new_name) + new_name = re.sub("encoder.downsample_convs", "encoder.ccfm.downsample_convs", new_name) + new_name = re.sub("encoder.fpn_blocks", "encoder.ccfm.fpn_blocks", new_name) + new_name = re.sub("encoder.lateral_convs", "encoder.ccfm.lateral_convs", new_name) + + # Backbone renaming + new_name = re.sub(".branch2b.", ".convs.branch2b.", new_name) + new_name = re.sub(".branch2a.", ".convs.branch2a.", new_name) + new_name = re.sub(".branch2c.", ".convs.branch2c.", new_name) + + return new_name + + def _state_dict_proc(state_dict: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + state_dict = state_dict["ema"]["module"] # type:ignore + new_state_dict = {} + + # Apply the regex-based mapping function to each key + for old_name in state_dict.keys(): + new_name = map_name(old_name) + new_state_dict[new_name] = state_dict[old_name] + + return new_state_dict + + model = RTDETR.from_name(model_name, num_classes=80) + + model.load_state_dict(_state_dict_proc(state_dict)) + return model + + @staticmethod + def from_name(model_name: str, num_classes: int = 80) -> RTDETR: + """Load model without pretrained weights. + + Args: + model_name: 'rtdetr_r18vd', 'rtdetr_r34vd', 'rtdetr_r50vd_m', 'rtdetr_r50vd', 'rtdetr_r101vd'. + num_classes: number of classes to detect. + + """ + model = RTDETR.from_config(RTDETRConfig.from_name(model_name, num_classes)) + return model + + def to_onnx( + self, + onnx_name: Optional[str] = None, + pseudo_shape: Optional[list[int]] = None, + save: bool = True, + **kwargs: Any, + ) -> Any: + """Export RT-DETR to ONNX with correct dual-output names and dynamic axes. + + RT-DETR's :meth:`forward` returns ``(logits, boxes)`` — a two-output model. + The base :class:`~kornia.core.mixin.onnx.ONNXExportMixin` defaults to a single + ``"output"`` name, which mismatches the actual graph. This override provides + ``output_names=["pred_logits", "pred_boxes"]`` and sets per-output dynamic axes + for the batch dimension. + + Args: + onnx_name: Path for the saved ``.onnx`` file. Defaults to + ``"Kornia-RTDETR.onnx"``. + pseudo_shape: Concrete input shape used to trace the model, e.g. + ``[1, 3, 640, 640]``. Defaults to ``[1, 3, 640, 640]``. + save: Whether to write the model to disk. Default ``True``. + **kwargs: Additional keyword arguments forwarded to + :func:`torch.onnx.export`. + + Returns: + ``onnx.ModelProto`` of the exported model. + + """ + kwargs.setdefault("output_names", ["pred_logits", "pred_boxes"]) + kwargs.setdefault( + "dynamic_axes", + { + "input": {0: "batch"}, + "pred_logits": {0: "batch"}, + "pred_boxes": {0: "batch"}, + }, + ) + if pseudo_shape is None: + pseudo_shape = [1, 3, 640, 640] + return super().to_onnx( + onnx_name=onnx_name, + input_shape=[-1, 3, pseudo_shape[2], pseudo_shape[3]], + pseudo_shape=pseudo_shape, + save=save, + **kwargs, + ) + + def forward(self, images: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Detect objects in an image. + + Args: + images: images to be detected. Shape :math:`(N, C, H, W)`. + + Returns: + - **logits** - Tensor of shape :math:`(N, Q, K)`, where :math:`Q` is the number of queries, + :math:`K` is the number of classes. + - **boxes** - Tensor of shape :math:`(N, Q, 4)`, where :math:`Q` is the number of queries. + + """ + feats = self.backbone(images) + feats_buf = self.encoder(feats) + logits, boxes = self.decoder(feats_buf) + return logits, boxes diff --git a/kornia/models/rt_detr/post_processor.py b/kornia/models/rt_detr/post_processor.py new file mode 100644 index 0000000..890830d --- /dev/null +++ b/kornia/models/rt_detr/post_processor.py @@ -0,0 +1,133 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Post-processor for the RT-DETR model.""" + +from __future__ import annotations + +from typing import Optional, Union + +import torch +from torch import nn + +from kornia.contrib.object_detection import BoxFiltering + + +def mod(a: torch.Tensor, b: int) -> torch.Tensor: + """Compute the element-wise remainder of tensor `a` divided by integer `b`. + + This function requires `a` to be a `torch.Tensor` and `b` to be an `int`. + It returns a `torch.Tensor` with the same shape/device as `a`. The + implementation uses `a % b` (equivalent to `torch.remainder(a, b)`). + + Args: + a (torch.Tensor): Dividend torch.tensor(any numeric dtype). + b (int): Divisor (must be non-zero). + + Returns: + torch.Tensor: Element-wise remainder of `a` divided by `b`. + + Examples: + >>> mod(torch.tensor(7), 3) + tensor(1) + >>> mod(torch.tensor([7, -1, 2]), 3) + tensor([1, 2, 2]) + """ + return a % b + + +# TODO: deprecate the confidence threshold and add the num_top_queries as a parameter and num_classes as a parameter +class DETRPostProcessor(nn.Module): + """Convert raw DETR model outputs into final bounding box detections. + + This module applies the softmax function to scores and transforms normalized + bounding box coordinates into the pixel coordinate system of the input image. + + Args: + num_classes: The number of object classes. + confidence_threshold: The threshold to filter out low-confidence detections. + num_top_queries: The number of top queries to consider for each image. + confidence_filtering: Whether to apply confidence-based filtering. + filter_as_zero: If True, boxes below the confidence threshold are set to zero instead of being removed. + """ + + def __init__( + self, + confidence_threshold: Optional[float] = None, + num_classes: int = 80, + num_top_queries: int = 300, + confidence_filtering: bool = True, + filter_as_zero: bool = False, + ) -> None: + super().__init__() + self.confidence_threshold = confidence_threshold + self.num_classes = num_classes + self.confidence_filtering = confidence_filtering + self.num_top_queries = num_top_queries + self.box_filtering = BoxFiltering( + torch.tensor(confidence_threshold) if confidence_threshold is not None else None, + filter_as_zero=filter_as_zero, + ) + + def forward( + self, logits: torch.Tensor, boxes: torch.Tensor, original_sizes: torch.Tensor + ) -> Union[torch.Tensor, list[torch.Tensor]]: + """Post-process outputs from DETR. + + Args: + logits: tensor with shape :math:`(N, Q, K)`, where :math:`N` is the batch size, :math:`Q` is the number of + queries, :math:`K` is the number of classes. + boxes: tensor with shape :math:`(N, Q, 4)`, where :math:`N` is the batch size, :math:`Q` is the number of + queries. + original_sizes: tensor with shape :math:`(N, 2)`, where :math:`N` is the batch size and each element + represents the image size of (img_height, img_width). + + Returns: + Processed detections. For each image, the detections have shape (D, 6), where D is the number of detections + in that image, 6 represent (class_id, confidence_score, x, y, w, h). + + """ + # NOTE: consider using kornia BoundingBox + # NOTE: consider having a separate function to convert the detections to a list of bounding boxes + + # https://github.com/PaddlePaddle/PaddleDetection/blob/5d1f888362241790000950e2b63115dc8d1c6019/ppdet/modeling/post_process.py#L446 + # box format is cxcywh + # convert to xywh + # bboxes[..., :2] -= bboxes[..., 2:] * 0.5 # in-place operation is not torch.compile()-friendly + # TODO: replace using kornia BoundingBox + cxcy, wh = boxes[..., :2], boxes[..., 2:] + boxes_xy = torch.cat([cxcy - wh * 0.5, wh], -1) + + # Get dynamic size from the input tensor itself + sizes_wh = original_sizes[0].flip(0).unsqueeze(0).unsqueeze(0).repeat(1, 1, 2) + + boxes_xy = boxes_xy * sizes_wh + scores = logits.sigmoid() # RT-DETR was trained with focal loss. thus sigmoid is used instead of softmax + + # retrieve the boxes with the highest score for each class + # https://github.com/lyuwenyu/RT-DETR/blob/b6bf0200b249a6e35b44e0308b6058f55b99696b/rtdetrv2_pytorch/src/zoo/rtdetr/rtdetr_postprocessor.py#L55-L62 + scores, index = torch.topk(scores.flatten(1), self.num_top_queries, dim=-1) + labels = mod(index, self.num_classes) + index = index // self.num_classes + boxes = boxes_xy.gather(dim=1, index=index.unsqueeze(-1).repeat(1, 1, boxes_xy.shape[-1])) + + all_boxes = torch.cat([labels[..., None], scores[..., None], boxes], -1) + + if not self.confidence_filtering or self.confidence_threshold == 0: + return all_boxes + + return self.box_filtering(all_boxes, self.confidence_threshold) diff --git a/kornia/models/sam/__init__.py b/kornia/models/sam/__init__.py new file mode 100644 index 0000000..681ea94 --- /dev/null +++ b/kornia/models/sam/__init__.py @@ -0,0 +1,23 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Contrib SAM — Segment Anything Model (SAM) integration for Kornia. + +This subpackage provides SAM model classes and configuration for segmentation tasks. +""" + +from kornia.models.sam.model import Sam, SamConfig, SamModelType diff --git a/kornia/models/sam/architecture/__init__.py b/kornia/models/sam/architecture/__init__.py new file mode 100644 index 0000000..32f3f1d --- /dev/null +++ b/kornia/models/sam/architecture/__init__.py @@ -0,0 +1,21 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Kornia Contrib SAM Architecture — Model architecture components for SAM. + +This subpackage provides building blocks for the Segment Anything Model (SAM). +""" diff --git a/kornia/models/sam/architecture/common.py b/kornia/models/sam/architecture/common.py new file mode 100644 index 0000000..c875054 --- /dev/null +++ b/kornia/models/sam/architecture/common.py @@ -0,0 +1,65 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Based from the original code from Meta Platforms, Inc. and affiliates. + +https://github.com/facebookresearch/segment- +anything/blob/3518c86b78b3bc9cf4fbe3d18e682fad1c79dc51/segment_anything/modeling/common.py +""" + +from __future__ import annotations + +from typing import Any + +import torch +from torch import nn + + +class MLPBlock(nn.Module): + """Implement a standard Multi-Layer Perceptron (MLP) block. + + Args: + embedding_dim: The size of the input and output embeddings. + mlp_dim: The size of the hidden layer. + act: The activation function class to use. Default: nn.GELU. + """ + + def __init__(self, embedding_dim: int, mlp_dim: int, act: type[nn.Module] = nn.GELU) -> None: + super().__init__() + self.lin1 = nn.Linear(embedding_dim, mlp_dim) + self.lin2 = nn.Linear(mlp_dim, embedding_dim) + self.act = act() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply the two-layer MLP block to token embeddings. + + Args: + x: Token tensor with last dimension ``embedding_dim``. + + Returns: + Tensor with the same shape as ``x`` after hidden projection, + activation, and projection back to ``embedding_dim``. + """ + return self.lin2(self.act(self.lin1(x))) + + +class LayerNorm(nn.LayerNorm): + """Implement Layer Normalization with a small epsilon to avoid division by zero.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.eps = 1e-6 diff --git a/kornia/models/sam/architecture/image_encoder.py b/kornia/models/sam/architecture/image_encoder.py new file mode 100644 index 0000000..56c2bb6 --- /dev/null +++ b/kornia/models/sam/architecture/image_encoder.py @@ -0,0 +1,415 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Based from the original code from Meta Platforms, Inc. and affiliates. + +https://github.com/facebookresearch/segment- +anything/blob/3518c86b78b3bc9cf4fbe3d18e682fad1c79dc51/segment_anything/modeling/image_encoder.py +""" + +from __future__ import annotations + +from typing import Optional + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.models.common import LayerNorm2d, window_partition, window_unpartition +from kornia.models.sam.architecture.common import MLPBlock + + +# This class and its supporting functions below lightly adapted from the ViTDet backbone available at: https://github.com/facebookresearch/detectron2/blob/main/detectron2/modeling/backbone/vit.py # noqa +class ImageEncoderViT(nn.Module): + """Implement the Vision Transformer (ViT) based image encoder for SAM. + + Args: + img_size: The input image resolution. + patch_size: The size of the patches for the embedding. + in_chans: The number of input color channels. + embed_dim: The dimension of the patch embeddings. + depth: The number of transformer blocks. + num_heads: The number of attention heads. + mlp_ratio: The ratio of MLP hidden dim to embedding dim. + out_chans: The number of output feature channels. + qkv_bias: Whether to use bias in the QKV projection. + norm_layer: The normalization layer class. + act_layer: The activation layer class. + use_abs_pos: Whether to use absolute position embeddings. + use_rel_pos: Whether to use relative position embeddings. + rel_pos_zero_init: Whether to zero-initialize relative position. + window_size: The window size for windowed attention. + global_attn_indexes: Indexes of blocks that use global attention. + """ + + def __init__( + self, + img_size: int = 1024, + *, + patch_size: int = 16, + in_chans: int = 3, + embed_dim: int = 768, + depth: int = 12, + num_heads: int = 12, + mlp_ratio: float = 4.0, + out_chans: int = 256, + qkv_bias: bool = True, + norm_layer: type[nn.Module] = nn.LayerNorm, + act_layer: type[nn.Module] = nn.GELU, + use_abs_pos: bool = True, + use_rel_pos: bool = False, + rel_pos_zero_init: bool = True, + window_size: int = 0, + global_attn_indexes: tuple[int, ...] = (), + ) -> None: + """Construct Image Encoder ViT. + + Args: + img_size: Input image size. + patch_size: Patch size. + in_chans: Number of input image channels. + embed_dim: Patch embedding dimension. + depth: Depth of ViT. + num_heads: Number of attention heads in each ViT block. + mlp_ratio: Ratio of mlp hidden dim to embedding dim. + out_chans: Number of output channels. + qkv_bias: If True, add a learnable bias to query, key, value. + norm_layer: Normalization layer. + act_layer: Activation layer. + use_abs_pos: If True, use absolute positional embeddings. + use_rel_pos: If True, add relative positional embeddings to the attention map. + rel_pos_zero_init: If True, zero initialize relative positional parameters. + window_size: Window size for window attention blocks. + global_attn_indexes: Indexes for blocks using global attention. + + """ + super().__init__() + self.img_size = img_size + + self.patch_embed = PatchEmbed( + kernel_size=(patch_size, patch_size), + stride=(patch_size, patch_size), + in_chans=in_chans, + embed_dim=embed_dim, + ) + + self.pos_embed: Optional[nn.Parameter] = None + if use_abs_pos: + # Initialize absolute positional embedding with pretrain image size. + self.pos_embed = nn.Parameter(torch.zeros(1, img_size // patch_size, img_size // patch_size, embed_dim)) + + self.blocks = nn.ModuleList() + for i in range(depth): + block = Block( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + norm_layer=norm_layer, + act_layer=act_layer, + use_rel_pos=use_rel_pos, + rel_pos_zero_init=rel_pos_zero_init, + window_size=window_size if i not in global_attn_indexes else 0, + input_size=(img_size // patch_size, img_size // patch_size), + ) + self.blocks.append(block) + + self.neck = nn.Sequential( + nn.Conv2d(embed_dim, out_chans, kernel_size=1, bias=False), + LayerNorm2d(out_chans), + nn.Conv2d(out_chans, out_chans, kernel_size=3, padding=1, bias=False), + LayerNorm2d(out_chans), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Encode an image batch into SAM image embeddings. + + Args: + x: Image tensor with shape :math:`(B, C, H, W)`, where :math:`B` + is batch size, :math:`C` is channel count, and :math:`H`, + :math:`W` are image height and width. + + Returns: + Image embedding tensor with shape :math:`(B, C_{out}, H_e, W_e)` + after patch embedding, transformer blocks, and the neck. + """ + x = self.patch_embed(x) + if self.pos_embed is not None: + x = x + self.pos_embed + + for blk in self.blocks: + x = blk(x) + + x = self.neck(x.permute(0, 3, 1, 2)) + + return x + + +class Block(nn.Module): + """Transformer blocks with support of window attention and residual propagation blocks.""" + + def __init__( + self, + dim: int, + num_heads: int, + mlp_ratio: float = 4.0, + qkv_bias: bool = True, + norm_layer: type[nn.Module] = nn.LayerNorm, + act_layer: type[nn.Module] = nn.GELU, + use_rel_pos: bool = False, + rel_pos_zero_init: bool = True, + window_size: int = 0, + input_size: Optional[tuple[int, int]] = None, + ) -> None: + """Construct transformer block. + + Args: + dim: Number of input channels. + num_heads: Number of attention heads in each ViT block. + mlp_ratio: Ratio of mlp hidden dim to embedding dim. + qkv_bias: If True, add a learnable bias to query, key, value. + norm_layer: Normalization layer. + act_layer: Activation layer. + use_rel_pos: If True, add relative positional embeddings to the attention map. + rel_pos_zero_init: If True, zero initialize relative positional parameters. + window_size: Window size for window attention blocks. If it equals 0, then use global attention. + input_size: Input resolution for calculating the relative positional parameter size. + + """ + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + use_rel_pos=use_rel_pos, + rel_pos_zero_init=rel_pos_zero_init, + input_size=input_size if window_size == 0 else (window_size, window_size), + ) + + self.norm2 = norm_layer(dim) + self.mlp = MLPBlock(embedding_dim=dim, mlp_dim=int(dim * mlp_ratio), act=act_layer) + + self.window_size = window_size + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply one SAM image-encoder transformer block. + + Args: + x: Feature tensor with shape :math:`(B, H, W, C)`. + + Returns: + Tensor with the same shape as ``x`` after window or global + attention, MLP, and residual connections. + """ + shortcut = x + x = self.norm1(x) + # Window partition + if self.window_size > 0: + H, W = x.shape[1], x.shape[2] + x, pad_hw = window_partition(x, self.window_size) + + x = self.attn(x) + # Reverse window partition + if self.window_size > 0: + x = window_unpartition(x, self.window_size, pad_hw, (H, W)) + + x = shortcut + x + x = x + self.mlp(self.norm2(x)) + + return x + + +class Attention(nn.Module): + """Multi-head Attention block with relative position embeddings.""" + + def __init__( + self, + dim: int, + num_heads: int = 8, + qkv_bias: bool = True, + use_rel_pos: bool = False, + rel_pos_zero_init: bool = True, + input_size: Optional[tuple[int, int]] = None, + ) -> None: + """Construct attention block. + + Args: + dim: Number of input channels. + num_heads: Number of attention heads. + qkv_bias: If True, add a learnable bias to query, key, value. + use_rel_pos: If True, add relative positional embeddings to the attention map. + rel_pos_zero_init: If True, zero initialize relative positional parameters. + input_size: Input resolution for calculating the relative positional parameter size. + + """ + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim**-0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.proj = nn.Linear(dim, dim) + + self.use_rel_pos = use_rel_pos + if self.use_rel_pos and isinstance(input_size, tuple): + # initialize relative positional embeddings + self.rel_pos_h = nn.Parameter(torch.zeros(2 * input_size[0] - 1, head_dim)) + self.rel_pos_w = nn.Parameter(torch.zeros(2 * input_size[1] - 1, head_dim)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply multi-head attention over 2D image tokens. + + Args: + x: Token grid with shape :math:`(B, H, W, C)`. + + Returns: + Tensor with shape :math:`(B, H, W, C)` after attention, optional + decomposed relative positional bias, and output projection. + """ + B, H, W, _ = x.shape + # qkv with shape (3, B, nHead, H * W, C) + qkv = self.qkv(x).reshape(B, H * W, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + # q, k, v with shape (B * nHead, H * W, C) + q, k, v = qkv.reshape(3, B * self.num_heads, H * W, -1).unbind(0) + + attn = (q * self.scale) @ k.transpose(-2, -1) + + if self.use_rel_pos: + attn = add_decomposed_rel_pos(attn, q, self.rel_pos_h, self.rel_pos_w, (H, W), (H, W)) + + attn = attn.softmax(dim=-1) + x = (attn @ v).view(B, self.num_heads, H, W, -1).permute(0, 2, 3, 1, 4).reshape(B, H, W, -1) + x = self.proj(x) + + return x + + +def get_rel_pos(q_size: int, k_size: int, rel_pos: torch.Tensor) -> torch.Tensor: + """Get relative positional embeddings according to the relative positions of query and key sizes. + + Args: + q_size: size of query q. + k_size: size of key k. + rel_pos: relative position embeddings (L, C). + + Returns: + Extracted positional embeddings according to relative positions. + + """ + max_rel_dist = int(2 * max(q_size, k_size) - 1) + # Interpolate rel pos if needed. + if rel_pos.shape[0] != max_rel_dist: + # Interpolate rel pos. + rel_pos_resized = F.interpolate( + rel_pos.reshape(1, rel_pos.shape[0], -1).permute(0, 2, 1), size=max_rel_dist, mode="linear" + ) + rel_pos_resized = rel_pos_resized.reshape(-1, max_rel_dist).permute(1, 0) + else: + rel_pos_resized = rel_pos + + # Scale the coords with short length if shapes for q and k are different. + q_coords = torch.arange(q_size)[:, None] * max(k_size / q_size, 1.0) + k_coords = torch.arange(k_size)[None, :] * max(q_size / k_size, 1.0) + relative_coords = (q_coords - k_coords) + (k_size - 1) * max(q_size / k_size, 1.0) + + return rel_pos_resized[relative_coords.long()] + + +def add_decomposed_rel_pos( + attn: torch.Tensor, + q: torch.Tensor, + rel_pos_h: torch.Tensor, + rel_pos_w: torch.Tensor, + q_size: tuple[int, int], + k_size: tuple[int, int], +) -> torch.Tensor: + """Calculate decomposed Relative Positional Embeddings. + + from :paper:`mvitv2`. + https://github.com/facebookresearch/mvit/blob/19786631e330df9f3622e5402b4a419a263a2c80/mvit/models/attention.py + + + Args: + attn: attention map. + q: query q in the attention layer with shape (B, q_h * q_w, C). + rel_pos_h: relative position embeddings (Lh, C) for height axis. + rel_pos_w: relative position embeddings (Lw, C) for width axis. + q_size: spatial sequence size of query q with (q_h, q_w). + k_size: spatial sequence size of key k with (k_h, k_w). + + Returns: + att: attention map with added relative positional embeddings. + + """ + q_h, q_w = q_size + k_h, k_w = k_size + Rh = get_rel_pos(q_h, k_h, rel_pos_h) + Rw = get_rel_pos(q_w, k_w, rel_pos_w) + + B, _, dim = q.shape + r_q = q.reshape(B, q_h, q_w, dim) + rel_h = torch.einsum("bhwc,hkc->bhwk", r_q, Rh) + rel_w = torch.einsum("bhwc,wkc->bhwk", r_q, Rw) + + attn = (attn.view(B, q_h, q_w, k_h, k_w) + rel_h[:, :, :, :, None] + rel_w[:, :, :, None, :]).view( + B, q_h * q_w, k_h * k_w + ) + + return attn + + +class PatchEmbed(nn.Module): + """Image to Patch Embedding.""" + + def __init__( + self, + kernel_size: tuple[int, int] = (16, 16), + stride: tuple[int, int] = (16, 16), + padding: tuple[int, int] = (0, 0), + in_chans: int = 3, + embed_dim: int = 768, + ) -> None: + """Construct Patch Embedding. + + Args: + kernel_size: kernel size of the projection layer. + stride: stride of the projection layer. + padding: padding size of the projection layer. + in_chans: Number of input image channels. + embed_dim: Patch embedding dimension. + + """ + super().__init__() + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=kernel_size, stride=stride, padding=padding) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Project an image into patch embeddings. + + Args: + x: Image tensor with shape :math:`(B, C, H, W)`. + + Returns: + Patch embedding tensor with shape :math:`(B, H_p, W_p, D)`, where + :math:`H_p` and :math:`W_p` are patch-grid dimensions and + :math:`D` is embedding dimension. + """ + x = self.proj(x) + # B C H W -> B H W C + x = x.permute(0, 2, 3, 1) + return x diff --git a/kornia/models/sam/architecture/mask_decoder.py b/kornia/models/sam/architecture/mask_decoder.py new file mode 100644 index 0000000..07e2bbe --- /dev/null +++ b/kornia/models/sam/architecture/mask_decoder.py @@ -0,0 +1,159 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Based from the original code from Meta Platforms, Inc. and affiliates. + +https://github.com/facebookresearch/segment- +anything/blob/3518c86b78b3bc9cf4fbe3d18e682fad1c79dc51/segment_anything/modeling/mask_decoder.py +""" + +from __future__ import annotations + +import torch +from torch import nn + +from kornia.models.common import MLP, LayerNorm2d + + +class MaskDecoder(nn.Module): + """Predict object masks and quality scores from image and prompt embeddings.""" + + def __init__( + self, + *, + transformer_dim: int, + transformer: nn.Module, + num_multimask_outputs: int = 3, + activation: type[nn.Module] = nn.GELU, + iou_head_depth: int = 3, + iou_head_hidden_dim: int = 256, + ) -> None: + """Predicts masks given an image and prompt embeddings, using a transformer architecture. + + Args: + transformer_dim: the channel dimension of the transformer + transformer: the transformer used to predict masks + num_multimask_outputs: the number of masks to predict when disambiguating masks + activation: the type of activation to use when upscaling masks + iou_head_depth: the depth of the MLP used to predict mask quality + iou_head_hidden_dim: the hidden dimension of the MLP used to predict mask quality + + """ + super().__init__() + self.transformer_dim = transformer_dim + self.transformer = transformer + + self.num_multimask_outputs = num_multimask_outputs + + self.iou_token = nn.Embedding(1, transformer_dim) + self.num_mask_tokens = num_multimask_outputs + 1 + self.mask_tokens = nn.Embedding(self.num_mask_tokens, transformer_dim) + + self.output_upscaling = nn.Sequential( + nn.ConvTranspose2d(transformer_dim, transformer_dim // 4, kernel_size=2, stride=2), + LayerNorm2d(transformer_dim // 4), + activation(), + nn.ConvTranspose2d(transformer_dim // 4, transformer_dim // 8, kernel_size=2, stride=2), + activation(), + ) + self.output_hypernetworks_mlps = nn.ModuleList( + [MLP(transformer_dim, transformer_dim, transformer_dim // 8, 3) for i in range(self.num_mask_tokens)] + ) + + self.iou_prediction_head = MLP(transformer_dim, iou_head_hidden_dim, self.num_mask_tokens, iou_head_depth) + + def forward( + self, + image_embeddings: torch.Tensor, + image_pe: torch.Tensor, + sparse_prompt_embeddings: torch.Tensor, + dense_prompt_embeddings: torch.Tensor, + multimask_output: bool, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Predict masks given image and prompt embeddings. + + Args: + image_embeddings: the embeddings from the image encoder + image_pe: positional encoding with the shape of image_embeddings + sparse_prompt_embeddings: the embeddings of the points and boxes + dense_prompt_embeddings: the embeddings of the mask inputs + multimask_output: Whether to return multiple masks or a single mask. + + Returns: + batched predicted masks + batched predictions of mask quality + + """ + masks, iou_pred = self.predict_masks( + image_embeddings=image_embeddings, + image_pe=image_pe, + sparse_prompt_embeddings=sparse_prompt_embeddings, + dense_prompt_embeddings=dense_prompt_embeddings, + ) + + # Select the correct mask or masks for outptu + if multimask_output: + mask_slice = slice(1, None) + else: + mask_slice = slice(0, 1) + masks = masks[:, mask_slice, :, :] + iou_pred = iou_pred[:, mask_slice] + + # Prepare output + return masks, iou_pred + + def predict_masks( + self, + image_embeddings: torch.Tensor, + image_pe: torch.Tensor, + sparse_prompt_embeddings: torch.Tensor, + dense_prompt_embeddings: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Predicts masks. + + See 'forward' for more details. + """ + # Concatenate output tokens + output_tokens = torch.cat([self.iou_token.weight, self.mask_tokens.weight], dim=0) + output_tokens = output_tokens[None, ...].expand(sparse_prompt_embeddings.size(0), -1, -1) + tokens = torch.cat((output_tokens, sparse_prompt_embeddings), dim=1) + + # Expand per-image data in batch direction to be per-mask + src = torch.repeat_interleave(image_embeddings, tokens.shape[0], dim=0) + src = src + dense_prompt_embeddings + pos_src = torch.repeat_interleave(image_pe, tokens.shape[0], dim=0) + b, c, h, w = src.shape + + # Run the transformer + hs, src = self.transformer(src, pos_src, tokens) + iou_token_out = hs[:, 0, :] + mask_tokens_out = hs[:, 1 : (1 + self.num_mask_tokens), :] + + # Upscale mask embeddings and predict masks using the mask tokens + src = src.transpose(1, 2).view(b, c, h, w) + upscaled_embedding = self.output_upscaling(src) + hyper_in_list: list[torch.Tensor] = [] + for i in range(self.num_mask_tokens): + hyper_in_list.append(self.output_hypernetworks_mlps[i](mask_tokens_out[:, i, :])) + hyper_in = torch.stack(hyper_in_list, dim=1) + b, c, h, w = upscaled_embedding.shape + masks = (hyper_in @ upscaled_embedding.view(b, c, h * w)).view(b, -1, h, w) + + # Generate mask quality predictions + iou_pred = self.iou_prediction_head(iou_token_out) + + return masks, iou_pred diff --git a/kornia/models/sam/architecture/prompt_encoder.py b/kornia/models/sam/architecture/prompt_encoder.py new file mode 100644 index 0000000..0d3bd53 --- /dev/null +++ b/kornia/models/sam/architecture/prompt_encoder.py @@ -0,0 +1,224 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Based from the original code from Meta Platforms, Inc. and affiliates. + +https://github.com/facebookresearch/segment- +anything/blob/3518c86b78b3bc9cf4fbe3d18e682fad1c79dc51/segment_anything/modeling/prompt_encoder.py +""" + +from __future__ import annotations + +from math import pi +from typing import Optional + +import torch +from torch import nn + +from kornia.models.common import LayerNorm2d + + +class PromptEncoder(nn.Module): + """Encode sparse (points, boxes) and dense (masks) prompts into embeddings. + + This module transforms different types of prompts into a common embedding space + to be consumed by the mask decoder. + + Args: + embed_dim: The internal embedding dimension. + image_embedding_size: The spatial resolution of the image embedding as $(H, W)$. + input_image_size: The resolution of the input image as $(H, W)$. + mask_in_chans: The number of hidden channels used for encoding input masks. + activation: The activation function for the mask encoder. Default: :class:`nn.GELU`. + """ + + def __init__( + self, + embed_dim: int, + image_embedding_size: tuple[int, int], + input_image_size: tuple[int, int], + mask_in_chans: int, + activation: type[nn.Module] = nn.GELU, + ) -> None: + """Encode prompts for input to SAM's mask decoder. + + Args: + embed_dim: The prompts' embedding dimension + image_embedding_size: The spatial size of the image embedding, as (H, W). + input_image_size: The padded size of the image as input to the image encoder, as (H, W). + mask_in_chans: The number of hidden channels used for encoding input masks. + activation: The activation to use when encoding input masks. + + """ + super().__init__() + self.embed_dim = embed_dim + self.input_image_size = input_image_size + self.image_embedding_size = image_embedding_size + self.pe_layer = PositionEmbeddingRandom(embed_dim // 2) + + self.num_point_embeddings: int = 4 # pos/neg point + 2 box corners + point_embeddings = [nn.Embedding(1, embed_dim) for i in range(self.num_point_embeddings)] + self.point_embeddings = nn.ModuleList(point_embeddings) + self.not_a_point_embed = nn.Embedding(1, embed_dim) + + self.mask_input_size = (4 * image_embedding_size[0], 4 * image_embedding_size[1]) + self.mask_downscaling = nn.Sequential( + nn.Conv2d(1, mask_in_chans // 4, kernel_size=2, stride=2), + LayerNorm2d(mask_in_chans // 4), + activation(), + nn.Conv2d(mask_in_chans // 4, mask_in_chans, kernel_size=2, stride=2), + LayerNorm2d(mask_in_chans), + activation(), + nn.Conv2d(mask_in_chans, embed_dim, kernel_size=1), + ) + self.no_mask_embed = nn.Embedding(1, embed_dim) + + def get_dense_pe(self) -> torch.Tensor: + """Return the positional encoding used to encode point prompts, applied to a dense set of points the shape + of the image encoding. + + Returns: + Positional encoding with shape 1x(embed_dim)x(embedding_h)x(embedding_w) + + """ # noqa: D205 + return self.pe_layer(self.image_embedding_size)[None, ...] + + def _embed_points(self, points: torch.Tensor, labels: torch.Tensor, pad: bool) -> torch.Tensor: + """Embeds point prompts.""" + points = points + 0.5 # Shift to center of pixel + if pad: + padding_point = torch.zeros((points.shape[0], 1, 2), device=points.device) + padding_label = -torch.ones((labels.shape[0], 1), device=labels.device) + points = torch.cat([points, padding_point], dim=1) + labels = torch.cat([labels, padding_label], dim=1) + point_embedding = self.pe_layer.forward_with_coords(points, self.input_image_size) + point_embedding[labels == -1] = 0.0 + point_embedding[labels == -1] += self.not_a_point_embed.weight + point_embedding[labels == 0] += self.point_embeddings[0].weight + point_embedding[labels == 1] += self.point_embeddings[1].weight + return point_embedding + + def _embed_boxes(self, boxes: torch.Tensor) -> torch.Tensor: + """Embeds box prompts.""" + boxes = boxes + 0.5 # Shift to center of pixel + coords = boxes.reshape(-1, 2, 2) + corner_embedding = self.pe_layer.forward_with_coords(coords, self.input_image_size) + corner_embedding[:, 0, :] += self.point_embeddings[2].weight + corner_embedding[:, 1, :] += self.point_embeddings[3].weight + return corner_embedding + + def _embed_masks(self, masks: torch.Tensor) -> torch.Tensor: + """Embeds mask inputs.""" + mask_embedding = self.mask_downscaling(masks) + return mask_embedding + + def _get_batch_size( + self, + points: Optional[tuple[torch.Tensor, torch.Tensor]], + boxes: Optional[torch.Tensor], + masks: Optional[torch.Tensor], + ) -> int: + """Get the batch size of the output given the batch size of the input prompts.""" + if points is not None: + return points[0].shape[0] + elif boxes is not None: + return boxes.shape[0] + elif masks is not None: + return masks.shape[0] + else: + return 1 + + def _get_device(self) -> torch.device: + return self.point_embeddings[0].weight.device + + def forward( + self, + points: Optional[tuple[torch.Tensor, torch.Tensor]], + boxes: Optional[torch.Tensor], + masks: Optional[torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor]: + """Embeds different types of prompts, returning both sparse and dense embeddings. + + Args: + points: point coordinates and labels to embed. + boxes: boxes to embed + masks: masks to embed + + Returns: + - sparse embeddings for the points and boxes, with shape BxNx(embed_dim), where N is determined by the + number of input points and boxes. + - dense embeddings for the masks, in the shape Bx(embed_dim)x(embed_H)x(embed_W) + + """ + bs = self._get_batch_size(points, boxes, masks) + sparse_embeddings = torch.empty((bs, 0, self.embed_dim), device=self._get_device()) + if points is not None: + coords, labels = points + point_embeddings = self._embed_points(coords, labels, pad=(boxes is None)) + sparse_embeddings = torch.cat([sparse_embeddings, point_embeddings], dim=1) + if boxes is not None: + box_embeddings = self._embed_boxes(boxes) + sparse_embeddings = torch.cat([sparse_embeddings, box_embeddings], dim=1) + + if masks is not None: + dense_embeddings = self._embed_masks(masks) + else: + dense_embeddings = self.no_mask_embed.weight.reshape(1, -1, 1, 1).expand( + bs, -1, self.image_embedding_size[0], self.image_embedding_size[1] + ) + + return sparse_embeddings, dense_embeddings + + +class PositionEmbeddingRandom(nn.Module): + """Positional encoding using random spatial frequencies.""" + + def __init__(self, num_pos_feats: int = 64, scale: Optional[float] = None) -> None: + super().__init__() + if scale is None or scale <= 0.0: + scale = 1.0 + self.register_buffer("positional_encoding_gaussian_matrix", scale * torch.randn((2, num_pos_feats))) + + def _pe_encoding(self, coords: torch.Tensor) -> torch.Tensor: + """Positionally encode points that are normalized to [0,1].""" + # assuming coords are in [0, 1]^2 square and have d_1 x ... x d_n x 2 shape + coords = 2 * coords - 1 + coords = coords @ self.positional_encoding_gaussian_matrix + coords = 2 * pi * coords + # outputs d_1 x ... x d_n x C shape + return torch.cat([torch.sin(coords), torch.cos(coords)], dim=-1) + + def forward(self, size: tuple[int, int]) -> torch.Tensor: + """Generate positional encoding for a grid of the specified size.""" + h, w = size + _dev = self.positional_encoding_gaussian_matrix.device + device = _dev if isinstance(_dev, torch.device) else None + grid = torch.ones((h, w), device=device, dtype=torch.float32) + y_embed = grid.cumsum(dim=0) - 0.5 + x_embed = grid.cumsum(dim=1) - 0.5 + y_embed = y_embed / h + x_embed = x_embed / w + + pe = self._pe_encoding(torch.stack([x_embed, y_embed], dim=-1)) + return pe.permute(2, 0, 1) # C x H x W + + def forward_with_coords(self, coords_input: torch.Tensor, image_size: tuple[int, int]) -> torch.Tensor: + """Positionally encode points that are not normalized to [0,1].""" + coords = coords_input.clone() + coords[:, :, 0] = coords[:, :, 0] / image_size[1] + coords[:, :, 1] = coords[:, :, 1] / image_size[0] + return self._pe_encoding(coords.to(torch.float32)) # B x N x C diff --git a/kornia/models/sam/architecture/transformer.py b/kornia/models/sam/architecture/transformer.py new file mode 100644 index 0000000..f2d9f88 --- /dev/null +++ b/kornia/models/sam/architecture/transformer.py @@ -0,0 +1,291 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Based from the original code from Meta Platforms, Inc. and affiliates. + +https://github.com/facebookresearch/segment- +anything/blob/3518c86b78b3bc9cf4fbe3d18e682fad1c79dc51/segment_anything/modeling/transformer.py +""" + +from __future__ import annotations + +import math + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK +from kornia.models.sam.architecture.common import MLPBlock + + +class TwoWayTransformer(nn.Module): + """Implement a two-way transformer for interaction between image and prompt embeddings. + + This transformer allows for simultaneous attention between the image features + and the prompt tokens (e.g., points or boxes). + + Args: + depth: The number of transformer layers. + embedding_dim: The dimension of the embeddings. + num_heads: The number of heads in the multi-head attention. + mlp_dim: The dimension of the internal MLP layers. + activation: The activation function to use. Default: :class:`nn.GELU`. + attention_type: The type of attention mechanism to apply. Default: "unnorm". + """ + + def __init__( + self, + depth: int, + embedding_dim: int, + num_heads: int, + mlp_dim: int, + activation: type[nn.Module] = nn.ReLU, + attention_downsample_rate: int = 2, + ) -> None: + """Construct a transformer decoder that attends to an input image using queries whose positional embedding is + supplied. + + Args: + depth: number of layers in the transformer + embedding_dim: the channel dimension for the input embeddings + num_heads: the number of heads for multihead attention. Must divide embedding_dim + mlp_dim: the channel dimension internal to the MLP block + activation: the activation to use in the MLP block + attention_downsample_rate: downsampling rate from embedding dimension + + """ # noqa: D205 + super().__init__() + self.depth = depth + self.embedding_dim = embedding_dim + self.num_heads = num_heads + self.mlp_dim = mlp_dim + self.layers = nn.ModuleList() + + for i in range(depth): + self.layers.append( + TwoWayAttentionBlock( + embedding_dim=embedding_dim, + num_heads=num_heads, + mlp_dim=mlp_dim, + activation=activation, + attention_downsample_rate=attention_downsample_rate, + skip_first_layer_pe=(i == 0), + ) + ) + + self.final_attn_token_to_image = Attention(embedding_dim, num_heads, downsample_rate=attention_downsample_rate) + self.norm_final_attn = nn.LayerNorm(embedding_dim) + + def forward( + self, image_embedding: torch.Tensor, image_pe: torch.Tensor, point_embedding: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + """Run forward. + + Args: + image_embedding: image to attend to. Should be shape B x embedding_dim x h x w for any h and w. + image_pe: the positional encoding to add to the image. Must have the same shape as image_embedding. + point_embedding: the embedding to add to the query points. Must have shape B x N_points x embedding_dim + for any N_points. + + Returns: + - the processed point_embedding + - the processed image_embedding + + """ + # BxCxHxW -> BxHWxC == B x N_image_tokens x C + _bs, _c, _h, _w = image_embedding.shape + image_embedding = image_embedding.flatten(2).permute(0, 2, 1) + image_pe = image_pe.flatten(2).permute(0, 2, 1) + + # Prepare queries + queries = point_embedding + keys = image_embedding + + # Apply transformer blocks and final layernorm + for layer in self.layers: + queries, keys = layer(queries=queries, keys=keys, query_pe=point_embedding, key_pe=image_pe) + + # Apply the final attenion layer from the points to the image + q = queries + point_embedding + k = keys + image_pe + attn_out = self.final_attn_token_to_image(q=q, k=k, v=keys) + queries = queries + attn_out + queries = self.norm_final_attn(queries) + + return queries, keys + + +class TwoWayAttentionBlock(nn.Module): + """Implement a single block for the Two-Way Transformer. + + This block performs self-attention on tokens, cross-attention from tokens to + image features, and cross-attention from image features to tokens. + + Args: + embedding_dim: The dimension of the embeddings. + num_heads: The number of attention heads. + mlp_dim: The dimension of the internal MLP layers. Default: 2048. + activation: The activation function to use. Default: :class:`nn.GELU`. + attention_type: The type of attention mechanism. Default: "unnorm". + skip_first_layer_pe: Whether to skip position encoding in the first layer. Default: False. + """ + + def __init__( + self, + embedding_dim: int, + num_heads: int, + mlp_dim: int = 2048, + activation: type[nn.Module] = nn.ReLU, + attention_downsample_rate: int = 2, + skip_first_layer_pe: bool = False, + ) -> None: + """Construct a transformer block with four layers. + + (1) self-attention of sparse inputs, (2) cross attention of + sparse inputs to dense inputs, (3) mlp block on sparse inputs, and (4) cross attention of dense inputs to sparse + inputs. + + Args: + embedding_dim: the channel dimension of the embeddings + num_heads: the number of heads in the attention layers + mlp_dim: the hidden dimension of the mlp block + activation: the activation of the mlp block + skip_first_layer_pe: skip the PE on the first layer + attention_downsample_rate: downsampling rate from embedding dimension + + """ + super().__init__() + self.self_attn = Attention(embedding_dim, num_heads) + self.norm1 = nn.LayerNorm(embedding_dim) + + self.cross_attn_token_to_image = Attention(embedding_dim, num_heads, downsample_rate=attention_downsample_rate) + self.norm2 = nn.LayerNorm(embedding_dim) + + self.mlp = MLPBlock(embedding_dim, mlp_dim, activation) + self.norm3 = nn.LayerNorm(embedding_dim) + + self.norm4 = nn.LayerNorm(embedding_dim) + self.cross_attn_image_to_token = Attention(embedding_dim, num_heads, downsample_rate=attention_downsample_rate) + + self.skip_first_layer_pe = skip_first_layer_pe + + def forward( + self, queries: torch.Tensor, keys: torch.Tensor, query_pe: torch.Tensor, key_pe: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + """Run one two-way transformer block for prompt and image tokens. + + Args: + queries: Prompt token tensor with shape :math:`(B, N_q, D)`. + keys: Image token tensor with shape :math:`(B, N_k, D)`. + query_pe: Positional encoding for ``queries`` with the same shape. + key_pe: Positional encoding for ``keys`` with the same shape. + + Returns: + Tuple ``(queries, keys)`` after prompt self-attention, + prompt-to-image cross-attention, MLP update, and image-to-prompt + cross-attention. + """ + # Self attention block + if self.skip_first_layer_pe: + queries = self.self_attn(q=queries, k=queries, v=queries) + else: + q = queries + query_pe + attn_out = self.self_attn(q=q, k=q, v=queries) + queries = queries + attn_out + queries = self.norm1(queries) + + # Cross attention block, tokens attending to image embedding + q = queries + query_pe + k = keys + key_pe + attn_out = self.cross_attn_token_to_image(q=q, k=k, v=keys) + queries = queries + attn_out + queries = self.norm2(queries) + + # MLP block + mlp_out = self.mlp(queries) + queries = queries + mlp_out + queries = self.norm3(queries) + + # Cross attention block, image embedding attending to tokens + q = queries + query_pe + k = keys + key_pe + attn_out = self.cross_attn_image_to_token(q=k, k=q, v=queries) + keys = keys + attn_out + keys = self.norm4(keys) + + return queries, keys + + +class Attention(nn.Module): + """Attention layer that allows for downscaling the embedding after projection to queries, keys, and values.""" + + def __init__(self, embedding_dim: int, num_heads: int, downsample_rate: int = 1) -> None: + super().__init__() + self.embedding_dim = embedding_dim + self.internal_dim = embedding_dim // downsample_rate + self.num_heads = num_heads + KORNIA_CHECK(self.internal_dim % num_heads == 0, "num_heads must divide embedding_dim.") + + self.q_proj = nn.Linear(embedding_dim, self.internal_dim) + self.k_proj = nn.Linear(embedding_dim, self.internal_dim) + self.v_proj = nn.Linear(embedding_dim, self.internal_dim) + self.out_proj = nn.Linear(self.internal_dim, embedding_dim) + + def _separate_heads(self, x: torch.Tensor, num_heads: int) -> torch.Tensor: + b, n, c = x.shape + x = x.reshape(b, n, num_heads, c // num_heads) + return x.transpose(1, 2) # B x N_heads x N_tokens x C_per_head + + def _recombine_heads(self, x: torch.Tensor) -> torch.Tensor: + b, n_heads, n_tokens, c_per_head = x.shape + x = x.transpose(1, 2) + return x.reshape(b, n_tokens, n_heads * c_per_head) # B x N_tokens x C + + def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor: + """Apply scaled dot-product attention to query, key, and value tokens. + + Args: + q: Query tensor with shape :math:`(B, N_q, D)`. + k: Key tensor with shape :math:`(B, N_k, D)`. + v: Value tensor with shape :math:`(B, N_k, D)`. + + Returns: + Tensor with shape :math:`(B, N_q, D)` after multi-head attention + and output projection. + """ + # Input projections + q = self.q_proj(q) + k = self.k_proj(k) + v = self.v_proj(v) + + # Separate into heads + q = self._separate_heads(q, self.num_heads) + k = self._separate_heads(k, self.num_heads) + v = self._separate_heads(v, self.num_heads) + + # Attention + _, _, _, c_per_head = q.shape + attn = q @ k.permute(0, 1, 3, 2) # B x N_heads x N_tokens x N_tokens + attn = attn / math.sqrt(c_per_head) + attn = attn.softmax(dim=-1) + + # Get output + out = attn @ v + out = self._recombine_heads(out) + out = self.out_proj(out) + + return out diff --git a/kornia/models/sam/model.py b/kornia/models/sam/model.py new file mode 100644 index 0000000..c7d31a4 --- /dev/null +++ b/kornia/models/sam/model.py @@ -0,0 +1,409 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Based from the original code from Meta Platforms, Inc. and affiliates. + +https://github.com/facebookresearch/segment- +anything/blob/3518c86b78b3bc9cf4fbe3d18e682fad1c79dc51/segment_anything/build_sam.py + +https://github.com/facebookresearch/segment- +anything/blob/3518c86b78b3bc9cf4fbe3d18e682fad1c79dc51/segment_anything/modeling/sam.py +""" + +from __future__ import annotations + +import warnings +from dataclasses import dataclass +from enum import Enum +from typing import Any, Optional + +import torch + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SHAPE +from kornia.core.mixin.onnx import ONNXExportMixin +from kornia.models.base import ModelBase +from kornia.models.sam.architecture.common import LayerNorm +from kornia.models.sam.architecture.image_encoder import ImageEncoderViT +from kornia.models.sam.architecture.mask_decoder import MaskDecoder +from kornia.models.sam.architecture.prompt_encoder import PromptEncoder +from kornia.models.sam.architecture.transformer import TwoWayTransformer +from kornia.models.structures import SegmentationResults +from kornia.models.tiny_vit import TinyViT + + +class SamModelType(Enum): + """Map the SAM model types.""" + + vit_h = 0 + vit_l = 1 + vit_b = 2 + mobile_sam = 3 + + +@dataclass +class SamConfig: + """Encapsulate the Config to build a SAM model. + + Args: + model_type: the available models are: + + - 0, 'vit_h' or :func:`kornia.contrib.sam.SamModelType.vit_h` + - 1, 'vit_l' or :func:`kornia.contrib.sam.SamModelType.vit_l` + - 2, 'vit_b' or :func:`kornia.contrib.sam.SamModelType.vit_b` + - 3, 'mobile_sam', or :func:`kornia.contrib.sam.SamModelType.mobile_sam` + + checkpoint: URL or a path for a file with the weights of the model + encoder_embed_dim: Patch embedding dimension. + encoder_depth: Depth of ViT. + encoder_num_heads: Number of attention heads in each ViT block. + encoder_global_attn_indexes: Encoder indexes for blocks using global attention. + + """ + + model_type: Optional[str | int | SamModelType] = None + checkpoint: Optional[str] = None + pretrained: bool = False + + encoder_embed_dim: Optional[int] = None + encoder_depth: Optional[int] = None + encoder_num_heads: Optional[int] = None + encoder_global_attn_indexes: Optional[tuple[int, ...]] = None + + +class Sam(ONNXExportMixin, ModelBase[SamConfig]): + """Implement the Segment Anything Model (SAM) wrapper. + + This class coordinates the image encoder, prompt encoder, and mask decoder. + """ + + mask_threshold: float = 0.0 + + def __init__( + self, image_encoder: ImageEncoderViT | TinyViT, prompt_encoder: PromptEncoder, mask_decoder: MaskDecoder + ) -> None: + """SAM predicts object masks from an image and input prompts. + + Args: + image_encoder: The backbone used to encode the image into image embeddings that allow for efficient mask + prediction. + prompt_encoder: Encodes various types of input prompts. + mask_decoder: Predicts masks from the image embeddings and encoded prompts. + + """ + super().__init__() + self.image_encoder = image_encoder + self.prompt_encoder = prompt_encoder + self.mask_decoder = mask_decoder + + @staticmethod + def from_name(name: str) -> Sam: + """Build/load the SAM model based on it's name. + + Args: + name: The name of the SAM model. Valid names are: + - 'vit_b' + - 'vit_l' + - 'vit_h' + - 'mobile_sam' + + Returns: + The respective SAM model + + """ + if name in ["vit_b", "vit_l", "vit_h", "mobile_sam"]: + return Sam.from_config(SamConfig(name)) + else: + raise ValueError(f"Invalid SAM model name: {name}") + + @staticmethod + def from_config(config: SamConfig) -> Sam: + """Build/load the SAM model based on it's config. + + Args: + config: The SamConfig data structure. If the model_type is available, build from it, otherwise will use + the parameters set. + + Returns: + The respective SAM model + + Example: + >>> from kornia.models.sam import SamConfig + >>> sam_model = Sam.from_config(SamConfig('vit_b')) + + """ + model_type = config.model_type + + if isinstance(model_type, int): + model_type = SamModelType(model_type) + elif isinstance(model_type, str): + _map_sam_type = { + "vit_h": SamModelType.vit_h, + "vit_l": SamModelType.vit_l, + "vit_b": SamModelType.vit_b, + "mobile_sam": SamModelType.mobile_sam, + } + model_type = _map_sam_type[model_type] + + if model_type == SamModelType.vit_b: + model = _build_sam( + encoder_embed_dim=768, encoder_depth=12, encoder_num_heads=12, encoder_global_attn_indexes=(2, 5, 8, 11) + ) + + elif model_type == SamModelType.vit_l: + model = _build_sam( + encoder_embed_dim=1024, + encoder_depth=24, + encoder_num_heads=16, + encoder_global_attn_indexes=(5, 11, 17, 23), + ) + + elif model_type == SamModelType.vit_h: + model = _build_sam( + encoder_embed_dim=1280, + encoder_depth=32, + encoder_num_heads=16, + encoder_global_attn_indexes=(7, 15, 23, 31), + ) + + elif model_type == SamModelType.mobile_sam: + # TODO: merge this with _build_sam() + prompt_embed_dim = 256 + image_size = 1024 + vit_patch_size = 16 + image_embedding_size = image_size // vit_patch_size + + model = Sam( + image_encoder=TinyViT.from_config("5m", img_size=image_size, mobile_sam=True), + prompt_encoder=PromptEncoder( + embed_dim=prompt_embed_dim, + image_embedding_size=(image_embedding_size, image_embedding_size), + input_image_size=(image_size, image_size), + mask_in_chans=16, + ), + mask_decoder=MaskDecoder( + num_multimask_outputs=3, + transformer=TwoWayTransformer(depth=2, embedding_dim=prompt_embed_dim, mlp_dim=2048, num_heads=8), + transformer_dim=prompt_embed_dim, + iou_head_depth=3, + iou_head_hidden_dim=256, + ), + # pixel_mean=[123.675, 116.28, 103.53], + # pixel_std=[58.395, 57.12, 57.375], + ) + + elif ( + isinstance(config.encoder_embed_dim, int) + and isinstance(config.encoder_depth, int) + and isinstance(config.encoder_num_heads, int) + and isinstance(config.encoder_global_attn_indexes, int) + ): + model = _build_sam( + encoder_embed_dim=config.encoder_embed_dim, + encoder_depth=config.encoder_depth, + encoder_num_heads=config.encoder_num_heads, + encoder_global_attn_indexes=config.encoder_global_attn_indexes, + ) + + else: + raise NotImplementedError("Unexpected config. The model_type should be provide or the encoder configs.") + + checkpoint = config.checkpoint + if config.pretrained: + if checkpoint is None: + checkpoint = { + SamModelType.vit_b: "https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth", + SamModelType.vit_l: "https://dl.fbaipublicfiles.com/segment_anything/sam_vit_l_0b3195.pth", + SamModelType.vit_h: "https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth", + SamModelType.mobile_sam: "https://github.com/ChaoningZhang/MobileSAM/raw/a509aac54fdd7af59f843135f2f7cee307283c88/weights/mobile_sam.pt", + }[model_type] + else: + warnings.warn("checkpoint is not None. pretrained=True is ignored", stacklevel=1) + + if checkpoint: + model.load_checkpoint(checkpoint) + + return model + + def to_onnx( + self, + onnx_name: Optional[str] = None, + pseudo_shape: Optional[list[int]] = None, + save: bool = True, + **kwargs: Any, + ) -> Any: + """Export SAM's image encoder to ONNX. + + SAM's full :meth:`forward` signature accepts non-tensor inputs + (``batched_prompts: list[dict]``, ``multimask_output: bool``) and returns + a Python list of :class:`~kornia.models.structures.SegmentationResults`, + which cannot be directly exported via :func:`torch.onnx.export`. + + This override exports only the **image encoder** subgraph + (``self.image_encoder``), which is a pure ``(B, 3, H, W) -> (B, C, H', W')`` + tensor-to-tensor module and fully ONNX-compatible. The encoder embeddings + can then be fed into a separate prompt-encoder / mask-decoder pipeline. + + Args: + onnx_name: Path for the saved ``.onnx`` file. Defaults to + ``"Kornia-Sam-ImageEncoder.onnx"``. + pseudo_shape: Concrete input shape used to trace the encoder, e.g. + ``[1, 3, 1024, 1024]``. Defaults to ``[1, 3, 1024, 1024]``. + save: Whether to write the model to disk. Default ``True``. + **kwargs: Additional keyword arguments forwarded to + :func:`torch.onnx.export`. + + Returns: + ``onnx.ModelProto`` of the exported image encoder. + + """ + if onnx_name is None: + onnx_name = "Kornia-Sam-ImageEncoder.onnx" + if pseudo_shape is None: + pseudo_shape = [1, 3, 1024, 1024] + kwargs.setdefault("output_names", ["image_embeddings"]) + kwargs.setdefault( + "dynamic_axes", + { + "input": {0: "batch"}, + "image_embeddings": {0: "batch"}, + }, + ) + return super().to_onnx( + onnx_name=onnx_name, + input_shape=[-1, 3, -1, -1], + pseudo_shape=pseudo_shape, + model=self.image_encoder, + save=save, + **kwargs, + ) + + @torch.no_grad() + def forward( + self, images: torch.Tensor, batched_prompts: list[dict[str, Any]], multimask_output: bool + ) -> list[SegmentationResults]: + """Predicts masks end-to-end from provided images and prompts. + + This method expects that the images have already been pre-processed, at least been normalized, resized and + padded to be compatible with the `self.image_encoder`. + + .. note:: For each image :math:`(3, H, W)`, it is possible to input a batch (:math:`K`) of :math:`N` prompts, + the results are batched by the number of prompts batch. So given a prompt with :math:`K=5`, and + :math:`N=10`, the results will look like :math:`5xCxHxW` where :math:`C` is determined by + multimask_output. And within each of these masks :math:`(5xC)`, it should be possible to find + :math:`N` instances if the model succeed. + + Args: + images: The image as a torch tensor in :math:`(B, 3, H, W)` format, already transformed for input to the + model. + batched_prompts: A list over the batch of images (list length should be :math:`B`), each a dictionary with + the following keys. If it does not have the respective prompt, it should not be included + in this dictionary. The options are: + + - "points": tuple of (torch.Tensor, torch.Tensor) within the coordinate keypoints + and their respective labels. The tuple should look like (keypoints, labels), where the keypoints + (a tensor) are a batched point prompts for this image, with shape :math:`(K, N, 2)`. Already + transformed to the input frame of the model. The labels (a tensor) are a batched labels for point + prompts, with shape :math:`(K, N)`. Where 1 indicates a foreground point and 0 indicates a background + point. + + - "boxes": (torch.Tensor) Batched box inputs, with shape :math:`(K, 4)`. + Already transformed to the input frame of the model. + + - "mask_inputs": (torch.Tensor) Batched mask inputs to the model, in the form :math:`(K, 1, H, W)`. + + multimask_output: Whether the model should predict multiple disambiguating masks, or return a single mask. + + Returns: + A list over input images, where each element is as SegmentationResults the following: + + - logits: Low resolution logits with shape :math:`(K, C, H, W)`. Can be passed as mask input to + subsequent iterations of prediction. Where :math:`K` is the number of input prompts, + :math:`C` is determined by multimask_output, and :math:`H=W=256` are the model output size. + - scores: The model's predictions of mask quality (iou prediction), in shape BxC. + + """ + KORNIA_CHECK_SHAPE(images, ["B", "3", "H", "W"]) + KORNIA_CHECK( + images.shape[0] == len(batched_prompts), + "The number of images (`B`) should match with the length of prompts!", + ) + + image_embeddings = self.image_encoder(images) + + outputs = [] + for prompt_record, curr_embedding in zip(batched_prompts, image_embeddings): + # Embed prompts + sparse_embeddings, dense_embeddings = self.prompt_encoder( + points=prompt_record.get("points", None), + boxes=prompt_record.get("boxes", None), + masks=prompt_record.get("mask_inputs", None), + ) + + # Predict masks + low_res_logits, iou_predictions = self.mask_decoder( + image_embeddings=curr_embedding[None, ...], + image_pe=self.prompt_encoder.get_dense_pe(), + sparse_prompt_embeddings=sparse_embeddings, + dense_prompt_embeddings=dense_embeddings, + multimask_output=multimask_output, + ) + + # Save results + outputs.append(SegmentationResults(low_res_logits, iou_predictions, self.mask_threshold)) + + return outputs + + +def _build_sam( + encoder_embed_dim: int, encoder_depth: int, encoder_num_heads: int, encoder_global_attn_indexes: tuple[int, ...] +) -> Sam: + prompt_embed_dim = 256 + image_size = 1024 + vit_patch_size = 16 + image_embedding_size = image_size // vit_patch_size + + return Sam( + image_encoder=ImageEncoderViT( + depth=encoder_depth, + embed_dim=encoder_embed_dim, + img_size=image_size, + mlp_ratio=4, + norm_layer=LayerNorm, + num_heads=encoder_num_heads, + patch_size=vit_patch_size, + qkv_bias=True, + use_rel_pos=True, + global_attn_indexes=encoder_global_attn_indexes, + window_size=14, + out_chans=prompt_embed_dim, + ), + prompt_encoder=PromptEncoder( + embed_dim=prompt_embed_dim, + image_embedding_size=(image_embedding_size, image_embedding_size), + input_image_size=(image_size, image_size), + mask_in_chans=16, + ), + mask_decoder=MaskDecoder( + num_multimask_outputs=3, + transformer=TwoWayTransformer(depth=2, embedding_dim=prompt_embed_dim, mlp_dim=2048, num_heads=8), + transformer_dim=prompt_embed_dim, + iou_head_depth=3, + iou_head_hidden_dim=256, + ), + # pixel_mean=[123.675, 116.28, 103.53], + # pixel_std=[58.395, 57.12, 57.375], + ) diff --git a/kornia/models/sam3/__init__.py b/kornia/models/sam3/__init__.py new file mode 100644 index 0000000..a0c78c9 --- /dev/null +++ b/kornia/models/sam3/__init__.py @@ -0,0 +1,29 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""SAM-3 (Segment Anything Model v3) integration for Kornia. + +This module provides the Segment Anything Model v3 (SAM-3) implementation for image segmentation tasks. +SAM-3 is a foundation model for image segmentation that can segment any object in an image with zero-shot +capability. +""" + +from __future__ import annotations + +from .architecture import ImageEncoderHiera + +__all__ = ["ImageEncoderHiera"] diff --git a/kornia/models/sam3/architecture/__init__.py b/kornia/models/sam3/architecture/__init__.py new file mode 100644 index 0000000..a978019 --- /dev/null +++ b/kornia/models/sam3/architecture/__init__.py @@ -0,0 +1,37 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""SAM-3 architecture modules for Kornia.""" + +from __future__ import annotations + +from .common import Attention, MLPBlock +from .image_encoder import ImageEncoderHiera, PatchEmbedding, ViTBlock +from .mask_decoder import CrossAttentionTransformer, MaskDecoder +from .prompt_encoder import PositionalEncoding, PromptEncoder + +__all__ = [ + "Attention", + "CrossAttentionTransformer", + "ImageEncoderHiera", + "MLPBlock", + "MaskDecoder", + "PatchEmbedding", + "PositionalEncoding", + "PromptEncoder", + "ViTBlock", +] diff --git a/kornia/models/sam3/architecture/common.py b/kornia/models/sam3/architecture/common.py new file mode 100644 index 0000000..5e67510 --- /dev/null +++ b/kornia/models/sam3/architecture/common.py @@ -0,0 +1,115 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Common components for SAM-3 architecture. + +This module provides shared building blocks for SAM-3 including: +- Normalization layers +- Activation functions +- Attention primitives +- Helper utilities +""" + +from __future__ import annotations + +import torch +import torch.nn.functional as F +from torch import nn + + +class MLPBlock(nn.Module): + """Multi-layer Perceptron block. + + A simple feedforward network with two linear layers and GELU activation. + """ + + def __init__(self, embedding_dim: int, mlp_dim: int) -> None: + """Initialize MLPBlock. + + Args: + embedding_dim: Dimension of input and output features. + mlp_dim: Dimension of the hidden layer. + """ + super().__init__() + self.lin1 = nn.Linear(embedding_dim, mlp_dim) + self.lin2 = nn.Linear(mlp_dim, embedding_dim) + self.act = nn.GELU() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply MLP block. + + Args: + x: Input tensor. + + Returns: + Output tensor of the same shape as input. + """ + return self.lin2(self.act(self.lin1(x))) + + +class Attention(nn.Module): + """Multi-head attention block. + + Standard multi-head self-attention implementation following the Transformer architecture. + """ + + def __init__(self, dim: int, heads: int = 8, dim_head: int = 64, dropout: float = 0.0) -> None: + """Initialize Attention. + + Args: + dim: Dimension of the input features. Should be divisible by heads * dim_head if project_out=False. + heads: Number of attention heads. + dim_head: Dimension of each attention head. + dropout: Dropout probability. + """ + super().__init__() + inner_dim = dim_head * heads + project_out = not (heads == 1 and dim_head == dim) + + self.heads = heads + self.dropout = nn.Dropout(dropout) + + self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False) + + self.to_out = ( + nn.Sequential( + nn.Linear(inner_dim, dim), + nn.Dropout(dropout), + ) + if project_out + else nn.Identity() + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply multi-head attention. + + Args: + x: Input tensor of shape (B, N, D) where B is batch size, N is sequence length, D is feature dimension. + + Returns: + Output tensor of the same shape as input. + """ + qkv = self.to_qkv(x).chunk(3, dim=-1) + q, k, v = tuple(t.view(t.shape[0], t.shape[1], self.heads, -1).transpose(1, 2) for t in qkv) + + dropout_p = self.dropout.p if self.training and self.dropout.p > 0.0 else 0.0 + out = F.scaled_dot_product_attention(q, k, v, dropout_p=dropout_p) + out = out.transpose(1, 2).contiguous().view(x.shape[0], x.shape[1], -1) + return self.to_out(out) + + +__all__ = ["Attention", "MLPBlock"] diff --git a/kornia/models/sam3/architecture/image_encoder.py b/kornia/models/sam3/architecture/image_encoder.py new file mode 100644 index 0000000..cc1d25f --- /dev/null +++ b/kornia/models/sam3/architecture/image_encoder.py @@ -0,0 +1,233 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""SAM-3 Image Encoder (Hiera-based backbone). + +This module implements the Hiera-based image encoder for SAM-3. +The encoder processes input images and extracts multi-scale feature representations. +""" + +from __future__ import annotations + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SHAPE + +from .common import Attention, MLPBlock + + +class PatchEmbedding(nn.Module): + """Patch embedding layer for vision transformers. + + Converts image patches to embeddings through a convolutional layer. + """ + + def __init__(self, img_size: int, patch_size: int, in_channels: int, embed_dim: int) -> None: + """Initialize PatchEmbedding. + + Args: + img_size: Input image size (assumed square). + patch_size: Size of each patch. + in_channels: Number of input channels (typically 3 for RGB). + embed_dim: Embedding dimension. + """ + super().__init__() + self.img_size = img_size + self.patch_size = patch_size + self.num_patches = (img_size // patch_size) ** 2 + + self.proj = nn.Conv2d(in_channels, embed_dim, kernel_size=patch_size, stride=patch_size) + self.norm = nn.LayerNorm(embed_dim) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply patch embedding. + + Args: + x: Input tensor of shape (B, C, H, W). + + Returns: + Tensor of shape (B, num_patches, embed_dim). + + Raises: + ValueError: If image dimensions are not divisible by patch_size. + """ + KORNIA_CHECK_SHAPE(x, ["B", "C", "H", "W"]) + KORNIA_CHECK( + x.shape[2] % self.patch_size == 0 and x.shape[3] % self.patch_size == 0, + f"Image dimensions must be divisible by patch_size={self.patch_size}, got H={x.shape[2]}, W={x.shape[3]}", + ) + x = self.proj(x) # (B, embed_dim, H//patch_size, W//patch_size) + x = x.flatten(2).transpose(1, 2) # (B, H*W, embed_dim) + x = self.norm(x) + return x + + +class ViTBlock(nn.Module): + """Vision Transformer block with self-attention and MLP. + + Standard transformer block used in Vision Transformers. + """ + + def __init__(self, dim: int, num_heads: int, mlp_ratio: float = 4.0, dropout: float = 0.0) -> None: + """Initialize ViTBlock. + + Args: + dim: Embedding dimension. + num_heads: Number of attention heads. + mlp_ratio: Ratio of mlp hidden dim to embedding dim. + dropout: Dropout probability. + """ + super().__init__() + self.norm1 = nn.LayerNorm(dim) + self.attn = Attention(dim, heads=num_heads, dropout=dropout) + self.norm2 = nn.LayerNorm(dim) + mlp_dim = int(dim * mlp_ratio) + self.mlp = MLPBlock(dim, mlp_dim) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply ViT block. + + Args: + x: Input tensor of shape (B, N, D). + + Returns: + Output tensor of the same shape. + """ + x = x + self.attn(self.norm1(x)) + x = x + self.mlp(self.norm2(x)) + return x + + +class ImageEncoderHiera(nn.Module): + """Hiera-based image encoder for SAM-3. + + This encoder extracts multi-scale features from input images using a Hiera backbone. + The Hiera architecture is designed for efficient hierarchical feature extraction. + """ + + def __init__( + self, + img_size: int = 1024, + patch_size: int = 16, + in_channels: int = 3, + embed_dim: int = 768, + depth: int = 12, + num_heads: int = 12, + mlp_ratio: float = 4.0, + dropout: float = 0.0, + ) -> None: + """Initialize ImageEncoderHiera. + + Args: + img_size: Input image size (assumed square). Default: 1024 + patch_size: Patch size for patch embedding. Default: 16 + in_channels: Number of input channels. Default: 3 + embed_dim: Embedding dimension. Default: 768 + depth: Number of transformer blocks. Default: 12 + num_heads: Number of attention heads. Default: 12 + mlp_ratio: Ratio of mlp hidden dim to embedding dim. Default: 4.0 + dropout: Dropout probability. Default: 0.0 + """ + super().__init__() + self.img_size = img_size + self.patch_size = patch_size + self.embed_dim = embed_dim + self.depth = depth + self.num_heads = num_heads + + # Patch embedding + self.patch_embed = PatchEmbedding(img_size, patch_size, in_channels, embed_dim) + + # Positional embedding + num_patches = (img_size // patch_size) ** 2 + self.pos_embed = nn.Parameter(torch.zeros(1, num_patches, embed_dim)) + nn.init.trunc_normal_(self.pos_embed, std=0.02) + + # Transformer blocks + self.blocks = nn.ModuleList([ViTBlock(embed_dim, num_heads, mlp_ratio, dropout) for _ in range(depth)]) + + # Final norm + self.norm = nn.LayerNorm(embed_dim) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Forward pass of the image encoder. + + Args: + x: Input tensor of shape (B, C, H, W) where C should match in_channels and H, W should match img_size. + + Returns: + Tensor of shape (B, num_patches, embed_dim) containing the encoded features. + + Raises: + ValueError: If input channels don't match in_channels or image size doesn't match img_size. + + Example: + >>> encoder = ImageEncoderHiera(img_size=1024, patch_size=16, embed_dim=768) + >>> images = torch.randn(1, 3, 1024, 1024) + >>> features = encoder(images) + >>> features.shape + torch.Size([1, 4096, 768]) + """ + expected_channels = self.patch_embed.proj.in_channels + KORNIA_CHECK_SHAPE(x, ["B", str(expected_channels), "H", "W"]) + KORNIA_CHECK( + x.shape[2] == self.img_size and x.shape[3] == self.img_size, + f"Input image size must be {self.img_size}x{self.img_size}, got {x.shape[2]}x{x.shape[3]}", + ) + + # Patch embedding + x = self.patch_embed(x) # (B, num_patches, embed_dim) + + # Add positional embedding + x = x + self.pos_embed + + # Apply transformer blocks + for block in self.blocks: + x = block(x) + + # Final normalization + x = self.norm(x) + + return x + + def get_output_shape(self, input_shape: tuple[int, ...]) -> tuple[int, ...]: + """Get output shape given input shape. + + Args: + input_shape: Input tensor shape (B, C, H, W). + + Returns: + Output tensor shape (B, num_patches, embed_dim). + + Raises: + ValueError: If input_shape doesn't have 4 dimensions or dimensions are not divisible by patch_size. + """ + KORNIA_CHECK( + len(input_shape) == 4, + f"Input shape must have 4 dimensions (B, C, H, W), got {len(input_shape)}", + ) + B, _, H, W = input_shape + KORNIA_CHECK( + H % self.patch_size == 0 and W % self.patch_size == 0, + f"Image dimensions must be divisible by patch_size={self.patch_size}, got H={H}, W={W}", + ) + num_patches = (H // self.patch_size) * (W // self.patch_size) + return (B, num_patches, self.embed_dim) + + +__all__ = ["ImageEncoderHiera", "PatchEmbedding", "ViTBlock"] diff --git a/kornia/models/sam3/architecture/mask_decoder.py b/kornia/models/sam3/architecture/mask_decoder.py new file mode 100644 index 0000000..8d48b64 --- /dev/null +++ b/kornia/models/sam3/architecture/mask_decoder.py @@ -0,0 +1,253 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""SAM-3 Mask Decoder for generating segmentation masks. + +This module implements the mask decoder for SAM-3 which takes image embeddings +and prompt embeddings to generate segmentation masks and IoU predictions. +""" + +from __future__ import annotations + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK + +from .common import Attention, MLPBlock + + +class CrossAttentionTransformer(nn.Module): + """Transformer block with cross-attention between image and prompt embeddings. + + Applies cross-attention from prompt embeddings to image embeddings, followed by self-attention. + """ + + def __init__(self, embed_dim: int, num_heads: int = 8, mlp_ratio: float = 4.0) -> None: + """Initialize CrossAttentionTransformer. + + Args: + embed_dim: Embedding dimension. + num_heads: Number of attention heads. + mlp_ratio: Ratio of mlp hidden dim to embedding dim. + """ + super().__init__() + self.norm1 = nn.LayerNorm(embed_dim) + self.cross_attn = nn.MultiheadAttention(embed_dim, num_heads, batch_first=True) + self.norm2 = nn.LayerNorm(embed_dim) + self.self_attn = Attention(embed_dim, heads=num_heads) + self.norm3 = nn.LayerNorm(embed_dim) + mlp_dim = int(embed_dim * mlp_ratio) + self.mlp = MLPBlock(embed_dim, mlp_dim) + + def forward( + self, + prompts: torch.Tensor, + image_embeddings: torch.Tensor, + ) -> torch.Tensor: + """Apply cross-attention transformer. + + Args: + prompts: Prompt embeddings of shape (B, M, D). + image_embeddings: Image embeddings of shape (B, N, D). + + Returns: + Updated prompt embeddings of shape (B, M, D). + """ + # Cross-attention: prompts attend to image embeddings + prompts_norm = self.norm1(prompts) + attn_out, _ = self.cross_attn(prompts_norm, image_embeddings, image_embeddings) + prompts = prompts + attn_out + + # Self-attention on prompts + prompts_norm = self.norm2(prompts) + self_attn_out = self.self_attn(prompts_norm) + prompts = prompts + self_attn_out + + # MLP + prompts_norm = self.norm3(prompts) + mlp_out = self.mlp(prompts_norm) + prompts = prompts + mlp_out + + return prompts + + +class MaskDecoder(nn.Module): + """Mask decoder for SAM-3 that generates segmentation masks. + + Takes image embeddings and prompt embeddings to produce segmentation masks + and IoU predictions for the prompted regions. + """ + + def __init__( + self, + embed_dim: int = 256, + num_multimask_outputs: int = 3, + activation: str = "gelu", + iou_head_depth: int = 3, + iou_head_hidden_dim: int = 256, + ) -> None: + """Initialize MaskDecoder. + + Args: + embed_dim: Embedding dimension. + num_multimask_outputs: Number of mask outputs per prompt. + activation: Activation function name. + iou_head_depth: Depth of IoU prediction head. + iou_head_hidden_dim: Hidden dimension of IoU head. + """ + super().__init__() + self.embed_dim = embed_dim + self.num_multimask_outputs = num_multimask_outputs + self.activation = activation + + # Transformer for processing prompts + self.transformer = CrossAttentionTransformer(embed_dim) + + # Mask prediction head + # NOTE: Phase 2 supports single-mask output only + # Multi-mask generation deferred to Phase 3 when mask_tokens and hypernetwork MLPs will be used + self.output_upscaling = nn.Sequential( + nn.ConvTranspose2d(embed_dim, embed_dim // 4, kernel_size=2, stride=2), + nn.GroupNorm(1, embed_dim // 4), + nn.ConvTranspose2d(embed_dim // 4, embed_dim // 8, kernel_size=2, stride=2), + ) + # NOTE: Phase 2 stub - mask_tokens and hypernetwork MLPs will be used in Phase 3 for multi-mask generation + + # IoU prediction head + self.iou_prediction_head = nn.Sequential( + nn.Linear(embed_dim, iou_head_hidden_dim), + nn.ReLU(), + nn.Linear(iou_head_hidden_dim, iou_head_hidden_dim), + nn.ReLU(), + nn.Linear(iou_head_hidden_dim, num_multimask_outputs), + ) + + def _predict_masks( + self, + image_embeddings: torch.Tensor, + sparse_prompt_embeddings: torch.Tensor, + dense_prompt_embeddings: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Predict masks from embeddings. + + Args: + image_embeddings: Image embeddings of shape (B, N, D). + sparse_prompt_embeddings: Sparse prompt embeddings of shape (B, M, D). + dense_prompt_embeddings: Dense prompt embeddings of shape (B, D, H, W). + + Returns: + Tuple of (masks, iou_pred) where: + - masks: Tensor of shape (B, num_masks, H, W). + - iou_pred: Tensor of shape (B, num_masks). + """ + B, N, _ = image_embeddings.shape + + # Infer spatial dimensions from sequence length + # image_embeddings: (B, N, D) where N = H*W for square grid (per batch) + H = W = int(N**0.5) + KORNIA_CHECK(H * W == N, f"image_embeddings must form a square grid. Got N={N}") + + # Reshape image embeddings to spatial form for processing + image_embeddings_spatial = image_embeddings.view(B, H, W, self.embed_dim).permute(0, 3, 1, 2) + + # Add dense prompts to image embeddings + if dense_prompt_embeddings.shape[1] > 0: + # Resize dense prompts to match image embedding spatial size + dense_resized = torch.nn.functional.interpolate( + dense_prompt_embeddings, + size=(H, W), + mode="bilinear", + align_corners=False, + ) + # Add to image embeddings + image_embeddings_spatial = image_embeddings_spatial + dense_resized + + # Reshape back to sequence form + image_with_prompts = image_embeddings_spatial.permute(0, 2, 3, 1).reshape(B, H * W, self.embed_dim) + + # Process sparse prompts through transformer + if sparse_prompt_embeddings.shape[1] > 0: + sparse_processed = self.transformer(sparse_prompt_embeddings, image_with_prompts) + else: + sparse_processed = sparse_prompt_embeddings + + # Upscale for mask prediction + masks = self.output_upscaling(image_embeddings_spatial) # (B, D/8, H_out, W_out) + + # Predict IoU scores + if sparse_processed.shape[1] > 0: + iou_input = sparse_processed.mean(dim=1) # (B, D) + else: + iou_input = torch.zeros(B, self.embed_dim, device=image_embeddings.device, dtype=image_embeddings.dtype) + + iou_pred = self.iou_prediction_head(iou_input) # (B, num_masks) + + return masks, iou_pred + + def forward( + self, + image_embeddings: torch.Tensor, + sparse_prompt_embeddings: torch.Tensor, + dense_prompt_embeddings: torch.Tensor, + multimask_output: bool = True, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Generate segmentation masks from image and prompt embeddings. + + Args: + image_embeddings: Image embeddings of shape (B, N, D) from image encoder. + sparse_prompt_embeddings: Sparse prompt embeddings of shape (B, M, D) from prompt encoder. + dense_prompt_embeddings: Dense prompt embeddings of shape (B, D, H, W) from prompt encoder. + multimask_output: If True, output multiple masks per prompt. Otherwise, output single mask. + + Returns: + Tuple of (masks, iou_pred) where: + - masks: Segmentation masks of shape (B, num_masks, H, W). + - iou_pred: IoU predictions of shape (B, num_masks). + + Raises: + ValueError: If image_embeddings shape is invalid. + """ + KORNIA_CHECK( + image_embeddings.ndim == 3, + f"image_embeddings must be 3D (B, N, D), got shape {image_embeddings.shape}", + ) + KORNIA_CHECK( + sparse_prompt_embeddings.ndim == 3, + f"sparse_prompt_embeddings must be 3D (B, M, D), got shape {sparse_prompt_embeddings.shape}", + ) + KORNIA_CHECK( + dense_prompt_embeddings.ndim == 4, + f"dense_prompt_embeddings must be 4D (B, D, H, W), got shape {dense_prompt_embeddings.shape}", + ) + + # Predict masks + # NOTE: positional encoding intentionally omitted in Phase 2 + masks, iou_pred = self._predict_masks( + image_embeddings, + sparse_prompt_embeddings, + dense_prompt_embeddings, + ) + + # NOTE: Phase 2 generates single mask only + # multimask_output parameter kept for forward compatibility but currently ignored + # Multi-mask generation deferred to Phase 3 + + return masks, iou_pred + + +__all__ = ["CrossAttentionTransformer", "MaskDecoder"] diff --git a/kornia/models/sam3/architecture/prompt_encoder.py b/kornia/models/sam3/architecture/prompt_encoder.py new file mode 100644 index 0000000..99c9582 --- /dev/null +++ b/kornia/models/sam3/architecture/prompt_encoder.py @@ -0,0 +1,268 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""SAM-3 Prompt Encoder for encoding user prompts. + +This module implements the prompt encoder for SAM-3 which processes point prompts, +bounding boxes, and mask prompts into dense and sparse embeddings. +""" + +from __future__ import annotations + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SHAPE + + +class PositionalEncoding(nn.Module): + """Positional encoding for 2D coordinates. + + Encodes 2D coordinates using sinusoidal positional encoding. + """ + + def __init__(self, embed_dim: int) -> None: + """Initialize PositionalEncoding. + + Args: + embed_dim: Embedding dimension (must be even). + """ + super().__init__() + self.embed_dim = embed_dim + KORNIA_CHECK( + embed_dim % 2 == 0, + f"embed_dim must be even, got {embed_dim}", + ) + + def forward(self, coords: torch.Tensor) -> torch.Tensor: + """Apply positional encoding to coordinates using sinusoidal basis functions. + + Encodes coordinates (x, y) using sin/cos at multiple frequency scales. + Implementation note: builds 2*embed_dim intermediate representation from x,y encoding, + then truncates to embed_dim to maintain dimensional consistency with input embeddings. + + Args: + coords: Coordinate tensor of shape (B, N, 2) where last dimension is (x, y). + + Returns: + Encoded tensor of shape (B, N, embed_dim). + + Raises: + ValueError: If coords does not have shape (B, N, 2). + """ + KORNIA_CHECK_SHAPE(coords, ["B", "N", "2"]) + B, N, _ = coords.shape + + # Create frequency bands + freqs = torch.arange(0, self.embed_dim // 2, dtype=torch.float32, device=coords.device) + freqs = 2.0 ** (freqs / (self.embed_dim // 2)) * torch.pi + + # Expand coordinates and frequency bands for broadcasting + coords_expanded = coords.unsqueeze(-1) # (B, N, 2, 1) + freqs_expanded = freqs.view(1, 1, 1, -1) # (1, 1, 1, embed_dim//2) + + # Compute sin and cos components + args = coords_expanded * freqs_expanded # (B, N, 2, embed_dim//2) + sin_part = torch.sin(args) # (B, N, 2, embed_dim//2) + cos_part = torch.cos(args) # (B, N, 2, embed_dim//2) + + # Interleave sin and cos for each coordinate + encoded = torch.stack([sin_part, cos_part], dim=-1) # (B, N, 2, embed_dim//2, 2) + encoded = encoded.view(B, N, 2, self.embed_dim) # (B, N, 2, embed_dim) + + # Separate x and y encodings and concatenate + x_encoded = encoded[:, :, 0, :] # (B, N, embed_dim) + y_encoded = encoded[:, :, 1, :] # (B, N, embed_dim) + output = torch.cat([x_encoded, y_encoded], dim=-1) # (B, N, 2*embed_dim) + + # Truncate to embed_dim for dimensional consistency with input embeddings + return output[:, :, : self.embed_dim] + + +class PromptEncoder(nn.Module): + """Encoder for SAM-3 prompts (points, boxes, masks). + + Encodes user prompts into sparse and dense embeddings that can be used + by the mask decoder. + """ + + def __init__( + self, + embed_dim: int = 256, + input_image_size: int = 1024, + mask_in_chans: int = 16, + ) -> None: + """Initialize PromptEncoder. + + Args: + embed_dim: Embedding dimension. + input_image_size: Size of input image (assumed square). + mask_in_chans: Number of input channels for mask encoding. + """ + super().__init__() + self.embed_dim = embed_dim + self.input_image_size = input_image_size + self.mask_in_chans = mask_in_chans + + # Point embedding + self.pe_layer = PositionalEncoding(embed_dim) + self.point_embeddings = nn.ModuleList( + [nn.Embedding(1, embed_dim) for _ in range(4)] + ) # (foreground, background, box top-left, box bottom-right) + + # Dense embedding (for masks) + self.mask_downscaling = nn.Sequential( + nn.Conv2d(1, mask_in_chans // 4, kernel_size=2, stride=2), + nn.GroupNorm(1, mask_in_chans // 4), + nn.Conv2d(mask_in_chans // 4, mask_in_chans, kernel_size=2, stride=2), + nn.GroupNorm(1, mask_in_chans), + ) + self.no_mask_embed = nn.Embedding(1, embed_dim) + + def _encode_points( + self, + points: tuple[torch.Tensor, torch.Tensor], + ) -> torch.Tensor: + """Encode point prompts into embeddings. + + Args: + points: Tuple of (coords, labels) where: + - coords: Tensor of shape (B, N, 2) with normalized coordinates. + - labels: Tensor of shape (B, N) with labels (0=background, 1=foreground). + + Returns: + Sparse embeddings of shape (B, N, embed_dim). + + Raises: + ValueError: If coords and labels have incompatible shapes. + """ + coords, labels = points + KORNIA_CHECK_SHAPE(coords, ["B", "N", "2"]) + KORNIA_CHECK_SHAPE(labels, ["B", "N"]) + KORNIA_CHECK( + coords.shape[:2] == labels.shape, + f"coords and labels must have matching batch and point dimensions, " + f"got {coords.shape[:2]} vs {labels.shape}", + ) + + B, N, _ = coords.shape + + # Encode coordinates using positional encoding + pe = self.pe_layer(coords) # (B, N, embed_dim) + + # Simple approach: use label to select embedding + # NOTE: loop-based implementation for clarity; can be vectorized in future optimization + label_embeddings = torch.zeros(B, N, self.embed_dim, device=coords.device, dtype=coords.dtype) + for b in range(B): + for i in range(N): + label = int(labels[b, i].item()) + label_idx = min(label, 1) # 0 or 1 for background/foreground + label_embeddings[b, i] = self.point_embeddings[label_idx].weight[0] + + output = pe + label_embeddings + return output + + def forward( + self, + *, + points: tuple[torch.Tensor, torch.Tensor] | None = None, + boxes: torch.Tensor | None = None, + masks: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Encode prompts into sparse and dense embeddings. + + Args: + points: Optional tuple of (coords, labels) where: + - coords: Tensor of shape (B, N, 2) with normalized coordinates in [0, 1]. + - labels: Tensor of shape (B, N) with binary labels (0 or 1). + boxes: Optional tensor of shape (B, num_boxes, 4) with normalized bbox coordinates. + Currently not implemented (returns zero embeddings). + masks: Optional tensor of shape (B, 1, H, W) with binary masks. + Currently not implemented (returns zero embeddings). + + Returns: + Tuple of (sparse_embeddings, dense_embeddings) where: + - sparse_embeddings: Tensor of shape (B, num_sparse, embed_dim) or (B, 0, embed_dim) if no prompts. + - dense_embeddings: Tensor of shape (B, embed_dim, H, W) or zeros if no masks. + + Raises: + ValueError: If no prompts are provided or point shapes are invalid. + """ + sparse_embeddings = [] + B = 1 # Default batch size + device = None + + # Determine batch size and device from inputs + if points is not None: + coords, labels = points + B = coords.shape[0] + device = coords.device + elif boxes is not None: + B = boxes.shape[0] + device = boxes.device + elif masks is not None: + B = masks.shape[0] + device = masks.device + + # Process point prompts + if points is not None: + coords, labels = points + point_embeddings = self._encode_points((coords, labels)) + sparse_embeddings.append(point_embeddings) + + # Process box prompts (stub) + if boxes is not None: + KORNIA_CHECK_SHAPE(boxes, ["B", "num_boxes", "4"]) + # Boxes are intentionally stubbed in Phase 2 + # Full implementation (box corner encoding + corner embedding lookup) deferred to Phase 3 + num_boxes = boxes.shape[1] + box_embeddings = torch.zeros(B, num_boxes, self.embed_dim, device=boxes.device, dtype=boxes.dtype) + sparse_embeddings.append(box_embeddings) + + # Concatenate sparse embeddings + if sparse_embeddings: + sparse_embeddings = torch.cat(sparse_embeddings, dim=1) + else: + if device is None: + device = torch.device("cpu") + sparse_embeddings = torch.zeros(B, 0, self.embed_dim, device=device) + + # Process mask prompts (stub) + if masks is not None: + KORNIA_CHECK_SHAPE(masks, ["B", "1", "H", "W"]) + dense_embeddings = self.mask_downscaling(masks) + # Resize to match expected output size + dense_embeddings = torch.nn.functional.interpolate( + dense_embeddings, + size=(self.input_image_size // 4, self.input_image_size // 4), + mode="bilinear", + align_corners=False, + ) + else: + # No mask: use no_mask embedding + if device is None: + device = torch.device("cpu") + dense_embeddings = self.no_mask_embed.weight.view(1, self.embed_dim, 1, 1) + dense_embeddings = dense_embeddings.expand( + B, self.embed_dim, self.input_image_size // 4, self.input_image_size // 4 + ) + dense_embeddings = dense_embeddings.to(device) + + return sparse_embeddings, dense_embeddings + + +__all__ = ["PositionalEncoding", "PromptEncoder"] diff --git a/kornia/models/segmentation/__init__.py b/kornia/models/segmentation/__init__.py new file mode 100644 index 0000000..bb89491 --- /dev/null +++ b/kornia/models/segmentation/__init__.py @@ -0,0 +1,24 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Segmentation models submodule for Kornia. + +This package provides model architectures and utilities for image segmentation tasks. +""" + +from .base import * +from .segmentation_models import * diff --git a/kornia/models/segmentation/base.py b/kornia/models/segmentation/base.py new file mode 100644 index 0000000..988635f --- /dev/null +++ b/kornia/models/segmentation/base.py @@ -0,0 +1,235 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import ClassVar, Optional, Union + +import torch + +import kornia +from kornia.core.external import PILImage as Image +from kornia.models.base import ModelBase + +__all__ = ["SemanticSegmentation"] + + +class SemanticSegmentation(ModelBase): + """Semantic Segmentation is a module that wraps a semantic segmentation model. + + This module uses SegmentationModel library for semantic segmentation. + """ + + ONNX_DEFAULT_INPUTSHAPE: ClassVar[list[int]] = [-1, 3, -1, -1] + ONNX_DEFAULT_OUTPUTSHAPE: ClassVar[list[int]] = [-1, -1, -1, -1] + + @torch.inference_mode() + def forward(self, images: Union[torch.Tensor, list[torch.Tensor]]) -> Union[torch.Tensor, list[torch.Tensor]]: + """Forward pass of the semantic segmentation model. + + Args: + images: If list of RGB images. Each image is a torch.Tensor with shape :math:`(3, H, W)`. + If torch.Tensor, a torch.Tensor with shape :math:`(B, 3, H, W)`. + + Returns: + output tensor. + + """ + outputs: Union[torch.Tensor, list[torch.Tensor]] + + if isinstance( + images, + ( + list, + tuple, + ), + ): + outputs = [] + for image in images: + image = self.pre_processor(image[None]) + output = self.model(image) + output = self.post_processor(output) + outputs.append(output[0]) + else: + images = self.pre_processor(images) + outputs = self.model(images) + outputs = self.post_processor(outputs) + + return outputs + + def get_colormap(self, num_classes: int, colormap: str = "random", manual_seed: int = 2147) -> torch.Tensor: + """Get a color map of size num_classes. + + Args: + num_classes: The number of colors in the color map. + colormap: The colormap to use, can be "random" or a custom color map. + manual_seed: The manual seed to use for the colormap. + + Returns: + A tensor of shape (num_classes, 3) representing the color map. + + """ + if colormap == "random": + # Generate a color for each class + g_cpu = torch.Generator() + g_cpu.manual_seed(manual_seed) + colors = torch.rand(num_classes, 3, generator=g_cpu) + else: + raise ValueError(f"Unsupported colormap: {colormap}") + + return colors + + def visualize_output(self, semantic_mask: torch.Tensor, colors: torch.Tensor) -> torch.Tensor: + """Visualize the output of the segmentation model. + + Args: + semantic_mask: The output of the segmentation model. Shape should be (C, H, W) or (B, C, H, W). + colors: The color map to use for visualizing the output of the segmentation model. + Shape should be (num_classes, 3). + + Returns: + A tensor of shape (3, H, W) or (B, 3, H, W) representing the visualized output of the segmentation model. + + Raises: + ValueError: If the shape of the semantic mask is not of shape (C, H, W) or (B, C, H, W). + ValueError: If the shape of the colors is not of shape (num_classes, 3). + ValueError: If only muliclass segmentation is supported. Please ensure a softmax is used, or submit a PR. + + """ + if semantic_mask.dim() == 3: + channel_dim = 0 + elif semantic_mask.dim() == 4: + channel_dim = 1 + else: + raise ValueError(f"Semantic mask must be of shape (C, H, W) or (B, C, H, W), got {semantic_mask.shape}.") + + if torch.allclose( + semantic_mask.sum(dim=channel_dim), torch.tensor(1, dtype=semantic_mask.dtype, device=semantic_mask.device) + ): + # Softmax is used, thus, muliclass segmentation + semantic_mask = semantic_mask.argmax(dim=channel_dim, keepdim=True) + # Create a colormap for each pixel based on the class with the highest probability + output = colors[semantic_mask.squeeze(channel_dim)] + if semantic_mask.dim() == 3: + output = output.permute(2, 0, 1) + elif semantic_mask.dim() == 4: + output = output.permute(0, 3, 1, 2) + else: + raise ValueError( + f"Semantic mask must be of shape (C, H, W) or (B, C, H, W), got {semantic_mask.shape}." + ) + else: + raise ValueError( + "Only muliclass segmentation is supported. Please ensure a softmax is used, or submit a PR." + ) + + return output + + def visualize( + self, + images: Union[torch.Tensor, list[torch.Tensor]], + semantic_masks: Optional[Union[torch.Tensor, list[torch.Tensor]]] = None, + output_type: str = "torch", + colormap: str = "random", + manual_seed: int = 2147, + ) -> Union[torch.Tensor, list[torch.Tensor], list[Image.Image]]: # type: ignore + """Visualize the segmentation masks. + + Args: + images: If list of RGB images. Each image is a torch.Tensor with shape :math:`(3, H, W)`. + If torch.Tensor, a torch.Tensor with shape :math:`(B, 3, H, W)`. + semantic_masks: If list of segmentation masks. Each mask is a torch.Tensor with shape :math:`(C, H, W)`. + If torch.Tensor, a torch.Tensor with shape :math:`(B, C, H, W)`. + output_type: The type of output, can be "torch" or "PIL". + colormap: The colormap to use, can be "random" or a custom color map. + manual_seed: The manual seed to use for the colormap. + + """ + if semantic_masks is None: + semantic_masks = self.forward(images) + + outputs: Union[torch.Tensor, list[torch.Tensor]] + if isinstance( + semantic_masks, + ( + list, + tuple, + ), + ): + outputs = [] + for semantic_mask in semantic_masks: + if semantic_mask.ndim != 3: + raise ValueError(f"Semantic mask must be of shape (C, H, W), got {semantic_mask.shape}.") + # Generate a color for each class + colors = self.get_colormap(semantic_mask.size(0), colormap, manual_seed=manual_seed) + outputs.append(self.visualize_output(semantic_mask, colors)) + + else: + # Generate a color for each class + colors = self.get_colormap(semantic_masks.size(1), colormap, manual_seed=manual_seed) + outputs = self.visualize_output(semantic_masks, colors) + + return self._tensor_to_type(outputs, output_type, is_batch=True if isinstance(outputs, torch.Tensor) else False) + + def save( + self, + images: Union[torch.Tensor, list[torch.Tensor]], + semantic_masks: Optional[Union[torch.Tensor, list[torch.Tensor]]] = None, + directory: Optional[str] = None, + output_type: str = "torch", + colormap: str = "random", + manual_seed: int = 2147, + ) -> None: + """Save the segmentation results. + + Args: + images: If list of RGB images. Each image is a torch.Tensor with shape :math:`(3, H, W)`. + If torch.Tensor, a torch.Tensor with shape :math:`(B, 3, H, W)`. + semantic_masks: If list of segmentation masks. Each mask is a torch.Tensor with shape :math:`(C, H, W)`. + If torch.Tensor, a torch.Tensor with shape :math:`(B, C, H, W)`. + directory: The directory to save the results. + output_type: The type of output, can be "torch" or "PIL". + colormap: The colormap to use, can be "random" or a custom color map. + manual_seed: The manual seed to use for the colormap. + + """ + colored_masks = self.visualize(images, semantic_masks, output_type, colormap=colormap, manual_seed=manual_seed) + overlaid: Union[torch.Tensor, list[torch.Tensor]] + if isinstance(images, torch.Tensor) and isinstance(colored_masks, torch.Tensor): + overlaid = kornia.enhance.add_weighted(images, 0.5, colored_masks, 0.5, 1.0) + elif isinstance( + images, + ( + list, + tuple, + ), + ) and isinstance( + colored_masks, + ( + list, + tuple, + ), + ): + overlaid = [] + for i in range(len(images)): + overlaid.append(kornia.enhance.add_weighted(images[i][None], 0.5, colored_masks[i][None], 0.5, 1.0)[0]) + else: + raise ValueError(f"`images` should be a torch.Tensor or a list of Tensors. Got {type(images)}") + + self._save_outputs(images, directory, suffix="_src") + self._save_outputs(colored_masks, directory, suffix="_mask") + self._save_outputs(overlaid, directory, suffix="_overlay") diff --git a/kornia/models/segmentation/segmentation_models.py b/kornia/models/segmentation/segmentation_models.py new file mode 100644 index 0000000..d30c655 --- /dev/null +++ b/kornia/models/segmentation/segmentation_models.py @@ -0,0 +1,134 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Any, Optional + +import torch +from torch import nn + +import kornia +from kornia.core.external import segmentation_models_pytorch as smp + +from .base import SemanticSegmentation + +__all__ = ["SegmentationModelsBuilder"] + + +class SegmentationModelsBuilder: + """Provide a factory to build various semantic segmentation models. + + This builder simplifies the creation of models like UNet or DeepLabV3 + by providing a unified interface for configuration and weight loading. + """ + + @staticmethod + def build( + model_name: str = "Unet", + encoder_name: str = "resnet34", + encoder_weights: Optional[str] = "imagenet", + in_channels: int = 3, + classes: int = 1, + activation: str = "softmax", + **kwargs: Any, + ) -> SemanticSegmentation: + """SegmentationModel is a module that wraps a segmentation model. + + This module uses SegmentationModel library for segmentation. + + Args: + model_name: Name of the model to use. Valid options are: + "Unet", "UnetPlusPlus", "MAnet", "LinkNet", "FPN", "PSPNet", "PAN", "DeepLabV3", "DeepLabV3Plus". + encoder_name: Name of the encoder to use. + encoder_depth: Depth of the encoder. + encoder_weights: Weights of the encoder. + decoder_channels: Number of channels in the decoder. + in_channels: Number of channels in the input. + classes: Number of classes to predict. + activation: Type of activation layer. + **kwargs: Additional arguments to pass to the model. Detailed arguments can be found at: + https://github.com/qubvel-org/segmentation_models.pytorch/tree/main/segmentation_models_pytorch/decoders + + Note: + Only encoder weights are available. + Pretrained weights for the whole model are not available. + + """ + preproc_params = smp.encoders.get_preprocessing_params(encoder_name) # type: ignore + preprocessor = SegmentationModelsBuilder.get_preprocessing_pipeline(preproc_params) + segmentation_model = getattr(smp, model_name)( + encoder_name=encoder_name, + encoder_weights=encoder_weights, + in_channels=in_channels, + classes=classes, + activation=activation, + **kwargs, + ) + + return SemanticSegmentation( + model=segmentation_model, + pre_processor=preprocessor, + post_processor=nn.Identity(), + name=f"{model_name}_{encoder_name}", + ) + + @staticmethod + def get_preprocessing_pipeline(preproc_params: dict[str, Any]) -> kornia.augmentation.container.ImageSequential: + """Build the preprocessing pipeline expected by a segmentation model. + + Args: + preproc_params: Dictionary from the segmentation-model metadata. + It must describe the input color space, value range, mean, and + standard deviation used by the pretrained encoder. + + Returns: + :class:`~kornia.augmentation.container.ImageSequential` containing + ONNX-friendly color conversion, rescaling, and normalization steps. + """ + # Ensure the color space transformation is ONNX-friendly + proc_sequence: list[nn.Module] = [] + input_space = preproc_params["input_space"] + if input_space == "BGR": + proc_sequence.append(kornia.color.BgrToRgb()) + elif input_space == "RGB": + pass + else: + raise ValueError(f"Unsupported input space: {input_space}") + + # Normalize input range if needed + input_range = preproc_params["input_range"] + if input_range[1] == 255: + proc_sequence.append(kornia.enhance.Normalize(mean=0.0, std=1 / 255.0)) + elif input_range[1] == 1: + pass + else: + raise ValueError(f"Unsupported input range: {input_range}") + + # Handle mean and std normalization + if preproc_params["mean"] is not None: + mean = torch.tensor([preproc_params["mean"]]) + else: + mean = torch.tensor(0.0) + + if preproc_params["std"] is not None: + std = torch.tensor([preproc_params["std"]]) + else: + std = torch.tensor(1.0) + proc_sequence.append(kornia.enhance.Normalize(mean=mean, std=std)) + + return kornia.augmentation.container.ImageSequential(*proc_sequence) diff --git a/kornia/models/siglip2/__init__.py b/kornia/models/siglip2/__init__.py new file mode 100644 index 0000000..4c2af72 --- /dev/null +++ b/kornia/models/siglip2/__init__.py @@ -0,0 +1,31 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""SigLip2 vision-language model implementation.""" + +from .builder import SigLip2Builder +from .config import SigLip2Config +from .model import SigLip2Model, SigLip2Result +from .preprocessor import SigLip2ImagePreprocessor + +__all__ = [ + "SigLip2Builder", + "SigLip2Config", + "SigLip2ImagePreprocessor", + "SigLip2Model", + "SigLip2Result", +] diff --git a/kornia/models/siglip2/attention.py b/kornia/models/siglip2/attention.py new file mode 100644 index 0000000..4ec629b --- /dev/null +++ b/kornia/models/siglip2/attention.py @@ -0,0 +1,141 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Attention modules for SigLip2.""" + +from __future__ import annotations + +import math +from typing import Optional + +import torch +import torch.nn.functional as F +from torch import nn + +from kornia.core.check import KORNIA_CHECK + +__all__ = ["SigLip2Attention"] + + +class SigLip2Attention(nn.Module): + """Multi-head self-attention mechanism for SigLip2. + + This module implements the standard multi-head self-attention used in + transformer architectures, with support for attention masks. + + Args: + hidden_size: Hidden dimension size. + num_heads: Number of attention heads. + dropout: Dropout probability for attention weights. + head_dim: Dimension of each attention head. If None, computed as hidden_size // num_heads. + """ + + def __init__( + self, + hidden_size: int, + num_heads: int, + dropout_p: float = 0.0, + head_dim: Optional[int] = None, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + self.num_heads = num_heads + + if head_dim is None: + head_dim = hidden_size // num_heads + self.head_dim = head_dim + + KORNIA_CHECK( + hidden_size % num_heads == 0, + f"hidden_size ({hidden_size}) must be divisible by num_heads ({num_heads})", + ) + + self.scale = 1.0 / math.sqrt(self.head_dim) + + # Separate Q/K/V projections (matching HF structure) + self.q_proj = nn.Linear(hidden_size, hidden_size) + self.k_proj = nn.Linear(hidden_size, hidden_size) + self.v_proj = nn.Linear(hidden_size, hidden_size) + self.out_proj = nn.Linear(hidden_size, hidden_size) + self.dropout = nn.Dropout(dropout_p) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Forward pass of attention. + + Args: + hidden_states: Input tensor of shape (batch_size, seq_len, hidden_size). + attention_mask: Optional attention mask. Can be: + - (batch_size, seq_len): 1D mask where 1 = attend, 0 = mask + - (batch_size, seq_len, seq_len): 2D mask where 1 = attend, 0 = mask + - (batch_size, 1, seq_len, seq_len): 4D mask (broadcastable) + + Returns: + Output tensor of shape (batch_size, seq_len, hidden_size). + """ + batch_size, seq_len, _ = hidden_states.shape + + # compute Q, K, V separately + query = self.q_proj(hidden_states) + key = self.k_proj(hidden_states) + value = self.v_proj(hidden_states) + + # reshape to (batch_size, seq_len, num_heads, head_dim) + query = query.reshape(batch_size, seq_len, self.num_heads, self.head_dim) + key = key.reshape(batch_size, seq_len, self.num_heads, self.head_dim) + value = value.reshape(batch_size, seq_len, self.num_heads, self.head_dim) + + # transpose to (batch_size, num_heads, seq_len, head_dim) + query = query.transpose(1, 2) + key = key.transpose(1, 2) + value = value.transpose(1, 2) + + # Convert attention mask to format expected by scaled_dot_product_attention + attn_mask = None + if attention_mask is not None: + # Handle different mask formats + if attention_mask.dim() == 2: + # (batch_size, seq_len) -> (batch_size, 1, seq_len, seq_len) + # Create 2D mask: both query and key positions must be valid + # Convert to boolean: 1 = attend (False = don't mask), 0 = mask (True = mask out) + mask_bool = attention_mask.bool() + # Expand to (batch_size, 1, seq_len, seq_len) where True means mask out + # We need: if either query or key is masked, then mask that attention + attn_mask = ~(mask_bool.unsqueeze(1).unsqueeze(2) & mask_bool.unsqueeze(1).unsqueeze(3)) + elif attention_mask.dim() == 3: + # (batch_size, seq_len, seq_len) -> (batch_size, 1, seq_len, seq_len) + attn_mask = ~attention_mask.bool().unsqueeze(1) + elif attention_mask.dim() == 4: + # Already in correct format (batch_size, 1, seq_len, seq_len) + attn_mask = ~attention_mask.bool() + else: + raise ValueError(f"Unsupported attention_mask dimension: {attention_mask.dim()}") + + dropout_p = self.dropout.p if self.training and self.dropout.p > 0.0 else 0.0 + attention_output = F.scaled_dot_product_attention( + query, + key, + value, + attn_mask=attn_mask, + dropout_p=dropout_p, + ) + + attention_output = attention_output.transpose(1, 2).contiguous().view(batch_size, seq_len, self.hidden_size) + return self.out_proj(attention_output) diff --git a/kornia/models/siglip2/builder.py b/kornia/models/siglip2/builder.py new file mode 100644 index 0000000..ed8675a --- /dev/null +++ b/kornia/models/siglip2/builder.py @@ -0,0 +1,166 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Builder for SigLip2 models.""" + +from __future__ import annotations + +import logging +from typing import Optional + +import torch + +from .config import SigLip2Config +from .model import SigLip2Model + +logger = logging.getLogger(__name__) + +__all__ = ["SigLip2Builder"] + + +def _download_weights(model_name: str, cache_dir: Optional[str]) -> dict[str, torch.Tensor]: + """Download model weights from HuggingFace Hub.""" + try: + from huggingface_hub import hf_hub_download + from safetensors import safe_open + except ImportError as e: + error_msg = ( + "safetensors library is required for loading model weights. Install it with: pip install safetensors" + ) + logger.error(error_msg) + raise ImportError(error_msg) from e + + try: + weights_path = hf_hub_download(repo_id=model_name, filename="model.safetensors", cache_dir=cache_dir) + state_dict = {} + with safe_open(weights_path, framework="pt", device="cpu") as f: + for key in f.keys(): + state_dict[key] = f.get_tensor(key) + return state_dict + except FileNotFoundError as e: + error_msg = ( + f"Could not find model.safetensors for {model_name}. The model must be available in safetensors format." + ) + logger.error(error_msg) + raise FileNotFoundError(error_msg) from e + + +def _infer_max_position_embeddings(config: SigLip2Config, state_dict: dict[str, torch.Tensor]) -> SigLip2Config: + """Infer max_position_embeddings from checkpoint if not in config.""" + pos_emb_key = "text_model.embeddings.position_embedding.weight" + if pos_emb_key in state_dict: + pos_emb_size = state_dict[pos_emb_key].shape[0] + if config.text_config.max_position_embeddings != pos_emb_size: + config.text_config.max_position_embeddings = pos_emb_size + return config + + +class SigLip2Builder: + """Builder for SigLip2 models. + + Provides convenient methods to create SigLip2 models from configs or + load pretrained weights from HuggingFace. + """ + + @staticmethod + def from_name(model_name: str) -> SigLip2Model: + """Build model from model name without loading pretrained weights. + + Supports the same models as from_pretrained_hf(). + + Args: + model_name: HuggingFace model identifier. + + Returns: + SigLip2Model instance with random initialization. + """ + config = SigLip2Config.from_name(model_name) + return SigLip2Model(config) + + @staticmethod + def from_config(config: SigLip2Config) -> SigLip2Model: + """Build model from configuration. + + Args: + config: Model configuration. + + Returns: + SigLip2Model instance. + """ + return SigLip2Model(config) + + @staticmethod + def from_pretrained_hf( + model_name: str = "google/siglip2-base-patch16-224", + cache_dir: Optional[str] = None, + ) -> SigLip2Model: + """Load pretrained model from HuggingFace Hub. + + Downloads model weights and config from HuggingFace Hub and loads them + using pure PyTorch (no transformers dependency). + + Supports the following models: + - google/siglip-base-patch16-224 (V1) + - google/siglip2-base-patch16-224 + - google/siglip2-base-patch16-256 + - google/siglip2-base-patch16-384 + - google/siglip2-base-patch16-512 + - google/siglip2-large-patch16-256 + - google/siglip2-large-patch16-384 + - google/siglip2-large-patch16-512 + + Args: + model_name: HuggingFace model identifier. Default: "google/siglip2-base-patch16-224". + cache_dir: Optional cache directory for model files. + + Returns: + SigLip2Model instance with pretrained weights. + + .. note:: + This method requires the `huggingface_hub` library to download files. + Install it with: ``pip install huggingface_hub`` + For safetensors files, also install: ``pip install safetensors`` + """ + # check for huggingface_hub dependency + try: + import huggingface_hub # noqa: F401 + except ImportError as e: + raise ImportError( + "huggingface_hub library is required for downloading pretrained models. " + "Install it with: pip install huggingface_hub" + ) from e + + # create config from model name + config = SigLip2Config.from_name(model_name) + + # download model weights + state_dict = _download_weights(model_name, cache_dir) + + # infer max_position_embeddings from checkpoint if not in config + config = _infer_max_position_embeddings(config, state_dict) + + # handle vision position embedding: HF has position_embedding.weight, we use position_embedding + if "vision_model.embeddings.position_embedding.weight" in state_dict: + state_dict["vision_model.embeddings.position_embedding"] = state_dict.pop( + "vision_model.embeddings.position_embedding.weight" + ) + + # create model and load weights directly (no transformation needed) + model = SigLip2Model(config) + model.load_state_dict(state_dict, strict=True) + + return model diff --git a/kornia/models/siglip2/config.py b/kornia/models/siglip2/config.py new file mode 100644 index 0000000..00c58d8 --- /dev/null +++ b/kornia/models/siglip2/config.py @@ -0,0 +1,185 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Configuration classes for SigLip2 model.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class SigLip2VisionConfig: + """Configuration for SigLip2 vision encoder. + + Args: + image_size: Size of input images. + patch_size: Size of image patches. + num_channels: Number of input channels. + hidden_size: Hidden dimension size. + num_hidden_layers: Number of transformer layers. + num_attention_heads: Number of attention heads. + intermediate_size: Intermediate size in feed-forward network. + hidden_act: Activation function (typically 'gelu'). + layer_norm_eps: Epsilon for layer normalization. + dropout_p: Dropout probability. + attention_dropout_p: Attention dropout probability. + """ + + image_size: int = 224 + patch_size: int = 16 + num_channels: int = 3 + hidden_size: int = 768 + num_hidden_layers: int = 12 + num_attention_heads: int = 12 + intermediate_size: int = 3072 + hidden_act: str = "gelu" + layer_norm_eps: float = 1e-6 + dropout_p: float = 0.0 + attention_dropout_p: float = 0.0 + + +@dataclass +class SigLip2TextConfig: + """Configuration for SigLip2 text encoder. + + Args: + vocab_size: Vocabulary size. + hidden_size: Hidden dimension size. + num_hidden_layers: Number of transformer layers. + num_attention_heads: Number of attention heads. + intermediate_size: Intermediate size in feed-forward network. + max_position_embeddings: Maximum sequence length. + hidden_act: Activation function (typically 'gelu'). + layer_norm_eps: Epsilon for layer normalization. + dropout_p: Dropout probability. + attention_dropout_p: Attention dropout probability. + """ + + vocab_size: int = 256000 + hidden_size: int = 768 + num_hidden_layers: int = 12 + num_attention_heads: int = 12 + intermediate_size: int = 3072 + max_position_embeddings: int = 512 + hidden_act: str = "gelu" + layer_norm_eps: float = 1e-6 + dropout_p: float = 0.0 + attention_dropout_p: float = 0.0 + + +@dataclass +class SigLip2Config: + """Configuration for SigLip2 model. + + Args: + vision_config: Vision encoder configuration. + text_config: Text encoder configuration. + projection_dim: Dimension of projection layers. + logit_scale_init_value: Initial value for logit scale (temperature). + logit_scale_max: Maximum value for logit_scale to prevent overflow. Default: 100.0 + """ + + vision_config: Optional[SigLip2VisionConfig] = None + text_config: Optional[SigLip2TextConfig] = None + projection_dim: int = 768 + logit_scale_init_value: float = 2.6592 + logit_scale_max: float = 100.0 + + def __post_init__(self) -> None: + """Initialize default configs if not provided.""" + if self.vision_config is None: + self.vision_config = SigLip2VisionConfig() + if self.text_config is None: + self.text_config = SigLip2TextConfig() + + @staticmethod + def from_name(model_name: str) -> SigLip2Config: + """Create config from model name. + + Supports the following models: + - google/siglip-base-patch16-224 (V1) + - google/siglip2-base-patch16-224 + - google/siglip2-base-patch16-256 + - google/siglip2-base-patch16-384 + - google/siglip2-base-patch16-512 + - google/siglip2-large-patch16-256 + - google/siglip2-large-patch16-384 + - google/siglip2-large-patch16-512 + + Args: + model_name: HuggingFace model identifier. + + Returns: + SigLip2Config instance configured for the specified model. + """ + # Parse model name + is_v1 = "google/siglip-base-patch16-224" in model_name + is_large = "large" in model_name + + # Extract image size from model name + image_size = 224 # default + if "224" in model_name: + image_size = 224 + elif "256" in model_name: + image_size = 256 + elif "384" in model_name: + image_size = 384 + elif "512" in model_name: + image_size = 512 + + # Base vs Large configurations + if is_large: + hidden_size = 1024 + num_layers = 24 + num_heads = 16 + projection_dim = 1024 + else: # base + hidden_size = 768 + num_layers = 12 + num_heads = 12 + projection_dim = 768 + + # Intermediate size is typically 4x hidden_size + intermediate_size = 4 * hidden_size + + # V1 vs V2 vocab size + vocab_size = 32000 if is_v1 else 256000 + + vision_config = SigLip2VisionConfig( + image_size=image_size, + patch_size=16, + hidden_size=hidden_size, + num_hidden_layers=num_layers, + num_attention_heads=num_heads, + intermediate_size=intermediate_size, + ) + + text_config = SigLip2TextConfig( + vocab_size=vocab_size, + hidden_size=hidden_size, + num_hidden_layers=num_layers, + num_attention_heads=num_heads, + intermediate_size=intermediate_size, + ) + + return SigLip2Config( + vision_config=vision_config, + text_config=text_config, + projection_dim=projection_dim, + ) diff --git a/kornia/models/siglip2/model.py b/kornia/models/siglip2/model.py new file mode 100644 index 0000000..09cab51 --- /dev/null +++ b/kornia/models/siglip2/model.py @@ -0,0 +1,248 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Main SigLip2 model.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Optional + +import torch +import torch.nn.functional as F +from torch import nn + +from .config import SigLip2Config +from .text_encoder import SigLip2TextModel +from .vision_encoder import SigLip2VisionModel + +__all__ = ["SigLip2Model", "SigLip2Result"] + + +@dataclass +class SigLip2Result: + """Result from SigLip2 model forward pass. + + Attributes: + image_embeds: Image embeddings of shape (batch_size, projection_dim) or None. + text_embeds: Text embeddings of shape (batch_size, projection_dim) or None. + logit_scale: Logit scale parameter (temperature) + logits_per_image: Logits for image-to-text matching of shape (batch_size, batch_size) or None. + logits_per_text: Logits for text-to-image matching of shape (batch_size, batch_size) or None. + loss: Contrastive loss or None. + """ + + logit_scale: torch.Tensor + image_embeds: Optional[torch.Tensor] = None + text_embeds: Optional[torch.Tensor] = None + logits_per_image: Optional[torch.Tensor] = None + logits_per_text: Optional[torch.Tensor] = None + loss: Optional[torch.Tensor] = None + + +class SigLip2Model(nn.Module): + """SigLip2 vision-language model. + + This model combines a vision encoder and text encoder with projection layers + to produce aligned embeddings in a shared space. + + Args: + config: Model configuration. + + Note: + Image preprocessing: Images should be preprocessed to match SigLip2ImagePreprocessor. + + Example: + >>> import torch + >>> from kornia.models.siglip2 import SigLip2Model, SigLip2Config, SigLip2ImagePreprocessor + >>> + >>> # Create model + >>> config = SigLip2Config() + >>> model = SigLip2Model(config) + >>> + >>> # Create preprocessor and process images + >>> preprocessor = SigLip2ImagePreprocessor(image_size=(224, 224)) + >>> images = torch.randint(0, 255, (2, 3, 256, 256), dtype=torch.float32) + >>> pixel_values = preprocessor(images) + >>> + >>> # Process image features + >>> image_features = model.get_image_features(pixel_values) + >>> + >>> # Process text features + >>> input_ids = torch.randint(0, 32000, (2, 10)) + >>> text_features = model.get_text_features(input_ids) + >>> + >>> # Joint processing + >>> output = model(pixel_values=pixel_values, input_ids=input_ids) + >>> logits = output.logits_per_image # Image-text similarity scores + """ + + def __init__(self, config: SigLip2Config) -> None: + super().__init__() + self.config = config + + if config.vision_config is None or config.text_config is None: + raise ValueError("vision_config and text_config must be provided") + + # vision and text encoders + self.vision_model = SigLip2VisionModel(config.vision_config) + self.text_model = SigLip2TextModel(config.text_config) + + # projection layers (use Identity when projection_dim == hidden_size) + vision_hidden_size = config.vision_config.hidden_size + text_hidden_size = config.text_config.hidden_size + projection_dim = config.projection_dim + + if projection_dim != vision_hidden_size: + self.vision_projection = nn.Linear(vision_hidden_size, projection_dim) + else: + self.vision_projection = nn.Identity() + + if projection_dim != text_hidden_size: + self.text_projection = nn.Linear(text_hidden_size, projection_dim) + else: + self.text_projection = nn.Identity() + + # logit scale (temperature parameter) + self.logit_scale = nn.Parameter(torch.tensor(config.logit_scale_init_value)) + # cache log of max scale for clamping (constant value) + self.logit_scale_max_log = math.log(config.logit_scale_max) + + # logit bias + self.logit_bias = nn.Parameter(torch.tensor(0.0)) + + def get_image_features( + self, + pixel_values: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + normalize: bool = True, + ) -> torch.Tensor: + """Get image features. + + Args: + pixel_values: Input images of shape (batch_size, num_channels, height, width). + attention_mask: Optional attention mask for vision encoder of shape (batch_size, seq_len). + normalize: Whether to normalize the output features. + + Returns: + Image features of shape (batch_size, projection_dim). + """ + vision_outputs = self.vision_model(pixel_values, attention_mask=attention_mask) + image_features = vision_outputs[0] # Pooled output + + # Apply projection (will be identity if projection_dim == hidden_size) + image_features = self.vision_projection(image_features) + + if normalize: + image_features = image_features / image_features.norm(dim=-1, keepdim=True) + + return image_features + + def get_text_features( + self, + input_ids: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + normalize: bool = True, + ) -> torch.Tensor: + """Get text features. + + Args: + input_ids: Token IDs of shape (batch_size, seq_len). + attention_mask: Optional attention mask of shape (batch_size, seq_len). + position_ids: Optional position IDs of shape (batch_size, seq_len). + normalize: Whether to normalize the output features. + + Returns: + Text features of shape (batch_size, projection_dim). + """ + text_outputs = self.text_model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + ) + text_features = text_outputs[0] # Pooled output + + # apply projection (will be identity if projection_dim == hidden_size) + text_features = self.text_projection(text_features) + + if normalize: + text_features = text_features / text_features.norm(dim=-1, keepdim=True) + + return text_features + + def forward( + self, + pixel_values: Optional[torch.Tensor] = None, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + return_loss: bool = False, + ) -> SigLip2Result: + """Forward pass. + + Args: + pixel_values: Input images of shape (batch_size, num_channels, height, width). + input_ids: Token IDs of shape (batch_size, seq_len). + attention_mask: Optional attention mask for text encoder of shape (batch_size, seq_len). + position_ids: Optional position IDs for text encoder. + return_loss: Whether to compute and return contrastive loss. + + Returns: + SigLip2Result containing: + - image_embeds: Image embeddings (if pixel_values provided) + - text_embeds: Text embeddings (if input_ids provided) + - logit_scale: Logit scale parameter + - logits_per_image: Logits for image-to-text matching (if both provided) + - logits_per_text: Logits for text-to-image matching (if both provided) + - loss: Contrastive loss (if return_loss=True and both provided) + """ + # get embeddings + image_embeds = self.get_image_features(pixel_values, normalize=True) if pixel_values is not None else None + text_embeds = ( + self.get_text_features(input_ids, attention_mask=attention_mask, position_ids=position_ids, normalize=True) + if input_ids is not None + else None + ) + + logit_scale = self.logit_scale.clamp(min=0.0, max=self.logit_scale_max_log).exp() + logits_per_image = None + logits_per_text = None + loss = None + + # compute similarity logits if both embeddings available + if image_embeds is not None and text_embeds is not None: + logits_per_text = torch.matmul(text_embeds, image_embeds.t()) * logit_scale + self.logit_bias + logits_per_image = logits_per_text.t() + + # compute loss if requested for training or evaluation + if return_loss: + batch_size = image_embeds.shape[0] + labels = torch.arange(batch_size, device=image_embeds.device) + loss_img = -F.logsigmoid(logits_per_image[labels, labels]).mean() + loss_txt = -F.logsigmoid(logits_per_text[labels, labels]).mean() + loss = (loss_img + loss_txt) / 2.0 + + return SigLip2Result( + image_embeds=image_embeds, + text_embeds=text_embeds, + logit_scale=logit_scale, + logits_per_image=logits_per_image, + logits_per_text=logits_per_text, + loss=loss, + ) diff --git a/kornia/models/siglip2/preprocessor.py b/kornia/models/siglip2/preprocessor.py new file mode 100644 index 0000000..c207bbc --- /dev/null +++ b/kornia/models/siglip2/preprocessor.py @@ -0,0 +1,115 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""SigLip2 image preprocessor.""" + +from __future__ import annotations + +import torch +from torch import nn + +from kornia.enhance.normalize import Normalize +from kornia.enhance.rescale import Rescale +from kornia.geometry.transform import Resize + + +class SigLip2ImagePreprocessor(nn.Module): + """Image preprocessor for SigLip2 models. + + This preprocessor applies the following steps: + - Rescales pixel values from [0, 255] to [0, 1] + - Resizes images to the specified size (with bicubic interpolation and antialiasing) + - Normalizes with mean=[0.5, 0.5, 0.5] and std=[0.5, 0.5, 0.5] (converts [0, 1] to [-1, 1]) + + Args: + image_size: Target image size (height, width). Default: (224, 224) + mean: Normalization mean. Default: [0.5, 0.5, 0.5] + std: Normalization std. Default: [0.5, 0.5, 0.5] + rescale_factor: Rescaling factor. Default: 1/255 + + Example: + >>> import torch + >>> from kornia.models.siglip2 import SigLip2ImagePreprocessor + >>> + >>> # Create preprocessor + >>> preprocessor = SigLip2ImagePreprocessor(image_size=(224, 224)) + >>> + >>> # Process image (assumes input in [0, 255] range) + >>> image = torch.randint(0, 255, (3, 300, 400), dtype=torch.float32) + >>> processed = preprocessor(image) # Shape: (3, 224, 224), range: [-1, 1] + """ + + def __init__( + self, + image_size: tuple[int, int] = (224, 224), + mean: list[float] | tuple[float, float, float] = (0.5, 0.5, 0.5), + std: list[float] | tuple[float, float, float] = (0.5, 0.5, 0.5), + rescale_factor: float = 1.0 / 255.0, + ) -> None: + super().__init__() + self.image_size = image_size + self.mean = torch.tensor([mean]) if isinstance(mean, list | tuple) else mean + self.std = torch.tensor([std]) if isinstance(std, list | tuple) else std + self.rescale_factor = rescale_factor + + # build preprocessing pipeline + preproc_list: list[nn.Module] = [] + + # rescale first (convert [0, 255] to [0, 1]) + if rescale_factor != 1.0: + preproc_list.append(Rescale(factor=rescale_factor)) + + # resize (on [0, 1] range) + preproc_list.append(Resize(size=image_size, interpolation="bicubic", align_corners=False, antialias=True)) + + # normalize (convert [0, 1] to [-1, 1]) + preproc_list.append(Normalize(mean=self.mean, std=self.std)) + + self.preprocessor = nn.Sequential(*preproc_list) + + def forward(self, images: torch.Tensor) -> torch.Tensor: + """Prepare images for SigLIP2 vision encoding. + + Args: + images: Image tensor with shape :math:`(C, H, W)` or + :math:`(B, C, H, W)`. Values are expected in the range used by + the configured rescale factor. + + Returns: + Batched tensor resized to the configured image size and normalized + with the stored mean and standard deviation. + """ + # ensure batch dimension + if images.dim() == 3: + images = images.unsqueeze(0) + + # process through pipeline + return self.preprocessor(images) + + @classmethod + def from_config(cls, image_size: int | tuple[int, int]) -> SigLip2ImagePreprocessor: + """Create preprocessor from image size configuration. + + Args: + image_size: Image size (single int for square, or tuple for (height, width)) + + Returns: + Preprocessor instance + """ + if isinstance(image_size, int): + image_size = (image_size, image_size) + return cls(image_size=image_size) diff --git a/kornia/models/siglip2/text_encoder.py b/kornia/models/siglip2/text_encoder.py new file mode 100644 index 0000000..2e59a1d --- /dev/null +++ b/kornia/models/siglip2/text_encoder.py @@ -0,0 +1,256 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Text encoder for SigLip2.""" + +from __future__ import annotations + +from typing import Optional + +import torch +from torch import nn + +from .attention import SigLip2Attention +from .config import SigLip2TextConfig + +__all__ = ["SigLip2TextEmbeddings", "SigLip2TextEncoder", "SigLip2TextLayer", "SigLip2TextModel"] + + +class SigLip2TextEmbeddings(nn.Module): + """Text embeddings for SigLip2. + + Combines token embeddings and position embeddings. + + Args: + config: Text encoder configuration. + """ + + def __init__(self, config: SigLip2TextConfig) -> None: + super().__init__() + self.config = config + self.vocab_size = config.vocab_size + self.hidden_size = config.hidden_size + self.max_position_embeddings = config.max_position_embeddings + + # token embeddings - [vocab_size, hidden_size] + self.token_embedding = nn.Embedding(config.vocab_size, config.hidden_size) + + # position embeddings - [max_position_embeddings, hidden_size] + self.position_embedding = nn.Embedding(config.max_position_embeddings, config.hidden_size) + + def forward(self, input_ids: torch.Tensor, position_ids: Optional[torch.Tensor] = None) -> torch.Tensor: + """Forward pass. + + Args: + input_ids: Token IDs of shape (batch_size, seq_len). + position_ids: Optional position IDs of shape (batch_size, seq_len). + If None, uses sequential positions. + + Returns: + Embedded tokens of shape (batch_size, seq_len, hidden_size). + """ + batch_size, seq_len = input_ids.shape + + # token embeddings - [batch_size, seq_len, hidden_size] + token_embeddings = self.token_embedding(input_ids) + + # position embeddings - [batch_size, seq_len, hidden_size] + if position_ids is None: + position_ids = torch.arange(seq_len, device=input_ids.device).unsqueeze(0).expand(batch_size, -1) + position_embeddings = self.position_embedding(position_ids) + + # combine embeddings - [batch_size, seq_len, hidden_size] + embeddings = token_embeddings + position_embeddings + + return embeddings + + +class SigLip2TextMLP(nn.Module): + """MLP (feed-forward network) for text encoder. + + Args: + config: Text encoder configuration. + """ + + def __init__(self, config: SigLip2TextConfig) -> None: + super().__init__() + self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size) + self.activation = nn.GELU() + self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size) + self.dropout = nn.Dropout(config.dropout_p) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + # linear transformation - [batch_size, seq_len, intermediate_size] + hidden_states = self.fc1(hidden_states) + # activation function - [batch_size, seq_len, intermediate_size] + hidden_states = self.activation(hidden_states) + # linear transformation - [batch_size, seq_len, hidden_size] + hidden_states = self.fc2(hidden_states) + hidden_states = self.dropout(hidden_states) + return hidden_states + + +class SigLip2TextLayer(nn.Module): + """Transformer layer for SigLip2 text encoder. + + Implements pre-norm architecture with residual connections. + + Args: + config: Text encoder configuration. + """ + + def __init__(self, config: SigLip2TextConfig) -> None: + super().__init__() + self.self_attn = SigLip2Attention( + hidden_size=config.hidden_size, + num_heads=config.num_attention_heads, + dropout_p=config.attention_dropout_p, + ) + self.mlp = SigLip2TextMLP(config) + self.layer_norm1 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.layer_norm2 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + def forward(self, hidden_states: torch.Tensor, attention_mask: Optional[torch.Tensor] = None) -> torch.Tensor: + """Forward pass. + + Args: + hidden_states: Input tensor of shape (batch_size, seq_len, hidden_size). + attention_mask: Optional attention mask. + + Returns: + Output tensor of shape (batch_size, seq_len, hidden_size). + """ + # self-attention with pre-norm + residual = hidden_states + hidden_states = self.layer_norm1(hidden_states) + hidden_states = self.self_attn(hidden_states, attention_mask=attention_mask) + hidden_states = residual + hidden_states + + # MLP with pre-norm - [batch_size, seq_len, hidden_size] + residual = hidden_states + hidden_states = self.layer_norm2(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + # return the hidden states - [batch_size, seq_len, hidden_size] + return hidden_states + + +class SigLip2TextEncoder(nn.Module): + """Text encoder stack for SigLip2. + + Args: + config: Text encoder configuration. + """ + + def __init__(self, config: SigLip2TextConfig) -> None: + super().__init__() + self.layers = nn.ModuleList([SigLip2TextLayer(config) for _ in range(config.num_hidden_layers)]) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + output_hidden_states: bool = False, + ) -> tuple[torch.Tensor, ...]: + """Forward pass through encoder layers. + + Args: + hidden_states: Input embeddings of shape (batch_size, seq_len, hidden_size). + attention_mask: Optional attention mask of shape (batch_size, seq_len). + output_hidden_states: Whether to return hidden states from all layers. + + Returns: + Tuple of (last_hidden_state,) or (last_hidden_state, all_hidden_states). + """ + all_hidden_states: list[torch.Tensor] = [] + + for layer in self.layers: + if output_hidden_states: + all_hidden_states.append(hidden_states) + hidden_states = layer(hidden_states, attention_mask=attention_mask) + + if output_hidden_states: + all_hidden_states.append(hidden_states) + return (hidden_states, tuple(all_hidden_states)) + + return (hidden_states,) + + +class SigLip2TextModel(nn.Module): + """Complete text encoder model for SigLip2. + + Args: + config: Text encoder configuration. + """ + + def __init__(self, config: SigLip2TextConfig) -> None: + super().__init__() + self.config = config + self.embeddings = SigLip2TextEmbeddings(config) + self.encoder = SigLip2TextEncoder(config) + # final layer norm - [batch_size, hidden_size] + self.final_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + # head layer - [batch_size, hidden_size] + self.head = nn.Linear(config.hidden_size, config.hidden_size) + + def forward( + self, + input_ids: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + output_hidden_states: bool = False, + ) -> tuple[torch.Tensor, ...]: + """Forward pass. + + Args: + input_ids: Token IDs of shape (batch_size, seq_len). + attention_mask: Optional attention mask of shape (batch_size, seq_len). + Values should be 1 for unmasked tokens and 0 for masked tokens. + position_ids: Optional position IDs. + output_hidden_states: Whether to return hidden states from all layers. + + Returns: + Tuple containing: + - Pooled output (EOS token) + - Last hidden state + - All hidden states (if output_hidden_states=True) + """ + # get embeddings - [batch_size, seq_len, hidden_size] + hidden_states = self.embeddings(input_ids, position_ids=position_ids) + + # encode - [batch_size, seq_len, hidden_size] + encoder_outputs = self.encoder( + hidden_states, + attention_mask=attention_mask, + output_hidden_states=output_hidden_states, + ) + + last_hidden_state = encoder_outputs[0] + + # apply final layer norm - [batch_size, hidden_size] + last_hidden_state = self.final_layer_norm(last_hidden_state) + + # pool: use the last token - [batch_size, hidden_size] + pooled_output = last_hidden_state[:, -1] + + # apply head layer - [batch_size, hidden_size] + pooled_output = self.head(pooled_output) + + if output_hidden_states: + return (pooled_output, last_hidden_state, encoder_outputs[1]) + return (pooled_output, last_hidden_state) diff --git a/kornia/models/siglip2/vision_encoder.py b/kornia/models/siglip2/vision_encoder.py new file mode 100644 index 0000000..44d4140 --- /dev/null +++ b/kornia/models/siglip2/vision_encoder.py @@ -0,0 +1,314 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Vision encoder for SigLip2.""" + +from __future__ import annotations + +from typing import Optional + +import torch +from torch import nn + +from .attention import SigLip2Attention +from .config import SigLip2VisionConfig + +__all__ = [ + "SigLip2MultiheadAttentionPoolingHead", + "SigLip2VisionEmbeddings", + "SigLip2VisionEncoder", + "SigLip2VisionLayer", + "SigLip2VisionModel", +] + + +class SigLip2VisionEmbeddings(nn.Module): + """Vision embeddings for SigLip2. + + Combines patch embedding and position embeddings. + + Args: + config: Vision encoder configuration. + """ + + def __init__(self, config: SigLip2VisionConfig) -> None: + super().__init__() + self.config = config + self.patch_size = config.patch_size + self.image_size = config.image_size + self.num_channels = config.num_channels + self.hidden_size = config.hidden_size + + # extract patches from the image using a convolutional layer + self.patch_embedding = nn.Conv2d( + in_channels=config.num_channels, + out_channels=config.hidden_size, + kernel_size=config.patch_size, + stride=config.patch_size, + bias=True, + ) + + # calculate the number of patches in the image + self.num_patches = (config.image_size // config.patch_size) ** 2 + + # position embeddings - [num_patches, hidden_size] + self.position_embedding = nn.Parameter(torch.randn(self.num_patches, config.hidden_size)) + + # dropout or identity + self.dropout = nn.Dropout(config.dropout_p) + + def forward(self, pixel_values: torch.Tensor) -> torch.Tensor: + """Forward pass. + + Args: + pixel_values: Input images of shape (batch_size, num_channels, height, width). + + Returns: + Embedded patches of shape (batch_size, num_patches, hidden_size). + """ + # extract patches from the image + embeddings = self.patch_embedding(pixel_values) # (batch_size, hidden_size, H', W') + embeddings = embeddings.flatten(2).transpose(1, 2) # (batch_size, num_patches, hidden_size) + + # add position embeddings to the embeddings + embeddings = embeddings + self.position_embedding.unsqueeze(0) + + return embeddings + + +class SigLip2VisionMLP(nn.Module): + """MLP (feed-forward network) for vision encoder. + + Args: + config: Vision encoder configuration. + """ + + def __init__(self, config: SigLip2VisionConfig) -> None: + super().__init__() + self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size) + self.activation = nn.GELU() + self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size) + self.dropout = nn.Dropout(config.dropout_p) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.fc1(hidden_states) + hidden_states = self.activation(hidden_states) + hidden_states = self.fc2(hidden_states) + hidden_states = self.dropout(hidden_states) + return hidden_states + + +class SigLip2VisionLayer(nn.Module): + """Transformer layer for vision encoder. + + Implements pre-norm architecture with residual connections. + + Args: + config: Vision encoder configuration. + """ + + def __init__(self, config: SigLip2VisionConfig) -> None: + super().__init__() + self.self_attn = SigLip2Attention( + hidden_size=config.hidden_size, + num_heads=config.num_attention_heads, + dropout_p=config.attention_dropout_p, + ) + self.mlp = SigLip2VisionMLP(config) + self.layer_norm1 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.layer_norm2 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + def forward(self, hidden_states: torch.Tensor, attention_mask: Optional[torch.Tensor] = None) -> torch.Tensor: + """Forward pass. + + Args: + hidden_states: Input tensor of shape (batch_size, seq_len, hidden_size). + attention_mask: Optional attention mask. + + Returns: + Output tensor of shape (batch_size, seq_len, hidden_size). + """ + # Self-attention with pre-norm + residual = hidden_states + hidden_states = self.layer_norm1(hidden_states) + hidden_states = self.self_attn(hidden_states, attention_mask=attention_mask) + hidden_states = residual + hidden_states + + # MLP with pre-norm + residual = hidden_states + hidden_states = self.layer_norm2(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + return hidden_states + + +class SigLip2MultiheadAttentionPoolingHead(nn.Module): + """Multi-head attention pooling head for vision encoder. + + This implements the pooling mechanism used by HF SigLip vision model. + Uses a learnable probe token with multi-head attention to pool the sequence. + + Args: + config: Vision encoder configuration. + """ + + def __init__(self, config: SigLip2VisionConfig) -> None: + super().__init__() + # Learnable probe (query token) + self.probe = nn.Parameter(torch.randn(1, 1, config.hidden_size)) + + # Multi-head attention (using PyTorch's built-in for compatibility) + self.attention = nn.MultiheadAttention( + embed_dim=config.hidden_size, + num_heads=config.num_attention_heads, + batch_first=True, + ) + self.layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.mlp = SigLip2VisionMLP(config) + + def forward(self, hidden_state: torch.Tensor) -> torch.Tensor: + """Pool vision tokens with a learned probe query. + + Args: + hidden_state: Vision token tensor with shape :math:`(B, N, D)`, + where :math:`B` is batch size, :math:`N` is token count, and + :math:`D` is hidden size. + + Returns: + Tensor with shape :math:`(B, 1, D)` containing the pooled visual + representation after attention, layer normalization, and MLP + refinement. + """ + # repeat the probe token for the batch size + batch_size = hidden_state.shape[0] + probe = self.probe.repeat(batch_size, 1, 1) # (batch_size, 1, hidden_size) + + # multi-head attention: probe as query, hidden_state as key/value + hidden_state, _ = self.attention(probe, hidden_state, hidden_state) + + # residual connection with layer norm and MLP + residual = hidden_state + hidden_state = self.layernorm(hidden_state) + hidden_state = residual + self.mlp(hidden_state) + + # return the first (and only) token - [batch_size, hidden_size] + return hidden_state[:, 0] + + +class SigLip2VisionEncoder(nn.Module): + """Vision encoder stack for SigLip2. + + Args: + config: Vision encoder configuration. + """ + + def __init__(self, config: SigLip2VisionConfig) -> None: + super().__init__() + self.layers = nn.ModuleList([SigLip2VisionLayer(config) for _ in range(config.num_hidden_layers)]) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + output_hidden_states: bool = False, + ) -> tuple[torch.Tensor, ...]: + """Forward pass through encoder layers. + + Args: + hidden_states: Input embeddings of shape (batch_size, seq_len, hidden_size). + attention_mask: Optional attention mask. + output_hidden_states: Whether to return hidden states from all layers. + + Returns: + Tuple of (last_hidden_state,) or (last_hidden_state, all_hidden_states). + """ + all_hidden_states: list[torch.Tensor] = [] + + for layer in self.layers: + if output_hidden_states: + all_hidden_states.append(hidden_states) + hidden_states = layer(hidden_states, attention_mask=attention_mask) + + if output_hidden_states: + all_hidden_states.append(hidden_states) + return (hidden_states, tuple(all_hidden_states)) + + return (hidden_states,) + + +class SigLip2VisionModel(nn.Module): + """Complete vision encoder model for SigLip2. + + Args: + config: Vision encoder configuration. + """ + + def __init__(self, config: SigLip2VisionConfig) -> None: + super().__init__() + self.config = config + self.embeddings = SigLip2VisionEmbeddings(config) + self.encoder = SigLip2VisionEncoder(config) + + # post layer norm - [batch_size, hidden_size] + self.post_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + # head: multi-head attention pooling - [batch_size, hidden_size] + self.head = SigLip2MultiheadAttentionPoolingHead(config) + + def forward( + self, + pixel_values: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + output_hidden_states: bool = False, + ) -> tuple[torch.Tensor, ...]: + """Forward pass. + + Args: + pixel_values: Input images of shape (batch_size, num_channels, height, width). + attention_mask: Optional attention mask. + output_hidden_states: Whether to return hidden states from all layers. + + Returns: + Tuple containing: + - Pooled output (mean pooled) + - Last hidden state + - All hidden states (if output_hidden_states=True) + """ + # Get embeddings + hidden_states = self.embeddings(pixel_values) + + # Encode + encoder_outputs = self.encoder( + hidden_states, + attention_mask=attention_mask, + output_hidden_states=output_hidden_states, + ) + + last_hidden_state = encoder_outputs[0] + + # apply post layer norm - [batch_size, hidden_size] + last_hidden_state = self.post_layernorm(last_hidden_state) + + # pool: multi-head attention pooling - [batch_size, hidden_size] + pooled_output = self.head(last_hidden_state) + + if output_hidden_states: + return (pooled_output, last_hidden_state, encoder_outputs[1]) + + # return the pooled output and the last hidden state - [batch_size, hidden_size] + return (pooled_output, last_hidden_state) diff --git a/kornia/models/small_sr.py b/kornia/models/small_sr.py new file mode 100644 index 0000000..ed2211e --- /dev/null +++ b/kornia/models/small_sr.py @@ -0,0 +1,133 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch +from torch import nn + +from kornia.color.ycbcr import RgbToYcbcr, YcbcrToRgb +from kornia.config import kornia_config +from kornia.onnx.download import CachedDownloader + +url = "https://s3.amazonaws.com/pytorch/test_data/export/superres_epoch100-44c6958e.pth" + + +class SmallSRNet(nn.Module): + """A small super-resolution model. + + This model uses the efficient sub-pixel convolution layer described in + "Real-Time Single Image and Video Super-Resolution Using an Efficient Sub-Pixel Convolutional Neural Network" + - Shi et al for increasing the resolution of an image by an upscale factor. + Taken from https://pytorch.org/tutorials/advanced/super_resolution_with_onnxruntime.html. + """ + + def __init__(self, upscale_factor: int, inplace: bool = False, pretrained: bool = True) -> None: + super().__init__() + + self.relu = nn.ReLU(inplace=inplace) + self.conv1 = nn.Conv2d(1, 64, (5, 5), (1, 1), (2, 2)) + self.conv2 = nn.Conv2d(64, 64, (3, 3), (1, 1), (1, 1)) + self.conv3 = nn.Conv2d(64, 32, (3, 3), (1, 1), (1, 1)) + self.conv4 = nn.Conv2d(32, upscale_factor**2, (3, 3), (1, 1), (1, 1)) + self.pixel_shuffle = nn.PixelShuffle(upscale_factor) + + if pretrained: + self.load_from_file(url) + else: + with torch.no_grad(): + weight_init(self) + + def load_from_file(self, path_file: str) -> None: + """Load pretrained super-resolution weights from the model cache. + + Args: + path_file: Checkpoint URL or path passed to the cached downloader. + The downloaded state dictionary is loaded strictly and the + module is switched to evaluation mode. + """ + # use torch.hub to load pretrained model + model_path = CachedDownloader.download_to_cache( + path_file, "small_sr.pth", download=True, suffix=".pth", cache_dir=kornia_config.hub_onnx_dir + ) + pretrained_dict = torch.load(model_path, map_location=torch.device("cpu")) + self.load_state_dict(pretrained_dict, strict=True) + self.eval() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Super-resolve a single-channel image tensor. + + Args: + x: Luminance image tensor with shape :math:`(B, 1, H, W)`, where + :math:`B` is batch size, :math:`H` is height, and :math:`W` is + width. + + Returns: + Upscaled luminance tensor with shape + :math:`(B, 1, H * r, W * r)`, where ``r`` is the configured + upscale factor. + """ + x = self.relu(self.conv1(x)) + x = self.relu(self.conv2(x)) + x = self.relu(self.conv3(x)) + x = self.pixel_shuffle(self.conv4(x)) + return x + + +def weight_init(module: nn.Module) -> None: + """Initialize model weights for Conv2d layers.""" + if isinstance(module, nn.Conv2d): + # Use orthogonal initialization with gain for all conv layers + # conv4 (the last layer before pixel shuffle) uses default gain + if module.out_channels in {64, 32}: # conv1, conv2, conv3 + torch.nn.init.orthogonal_(module.weight, torch.nn.init.calculate_gain("relu")) + else: # conv4 (upscale_factor**2 channels) + torch.nn.init.orthogonal_(module.weight) + + +class SmallSRNetWrapper(nn.Module): + """Wrap a Super-Resolution model with pre-processing and post-processing. + + Args: + upscale_factor: The factor by which the image resolution is increased. + pretrained: Whether to load weights from a pre-trained model. Default: True. + """ + + def __init__(self, upscale_factor: int = 3, pretrained: bool = True) -> None: + super().__init__() + self.rgb_to_ycbcr = RgbToYcbcr() + self.ycbcr_to_rgb = YcbcrToRgb() + self.model = SmallSRNet(upscale_factor=upscale_factor, pretrained=pretrained) + self.upscale_factor = upscale_factor + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Apply RGB super-resolution through a luminance-only SR model. + + Args: + input: RGB image tensor with shape :math:`(B, 3, H, W)`. + + Returns: + RGB tensor upscaled by ``self.upscale_factor``. The Y channel is + predicted by the super-resolution model, while Cb and Cr channels + are resized with bicubic interpolation before conversion back to + RGB. + """ + ycbcr = self.rgb_to_ycbcr(input) + y, cb, cr = ycbcr.split(1, dim=1) + out_y = self.model(y) + out_cb = torch.nn.functional.interpolate(cb, scale_factor=self.upscale_factor, mode="bicubic") + out_cr = torch.nn.functional.interpolate(cr, scale_factor=self.upscale_factor, mode="bicubic") + out = torch.cat([out_y, out_cb, out_cr], dim=1) + return self.ycbcr_to_rgb(out) diff --git a/kornia/models/smolvlm2/__init__.py b/kornia/models/smolvlm2/__init__.py new file mode 100644 index 0000000..d4efd55 --- /dev/null +++ b/kornia/models/smolvlm2/__init__.py @@ -0,0 +1,20 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .smolvlm2 import SmolVLM2 + +__all__ = ["SmolVLM2"] diff --git a/kornia/models/smolvlm2/smolvlm2.py b/kornia/models/smolvlm2/smolvlm2.py new file mode 100644 index 0000000..2b19c11 --- /dev/null +++ b/kornia/models/smolvlm2/smolvlm2.py @@ -0,0 +1,45 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch +from torch import nn + + +class SmolVLM2(nn.Module): + """SmolVLM2 scaffold. This is a placeholder implementation.""" + + def __init__(self, vision_dim: int = 768, text_dim: int = 768) -> None: + super().__init__() + self.vision_proj = nn.Linear(vision_dim, vision_dim) + self.text_proj = nn.Linear(text_dim, text_dim) + + def forward(self, image_features: torch.Tensor, text_features: torch.Tensor) -> torch.Tensor: + """Combine projected image and text feature tensors. + + Args: + image_features: Visual feature tensor with last dimension matching + the configured vision dimension. + text_features: Text feature tensor with the same broadcastable + leading shape and last dimension matching the configured text + dimension. + + Returns: + Tensor containing the sum of projected image and text features. + """ + v = self.vision_proj(image_features) + t = self.text_proj(text_features) + return v + t diff --git a/kornia/models/structures.py b/kornia/models/structures.py new file mode 100644 index 0000000..4a5b0d9 --- /dev/null +++ b/kornia/models/structures.py @@ -0,0 +1,130 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +import torch + +from kornia.core.check import KORNIA_CHECK +from kornia.geometry.transform import resize + + +@dataclass +class SegmentationResults: + """Encapsulate the results obtained by a Segmentation model. + + Args: + logits: Results logits with shape :math:`(B, C, H, W)`, where :math:`C` refers to the number of predicted masks + scores: The scores from the logits. Shape :math:`(B, C)` + mask_threshold: The threshold value to generate the `binary_masks` from the `logits` + + """ + + logits: torch.Tensor + scores: torch.Tensor + mask_threshold: float = 0.0 + _original_res_logits: Optional[torch.Tensor] = None + + @property + def binary_masks(self) -> torch.Tensor: + """Binary mask generated from logits considering the mask_threshold. + + Shape will be the same of logits :math:`(B, C, H, W)` where :math:`C` is the number masks predicted. + + .. note:: If you run `original_res_logits`, this will generate the masks + based on the original resolution logits. + Otherwise, this will use the low resolution logits (self.logits). + """ + if self._original_res_logits is not None: + x = self._original_res_logits + else: + x = self.logits + + return x > self.mask_threshold + + def original_res_logits( + self, input_size: tuple[int, int], original_size: tuple[int, int], image_size_encoder: Optional[tuple[int, int]] + ) -> torch.Tensor: + """Remove padding and upscale the logits to the original image size. + + Resize to image encoder input -> remove padding (bottom and right) -> Resize to original size + + .. note:: This method set a internal `original_res_logits` which will be used if available for the binary masks. + + Args: + input_size: The size of the image input to the model, in (H, W) format. Used to remove padding. + original_size: The original size of the image before resizing for input to the model, in (H, W) format. + image_size_encoder: The size of the input image for image encoder, in (H, W) format. Used to resize the + logits back to encoder resolution before remove the padding. + + Returns: + Batched logits in :math:`(K, C, H, W)` format, where (H, W) is given by original_size. + + """ + x = self.logits + + if isinstance(image_size_encoder, tuple): + x = resize(x, size=image_size_encoder, interpolation="bilinear", align_corners=False, antialias=False) + x = x[..., : input_size[0], : input_size[1]] + + x = resize(x, size=original_size, interpolation="bilinear", align_corners=False, antialias=False) + + self._original_res_logits = x + return self._original_res_logits + + def squeeze(self, dim: int = 0) -> SegmentationResults: + """Realize a squeeze for the dim given for all properties.""" + self.logits = self.logits.squeeze(dim) + self.scores = self.scores.squeeze(dim) + if isinstance(self._original_res_logits, torch.Tensor): + self._original_res_logits = self._original_res_logits.squeeze(dim) + + return self + + +@dataclass +class Prompts: + """Encapsulate the prompts inputs for a Model. + + Args: + points: A tuple with the keypoints (coordinates x, y) and their respective labels. Shape :math:`(K, N, 2)` for + the keypoints, and :math:`(K, N)` + boxes: Batched box inputs, with shape :math:`(K, 4)`. Expected to be into xyxy format. + masks: Batched mask prompts to the model with shape :math:`(K, 1, H, W)` + + """ + + points: Optional[tuple[torch.Tensor, torch.Tensor]] = None + boxes: Optional[torch.Tensor] = None + masks: Optional[torch.Tensor] = None + + def __post_init__(self) -> None: + if isinstance(self.keypoints, torch.Tensor) and isinstance(self.boxes, torch.Tensor): + KORNIA_CHECK(self.keypoints.shape[0] == self.boxes.shape[0], "The prompts should have the same batch size!") + + @property + def keypoints(self) -> Optional[torch.Tensor]: + """The keypoints from the `points`.""" + return self.points[0] if isinstance(self.points, tuple) else None + + @property + def keypoints_labels(self) -> Optional[torch.Tensor]: + """The keypoints labels from the `points`.""" + return self.points[1] if isinstance(self.points, tuple) else None diff --git a/kornia/models/tiny_vit.py b/kornia/models/tiny_vit.py new file mode 100644 index 0000000..aa12ba3 --- /dev/null +++ b/kornia/models/tiny_vit.py @@ -0,0 +1,772 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# https://github.com/microsoft/Cream/blob/8dc38822b99fff8c262c585a32a4f09ac504d693/TinyViT/models/tiny_vit.py +# https://github.com/ChaoningZhang/MobileSAM/blob/01ea8d0f5590082f0c1ceb0a3e2272593f20154b/mobile_sam/modeling/tiny_vit_sam.py + +from __future__ import annotations + +import warnings +from typing import Any, Optional, Sequence + +import torch +import torch.nn.functional as F +from torch import nn +from torch.utils import checkpoint + +from kornia.core.check import KORNIA_CHECK +from kornia.models.common import DropPath, LayerNorm2d, window_partition, window_unpartition + + +def _make_pair(x: int | tuple[int, int]) -> tuple[int, int]: + return (x, x) if isinstance(x, int) else x + + +class ConvBN(nn.Sequential): + """Implement a sequential block containing a convolution followed by Batch Normalization. + + Args: + in_channels: The number of input channels. + out_channels: The number of output channels. + kernel_size: The size of the convolving kernel. Default: 1. + stride: The stride of the convolution. Default: 1. + padding: The zero-padding added to both sides of the input. Default: 0. + groups: The number of blocked connections from input to output. Default: 1. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + stride: int = 1, + padding: int = 0, + groups: int = 1, + activation: type[nn.Module] = nn.Identity, + ) -> None: + super().__init__() + self.c = nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding, groups=groups, bias=False) + self.bn = nn.BatchNorm2d(out_channels) + self.act = activation() + + +class PatchEmbed(nn.Sequential): + """Perform patch embedding using a series of convolutions. + + This module divides the input image into patches and projects them into + the embedding dimension. + + Args: + in_channels: The number of input image channels. + embed_dim: The dimension of the resulting embeddings. + activation: The activation layer to use. Default: :class:`nn.GELU`. + """ + + def __init__(self, in_channels: int, embed_dim: int, activation: type[nn.Module] = nn.GELU) -> None: + super().__init__() + self.seq = nn.Sequential( + ConvBN(in_channels, embed_dim // 2, 3, 2, 1), activation(), ConvBN(embed_dim // 2, embed_dim, 3, 2, 1) + ) + + +class MBConv(nn.Module): + """Implement the Mobile Inverted Residual Bottleneck Convolution layer. + + Args: + in_channels: The number of input channels. + out_channels: The number of output channels. + expansion: The expansion ratio for the hidden dimension. + kernel_size: The convolution kernel size. + stride: The stride of the convolution. + dropout: The dropout rate for the stochastic depth. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + expansion_ratio: float, + activation: type[nn.Module] = nn.GELU, + drop_path: float = 0.0, + ) -> None: + super().__init__() + hidden_channels = int(in_channels * expansion_ratio) + self.conv1 = ConvBN(in_channels, hidden_channels, 1, activation=activation) # point-wise + self.conv2 = ConvBN(hidden_channels, hidden_channels, 3, 1, 1, hidden_channels, activation) # depth-wise + self.conv3 = ConvBN(hidden_channels, out_channels, 1) + self.drop_path = DropPath(drop_path) + self.act = activation() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply the MBConv residual block to a feature map. + + Args: + x: Feature map tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is batch size, :math:`C` is channels, :math:`H` is + height, and :math:`W` is width. + + Returns: + Tensor with the same shape as ``x`` after pointwise expansion, + depthwise convolution, projection, stochastic depth, and residual + activation. + """ + return self.act(x + self.drop_path(self.conv3(self.conv2(self.conv1(x))))) + + +class PatchMerging(nn.Module): + """Implement the patch merging layer for downsampling in TinyViT. + + Args: + input_resolution: The height and width of the input feature map. + dim: The number of input channels. + out_dim: The number of output channels. + act: The activation function class. Default: nn.GELU. + """ + + def __init__( + self, + input_resolution: int | tuple[int, int], + dim: int, + out_dim: int, + stride: int, + activation: type[nn.Module] = nn.GELU, + ) -> None: + KORNIA_CHECK(stride in (1, 2), "stride must be either 1 or 2") + super().__init__() + self.input_resolution = _make_pair(input_resolution) + self.conv1 = ConvBN(dim, out_dim, 1, activation=activation) + self.conv2 = ConvBN(out_dim, out_dim, 3, stride, 1, groups=out_dim, activation=activation) + self.conv3 = ConvBN(out_dim, out_dim, 1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Merge or project patch tokens for TinyViT downsampling. + + Args: + x: Either token tensor with shape :math:`(B, H * W, C)` or feature + map tensor with shape :math:`(B, C, H, W)`. + + Returns: + Token tensor with shape :math:`(B, H_{out} * W_{out}, C_{out})`. + The spatial size is reduced when this layer was configured with + stride ``2``. + """ + if x.ndim == 3: + x = x.transpose(1, 2).unflatten(2, self.input_resolution) # (B, H * W, C) -> (B, C, H, W) + x = self.conv3(self.conv2(self.conv1(x))) + x = x.flatten(2).transpose(1, 2) # (B, C, H, W) -> (B, H * W, C) + return x + + +class ConvLayer(nn.Module): + """Implement a convolutional layer with optional checkpointing and downsample. + + Args: + dim: The number of input channels. + depth: The number of blocks in the convolutional layer. + activation: The activation function to use. Default: :class:`nn.GELU`. + drop_path: The dropout rate for the stochastic depth. Default: 0.0. + downsample: The downsample module to use. Default: None. + use_checkpoint: Whether to use checkpointing for memory efficiency. Default: False. + conv_expand_ratio: The expansion ratio for the hidden dimension. Default: 4.0. + """ + + def __init__( + self, + dim: int, + depth: int, + activation: type[nn.Module] = nn.GELU, + drop_path: float | list[float] = 0.0, + downsample: Optional[nn.Module] = None, + use_checkpoint: bool = False, + conv_expand_ratio: float = 4.0, + ) -> None: + super().__init__() + self.use_checkpoint = use_checkpoint + + # build blocks + if not isinstance(drop_path, list): + drop_path = [drop_path] * depth + self.blocks = nn.ModuleList( + [MBConv(dim, dim, conv_expand_ratio, activation, drop_path[i]) for i in range(depth)] + ) + + # patch merging layer + self.downsample = downsample + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply a sequence of convolutional TinyViT blocks. + + Args: + x: Feature tensor passed through the layer blocks. + + Returns: + Tensor after all MBConv blocks and the optional downsample module. + The output may be a token tensor when downsampling is configured. + """ + for blk in self.blocks: + x = checkpoint.checkpoint(blk, x) if self.use_checkpoint else blk(x) + if self.downsample is not None: + x = self.downsample(x) + return x + + +class MLP(nn.Sequential): + """Implement a multi-layer perceptron (MLP) with optional dropout. + + Args: + in_features: The number of input features. + hidden_features: The number of hidden features. + out_features: The number of output features. + activation: The activation function to use. Default: :class:`nn.GELU`. + drop: The dropout rate. Default: 0.0. + """ + + def __init__( + self, + in_features: int, + hidden_features: int, + out_features: int, + activation: type[nn.Module] = nn.GELU, + drop: float = 0.0, + ) -> None: + super().__init__() + self.norm = nn.LayerNorm(in_features) + self.fc1 = nn.Linear(in_features, hidden_features) + self.act1 = activation() + self.drop1 = nn.Dropout(drop) + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop2 = nn.Dropout(drop) + + +# NOTE: differences from image_encoder.Attention: +# - different relative position encoding mechanism (separable/decomposed vs joint) +# - this impl supports attn_ratio (increase output size for value), though it is not used +class Attention(nn.Module): + """Implement an attention mechanism with optional relative position encoding. + + Args: + dim: The number of input channels. + key_dim: The dimension of the key. + num_heads: The number of attention heads. Default: 8. + attn_ratio: The ratio of the attention dimension. Default: 4.0. + resolution: The resolution of the input feature map. Default: (14, 14). + """ + + def __init__( + self, + dim: int, + key_dim: int, + num_heads: int = 8, + attn_ratio: float = 4.0, + resolution: tuple[int, int] = (14, 14), + ) -> None: + super().__init__() + self.num_heads = num_heads + self.scale = key_dim**-0.5 + self.key_dim = key_dim + self.nh_kd = key_dim * num_heads + self.d = int(attn_ratio * key_dim) + self.dh = int(attn_ratio * key_dim) * num_heads + self.attn_ratio = attn_ratio + h = self.dh + self.nh_kd * 2 + + self.norm = nn.LayerNorm(dim) + self.qkv = nn.Linear(dim, h) + self.proj = nn.Linear(self.dh, dim) + + indices, attn_offset_size = self.build_attention_bias(resolution) + self.attention_biases = nn.Parameter(torch.zeros(num_heads, attn_offset_size)) + self.register_buffer("attention_bias_idxs", indices, persistent=False) + self.attention_bias_idxs: torch.Tensor + self.ab: Optional[torch.Tensor] = None + + @staticmethod + def build_attention_bias(resolution: tuple[int, int]) -> tuple[torch.Tensor, int]: + """Build lookup indices for relative attention bias. + + Args: + resolution: Window resolution ``(H, W)`` used by the attention + block. + + Returns: + Tuple ``(indices, size)``. ``indices`` maps each token pair in the + window to a relative-position bias entry, and ``size`` is the + number of unique bias entries. + """ + H, W = resolution + rows = torch.arange(H) + cols = torch.arange(W) + rr = rows.repeat_interleave(W) + cc = cols.repeat(H) + dr = (rr[:, None] - rr[None, :]).abs() + dc = (cc[:, None] - cc[None, :]).abs() + keys = dr * W + dc + unique_keys, inverse = torch.unique(keys, return_inverse=True) + indices = inverse.view(H * W, H * W) + attn_offset_size = unique_keys.numel() + return indices, attn_offset_size + + # is this really necessary? + @torch.no_grad() + def train(self, mode: bool = True) -> Attention: + """Switch training/evaluation mode and refresh cached attention bias. + + Args: + mode: ``True`` for training mode, ``False`` for evaluation mode. + + Returns: + ``self`` after updating the module mode. In evaluation mode the + relative attention bias lookup is cached for reuse. + """ + super().train(mode) + self.ab = None if (mode and self.ab is not None) else self.attention_biases[:, self.attention_bias_idxs] + return self + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply relative-position self-attention to token features. + + Args: + x: Token tensor with shape :math:`(B, N, C)`, where :math:`B` is + batch size, :math:`N` is number of tokens, and :math:`C` is + embedding dimension. + + Returns: + Tensor with shape :math:`(B, N, C)` after attention and projection. + """ + B, N, _ = x.shape + x = self.norm(x) + qkv = self.qkv(x) + qkv = qkv.view(B, N, self.num_heads, -1).permute(0, 2, 1, 3) + q, k, v = qkv.split([self.key_dim, self.key_dim, self.d], dim=3) + + bias = self.attention_biases[:, self.attention_bias_idxs] if self.training else self.ab + attn = (q @ k.transpose(-2, -1)) * self.scale + bias + + attn = attn.softmax(dim=-1) + x = (attn @ v).transpose(1, 2).reshape(B, N, self.dh) + x = self.proj(x) + return x + + +class TinyViTBlock(nn.Module): + """Implement a single block of the TinyViT architecture. + + This block consists of multi-head self-attention and a feed-forward network + with residual connections. + + Args: + dim: The input dimension size. + input_resolution: The height and width of the input feature map. + num_heads: The number of attention heads. + window_size: The size of the local attention window. + mlp_ratio: The ratio of MLP hidden dimension to embedding dimension. + drop: The dropout rate. Default: 0.0. + drop_path: The stochastic depth rate. Default: 0.0. + layer_scale_init_value: The initial value for layer scaling. Default: 1e-5. + """ + + def __init__( + self, + dim: int, + input_resolution: int | tuple[int, int], + num_heads: int, + window_size: int = 7, + mlp_ratio: float = 4.0, + drop: float = 0.0, + drop_path: float = 0.0, + local_conv_size: int = 3, + activation: type[nn.Module] = nn.GELU, + ) -> None: + KORNIA_CHECK(dim % num_heads == 0, "dim must be divislbe by num_heads") + super().__init__() + self.input_resolution = _make_pair(input_resolution) + self.window_size = window_size + head_dim = dim // num_heads + + self.attn = Attention(dim, head_dim, num_heads, 1.0, (window_size, window_size)) + self.drop_path1 = DropPath(drop_path) + self.local_conv = ConvBN(dim, dim, local_conv_size, 1, local_conv_size // 2, dim) + self.mlp = MLP(dim, int(dim * mlp_ratio), dim, activation, drop) + self.drop_path2 = DropPath(drop_path) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply one TinyViT window-attention block. + + Args: + x: Token tensor with shape :math:`(B, H * W, C)`, where + :math:`H` and :math:`W` match ``self.input_resolution``. + + Returns: + Tensor with the same shape as ``x`` after window attention, local + convolution, MLP, and residual connections. + """ + H, W = self.input_resolution + B, L, C = x.shape + res_x = x + + x = x.view(B, H, W, C) + x, pad_hw = window_partition(x, self.window_size) # (B * num_windows, window_size, window_size, C) + x = self.attn(x.flatten(1, 2)) + x = window_unpartition(x, self.window_size, pad_hw, (H, W)) + x = x.view(B, L, C) + + x = res_x + self.drop_path1(x) + + x = x.transpose(1, 2).reshape(B, C, H, W) + x = self.local_conv(x) + x = x.view(B, C, L).transpose(1, 2) + + x = x + self.drop_path2(self.mlp(x)) + return x + + +class BasicLayer(nn.Module): + """Implement a basic layer of the TinyViT architecture. + + This layer consists of a series of TinyViT blocks, each followed by a patch + merging operation. + + Args: + dim: The input dimension size. + input_resolution: The height and width of the input feature map. + depth: The number of blocks in the layer. + num_heads: The number of attention heads. + window_size: The size of the local attention window. + mlp_ratio: The ratio of MLP hidden dimension to embedding dimension. + drop: The dropout rate. Default: 0.0. + drop_path: The stochastic depth rate. Default: 0.0. + downsample: The downsample module to use. Default: None. + use_checkpoint: Whether to use checkpointing for memory efficiency. Default: False. + local_conv_size: The size of the local convolution kernel. Default: 3. + activation: The activation function to use. Default: :class:`nn.GELU`. + """ + + def __init__( + self, + dim: int, + input_resolution: int | tuple[int, int], + depth: int, + num_heads: int, + window_size: int, + mlp_ratio: float = 4.0, + drop: float = 0.0, + drop_path: float | list[float] = 0.0, + downsample: Optional[nn.Module] = None, + use_checkpoint: bool = False, + local_conv_size: int = 3, + activation: type[nn.Module] = nn.GELU, + ) -> None: + super().__init__() + self.use_checkpoint = use_checkpoint + + self.blocks = nn.ModuleList( + [ + TinyViTBlock( + dim, + input_resolution, + num_heads, + window_size, + mlp_ratio, + drop, + drop_path[i] if isinstance(drop_path, list) else drop_path, + local_conv_size, + activation, + ) + for i in range(depth) + ] + ) + + # patch merging layer + self.downsample = downsample + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run a TinyViT stage made of several blocks. + + Args: + x: Token tensor with shape :math:`(B, H * W, C)`. + + Returns: + Tensor after all TinyViT blocks and optional patch merging. The + channel count or token resolution may change if ``downsample`` is + configured. + """ + for blk in self.blocks: + x = checkpoint.checkpoint(blk, x) if self.use_checkpoint else blk(x) + if self.downsample is not None: + x = self.downsample(x) + return x + + +class TinyViT(nn.Module): + """TinyViT model, as described in https://arxiv.org/abs/2207.10666. + + Args: + img_size: Size of input image. + in_chans: Number of input image's channels. + num_classes: Number of output classes. + embed_dims: List of embedding dimensions. + depths: List of block count for each downsampling stage + num_heads: List of attention heads used in self-attention for each downsampling stage. + window_sizes: List of self-attention's window size for each downsampling stage. + mlp_ratio: Ratio of MLP dimension to embedding dimension in self-attention. + drop_rate: Dropout rate. + drop_path_rate: Stochastic depth rate. + use_checkpoint: Whether to use activation checkpointing to trade compute for memory. + mbconv_expand_ratio: Expansion ratio used in MBConv block. + local_conv_size: Kernel size of convolution used in TinyViTBlock + activation: activation function. + mobile_same: Whether to use modifications for MobileSAM. + + """ + + def __init__( + self, + img_size: int = 224, + in_chans: int = 3, + num_classes: int = 1000, + embed_dims: Sequence[int] = (96, 192, 384, 768), + depths: Sequence[int] = (2, 2, 6, 2), + num_heads: Sequence[int] = (3, 6, 12, 24), + window_sizes: Sequence[int] = (7, 7, 14, 7), + mlp_ratio: float = 4.0, + drop_rate: float = 0.0, + drop_path_rate: float = 0.0, + use_checkpoint: bool = False, + mbconv_expand_ratio: float = 4.0, + local_conv_size: int = 3, + # layer_lr_decay: float = 1.0, + activation: type[nn.Module] = nn.GELU, + mobile_sam: bool = False, + ) -> None: + super().__init__() + self.img_size = img_size + self.mobile_sam = mobile_sam + self.neck: Optional[nn.Module] + if mobile_sam: + # MobileSAM adjusts the stride to match the total stride of other ViT backbones + # used in the original SAM (stride 16) + strides = [2, 2, 1, 1] + self.neck = nn.Sequential( + nn.Conv2d(embed_dims[-1], 256, 1, bias=False), + LayerNorm2d(256), + nn.Conv2d(256, 256, 3, 1, 1, bias=False), + LayerNorm2d(256), + ) + else: + strides = [2, 2, 2, 1] + self.neck = None + + self.patch_embed = PatchEmbed(in_chans, embed_dims[0], activation) + input_resolution = img_size // 4 + + # NOTE: if we don't support training, this might be unimportant + # stochastic depth decay rule + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))] + + # build layers + n_layers = len(depths) + layers = [] + for i_layer, (embed_dim, depth, num_heads_i, window_size, stride) in enumerate( + zip(embed_dims, depths, num_heads, window_sizes, strides) + ): + out_dim = embed_dims[min(i_layer + 1, len(embed_dims) - 1)] + downsample = ( + PatchMerging(input_resolution, embed_dim, out_dim, stride, activation) + if (i_layer < n_layers - 1) + else None + ) + kwargs: dict[str, Any] = { + "dim": embed_dim, + "depth": depth, + "drop_path": dpr[sum(depths[:i_layer]) : sum(depths[: i_layer + 1])], + "downsample": downsample, + "use_checkpoint": use_checkpoint, + "activation": activation, + } + layer: ConvLayer | BasicLayer + if i_layer == 0: + layer = ConvLayer(conv_expand_ratio=mbconv_expand_ratio, **kwargs) + else: + layer = BasicLayer( + input_resolution=input_resolution, + num_heads=num_heads_i, + window_size=window_size, + mlp_ratio=mlp_ratio, + drop=drop_rate, + local_conv_size=local_conv_size, + **kwargs, + ) + layers.append(layer) + input_resolution //= stride + self.layers = nn.Sequential(*layers) + self.feat_size = input_resolution # final feature map size + + # Classifier head + # NOTE: this is redundant for MobileSAM, but we still need it + # to load pre-trained weights with strict=True + # TODO: enable strict=False, or host our own weights + self.norm_head = nn.LayerNorm(embed_dims[-1]) + self.head = nn.Linear(embed_dims[-1], num_classes) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Classify images if ``mobile_sam=False``, produce feature maps if ``mobile_sam=True``.""" + x = self.patch_embed(x) + x = self.layers(x) + + if self.mobile_sam: + # MobileSAM + x = x.unflatten(1, (self.feat_size, self.feat_size)).permute(0, 3, 1, 2) + x = self.neck(x) # type: ignore + else: + # classification + x = x.mean(1) + x = self.head(self.norm_head(x)) + return x + + @staticmethod + def from_config(variant: str, pretrained: bool | str = False, **kwargs: Any) -> TinyViT: + """Create a TinyViT model from pre-defined variants. + + Args: + variant: TinyViT variant. Possible values: ``'5m'``, ``'11m'``, ``'21m'``. + pretrained: whether to use pre-trained weights. Possible values: ``False``, ``True``, ``'in22k'``, + ``'in1k'``. For TinyViT-21M (``variant='21m'``), ``'in1k_384'``, ``'in1k_512'`` are also available. + **kwargs: other keyword arguments that will be passed to :class:`TinyViT`. + + .. note:: + When ``img_size`` is different from the pre-trained size, bicubic interpolation will be performed on + attention biases. When using ``pretrained=True``, ImageNet-1k checkpoint (``'in1k'``) is used. + For feature extraction or fine-tuning, ImageNet-22k checkpoint (``'in22k'``) is preferred. + + """ + KORNIA_CHECK(variant in ("5m", "11m", "21m"), "Only variant 5m, 11m, and 21m are supported") + return {"5m": _tiny_vit_5m, "11m": _tiny_vit_11m, "21m": _tiny_vit_21m}[variant](pretrained, **kwargs) + + +def _load_pretrained(model: TinyViT, url: str) -> TinyViT: + model_state_dict = model.state_dict() + state_dict = torch.hub.load_state_dict_from_url(url) + + # official checkpoint has "model" key + if "model" in state_dict: + state_dict = state_dict["model"] + + # https://github.com/microsoft/Cream/blob/8dc38822b99fff8c262c585a32a4f09ac504d693/TinyViT/utils.py#L163 + # bicubic interpolate attention biases + ab_keys = [k for k in state_dict.keys() if "attention_biases" in k] + for k in ab_keys: + n_heads1, L1 = state_dict[k].shape + n_heads2, L2 = model_state_dict[k].shape + KORNIA_CHECK(n_heads1 == n_heads2, f"Fail to load {k}. Pre-trained checkpoint should have num_heads={n_heads1}") + + if L1 != L2: + S1 = int(L1**0.5) + S2 = int(L2**0.5) + attention_biases = state_dict[k].view(1, n_heads1, S1, S1) + attention_biases = F.interpolate(attention_biases, size=(S2, S2), mode="bicubic") + state_dict[k] = attention_biases.view(n_heads2, L2) + + if state_dict["head.weight"].shape[0] != model.head.out_features: + msg = "Number of classes does not match pre-trained checkpoint's. Resetting classification head to zeros" + warnings.warn(msg, stacklevel=1) + state_dict["head.weight"] = torch.zeros_like(model.head.weight) + state_dict["head.bias"] = torch.zeros_like(model.head.bias) + + model.load_state_dict(state_dict) + return model + + +def _tiny_vit_5m(pretrained: bool | str = False, **kwargs: Any) -> TinyViT: + model = TinyViT( + embed_dims=[64, 128, 160, 320], + depths=[2, 2, 6, 2], + num_heads=[2, 4, 5, 10], + window_sizes=[7, 7, 14, 7], + drop_path_rate=0.0, + **kwargs, + ) + + if pretrained: + if pretrained is True: + pretrained = "in1k" + + url = { + "in22k": ( + "https://github.com/wkcn/TinyViT-model-zoo/releases/download/checkpoints/tiny_vit_5m_22k_distill.pth" + ), + "in1k": "https://github.com/wkcn/TinyViT-model-zoo/releases/download/checkpoints/tiny_vit_5m_22kto1k_distill.pth", + }[pretrained] + model = _load_pretrained(model, url) + + return model + + +def _tiny_vit_11m(pretrained: bool | str = False, **kwargs: Any) -> TinyViT: + model = TinyViT( + embed_dims=[64, 128, 256, 448], + depths=[2, 2, 6, 2], + num_heads=[2, 4, 8, 14], + window_sizes=[7, 7, 14, 7], + drop_path_rate=0.1, + **kwargs, + ) + + if pretrained: + if pretrained is True: + pretrained = "in1k" + + url = { + "in22k": ( + "https://github.com/wkcn/TinyViT-model-zoo/releases/download/checkpoints/tiny_vit_11m_22k_distill.pth" + ), + "in1k": "https://github.com/wkcn/TinyViT-model-zoo/releases/download/checkpoints/tiny_vit_11m_22kto1k_distill.pth", + }[pretrained] + model = _load_pretrained(model, url) + + return model + + +def _tiny_vit_21m(pretrained: bool | str = False, **kwargs: Any) -> TinyViT: + model = TinyViT( + embed_dims=[96, 192, 384, 576], + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 18], + window_sizes=[7, 7, 14, 7], + drop_path_rate=0.2, + **kwargs, + ) + + if pretrained: + if pretrained is True: + pretrained = "in1k" + img_size = kwargs.get("img_size", 224) + if img_size >= 384: + pretrained = "in1k_384" + if img_size >= 512: + pretrained = "in1k_512" + + url = { + "in22k": ( + "https://github.com/wkcn/TinyViT-model-zoo/releases/download/checkpoints/tiny_vit_21m_22k_distill.pth" + ), + "in1k": "https://github.com/wkcn/TinyViT-model-zoo/releases/download/checkpoints/tiny_vit_21m_22kto1k_distill.pth", + "in1k_384": "https://github.com/wkcn/TinyViT-model-zoo/releases/download/checkpoints/tiny_vit_21m_22kto1k_384_distill.pth", + "in1k_512": "https://github.com/wkcn/TinyViT-model-zoo/releases/download/checkpoints/tiny_vit_21m_22kto1k_512_distill.pth", + }[pretrained] + model = _load_pretrained(model, url) + + return model diff --git a/kornia/models/vit.py b/kornia/models/vit.py new file mode 100644 index 0000000..f0b7cf1 --- /dev/null +++ b/kornia/models/vit.py @@ -0,0 +1,336 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Module that implement Vision Transformer (ViT). + +Paper: https://paperswithcode.com/paper/an-image-is-worth-16x16-words-transformers-1 + +Based on: `https://towardsdatascience.com/implementing-visualttransformer-in-pytorch-184f9f16f632` + +Added some tricks from: `https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py` +""" + +from __future__ import annotations + +from typing import Any, Callable + +import torch +from torch import nn + +from kornia.core.check import KORNIA_CHECK + +__all__ = ["VisionTransformer"] + + +class ResidualAdd(nn.Module): + def __init__(self, fn: Callable[..., torch.Tensor]) -> None: + super().__init__() + self.fn = fn + + def forward(self, x: torch.Tensor, **kwargs: Any) -> torch.Tensor: + res = x + x = self.fn(x, **kwargs) + x += res + return x + + +class FeedForward(nn.Sequential): + def __init__(self, in_features: int, hidden_features: int, out_features: int, dropout_rate: float = 0.0) -> None: + super().__init__( + nn.Linear(in_features, hidden_features), + nn.GELU(), + nn.Dropout(dropout_rate), + nn.Linear(hidden_features, out_features), + nn.Dropout(dropout_rate), # added one extra as in timm + ) + + +class MultiHeadAttention(nn.Module): + def __init__(self, emb_size: int, num_heads: int, att_drop: float, proj_drop: float) -> None: + super().__init__() + self.emb_size = emb_size + self.num_heads = num_heads + head_size = emb_size // num_heads # from timm + self.scale = head_size**-0.5 # from timm + + if self.emb_size % self.num_heads: + raise ValueError( + f"Size of embedding inside the transformer decoder must be visible by number of heads" + f"for correct multi-head attention " + f"Got: {self.emb_size} embedding size and {self.num_heads} numbers of heads" + ) + + # fuse the queries, keys and values in one matrix + self.qkv = nn.Linear(emb_size, emb_size * 3) + self.att_drop = nn.Dropout(att_drop) + self.projection = nn.Linear(emb_size, emb_size) + self.projection_drop = nn.Dropout(proj_drop) # added timm trick + + def forward(self, x: torch.Tensor) -> torch.Tensor: + B, N, C = x.shape + # split keys, queries and values in num_heads + # NOTE: the line below differs from timm + # timm: qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] + + # sum up over the last axis + att = torch.einsum("bhqd, bhkd -> bhqk", q, k) * self.scale + att = att.softmax(dim=-1) + att = self.att_drop(att) + + # sum up over the third axis + out = torch.einsum("bhal, bhlv -> bhav ", att, v) + out = out.permute(0, 2, 1, 3).contiguous().view(B, N, -1) + out = self.projection(out) + out = self.projection_drop(out) + return out + + +class TransformerEncoderBlock(nn.Sequential): + def __init__(self, embed_dim: int, num_heads: int, dropout_rate: float, dropout_attn: float) -> None: + super().__init__( + ResidualAdd( + nn.Sequential( + nn.LayerNorm(embed_dim, 1e-6), + MultiHeadAttention(embed_dim, num_heads, dropout_attn, dropout_rate), + nn.Dropout(dropout_rate), + ) + ), + ResidualAdd( + nn.Sequential( + nn.LayerNorm(embed_dim, 1e-6), + FeedForward(embed_dim, embed_dim * 4, embed_dim, dropout_rate=dropout_rate), + nn.Dropout(dropout_rate), + ) + ), + ) + + +class TransformerEncoder(nn.Module): + def __init__( + self, + embed_dim: int = 768, + depth: int = 12, + num_heads: int = 12, + dropout_rate: float = 0.0, + dropout_attn: float = 0.0, + ) -> None: + super().__init__() + self.blocks = nn.Sequential( + *(TransformerEncoderBlock(embed_dim, num_heads, dropout_rate, dropout_attn) for _ in range(depth)) + ) + self.results: list[torch.Tensor] = [] + + def forward(self, x: torch.Tensor) -> torch.Tensor: + self.results = [] + out = x + for m in self.blocks.children(): + out = m(out) + self.results.append(out) + return out + + +class PatchEmbedding(nn.Module): + """Compute the 2d image patch embedding ready to pass to transformer encoder.""" + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 768, + patch_size: int = 16, + image_size: int = 224, + backbone: nn.Module | None = None, + ) -> None: + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.patch_size = patch_size + + # logic needed in case a backbone is passed + self.backbone = backbone or nn.Conv2d(in_channels, out_channels, kernel_size=patch_size, stride=patch_size) + if backbone is not None: + out_channels, feat_size = self._compute_feats_dims((in_channels, image_size, image_size)) + self.out_channels = out_channels + else: + feat_size = (image_size // patch_size) ** 2 + + self.cls_token = nn.Parameter(torch.randn(1, 1, out_channels)) + self.positions = nn.Parameter(torch.randn(feat_size + 1, out_channels)) + + def _compute_feats_dims(self, image_size: tuple[int, int, int]) -> tuple[int, int]: + out = self.backbone(torch.zeros(1, *image_size)).detach() + return out.shape[-3], out.shape[-2] * out.shape[-1] + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.backbone(x) + B, N, _, _ = x.shape + x = x.view(B, N, -1).permute(0, 2, 1) # BxNxE + cls_tokens = self.cls_token.repeat(B, 1, 1) # Bx1xE + # prepend the cls token to the input + x = torch.cat([cls_tokens, x], dim=1) # Bx(N+1)xE + # add position embedding + x += self.positions + return x + + +class VisionTransformer(nn.Module): + """Vision transformer (ViT) module. + + The module is expected to be used as operator for different vision tasks. + + The method is inspired from existing implementations of the paper :cite:`dosovitskiy2020vit`. + + .. warning:: + This is an experimental API subject to changes in favor of flexibility. + + Args: + image_size: the size of the input image. + patch_size: the size of the patch to compute the embedding. + in_channels: the number of channels for the input. + embed_dim: the embedding dimension inside the transformer encoder. + depth: the depth of the transformer. + num_heads: the number of attention heads. + dropout_rate: dropout rate. + dropout_attn: attention dropout rate. + backbone: an nn.Module to compute the image patches embeddings. + + Example: + >>> img = torch.rand(1, 3, 224, 224) + >>> vit = VisionTransformer(image_size=224, patch_size=16) + >>> vit(img).shape + torch.Size([1, 197, 768]) + + """ + + def __init__( + self, + image_size: int = 224, + patch_size: int = 16, + in_channels: int = 3, + embed_dim: int = 768, + depth: int = 12, + num_heads: int = 12, + dropout_rate: float = 0.0, + dropout_attn: float = 0.0, + backbone: nn.Module | None = None, + ) -> None: + super().__init__() + self.image_size = image_size + self.patch_size = patch_size + self.in_channels = in_channels + self.embed_size = embed_dim + + self.patch_embedding = PatchEmbedding(in_channels, embed_dim, patch_size, image_size, backbone) + hidden_dim = self.patch_embedding.out_channels + self.encoder = TransformerEncoder(hidden_dim, depth, num_heads, dropout_rate, dropout_attn) + self.norm = nn.LayerNorm(hidden_dim, 1e-6) + + @property + def encoder_results(self) -> list[torch.Tensor]: + """Return intermediate outputs captured by the transformer encoder. + + Returns: + List of tensors produced by the encoder blocks. Each tensor stores + token embeddings for a layer, typically shaped :math:`(B, N, D)`, + where :math:`B` is batch size, :math:`N` is token count, and + :math:`D` is embedding dimension. + """ + return self.encoder.results + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Encode an image batch into Vision Transformer token embeddings. + + Args: + x: Image tensor with shape :math:`(B, C, H, W)`, where + :math:`B` is batch size, :math:`C` must match + ``self.in_channels``, and :math:`H` and :math:`W` are expected + to match ``self.image_size``. + + Returns: + Normalized token embedding tensor produced by patch embedding and + the transformer encoder. The output shape follows the encoder + layout, usually :math:`(B, N, D)`. + """ + if not isinstance(x, torch.Tensor): + raise TypeError(f"Input x type is not a torch.Tensor. Got: {type(x)}") + + if self.image_size not in (*x.shape[-2:],) and x.shape[-3] != self.in_channels: + raise ValueError( + f"Input image shape must be Bx{self.in_channels}x{self.image_size}x{self.image_size}. Got: {x.shape}" + ) + + out = self.patch_embedding(x) + out = self.encoder(out) + out = self.norm(out) + return out + + @staticmethod + def from_config(variant: str, pretrained: bool = False, **kwargs: Any) -> VisionTransformer: + """Build ViT model based on the given config string. + + The format is ``vit_{size}/{patch_size}``. + E.g. ``vit_b/16`` means ViT-Base, patch size 16x16. If ``pretrained=True``, AugReg weights are loaded. + The weights are hosted on HuggingFace's model hub: https://huggingface.co/kornia. + + .. note:: + The available weights are: ``vit_l/16``, ``vit_b/16``, ``vit_s/16``, ``vit_ti/16``, + ``vit_b/32``, ``vit_s/32``. + + Args: + variant: ViT model variant e.g. ``vit_b/16``. + pretrained: whether to load pre-trained AugReg weights. + kwargs: other keyword arguments that will be passed to :func:`kornia.models.vit.VisionTransformer`. + + Returns: + The respective ViT model + + Example: + >>> from kornia.models.vit import VisionTransformer + >>> vit_model = VisionTransformer.from_config("vit_b/16", pretrained=True) + + """ + model_type, patch_size_str = variant.split("/") + patch_size = int(patch_size_str) + + model_config = { + "vit_ti": {"embed_dim": 192, "depth": 12, "num_heads": 3}, + "vit_s": {"embed_dim": 384, "depth": 12, "num_heads": 6}, + "vit_b": {"embed_dim": 768, "depth": 12, "num_heads": 12}, + "vit_l": {"embed_dim": 1024, "depth": 24, "num_heads": 16}, + "vit_h": {"embed_dim": 1280, "depth": 32, "num_heads": 16}, + }[model_type] + kwargs.update(model_config, patch_size=patch_size) + + model = VisionTransformer(**kwargs) + + if pretrained: + url = _get_weight_url(variant) + state_dict = torch.hub.load_state_dict_from_url(url) + model.load_state_dict(state_dict) + + return model + + +_AVAILABLE_WEIGHTS = ["vit_l/16", "vit_b/16", "vit_s/16", "vit_ti/16", "vit_b/32", "vit_s/32"] + + +def _get_weight_url(variant: str) -> str: + """Return the URL of the model weights.""" + KORNIA_CHECK(variant in _AVAILABLE_WEIGHTS, f"Variant {variant} does not have pre-trained checkpoint") + model_type, patch_size = variant.split("/") + return f"https://huggingface.co/kornia/{model_type}{patch_size}_augreg_i21k_r224/resolve/main/{model_type}-{patch_size}.pth" diff --git a/kornia/models/vit_mobile.py b/kornia/models/vit_mobile.py new file mode 100644 index 0000000..fc7408c --- /dev/null +++ b/kornia/models/vit_mobile.py @@ -0,0 +1,409 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict, Tuple + +import torch +from torch import nn + + +def conv_1x1_bn(inp: int, oup: int) -> nn.Module: + """Apply 1x1 Convolution with Batch Norm.""" + return nn.Sequential(nn.Conv2d(inp, oup, 1, 1, 0, bias=False), nn.BatchNorm2d(oup), nn.SiLU()) + + +def conv_nxn_bn(inp: int, oup: int, kernal_size: int = 3, stride: int = 1) -> nn.Module: + """Apply NxN Convolution with Batch Norm.""" + return nn.Sequential(nn.Conv2d(inp, oup, kernal_size, stride, 1, bias=False), nn.BatchNorm2d(oup), nn.SiLU()) + + +class PreNorm(nn.Module): + """Apply a normalization layer before the functional block. + + Args: + dim: The input dimension size. + fn: The module or function to be executed after normalization. + """ + + def __init__(self, dim: int, fn: nn.Module) -> None: + super().__init__() + self.norm = nn.LayerNorm(dim) + self.fn = fn + + def forward(self, x: torch.Tensor, **kwargs: Dict[str, Any]) -> torch.Tensor: + """Normalize token features before applying the wrapped block. + + Args: + x: Token tensor whose last dimension is ``dim``. + **kwargs: Extra keyword arguments forwarded to the wrapped module. + + Returns: + Output tensor returned by ``self.fn`` after layer normalization. + """ + return self.fn(self.norm(x), **kwargs) + + +class FeedForward(nn.Module): + """Implement the Feed-Forward network block for Vision Transformers. + + Args: + dim: The input dimension size. + hidden_dim: The dimension of the hidden layer. + dropout: The dropout probability. Default: 0.0. + """ + + def __init__(self, dim: int, hidden_dim: int, dropout: float = 0.0) -> None: + super().__init__() + self.net = nn.Sequential( + nn.Linear(dim, hidden_dim), nn.SiLU(), nn.Dropout(dropout), nn.Linear(hidden_dim, dim), nn.Dropout(dropout) + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply the feed-forward projection to token features. + + Args: + x: Token tensor with last dimension equal to ``dim``. + + Returns: + Tensor with the same shape as ``x`` after two linear projections, + activation, and dropout. + """ + return self.net(x) + + +class Attention(nn.Module): + """Implement the Multi-Head Attention module. + + Args: + dim: The input dimension size. + heads: The number of attention heads. Default: 8. + dim_head: The dimension of each head. Default: 64. + dropout: The dropout probability. Default: 0.0. + """ + + def __init__(self, dim: int, heads: int = 8, dim_head: int = 64, dropout: float = 0.0) -> None: + super().__init__() + inner_dim = dim_head * heads + project_out = not (heads == 1 and dim_head == dim) + + self.heads = heads + self.scale = dim_head**-0.5 + + self.attend = nn.Softmax(dim=-1) + self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False) + + self.to_out = nn.Sequential(nn.Linear(inner_dim, dim), nn.Dropout(dropout)) if project_out else nn.Identity() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply multi-head self-attention to MobileViT tokens. + + Args: + x: Token tensor with shape :math:`(B, P, N, D)`, where + :math:`B` is batch size, :math:`P` is patch area, + :math:`N` is number of patches, and :math:`D` is embedding + dimension. + + Returns: + Tensor with the same shape as ``x`` after attention and output + projection. + """ + qkv = self.to_qkv(x).chunk(3, dim=-1) + + b, p, n, hd = qkv[0].shape + q, k, v = (t.reshape(b, p, n, self.heads, hd // self.heads).transpose(2, 3) for t in qkv) + + dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale + attn = self.attend(dots) + out = torch.matmul(attn, v) + out = out.transpose(2, 3).reshape(b, p, n, hd) + return self.to_out(out) + + +class Transformer(nn.Module): + """Transformer block described in ViT. + + Paper: https://arxiv.org/abs/2010.11929 + Based on: https://github.com/lucidrains/vit-pytorch + + Args: + dim: input dimension. + depth: depth for transformer block. + heads: number of heads in multi-head attention layer. + dim_head: head size. + mlp_dim: dimension of the FeedForward layer. + dropout: dropout ratio, defaults to 0. + + """ + + def __init__(self, dim: int, depth: int, heads: int, dim_head: int, mlp_dim: int, dropout: float = 0.0) -> None: + super().__init__() + self.layers = nn.ModuleList([]) + for _ in range(depth): + self.layers.append( + nn.ModuleList( + [ + PreNorm(dim, Attention(dim, heads, dim_head, dropout)), + PreNorm(dim, FeedForward(dim, mlp_dim, dropout)), + ] + ) + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run stacked attention and feed-forward residual layers. + + Args: + x: Token tensor with shape :math:`(B, P, N, D)`. + + Returns: + Tensor with the same shape as ``x`` after all transformer layers. + """ + for attn, ff in self.layers: + x = attn(x) + x + x = ff(x) + x + return x + + +class MV2Block(nn.Module): + """MV2 block described in MobileNetV2. + + Paper: https://arxiv.org/pdf/1801.04381 + Based on: https://github.com/tonylins/pytorch-mobilenet-v2 + + Args: + inp: input channel. + oup: output channel. + stride: stride for convolution, defaults to 1, set to 2 if down-sample. + expansion: expansion ratio for hidden dimension, defaults to 4. + + """ + + def __init__(self, inp: int, oup: int, stride: int = 1, expansion: int = 4) -> None: + super().__init__() + self.stride = stride + + hidden_dim = int(inp * expansion) + self.use_res_connect = self.stride == 1 and inp == oup + + if expansion == 1: + self.conv = nn.Sequential( + # depthwise + nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False), + nn.BatchNorm2d(hidden_dim), + nn.SiLU(), + # pointwise + nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + ) + else: + self.conv = nn.Sequential( + # pointwise + nn.Conv2d(inp, hidden_dim, 1, 1, 0, bias=False), + nn.BatchNorm2d(hidden_dim), + nn.SiLU(), + # depthwise + nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False), + nn.BatchNorm2d(hidden_dim), + nn.SiLU(), + # pointwise + nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply an inverted residual MobileNetV2 block. + + Args: + x: Feature map tensor with shape :math:`(B, C_{in}, H, W)`. + + Returns: + Feature map tensor with output channels ``oup``. When stride is one + and input/output channels match, the block adds the residual input. + """ + if self.use_res_connect: + return x + self.conv(x) + else: + return self.conv(x) + + +class MobileViTBlock(nn.Module): + """MobileViT block mentioned in MobileViT. + + Args: + dim: input dimension of Transformer. + depth: depth of Transformer. + channel: input channel. + kernel_size: kernel size. + patch_size: patch size for folding and unfloding. + mlp_dim: dimension of the FeedForward layer in Transformer. + dropout: dropout ratio, defaults to 0. + + """ + + def __init__( + self, + dim: int, + depth: int, + channel: int, + kernel_size: int, + patch_size: Tuple[int, int], + mlp_dim: int, + dropout: float = 0.0, + ) -> None: + super().__init__() + self.ph, self.pw = patch_size + + self.conv1 = conv_nxn_bn(channel, channel, kernel_size) + self.conv2 = conv_1x1_bn(channel, dim) + + self.transformer = Transformer(dim, depth, 4, 8, mlp_dim, dropout) + + self.conv3 = conv_1x1_bn(dim, channel) + self.conv4 = conv_nxn_bn(2 * channel, channel, kernel_size) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Fuse local convolutions with global transformer context. + + Args: + x: Feature map tensor with shape :math:`(B, C, H, W)`. + + Returns: + Feature map tensor with the same spatial size as ``x``. Local + convolutional features are unfolded into patches, processed by a + transformer, folded back, and fused with the residual feature map. + """ + y = x.clone() + + # Local representations + x = self.conv1(x) + x = self.conv2(x) + + b, d, h, w = x.shape + nh, nw = h // self.ph, w // self.pw + + # Global representations + # [b, d, h, w] -> [b * d * nh, nw, ph, pw] + x = x.reshape(b * d * nh, self.ph, nw, self.pw).transpose(1, 2) + # [b * d * nh, nw, ph, pw] -> [b, (ph pw), (nh nw), d] + x = x.reshape(b, d, nh * nw, self.ph * self.pw).transpose(1, 3) + + x = self.transformer(x) + + # [b, (ph pw), (nh nw), d] -> [b * d * nh, nw, ph, pw] + x = x.transpose(1, 3).reshape(b * d * nh, nw, self.ph, self.pw) + # [b * d * nh, nw, ph, pw] -> [b, d, h, w] + x = x.transpose(1, 2).reshape(b, d, h, w) + + # Fusion + x = self.conv3(x) + x = torch.cat((x, y), 1) + x = self.conv4(x) + return x + + +class MobileViT(nn.Module): + """Module MobileViT. Default arguments is for MobileViT XXS. + + Paper: https://arxiv.org/abs/2110.02178 + Based on: https://github.com/chinhsuanwu/mobilevit-pytorch + + Args: + mode: 'xxs', 'xs' or 's', defaults to 'xxs'. + in_channels: the number of channels for the input image. + patch_size: image_size must be divisible by patch_size. + dropout: dropout ratio in Transformer. + + Example: + >>> img = torch.rand(1, 3, 256, 256) + >>> mvit = MobileViT(mode='xxs') + >>> mvit(img).shape + torch.Size([1, 320, 8, 8]) + + """ + + def __init__( + self, mode: str = "xxs", in_channels: int = 3, patch_size: Tuple[int, int] = (2, 2), dropout: float = 0.0 + ) -> None: + super().__init__() + if mode == "xxs": + expansion = 2 + dims = [64, 80, 96] + channels = [16, 16, 24, 24, 48, 48, 64, 64, 80, 80, 320] + elif mode == "xs": + expansion = 4 + dims = [96, 120, 144] + channels = [16, 32, 48, 48, 64, 64, 80, 80, 96, 96, 384] + elif mode == "s": + expansion = 4 + dims = [144, 192, 240] + channels = [16, 32, 64, 64, 96, 96, 128, 128, 160, 160, 640] + + kernel_size = 3 + depth = [2, 4, 3] + + self.conv1 = conv_nxn_bn(in_channels, channels[0], stride=2) + + self.mv2 = nn.ModuleList([]) + self.mv2.append(MV2Block(channels[0], channels[1], 1, expansion)) + self.mv2.append(MV2Block(channels[1], channels[2], 2, expansion)) + self.mv2.append(MV2Block(channels[2], channels[3], 1, expansion)) + self.mv2.append(MV2Block(channels[2], channels[3], 1, expansion)) # Repeat + self.mv2.append(MV2Block(channels[3], channels[4], 2, expansion)) + self.mv2.append(MV2Block(channels[5], channels[6], 2, expansion)) + self.mv2.append(MV2Block(channels[7], channels[8], 2, expansion)) + + self.mvit = nn.ModuleList([]) + self.mvit.append( + MobileViTBlock(dims[0], depth[0], channels[5], kernel_size, patch_size, int(dims[0] * 2), dropout=dropout) + ) + self.mvit.append( + MobileViTBlock(dims[1], depth[1], channels[7], kernel_size, patch_size, int(dims[1] * 4), dropout=dropout) + ) + self.mvit.append( + MobileViTBlock(dims[2], depth[2], channels[9], kernel_size, patch_size, int(dims[2] * 4), dropout=dropout) + ) + + self.conv2 = conv_1x1_bn(channels[-2], channels[-1]) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + r"""Run the MobileViT backbone and return final feature maps. + + Args: + x: Image tensor with shape :math:`(B, C, H, W)`, where :math:`C` + matches the configured input channel count. + + Returns: + Feature tensor from the final MobileViT stage. For the default + configuration, a :math:`256 \times 256` input produces + :math:`(B, 320, 8, 8)`. + """ + x = self.conv1(x) + x = self.mv2[0](x) + + x = self.mv2[1](x) + x = self.mv2[2](x) + x = self.mv2[3](x) # Repeat + + x = self.mv2[4](x) + x = self.mvit[0](x) + + x = self.mv2[5](x) + x = self.mvit[1](x) + + x = self.mv2[6](x) + x = self.mvit[2](x) + x = self.conv2(x) + return x diff --git a/kornia/models/yunet/__init__.py b/kornia/models/yunet/__init__.py new file mode 100644 index 0000000..12e9374 --- /dev/null +++ b/kornia/models/yunet/__init__.py @@ -0,0 +1,20 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .model import YuNet + +__all__ = ["YuNet"] diff --git a/kornia/models/yunet/model.py b/kornia/models/yunet/model.py new file mode 100644 index 0000000..cd4f1b5 --- /dev/null +++ b/kornia/models/yunet/model.py @@ -0,0 +1,161 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# based on: https://github.com/ShiqiYu/libfacedetection.train/blob/74f3aa77c63234dd954d21286e9a60703b8d0868/tasks/task1/yufacedetectnet.py # noqa +from typing import Dict + +import torch +import torch.nn.functional as F +from torch import nn + +__all__ = ["YuNet"] + +url: str = "https://github.com/kornia/data/raw/main/yunet_final.pth" + + +class ConvDPUnit(nn.Sequential): + def __init__(self, in_channels: int, out_channels: int, withBNRelu: bool = True) -> None: + super().__init__() + self.add_module("conv1", nn.Conv2d(in_channels, out_channels, 1, 1, 0, bias=True, groups=1)) + self.add_module("conv2", nn.Conv2d(out_channels, out_channels, 3, 1, 1, bias=True, groups=out_channels)) + if withBNRelu: + self.add_module("bn", nn.BatchNorm2d(out_channels)) + self.add_module("relu", nn.ReLU(inplace=True)) + + +class Conv_head(nn.Sequential): + def __init__(self, in_channels: int, mid_channels: int, out_channels: int) -> None: + super().__init__() + self.add_module("conv1", nn.Conv2d(in_channels, mid_channels, 3, 2, 1, bias=True, groups=1)) + self.add_module("bn1", nn.BatchNorm2d(mid_channels)) + self.add_module("relu", nn.ReLU(inplace=True)) + self.add_module("conv2", ConvDPUnit(mid_channels, out_channels)) + + +class Conv4layerBlock(nn.Sequential): + def __init__(self, in_channels: int, out_channels: int, withBNRelu: bool = True) -> None: + super().__init__() + self.add_module("conv1", ConvDPUnit(in_channels, in_channels, True)) + self.add_module("conv2", ConvDPUnit(in_channels, out_channels, withBNRelu)) + + +class YuNet(nn.Module): + r"""YuNet face detection model. + + This is the underlying CNN model used by :py:class:`kornia.contrib.FaceDetector` for face detection. + The model architecture is based on the method described in :cite:`facedetect-yu`. + + Args: + phase: the phase of the model, either "train" or "test". + pretrained: if True, loads pretrained weights from the official repository. + + Example: + >>> model = YuNet("test", pretrained=True) + >>> img = torch.rand(1, 3, 320, 320) + >>> out = model(img) + + """ + + def __init__(self, phase: str, pretrained: bool) -> None: + super().__init__() + self.phase = phase + self.num_classes = 2 + + self.model0 = Conv_head(3, 16, 16) + self.model1 = Conv4layerBlock(16, 64) + self.model2 = Conv4layerBlock(64, 64) + self.model3 = Conv4layerBlock(64, 64) + self.model4 = Conv4layerBlock(64, 64) + self.model5 = Conv4layerBlock(64, 64) + self.model6 = Conv4layerBlock(64, 64) + + self.head = nn.Sequential( + Conv4layerBlock(64, 3 * (14 + 2 + 1), False), + Conv4layerBlock(64, 2 * (14 + 2 + 1), False), + Conv4layerBlock(64, 2 * (14 + 2 + 1), False), + Conv4layerBlock(64, 3 * (14 + 2 + 1), False), + ) + + if self.phase == "train": + for m in self.modules(): + if isinstance(m, nn.Conv2d): + if m.bias is not None: + nn.init.xavier_normal_(m.weight.data) + m.bias.data.fill_(0.02) + else: + m.weight.data.normal_(0, 0.01) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + + # use torch.hub to load pretrained model + if pretrained: + pretrained_dict = torch.hub.load_state_dict_from_url(url, map_location=torch.device("cpu")) + self.load_state_dict(pretrained_dict, strict=True) + self.eval() + + def forward(self, x: torch.Tensor) -> Dict[str, torch.Tensor]: + """Run YuNet face detection heads over an input image batch. + + Args: + x: Image tensor with shape :math:`(B, C, H, W)`, where :math:`B` is + batch size, :math:`C` is channel count, :math:`H` is height, + and :math:`W` is width. + + Returns: + Dictionary containing head outputs for locations, confidences, and + landmarks at the configured detection scales. + """ + detection_sources, head_list = [], [] + + x = self.model0(x) + x = F.max_pool2d(x, 2) + x = self.model1(x) + x = self.model2(x) + x = F.max_pool2d(x, 2) + x = self.model3(x) + detection_sources.append(x) + + x = F.max_pool2d(x, 2) + x = self.model4(x) + detection_sources.append(x) + + x = F.max_pool2d(x, 2) + x = self.model5(x) + detection_sources.append(x) + + x = F.max_pool2d(x, 2) + x = self.model6(x) + detection_sources.append(x) + + for i, h in enumerate(self.head): + x_tmp = h(detection_sources[i]) + head_list.append(x_tmp.permute(0, 2, 3, 1).contiguous()) + + head_data = torch.cat([o.view(o.size(0), -1) for o in head_list], 1) + head_data = head_data.view(head_data.size(0), -1, 17) + + loc_data, conf_data, iou_data = head_data.split((14, 2, 1), dim=-1) + + if self.phase == "test": + conf_data = torch.softmax(conf_data, dim=-1) + else: + loc_data = loc_data.view(loc_data.size(0), -1, 14) + conf_data = conf_data.view(conf_data.size(0), -1, self.num_classes) + iou_data = iou_data.view(iou_data.size(0), -1, 1) + + return {"loc": loc_data, "conf": conf_data, "iou": iou_data} diff --git a/kornia/models/yunet/processors.py b/kornia/models/yunet/processors.py new file mode 100644 index 0000000..58e99a5 --- /dev/null +++ b/kornia/models/yunet/processors.py @@ -0,0 +1,119 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Adapted from https://github.com/Hakuyume/chainer-ssd +import math +from typing import List, Tuple + +import torch + +__all__ = ["PriorBox", "decode"] + + +def decode(loc: torch.Tensor, priors: torch.Tensor, variances: List[float]) -> torch.Tensor: + """Decode locations from predictions using priors to undo the encoding for offset regression at train time. + + Args: + loc: location predictions for loc layers. Shape: [num_priors,4]. + priors: Prior boxes in center-offset form. Shape: [num_priors,4]. + variances: (list[float]) Variances of priorboxes. + + Return: + torch.Tensor containing decoded bounding box predictions. + + """ + boxes = torch.cat( + ( + priors[:, 0:2] + loc[:, 0:2] * variances[0] * priors[:, 2:4], + priors[:, 2:4] * torch.exp(loc[:, 2:4] * variances[1]), + priors[:, 0:2] + loc[:, 4:6] * variances[0] * priors[:, 2:4], + priors[:, 0:2] + loc[:, 6:8] * variances[0] * priors[:, 2:4], + priors[:, 0:2] + loc[:, 8:10] * variances[0] * priors[:, 2:4], + priors[:, 0:2] + loc[:, 10:12] * variances[0] * priors[:, 2:4], + priors[:, 0:2] + loc[:, 12:14] * variances[0] * priors[:, 2:4], + ), + 1, + ) + # prepare final output + tmp = boxes[:, 0:2] - boxes[:, 2:4] / 2 + return torch.cat((tmp, boxes[:, 2:4] + tmp, boxes[:, 4:]), dim=-1) + + +class PriorBox: + """Generate prior boxes for YuNet face detection model.""" + + def __init__(self, min_sizes: List[List[int]], steps: List[int], clip: bool, image_size: Tuple[int, int]) -> None: + self.min_sizes = min_sizes + self.steps = steps + self.clip = clip + self.image_size = image_size + + self.device: torch.device = torch.device("cpu") + self.dtype: torch.dtype = torch.float32 + + for i in range(4): + if self.steps[i] != math.pow(2, (i + 3)): + raise ValueError("steps must be [8,16,32,64]") + + self.feature_map_2th = [int(int((self.image_size[0] + 1) / 2) / 2), int(int((self.image_size[1] + 1) / 2) / 2)] + self.feature_map_3th = [int(self.feature_map_2th[0] / 2), int(self.feature_map_2th[1] / 2)] + self.feature_map_4th = [int(self.feature_map_3th[0] / 2), int(self.feature_map_3th[1] / 2)] + self.feature_map_5th = [int(self.feature_map_4th[0] / 2), int(self.feature_map_4th[1] / 2)] + self.feature_map_6th = [int(self.feature_map_5th[0] / 2), int(self.feature_map_5th[1] / 2)] + + self.feature_maps = [self.feature_map_3th, self.feature_map_4th, self.feature_map_5th, self.feature_map_6th] + + def to(self, device: torch.device, dtype: torch.dtype) -> "PriorBox": + """Set the device and dtype used when creating prior boxes. + + Args: + device: Target device for the generated prior-box tensor. + dtype: Target dtype for the generated prior-box tensor. + + Returns: + ``self`` with updated tensor creation settings. + """ + self.device = device + self.dtype = dtype + return self + + def __call__(self) -> torch.Tensor: + """Generate YuNet prior boxes for all configured feature maps. + + Returns: + Tensor with shape :math:`(N, 4)`, where :math:`N` is the total + number of anchors and the four values are normalized center-x, + center-y, width, and height. + """ + anchors: List[float] = [] + for k, f in enumerate(self.feature_maps): + min_sizes: List[int] = self.min_sizes[k] + # NOTE: the nested loop it's to make torchscript happy + for i in range(f[0]): + for j in range(f[1]): + for min_size in min_sizes: + s_kx = min_size / self.image_size[1] + s_ky = min_size / self.image_size[0] + + cx = (j + 0.5) * self.steps[k] / self.image_size[1] + cy = (i + 0.5) * self.steps[k] / self.image_size[0] + anchors += [cx, cy, s_kx, s_ky] + # back to torch land + output = torch.tensor(anchors, device=self.device, dtype=self.dtype).view(-1, 4) + if self.clip: + output = output.clamp(max=1, min=0) + return output diff --git a/kornia/morphology/__init__.py b/kornia/morphology/__init__.py new file mode 100644 index 0000000..568ae5f --- /dev/null +++ b/kornia/morphology/__init__.py @@ -0,0 +1,23 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Morphology submodule for Kornia. + +This package provides morphological operations for image processing tasks. +""" + +from .morphology import * diff --git a/kornia/morphology/morphology.py b/kornia/morphology/morphology.py new file mode 100644 index 0000000..a88bab5 --- /dev/null +++ b/kornia/morphology/morphology.py @@ -0,0 +1,582 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import List, Optional + +import torch +import torch.nn.functional as F + +__all__ = ["bottom_hat", "closing", "dilation", "erosion", "gradient", "opening", "top_hat"] + + +def _neight2channels_like_kernel(kernel: torch.Tensor) -> torch.Tensor: + h, w = kernel.size() + kernel = torch.eye(h * w, dtype=kernel.dtype, device=kernel.device) + return kernel.view(h * w, 1, h, w) + + +def dilation( + tensor: torch.Tensor, + kernel: torch.Tensor, + structuring_element: Optional[torch.Tensor] = None, + origin: Optional[List[int]] = None, + border_type: str = "geodesic", + border_value: float = 0.0, + max_val: float = 1e4, + engine: str = "unfold", +) -> torch.Tensor: + r"""Return the dilated image applying the same kernel in each channel. + + .. image:: _static/img/dilation.png + + The kernel must have 2 dimensions. + + Args: + tensor: Image with shape :math:`(B, C, H, W)`. + kernel: Positions of non-infinite elements of a flat structuring element. Non-zero values give + the set of neighbors of the center over which the operation is applied. Its shape is :math:`(k_x, k_y)`. + For full structural elements use torch.ones_like(structural_element). + structuring_element: Structuring element used for the grayscale dilation. It may be a non-flat + structuring element. + origin: Origin of the structuring element. Default: ``None`` and uses the center of + the structuring element as origin (rounding towards zero). + border_type: It determines how the image borders are handled, where ``border_value`` is the value + when ``border_type`` is equal to ``constant``. Default: ``geodesic`` which ignores the values that are + outside the image when applying the operation. + border_value: Value to fill past edges of input if ``border_type`` is ``constant``. + max_val: The value of the infinite elements in the kernel. + engine: convolution is faster and less memory hungry, and unfold is more stable numerically + + Returns: + Dilated image with shape :math:`(B, C, H, W)`. + + .. note:: + See a working example `here `__. + + Example: + >>> tensor = torch.rand(1, 3, 5, 5) + >>> kernel = torch.ones(3, 3) + >>> dilated_img = dilation(tensor, kernel) + + """ + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(tensor)}") + + if len(tensor.shape) != 4: + raise ValueError(f"Input size must have 4 dimensions. Got {tensor.dim()}") + + if not isinstance(kernel, torch.Tensor): + raise TypeError(f"Kernel type is not a torch.Tensor. Got {type(kernel)}") + + if len(kernel.shape) != 2: + raise ValueError(f"Kernel size must have 2 dimensions. Got {kernel.dim()}") + + # origin + se_h, se_w = kernel.shape + if origin is None: + origin = [se_h // 2, se_w // 2] + + # pad + pad_e: List[int] = [origin[1], se_w - origin[1] - 1, origin[0], se_h - origin[0] - 1] + if border_type == "geodesic": + border_value = -max_val + border_type = "constant" + output: torch.Tensor = F.pad(tensor, pad_e, mode=border_type, value=border_value) + + # computation + if structuring_element is None: + neighborhood = torch.zeros_like(kernel) + neighborhood[kernel == 0] = -max_val + else: + neighborhood = structuring_element.clone() + neighborhood[kernel == 0] = -max_val + + if engine == "unfold": + output = output.unfold(2, se_h, 1).unfold(3, se_w, 1) + output, _ = torch.max(output + neighborhood.flip((0, 1)), 4) + output, _ = torch.max(output, 4) + elif engine == "convolution": + B, C, H, W = tensor.size() + h_pad, w_pad = output.shape[-2:] + reshape_kernel = _neight2channels_like_kernel(kernel) + output, _ = F.conv2d( + output.view(B * C, 1, h_pad, w_pad), reshape_kernel, padding=0, bias=neighborhood.view(-1).flip(0) + ).max(dim=1) + output = output.view(B, C, H, W) + else: + raise NotImplementedError(f"engine {engine} is unknown, use 'convolution' or 'unfold'") + return output.view_as(tensor) + + +def erosion( + tensor: torch.Tensor, + kernel: torch.Tensor, + structuring_element: Optional[torch.Tensor] = None, + origin: Optional[List[int]] = None, + border_type: str = "geodesic", + border_value: float = 0.0, + max_val: float = 1e4, + engine: str = "unfold", +) -> torch.Tensor: + r"""Return the eroded image applying the same kernel in each channel. + + .. image:: _static/img/erosion.png + + The kernel must have 2 dimensions. + + Args: + tensor: Image with shape :math:`(B, C, H, W)`. + kernel: Positions of non-infinite elements of a flat structuring element. Non-zero values give + the set of neighbors of the center over which the operation is applied. Its shape is :math:`(k_x, k_y)`. + For full structural elements use torch.ones_like(structural_element). + structuring_element (torch.Tensor, optional): Structuring element used for the grayscale dilation. + It may be a non-flat structuring element. + origin: Origin of the structuring element. Default: ``None`` and uses the center of + the structuring element as origin (rounding towards zero). + border_type: It determines how the image borders are handled, where ``border_value`` is the value + when ``border_type`` is equal to ``constant``. Default: ``geodesic`` which ignores the values that are + outside the image when applying the operation. + border_value: Value to fill past edges of input if border_type is ``constant``. + max_val: The value of the infinite elements in the kernel. + engine: ``convolution`` is faster and less memory hungry, and ``unfold`` is more stable numerically + + Returns: + Eroded image with shape :math:`(B, C, H, W)`. + + .. note:: + See a working example `here `__. + + Example: + >>> tensor = torch.rand(1, 3, 5, 5) + >>> kernel = torch.ones(5, 5) + >>> output = erosion(tensor, kernel) + + """ + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(tensor)}") + + if len(tensor.shape) != 4: + raise ValueError(f"Input size must have 4 dimensions. Got {tensor.dim()}") + + if not isinstance(kernel, torch.Tensor): + raise TypeError(f"Kernel type is not a torch.Tensor. Got {type(kernel)}") + + if len(kernel.shape) != 2: + raise ValueError(f"Kernel size must have 2 dimensions. Got {kernel.dim()}") + + # origin + se_h, se_w = kernel.shape + if origin is None: + origin = [se_h // 2, se_w // 2] + + # pad + pad_e: List[int] = [origin[1], se_w - origin[1] - 1, origin[0], se_h - origin[0] - 1] + if border_type == "geodesic": + border_value = max_val + border_type = "constant" + output: torch.Tensor = F.pad(tensor, pad_e, mode=border_type, value=border_value) + + # computation + if structuring_element is None: + neighborhood = torch.zeros_like(kernel) + neighborhood[kernel == 0] = -max_val + else: + neighborhood = structuring_element.clone() + neighborhood[kernel == 0] = -max_val + + if engine == "unfold": + output = output.unfold(2, se_h, 1).unfold(3, se_w, 1) + output, _ = torch.min(output - neighborhood, 4) + output, _ = torch.min(output, 4) + elif engine == "convolution": + B, C, H, W = tensor.size() + Hpad, Wpad = output.shape[-2:] + reshape_kernel = _neight2channels_like_kernel(kernel) + output, _ = F.conv2d( + output.view(B * C, 1, Hpad, Wpad), reshape_kernel, padding=0, bias=-neighborhood.view(-1) + ).min(dim=1) + output = output.view(B, C, H, W) + else: + raise NotImplementedError(f"engine {engine} is unknown, use 'convolution' or 'unfold'") + + return output + + +def opening( + tensor: torch.Tensor, + kernel: torch.Tensor, + structuring_element: Optional[torch.Tensor] = None, + origin: Optional[List[int]] = None, + border_type: str = "geodesic", + border_value: float = 0.0, + max_val: float = 1e4, + engine: str = "unfold", +) -> torch.Tensor: + r"""Return the opened image, (that means, dilation after an erosion) applying the same kernel in each channel. + + .. image:: _static/img/opening.png + + The kernel must have 2 dimensions. + + Args: + tensor: Image with shape :math:`(B, C, H, W)`. + kernel: Positions of non-infinite elements of a flat structuring element. Non-zero values give + the set of neighbors of the center over which the operation is applied. Its shape is :math:`(k_x, k_y)`. + For full structural elements use torch.ones_like(structural_element). + structuring_element: Structuring element used for the grayscale dilation. It may be a + non-flat structuring element. + origin: Origin of the structuring element. Default: ``None`` and uses the center of + the structuring element as origin (rounding towards zero). + border_type: It determines how the image borders are handled, where ``border_value`` is the value + when ``border_type`` is equal to ``constant``. Default: ``geodesic`` which ignores the values that are + outside the image when applying the operation. + border_value: Value to fill past edges of input if ``border_type`` is ``constant``. + max_val: The value of the infinite elements in the kernel. + engine: convolution is faster and less memory hungry, and unfold is more stable numerically + + Returns: + torch.Tensor: Opened image with shape :math:`(B, C, H, W)`. + + .. note:: + See a working example `here `__. + + Example: + >>> tensor = torch.rand(1, 3, 5, 5) + >>> kernel = torch.ones(3, 3) + >>> opened_img = opening(tensor, kernel) + + """ + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(tensor)}") + + if len(tensor.shape) != 4: + raise ValueError(f"Input size must have 4 dimensions. Got {tensor.dim()}") + + if not isinstance(kernel, torch.Tensor): + raise TypeError(f"Kernel type is not a torch.Tensor. Got {type(kernel)}") + + if len(kernel.shape) != 2: + raise ValueError(f"Kernel size must have 2 dimensions. Got {kernel.dim()}") + + return dilation( + erosion( + tensor, + kernel=kernel, + structuring_element=structuring_element, + origin=origin, + border_type=border_type, + border_value=border_value, + max_val=max_val, + engine=engine, + ), + kernel=kernel, + structuring_element=structuring_element, + origin=origin, + border_type=border_type, + border_value=border_value, + max_val=max_val, + engine=engine, + ) + + +def closing( + tensor: torch.Tensor, + kernel: torch.Tensor, + structuring_element: Optional[torch.Tensor] = None, + origin: Optional[List[int]] = None, + border_type: str = "geodesic", + border_value: float = 0.0, + max_val: float = 1e4, + engine: str = "unfold", +) -> torch.Tensor: + r"""Return the closed image, (that means, erosion after a dilation) applying the same kernel in each channel. + + .. image:: _static/img/closing.png + + The kernel must have 2 dimensions. + + Args: + tensor: Image with shape :math:`(B, C, H, W)`. + kernel: Positions of non-infinite elements of a flat structuring element. Non-zero values give + the set of neighbors of the center over which the operation is applied. Its shape is :math:`(k_x, k_y)`. + For full structural elements use torch.ones_like(structural_element). + structuring_element: Structuring element used for the grayscale dilation. It may be a + non-flat structuring element. + origin: Origin of the structuring element. Default is None and uses the center of + the structuring element as origin (rounding towards zero). + border_type: It determines how the image borders are handled, where ``border_value`` is the value + when ``border_type`` is equal to ``constant``. Default: ``geodesic`` which ignores the values that are + outside the image when applying the operation. + border_value: Value to fill past edges of input if ``border_type`` is ``constant``. + max_val: The value of the infinite elements in the kernel. + engine: convolution is faster and less memory hungry, and unfold is more stable numerically + + Returns: + Closed image with shape :math:`(B, C, H, W)`. + + .. note:: + See a working example `here `__. + + Example: + >>> tensor = torch.rand(1, 3, 5, 5) + >>> kernel = torch.ones(3, 3) + >>> closed_img = closing(tensor, kernel) + + """ + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(tensor)}") + + if len(tensor.shape) != 4: + raise ValueError(f"Input size must have 4 dimensions. Got {tensor.dim()}") + + if not isinstance(kernel, torch.Tensor): + raise TypeError(f"Kernel type is not a torch.Tensor. Got {type(kernel)}") + + if len(kernel.shape) != 2: + raise ValueError(f"Kernel size must have 2 dimensions. Got {kernel.dim()}") + + return erosion( + dilation( + tensor, + kernel=kernel, + structuring_element=structuring_element, + origin=origin, + border_type=border_type, + border_value=border_value, + max_val=max_val, + engine=engine, + ), + kernel=kernel, + structuring_element=structuring_element, + origin=origin, + border_type=border_type, + border_value=border_value, + max_val=max_val, + engine=engine, + ) + + +# Morphological Gradient +def gradient( + tensor: torch.Tensor, + kernel: torch.Tensor, + structuring_element: Optional[torch.Tensor] = None, + origin: Optional[List[int]] = None, + border_type: str = "geodesic", + border_value: float = 0.0, + max_val: float = 1e4, + engine: str = "unfold", +) -> torch.Tensor: + r"""Return the morphological gradient of an image. + + .. image:: _static/img/gradient.png + + That means, (dilation - erosion) applying the same kernel in each channel. + The kernel must have 2 dimensions. + + Args: + tensor: Image with shape :math:`(B, C, H, W)`. + kernel: Positions of non-infinite elements of a flat structuring element. Non-zero values give + the set of neighbors of the center over which the operation is applied. Its shape is :math:`(k_x, k_y)`. + For full structural elements use torch.ones_like(structural_element). + structuring_element: Structuring element used for the grayscale dilation. It may be a + non-flat structuring element. + origin: Origin of the structuring element. Default is None and uses the center of + the structuring element as origin (rounding towards zero). + border_type: It determines how the image borders are handled, where ``border_value`` is the value + when ``border_type`` is equal to ``constant``. Default: ``geodesic`` which ignores the values that are + outside the image when applying the operation. + border_value: Value to fill past edges of input if ``border_type`` is ``constant``. + max_val: The value of the infinite elements in the kernel. + engine: convolution is faster and less memory hungry, and unfold is more stable numerically + + Returns: + Gradient image with shape :math:`(B, C, H, W)`. + + .. note:: + See a working example `here `__. + + Example: + >>> tensor = torch.rand(1, 3, 5, 5) + >>> kernel = torch.ones(3, 3) + >>> gradient_img = gradient(tensor, kernel) + + """ + return dilation( + tensor, + kernel=kernel, + structuring_element=structuring_element, + origin=origin, + border_type=border_type, + border_value=border_value, + max_val=max_val, + engine=engine, + ) - erosion( + tensor, + kernel=kernel, + structuring_element=structuring_element, + origin=origin, + border_type=border_type, + border_value=border_value, + max_val=max_val, + engine=engine, + ) + + +def top_hat( + tensor: torch.Tensor, + kernel: torch.Tensor, + structuring_element: Optional[torch.Tensor] = None, + origin: Optional[List[int]] = None, + border_type: str = "geodesic", + border_value: float = 0.0, + max_val: float = 1e4, + engine: str = "unfold", +) -> torch.Tensor: + r"""Return the top hat transformation of an image. + + .. image:: _static/img/top_hat.png + + That means, (image - opened_image) applying the same kernel in each channel. + The kernel must have 2 dimensions. + + See :func:`~kornia.morphology.opening` for details. + + Args: + tensor: Image with shape :math:`(B, C, H, W)`. + kernel: Positions of non-infinite elements of a flat structuring element. Non-zero values give + the set of neighbors of the center over which the operation is applied. Its shape is :math:`(k_x, k_y)`. + For full structural elements use torch.ones_like(structural_element). + structuring_element: Structuring element used for the grayscale dilation. It may be a + non-flat structuring element. + origin: Origin of the structuring element. Default: ``None`` and uses the center of + the structuring element as origin (rounding towards zero). + border_type: It determines how the image borders are handled, where ``border_value`` is the value + when ``border_type`` is equal to ``constant``. Default: ``geodesic`` which ignores the values that are + outside the image when applying the operation. + border_value: Value to fill past edges of input if ``border_type`` is ``constant``. + max_val: The value of the infinite elements in the kernel. + engine: convolution is faster and less memory hungry, and unfold is more stable numerically + + Returns: + Top hat transformed image with shape :math:`(B, C, H, W)`. + + .. note:: + See a working example `here `__. + + Example: + >>> tensor = torch.rand(1, 3, 5, 5) + >>> kernel = torch.ones(3, 3) + >>> top_hat_img = top_hat(tensor, kernel) + + """ + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(tensor)}") + + if len(tensor.shape) != 4: + raise ValueError(f"Input size must have 4 dimensions. Got {tensor.dim()}") + + if not isinstance(kernel, torch.Tensor): + raise TypeError(f"Kernel type is not a torch.Tensor. Got {type(kernel)}") + + if len(kernel.shape) != 2: + raise ValueError(f"Kernel size must have 2 dimensions. Got {kernel.dim()}") + + return tensor - opening( + tensor, + kernel=kernel, + structuring_element=structuring_element, + origin=origin, + border_type=border_type, + border_value=border_value, + max_val=max_val, + engine=engine, + ) + + +def bottom_hat( + tensor: torch.Tensor, + kernel: torch.Tensor, + structuring_element: Optional[torch.Tensor] = None, + origin: Optional[List[int]] = None, + border_type: str = "geodesic", + border_value: float = 0.0, + max_val: float = 1e4, + engine: str = "unfold", +) -> torch.Tensor: + r"""Return the bottom hat transformation of an image. + + .. image:: _static/img/bottom_hat.png + + That means, (closed_image - image) applying the same kernel in each channel. + The kernel must have 2 dimensions. + + See :func:`~kornia.morphology.closing` for details. + + Args: + tensor: Image with shape :math:`(B, C, H, W)`. + kernel: Positions of non-infinite elements of a flat structuring element. Non-zero values give + the set of neighbors of the center over which the operation is applied. Its shape is :math:`(k_x, k_y)`. + For full structural elements use torch.ones_like(structural_element). + structuring_element: Structuring element used for the grayscale dilation. It may be a + non-flat structuring element. + origin: Origin of the structuring element. Default: ``None`` and uses the center of + the structuring element as origin (rounding towards zero). + border_type: It determines how the image borders are handled, where ``border_value`` is the value + when ``border_type`` is equal to ``constant``. Default: ``geodesic`` which ignores the values that are + outside the image when applying the operation. + border_value: Value to fill past edges of input if ``border_type`` is ``constant``. + max_val: The value of the infinite elements in the kernel. + engine: convolution is faster and less memory hungry, and unfold is more stable numerically + + Returns: + Top hat transformed image with shape :math:`(B, C, H, W)`. + + .. note:: + See a working example `here `__. + + Example: + >>> tensor = torch.rand(1, 3, 5, 5) + >>> kernel = torch.ones(3, 3) + >>> bottom_hat_img = bottom_hat(tensor, kernel) + + """ + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"Input type is not a torch.Tensor. Got {type(tensor)}") + + if len(tensor.shape) != 4: + raise ValueError(f"Input size must have 4 dimensions. Got {tensor.dim()}") + + if not isinstance(kernel, torch.Tensor): + raise TypeError(f"Kernel type is not a torch.Tensor. Got {type(kernel)}") + + if len(kernel.shape) != 2: + raise ValueError(f"Kernel size must have 2 dimensions. Got {kernel.dim()}") + + return ( + closing( + tensor, + kernel=kernel, + structuring_element=structuring_element, + origin=origin, + border_type=border_type, + border_value=border_value, + max_val=max_val, + engine=engine, + ) + - tensor + ) diff --git a/kornia/onnx/__init__.py b/kornia/onnx/__init__.py new file mode 100644 index 0000000..e29e783 --- /dev/null +++ b/kornia/onnx/__init__.py @@ -0,0 +1,25 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""ONNX submodule for Kornia. + +This package provides utilities and modules for ONNX export and integration. +""" + +from .module import * +from .sequential import * +from .utils import * diff --git a/kornia/onnx/download.py b/kornia/onnx/download.py new file mode 100644 index 0000000..3f8cfcc --- /dev/null +++ b/kornia/onnx/download.py @@ -0,0 +1,123 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import logging +import os +import urllib.request +from typing import Any, Optional + +from kornia.config import kornia_config + +__all__ = ["CachedDownloader"] + +logger = logging.getLogger(__name__) + + +class CachedDownloader: + """Downloads files from URLs to the local cache or .kornia_hub directory.""" + + @classmethod + def _get_file_path(cls, model_name: str, cache_dir: Optional[str], suffix: Optional[str] = None) -> str: + """Construct the file path for the ONNX model based on the model name and cache directory. + + Args: + model_name: The name of the model or operator, typically in the format 'operators/model_name'. + cache_dir: The directory where the model should be cached. + Defaults to None, which will use a default `kornia.config.hub_onnx_dir` directory. + suffix: Optional file suffix when the filename is the model name. + + Returns: + str: The full local path where the model should be stored or loaded from. + + """ + # Determine the local file path + if cache_dir is None: + cache_dir = kornia_config.hub_cache_dir + + # The filename is the model name (without directory path) + if suffix is not None and not model_name.endswith(suffix): + file_name = f"{os.path.split(model_name)[-1]}{suffix}" + else: + file_name = os.path.split(model_name)[-1] + file_path = os.path.join(*cache_dir.split(os.sep), *model_name.split(os.sep)[:-1], file_name) + return file_path + + @classmethod + def download_to_cache(cls, url: str, name: str, download: bool = True, **kwargs: Any) -> str: + """Resolve a remote file into Kornia's local cache and download it when needed. + + Args: + url: HTTP or HTTPS URL of the file to cache. + name: Logical model or operator name used to construct the cache + path. Directory components in ``name`` are preserved under the + cache directory. + download: If ``True``, download the file when it is not already + present in the cache. If ``False``, require the cached file to + exist. + kwargs: Optional cache parameters. Supported keys include + ``cache_dir`` for overriding the default cache root and + ``suffix`` for appending a filename suffix when needed. + + Returns: + Local filesystem path to the cached file. + + Raises: + ValueError: If ``url`` is not an HTTP or HTTPS URL, or if download is + disabled and the file is missing. + """ + if url.startswith(("http:", "https:")): + cache_dir = kwargs.get("cache_dir", None) + suffix = kwargs.get("suffix", None) + file_path = cls._get_file_path(name, cache_dir, suffix=suffix) + cls.download(url, file_path, download_if_not_exists=download) + return file_path + raise ValueError(f"URL must start with 'http:' or 'https:'. Got {url}") + + @classmethod + def download( + cls, + url: str, + file_path: str, + download_if_not_exists: bool = True, + ) -> None: + """Download an ONNX model from the specified URL and save it to the specified file path. + + Args: + url: The URL of the ONNX model to download. + file_path: The local path where the downloaded model should be saved. + download_if_not_exists: If True, the file will be downloaded if it's not already downloaded. + + """ + if os.path.exists(file_path): + logger.info(f"Loading `{url}` from `{file_path}`.") + return + + if not download_if_not_exists: + raise ValueError(f"`{file_path}` not found. You may set `download=True`.") + + os.makedirs(os.path.dirname(file_path), exist_ok=True) # Create the cache directory if it doesn't exist + + if url.startswith(("http:", "https:")): + try: + logger.info(f"Downloading `{url}` to `{file_path}`.") + urllib.request.urlretrieve(url, file_path) # noqa: S310 + except urllib.error.HTTPError as e: + raise ValueError(f"Error in resolving `{url}`.") from e + else: + raise ValueError("URL must start with 'http:' or 'https:'") diff --git a/kornia/onnx/module.py b/kornia/onnx/module.py new file mode 100644 index 0000000..fedcada --- /dev/null +++ b/kornia/onnx/module.py @@ -0,0 +1,118 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Any, Optional, Union + +from kornia.core.external import onnx +from kornia.core.external import onnxruntime as ort +from kornia.core.mixin.onnx import ONNXMixin, ONNXRuntimeMixin + +__all__ = ["ONNXModule", "load"] + + +class ONNXModule(ONNXMixin, ONNXRuntimeMixin): + """ONNXModule to wrap an ONNX operator. + + Args: + arg: A variable number of ONNX models (either ONNX ModelProto objects or file paths). + For Hugging Face-hosted models, use the format 'hf://model_name'. Valid `model_name` can be found on + https://huggingface.co/kornia/ONNX_models. Or a URL to the ONNX model. + providers: A list of execution providers for ONNXRuntime + (e.g., ['CUDAExecutionProvider', 'CPUExecutionProvider']). + session_options: Optional ONNXRuntime session options for optimizing the session. + cache_dir: The directory where ONNX models are cached locally (only for downloading from HuggingFace). + Defaults to None, which will use a default `kornia.config.hub_onnx_dir` directory. + target_ir_version: The target IR version to convert to. + target_opset_version: The target OPSET version to convert to. + + """ + + def __init__( + self, + op: Union[onnx.ModelProto, str], # type:ignore + providers: Optional[list[str]] = None, + session_options: Optional[ort.SessionOptions] = None, # type:ignore + cache_dir: Optional[str] = None, + target_ir_version: Optional[int] = None, + target_opset_version: Optional[int] = None, + ) -> None: + self.op = self._load_op(op, cache_dir) + if target_ir_version is not None or target_opset_version is not None: + self.op = self._onnx_version_conversion( + self.op, target_ir_version=target_ir_version, target_opset_version=target_opset_version + ) + session = self.create_session(providers=providers, session_options=session_options) + self.set_session(session=session) + + def create_session( + self, providers: list[str] | None = None, session_options: Any | None = None + ) -> ort.InferenceSession: # type: ignore + """Create an ONNX Runtime session for the wrapped model graph. + + Args: + providers: Optional ordered list of ONNX Runtime execution providers, + such as ``"CUDAExecutionProvider"`` followed by + ``"CPUExecutionProvider"``. The first available provider is used + according to ONNX Runtime's provider selection rules. + session_options: Optional ONNX Runtime session options controlling + graph optimization, threading, logging, and other runtime + execution settings. + + Returns: + ONNX Runtime inference session bound to ``self.op``, the wrapped + ONNX model graph. + """ + return super()._create_session(self.op, providers, session_options) + + def export(self, file_path: str, **kwargs: Any) -> None: + """Serialize the wrapped ONNX model graph to disk. + + Args: + file_path: Destination path for the exported ``.onnx`` file. + kwargs: Additional keyword arguments forwarded to the underlying + ONNX export helper. + """ + return super()._export(self.op, file_path, **kwargs) + + def add_metadata(self, additional_metadata: Optional[list[tuple[str, str]]] = None) -> onnx.ModelProto: # type:ignore + """Attach metadata entries to the wrapped ONNX model graph. + + Args: + additional_metadata: Optional list of ``(key, value)`` string pairs + to store in the ONNX model metadata properties. + + Returns: + The wrapped ONNX model graph after metadata has been added. + """ + return super()._add_metadata(self.op, additional_metadata) + + +def load(model_name: Union[onnx.ModelProto, str]) -> ONNXModule: # type:ignore + """Load an ONNX model from either a file path or HuggingFace. + + The loaded model is an ONNXModule object, of which you may run the model with + the `__call__` method, with less boilerplate. + + Args: + model_name: The name of the model to load. For Hugging Face-hosted models, + use the format 'hf://model_name'. Valid `model_name` can be found on + https://huggingface.co/kornia/ONNX_models. Or a URL to the ONNX model. + + """ + return ONNXModule(model_name) diff --git a/kornia/onnx/sequential.py b/kornia/onnx/sequential.py new file mode 100644 index 0000000..d4f4e2a --- /dev/null +++ b/kornia/onnx/sequential.py @@ -0,0 +1,153 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Any, Optional, Union + +from kornia.core.external import onnx +from kornia.core.external import onnxruntime as ort +from kornia.core.mixin import ONNXMixin, ONNXRuntimeMixin + +__all__ = ["ONNXSequential"] + + +class ONNXSequential(ONNXMixin, ONNXRuntimeMixin): + """ONNXSequential to chain multiple ONNX operators together. + + Args: + *args: A variable number of ONNX models (either ONNX ModelProto objects or file paths). + For Hugging Face-hosted models, use the format 'hf://model_name'. Valid `model_name` can be found on + https://huggingface.co/kornia/ONNX_models. Or a URL to the ONNX model. + providers: A list of execution providers for ONNXRuntime + (e.g., ['CUDAExecutionProvider', 'CPUExecutionProvider']). + session_options: Optional ONNXRuntime session options for optimizing the session. + io_maps: An optional list of list of tuples specifying input-output mappings for combining models. + If None, we assume the default input name and output name are "input" and "output" accordingly, and + only one input and output node for each graph. + If not None, `io_maps[0]` shall represent the `io_map` for combining the first and second ONNX models. + cache_dir: The directory where ONNX models are cached locally (only for downloading from HuggingFace). + Defaults to None, which will use a default `kornia.config.hub_onnx_dir` directory. + auto_ir_version_conversion: If True, automatically convert the model's IR version to 9, and OPSET version to 17. + Other versions may be pointed to by `target_ir_version` and `target_opset_version`. + target_ir_version: The target IR version to convert to. + target_opset_version: The target OPSET version to convert to. + + """ + + def __init__( + self, + *args: Union[onnx.ModelProto, str], # type:ignore + providers: Optional[list[str]] = None, + session_options: Optional[ort.SessionOptions] = None, # type:ignore + io_maps: Optional[list[tuple[str, str]]] = None, + cache_dir: Optional[str] = None, + auto_ir_version_conversion: bool = False, + target_ir_version: Optional[int] = None, + target_opset_version: Optional[int] = None, + ) -> None: + self.operators = self._load_ops(*args, cache_dir=cache_dir) + if auto_ir_version_conversion: + self.operators = self._auto_version_conversion( + *self.operators, target_ir_version=target_ir_version, target_opset_version=target_opset_version + ) + self._combined_op = self.combine(io_maps=io_maps) + session = self.create_session(providers=providers, session_options=session_options) + self.set_session(session=session) + + def _auto_version_conversion( + self, + *args: onnx.ModelProto, # type:ignore + target_ir_version: Optional[int] = None, + target_opset_version: Optional[int] = None, + ) -> list[onnx.ModelProto]: # type:ignore + """Automatic conversion of the model's IR/OPSET version to the given target version. + + If `target_ir_version` is not provided, the model is converted to 9 by default. + If `target_opset_version` is not provided, the model is converted to 17 by default. + + Args: + args: List of operations to convert. + target_ir_version: The target IR version to convert to. + target_opset_version: The target OPSET version to convert to. + + """ + # TODO: maybe another logic for versioning. + if target_ir_version is None: + target_ir_version = 9 + if target_opset_version is None: + target_opset_version = 17 + + op_list = [] + for op in args: + op = super()._onnx_version_conversion( + op, target_ir_version=target_ir_version, target_opset_version=target_opset_version + ) + op_list.append(op) + return op_list + + def combine(self, io_maps: list[tuple[str, str]] | None = None) -> onnx.ModelProto: # type: ignore + """Combine the stored ONNX operators into one executable graph. + + Args: + io_maps: Optional list of ``(source_output, target_input)`` name + pairs describing how adjacent graphs should be connected. When + omitted, the helper assumes the default single input/output + names used by the ONNX composition utility. + + Returns: + Combined ONNX model graph containing all operators in + ``self.operators`` connected in sequence. + """ + return super()._combine(*self.operators, io_maps=io_maps) + + def create_session( + self, providers: list[str] | None = None, session_options: Any | None = None + ) -> ort.InferenceSession: # type: ignore + """Create an ONNX Runtime session for the combined graph. + + Args: + providers: Optional ordered list of execution providers used by ONNX + Runtime, for example GPU first and CPU as fallback. + session_options: Optional ONNX Runtime session options controlling + optimization, threading, and runtime behavior. + + Returns: + ONNX Runtime inference session for ``self._combined_op``. + """ + return super()._create_session(self._combined_op, providers, session_options) + + def export(self, file_path: str, **kwargs: Any) -> None: + """Serialize the combined ONNX graph to an ``.onnx`` file. + + Args: + file_path: Destination path where the composed ONNX graph is saved. + kwargs: Additional keyword arguments forwarded to the export helper. + """ + return super()._export(self._combined_op, file_path, **kwargs) + + def add_metadata(self, additional_metadata: Optional[list[tuple[str, str]]] = None) -> onnx.ModelProto: # type:ignore + """Attach metadata entries to the combined ONNX graph. + + Args: + additional_metadata: Optional list of ``(key, value)`` pairs to add + to the ONNX model metadata properties. + + Returns: + Combined ONNX model graph after metadata has been attached. + """ + return super()._add_metadata(self._combined_op, additional_metadata) diff --git a/kornia/onnx/utils.py b/kornia/onnx/utils.py new file mode 100644 index 0000000..7ec644a --- /dev/null +++ b/kornia/onnx/utils.py @@ -0,0 +1,251 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import json +import logging +import os +import pprint +from typing import Any, Optional + +import kornia +from kornia.config import kornia_config +from kornia.core.external import numpy as np +from kornia.core.external import onnx, requests +from kornia.onnx.download import CachedDownloader + +__all__ = ["ONNXLoader", "add_metadata", "io_name_conversion"] + +logger = logging.getLogger(__name__) + + +class ONNXLoader(CachedDownloader): + """Manages ONNX models, handling local caching, downloading from Hugging Face, and loading models.""" + + @classmethod + def load_config(cls, url: str, download: bool = True, **kwargs: Any) -> dict[str, Any]: + """Load JSON config from the specified URL. + + Args: + url: The URL of the preprocessor config to load. + download: If True, the config will be downloaded if it's not already in the local cache. + kwargs: Additional download arguments. + + Returns: + dict[str, Any]: The loaded preprocessor config. + + """ + if url.startswith(("http:", "https:")): + file_path = cls.download_to_cache( + url, + os.path.split(url)[-1], + download=download, + suffix=".json", + **kwargs, + ) + with open(file_path) as f: + json_data = json.load(f) + return json_data + + if not download: + raise RuntimeError(f"File `{url}` not found. You may set `download=True`.") + + raise RuntimeError(f"File `{url}` not found.") + + @classmethod + def load_model(cls, model_name: str, download: bool = True, with_data: bool = False, **kwargs) -> onnx.ModelProto: # type:ignore + """Load an ONNX model from the local cache or downloads it from Hugging Face if necessary. + + Args: + model_name: The name of the ONNX model or operator. For Hugging Face-hosted models, + use the format 'hf://model_name'. Valid `model_name` can be found on + https://huggingface.co/kornia/ONNX_models. + Or a URL to the ONNX model. + download: If True, the model will be downloaded from Hugging Face if it's not already in the local cache. + cache_dir: The directory where the model should be cached. + Defaults to None, which will use a default `{kornia_config.hub_onnx_dir}` directory. + with_data: If True, the model will be loaded with its `.onnx_data` weights. + **kwargs: Additional arguments to pass to the download method, if needed. + + Returns: + onnx.ModelProto: The loaded ONNX model. + + """ + if model_name.startswith("hf://"): + model_name = model_name[len("hf://") :] + url = f"https://huggingface.co/kornia/ONNX_models/resolve/main/{model_name}.onnx" + cache_dir = kwargs.get("cache_dir", None) or os.path.join( + kornia_config.hub_onnx_dir, model_name.split("/")[0] + ) + kwargs.update({"cache_dir": cache_dir}) + file_path = cls.download_to_cache( + url, model_name.split("/")[1], download=download, suffix=".onnx", **kwargs + ) + if with_data: + url_data = f"https://huggingface.co/kornia/ONNX_models/resolve/main/{model_name}.onnx_data" + cls.download_to_cache( + url_data, model_name.split("/")[1], download=download, suffix=".onnx_data", **kwargs + ) + return onnx.load(file_path) # type:ignore + + elif model_name.startswith("http://") or model_name.startswith("https://"): + cache_dir = kwargs.get("cache_dir", None) or kornia_config.hub_onnx_dir + kwargs.update({"cache_dir": cache_dir}) + file_path = cls.download_to_cache( + model_name, + os.path.split(model_name)[-1], + download=download, + suffix=".onnx", + **kwargs, + ) + if with_data: + url_data = model_name[:-5] + ".onnx_data" + cls.download_to_cache( + url_data, + os.path.split(model_name)[-1][:-5] + ".onnx_data", + download=download, + suffix=".onnx_data", + **kwargs, + ) + return onnx.load(file_path) # type:ignore + + elif os.path.exists(model_name): + return onnx.load(model_name) # type:ignore + + raise ValueError(f"File {model_name} not found") + + @staticmethod + def _fetch_repo_contents(folder: str) -> list[dict[str, Any]]: + """Fetch the contents of the Hugging Face repository using the Hugging Face API. + + Returns: + A list of all files in the repository as dictionaries containing file details. + + """ + url = f"https://huggingface.co/api/models/kornia/ONNX_models/tree/main/{folder}" + + response = requests.get(url, timeout=10) # type:ignore + + if response.status_code == 200: + return response.json() # Returns the JSON content of the repo + else: + raise ValueError(f"Failed to fetch repository contents: {response.status_code}") + + @classmethod + def list_operators(cls) -> None: + """List all available ONNX operators in the 'operators' folder of the Hugging Face repository.""" + repo_contents = cls._fetch_repo_contents("operators") + + # Filter for operators in the 'operators' directory + operators = [file["path"] for file in repo_contents] + + pprint.pp(operators) + + @classmethod + def list_models(cls) -> None: + """List all available ONNX models in the 'models' folder of the Hugging Face repository.""" + repo_contents = cls._fetch_repo_contents("models") + + # Filter for models in the 'models' directory + models = [file["path"] for file in repo_contents] + + pprint.pp(models) + + +def io_name_conversion( + onnx_model: onnx.ModelProto, # type:ignore + io_name_mapping: dict[str, str], +) -> onnx.ModelProto: # type:ignore + """Convert the input and output names of an ONNX model to 'input' and 'output'. + + Args: + onnx_model: The ONNX model to convert. + io_name_mapping: A dictionary mapping the original input and output names to the new ones. + + """ + # Convert I/O nodes + for i in range(len(onnx_model.graph.input)): + in_name = onnx_model.graph.input[i].name + if in_name in io_name_mapping: + onnx_model.graph.input[i].name = io_name_mapping[in_name] + + for i in range(len(onnx_model.graph.output)): + out_name = onnx_model.graph.output[i].name + if out_name in io_name_mapping: + onnx_model.graph.output[i].name = io_name_mapping[out_name] + + # Convert intermediate nodes + for i in range(len(onnx_model.graph.node)): + for j in range(len(onnx_model.graph.node[i].input)): + node_in_name = onnx_model.graph.node[i].input[j] + if node_in_name in io_name_mapping: + onnx_model.graph.node[i].input[j] = io_name_mapping[node_in_name] + + for j in range(len(onnx_model.graph.node[i].output)): + node_out_name = onnx_model.graph.node[i].output[j] + if node_out_name in io_name_mapping: + onnx_model.graph.node[i].output[j] = io_name_mapping[node_out_name] + + return onnx_model + + +def add_metadata( + onnx_model: onnx.ModelProto, # type: ignore + additional_metadata: Optional[list[tuple[str, str]]] = None, +) -> onnx.ModelProto: # type: ignore + """Add metadata to an ONNX model. + + The metadata includes the source library (set to "kornia"), the version of kornia, + and any additional metadata provided as a list of key-value pairs. + + Args: + onnx_model: The ONNX model to add metadata to. + additional_metadata: A list of tuples, where each tuple contains a key and a value + for the additional metadata to add to the ONNX model. + + Returns: + The ONNX model with the added metadata. + + """ + if additional_metadata is None: + additional_metadata = [] + for key, value in [ + ("source", "kornia"), + ("version", kornia.__version__), + *additional_metadata, + ]: + metadata_props = onnx_model.metadata_props.add() + metadata_props.key = key + metadata_props.value = str(value) + return onnx_model + + +def onnx_type_to_numpy(onnx_type: str) -> Any: + type_mapping = { + "tensor(float)": np.float32, + "tensor(float16)": np.float16, + "tensor(double)": np.float64, + "tensor(int32)": np.int32, + "tensor(int64)": np.int64, + "tensor(uint8)": np.uint8, + "tensor(int8)": np.int8, + "tensor(bool)": np.bool_, + } + if onnx_type not in type_mapping: + raise TypeError(f"ONNX type {onnx_type} not understood") + return type_mapping[onnx_type] diff --git a/kornia/py.typed b/kornia/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/kornia/sensors/__init__.py b/kornia/sensors/__init__.py new file mode 100644 index 0000000..e6e1a3d --- /dev/null +++ b/kornia/sensors/__init__.py @@ -0,0 +1,21 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Sensors submodule for Kornia. + +This package provides utilities and modules for sensor data processing and integration. +""" diff --git a/kornia/sensors/camera/__init__.py b/kornia/sensors/camera/__init__.py new file mode 100644 index 0000000..321fa2c --- /dev/null +++ b/kornia/sensors/camera/__init__.py @@ -0,0 +1,41 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Camera models submodule for Kornia sensors. + +This package provides camera model definitions and utilities for camera calibration and projection. +""" + +from .camera_model import ( + BrownConradyModel, + CameraModel, + CameraModelBase, + CameraModelType, + KannalaBrandtK3, + Orthographic, + PinholeModel, +) + +__all__ = [ + "BrownConradyModel", + "CameraModel", + "CameraModelBase", + "CameraModelType", + "KannalaBrandtK3", + "Orthographic", + "PinholeModel", +] diff --git a/kornia/sensors/camera/camera_model.py b/kornia/sensors/camera/camera_model.py new file mode 100644 index 0000000..2635f7d --- /dev/null +++ b/kornia/sensors/camera/camera_model.py @@ -0,0 +1,367 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# adapted from: https://github.com/strasdat/Sophus/blob/sophus2/cpp/sophus/sensor/camera_model.h +from __future__ import annotations + +from enum import Enum +from typing import Any, Union + +import torch + +from kornia.geometry.vector import Vector2, Vector3 +from kornia.image import ImageSize +from kornia.sensors.camera.distortion_model import AffineTransform, BrownConradyTransform, KannalaBrandtK3Transform +from kornia.sensors.camera.projection_model import OrthographicProjection, Z1Projection + + +class CameraModelType(Enum): + """Represent the type of camera projection and distortion model. + + Supported types: + - PINHOLE: Standard perspective projection with no distortion. + - BROWN_CONRADY: Standard radial and tangential distortion model, often used for wide-angle lenses. + - KANNALA_BRANDT_K3: Fisheye distortion model using a 9th-order polynomial for equidistant projection. + - ORTHOGRAPHIC: Parallel projection where rays are perpendicular to the image plane, with no perspective effect. + """ + + PINHOLE = 0 + BROWN_CONRADY = 1 + KANNALA_BRANDT_K3 = 2 + ORTHOGRAPHIC = 3 + + +def get_model_from_type( + model_type: CameraModelType, image_size: ImageSize, params: torch.Tensor +) -> CameraModelVariants: + """Get camera model from model type.""" + if model_type == CameraModelType.PINHOLE: + return PinholeModel(image_size, params) + elif model_type == CameraModelType.BROWN_CONRADY: + return BrownConradyModel(image_size, params) + elif model_type == CameraModelType.KANNALA_BRANDT_K3: + return KannalaBrandtK3(image_size, params) + elif model_type == CameraModelType.ORTHOGRAPHIC: + return Orthographic(image_size, params) + else: + raise ValueError("Invalid Camera Model Type") + + +CameraDistortionType = Union[AffineTransform, BrownConradyTransform, KannalaBrandtK3Transform] +CameraProjectionType = Union[Z1Projection, OrthographicProjection] + + +class CameraModelBase: + r"""Base class to represent camera models based on distortion and projection types. + + Distortion is of 3 types: + - Affine + - Brown Conrady + - Kannala Brandt K3 + Projection is of 2 types: + - Z1 + - Orthographic + + Example: + >>> params = torch.Tensor([328., 328., 320., 240.]) + >>> cam = CameraModelBase(BrownConradyTransform(), Z1Projection(), ImageSize(480, 640), params) + >>> cam.params + tensor([328., 328., 320., 240.]) + + """ + + def __init__( + self, + distortion: CameraDistortionType, + projection: CameraProjectionType, + image_size: ImageSize, + params: torch.Tensor, + ) -> None: + """Construct CameraModelBase class. + + Args: + distortion: Distortion type + projection: Projection type + image_size: Image size + params: Camera parameters of shape :math:`(B, 4)` + for PINHOLE Camera, :math:`(B, 12)` + for Brown Conrady, :math:`(B, 8)` + for Kannala Brandt K3. + + """ + self.distortion = distortion + self.projection = projection + self._image_size = image_size + self._height = image_size.height + self._width = image_size.width + self._params = params + + @property + def image_size(self) -> ImageSize: + """Returns the image size of the camera model.""" + return self._image_size + + @property + def height(self) -> int | torch.Tensor: + """Returns the height of the image.""" + return self._height + + @property + def width(self) -> int | torch.Tensor: + """Returns the width of the image.""" + return self._width + + @property + def params(self) -> torch.Tensor: + """Returns the camera parameters.""" + return self._params + + @property + def fx(self) -> torch.Tensor: + """Returns the focal length in x direction.""" + return self._params[..., 0] + + @property + def fy(self) -> torch.Tensor: + """Returns the focal length in y direction.""" + return self._params[..., 1] + + @property + def cx(self) -> torch.Tensor: + """Returns the principal point in x direction.""" + return self._params[..., 2] + + @property + def cy(self) -> torch.Tensor: + """Returns the principal point in y direction.""" + return self._params[..., 3] + + def matrix(self) -> torch.Tensor: + """Return the camera matrix.""" + raise NotImplementedError + + def K(self) -> torch.Tensor: + """Return the camera matrix.""" + return self.matrix() + + def project(self, points: Vector3) -> Vector2: + """Projects 3D points to 2D camera plane. + + Args: + points: Vector3 representing 3D points. + + Returns: + Vector2 representing the projected 2D points. + + Example: + >>> points = Vector3(torch.Tensor([1.0, 1.0, 1.0])) + >>> cam = CameraModel(ImageSize(480, 640), CameraModelType.PINHOLE, torch.Tensor([328., 328., 320., 240.])) + >>> cam.project(points) + x: 648.0 + y: 568.0 + + """ + return self.distortion.distort(self.params, self.projection.project(points)) + + def unproject(self, points: Vector2, depth: torch.Tensor) -> Vector3: + """Unprojects 2D points from camera plane to 3D. + + Args: + points: Vector2 representing 2D points. + depth: Depth of the points. + + Returns: + Vector3 representing the unprojected 3D points. + + Example: + >>> points = Vector2(torch.Tensor([1.0, 1.0])) + >>> cam = CameraModel(ImageSize(480, 640), CameraModelType.PINHOLE, torch.Tensor([328., 328., 320., 240.])) + >>> cam.unproject(points, torch.Tensor([1.0])) + x: tensor([-0.9726]) + y: tensor([-0.7287]) + z: tensor([1.]) + + """ + return self.projection.unproject(self.distortion.undistort(self.params, points), depth) + + +class PinholeModel(CameraModelBase): + r"""Class to represent Pinhole Camera Model. + + The pinhole camera model describes the mathematical relationship between + the coordinates of a point in three-dimensional space and its projection + onto the image plane of an ideal pinhole camera, + where the camera aperture is described as a point and no lenses are used to focus light. + See more: https://en.wikipedia.org/wiki/Pinhole_camera_model + + Example: + >>> cam = CameraModel(ImageSize(480, 640), CameraModelType.PINHOLE, torch.Tensor([328., 328., 320., 240.])) + >>> cam + CameraModel(ImageSize(height=480, width=640), PinholeModel, tensor([328., 328., 320., 240.])) + + """ + + def __init__(self, image_size: ImageSize, params: torch.Tensor) -> None: + """Construct PinholeModel class. + + Args: + image_size: Image size + params: Camera parameters of shape :math:`(B, 4)` of the form :math:`(fx, fy, cx, cy)`. + + """ + if params.shape[-1] != 4 or len(params.shape) > 2: + raise ValueError("params must be of shape (B, 4) for PINHOLE Camera") + super().__init__(AffineTransform(), Z1Projection(), image_size, params) + + def matrix(self) -> torch.Tensor: + r"""Return the camera matrix. + + The matrix is of the form: + + .. math:: + \begin{bmatrix} fx & 0 & cx \\ + 0 & fy & cy \\ + 0 & 0 & 1\end{bmatrix} + + Example: + >>> cam = CameraModel(ImageSize(480, 640), CameraModelType.PINHOLE, torch.Tensor([1.0, 2.0, 3.0, 4.0])) + >>> cam.matrix() + tensor([[1., 0., 3.], + [0., 2., 4.], + [0., 0., 1.]]) + + """ + z = torch.zeros_like(self.fx) + row1 = torch.stack((self.fx, z, self.cx), -1) + row2 = torch.stack((z, self.fy, self.cy), -1) + row3 = torch.stack((z, z, z), -1) + K = torch.stack((row1, row2, row3), -2) + K[..., -1, -1] = 1.0 + return K + + def scale(self, scale_factor: torch.Tensor) -> PinholeModel: + """Scales the camera model by a scale factor. + + Args: + scale_factor: Scale factor to scale the camera model. + + Returns: + Scaled camera model. + + Example: + >>> cam = CameraModel(ImageSize(480, 640), CameraModelType.PINHOLE, torch.Tensor([328., 328., 320., 240.])) + >>> cam_scaled = cam.scale(2) + >>> cam_scaled.params + tensor([656., 656., 640., 480.]) + + """ + fx = self.fx * scale_factor + fy = self.fy * scale_factor + cx = self.cx * scale_factor + cy = self.cy * scale_factor + params = torch.stack((fx, fy, cx, cy), -1) + image_size = ImageSize(self.image_size.height * scale_factor, self.image_size.width * scale_factor) + return PinholeModel(image_size, params) + + +class BrownConradyModel(CameraModelBase): + """Brown Conrady Camera Model.""" + + def __init__(self, image_size: ImageSize, params: torch.Tensor) -> None: + """Construct BrownConradyModel class. + + Args: + image_size: Image size + params: Camera parameters of shape :math:`(B, 12)` of the form :math:`(fx, fy, cx, cy, kb0, kb1, kb2, kb3, + k1, k2, k3, k4)`. + + """ + if params.shape[-1] != 12 or len(params.shape) > 2: + raise ValueError("params must be of shape (B, 12) for BROWN_CONRADY Camera") + super().__init__(BrownConradyTransform(), Z1Projection(), image_size, params) + + +class KannalaBrandtK3(CameraModelBase): + """Kannala Brandt K3 Camera Model.""" + + def __init__(self, image_size: ImageSize, params: torch.Tensor) -> None: + """Construct KannalaBrandtK3 class. + + Args: + image_size: Image size + params: Camera parameters of shape :math:`(B, 8)` of the form :math:`(fx, fy, cx, cy, kb0, kb1, kb2, kb3)`. + + """ + if params.shape[-1] != 8 or len(params.shape) > 2: + raise ValueError("params must be of shape B, 8 for KANNALA_BRANDT_K3 Camera") + super().__init__(KannalaBrandtK3Transform(), Z1Projection(), image_size, params) + + +class Orthographic(CameraModelBase): + """Orthographic Camera Model.""" + + def __init__(self, image_size: ImageSize, params: torch.Tensor) -> None: + """Construct Orthographic class. + + Args: + image_size: Image size + params: Camera parameters of shape :math:`(B, 4)` of the form :math:`(fx, fy, cx, cy)`. + + """ + super().__init__(AffineTransform(), OrthographicProjection(), image_size, params) + if params.shape[-1] != 4 or len(params.shape) > 2: + raise ValueError("params must be of shape B, 4 for ORTHOGRAPHIC Camera") + + +CameraModelVariants = Union[PinholeModel, BrownConradyModel, KannalaBrandtK3, Orthographic] + + +class CameraModel: + r"""Class to represent camera models. + + Example: + >>> # Pinhole Camera Model + >>> cam = CameraModel(ImageSize(480, 640), CameraModelType.PINHOLE, torch.Tensor([328., 328., 320., 240.])) + >>> # Brown Conrady Camera Model + >>> cam = CameraModel(ImageSize(480, 640), CameraModelType.BROWN_CONRADY, torch.Tensor([1.0, 1.0, 1.0, 1.0, + ... 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0])) + >>> # Kannala Brandt K3 Camera Model + >>> cam = CameraModel(ImageSize(480, 640), CameraModelType.KANNALA_BRANDT_K3, torch.Tensor([1.0, 1.0, 1.0, + ... 1.0, 1.0, 1.0, 1.0, 1.0])) + >>> # Orthographic Camera Model + >>> cam = CameraModel(ImageSize(480, 640), CameraModelType.ORTHOGRAPHIC, torch.Tensor([328., 328., 320., 240.])) + >>> cam.params + tensor([328., 328., 320., 240.]) + + """ + + def __init__(self, image_size: ImageSize, model_type: CameraModelType, params: torch.Tensor) -> None: + """Construct CameraModel class. + + Args: + image_size: Image size + model_type: Camera model type + params: Camera parameters of shape :math:`(B, N)`. + + """ + self._model = get_model_from_type(model_type, image_size, params) + + def __getattr__(self, name: str) -> Any: + return getattr(self._model, name) + + def __repr__(self) -> str: + return f"CameraModel({self.image_size}, {self._model.__class__.__name__}, {self.params})" diff --git a/kornia/sensors/camera/distortion_model.py b/kornia/sensors/camera/distortion_model.py new file mode 100644 index 0000000..0323399 --- /dev/null +++ b/kornia/sensors/camera/distortion_model.py @@ -0,0 +1,171 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import torch + +from kornia.geometry.vector import Vector2 + + +class AffineTransform: + """Apply an affine transformation to a set of 2D points. + + This class handles the scaling and shifting of coordinates, typically used + to map normalized coordinates to pixel coordinates. + """ + + def distort(self, params: torch.Tensor, points: Vector2) -> Vector2: + """Distort one or more Vector2 points using the affine transform. + + Args: + params: torch.Tensor representing the affine transform parameters. + points: Vector2 representing the points to distort. + + Returns: + Vector2 representing the distorted points. + + Example: + >>> params = torch.Tensor([1., 2., 3., 4.]) + >>> points = Vector2.from_coords(1., 2.) + >>> AffineTransform().distort(params, points) + x: 4.0 + y: 8.0 + + """ + fx, fy, cx, cy = params[..., 0], params[..., 1], params[..., 2], params[..., 3] + u = points.x * fx + cx + v = points.y * fy + cy + return Vector2.from_coords(u, v) + + def undistort(self, params: torch.Tensor, points: Vector2) -> Vector2: + """Undistort one or more Vector2 points using the affine transform. + + Args: + params: torch.Tensor representing the affine transform parameters. + points: Vector2 representing the points to undistort. + + Returns: + Vector2 representing the undistorted points. + + Example: + >>> params = torch.Tensor([1., 2., 3., 4.]) + >>> points = Vector2.from_coords(1., 2.) + >>> AffineTransform().undistort(params, points) + x: -2.0 + y: -1.0 + + """ + fx, fy, cx, cy = params[..., 0], params[..., 1], params[..., 2], params[..., 3] + x = (points.x - cx) / fx + y = (points.y - cy) / fy + return Vector2.from_coords(x, y) + + +class BrownConradyTransform: + """Implement the Brown-Conrady model for lens distortion and undistortion. + + The model accounts for radial distortion (due to lens shape) and tangential + distortion (due to lens misalignment). It is commonly used to transform + points between ideal pinhole projections and distorted image coordinates. + + Args: + params: A tensor containing the distortion coefficients + (usually k1, k2, p1, p2, k3). + points: A :class:`Vector2` representing the 2D coordinates to be transformed. + """ + + def distort(self, params: torch.Tensor, points: Vector2) -> Vector2: + """Apply Brown-Conrady lens distortion to ideal normalized points. + + Args: + params: Distortion parameter tensor, typically containing radial + coefficients such as ``k1``, ``k2``, ``k3`` and tangential + coefficients such as ``p1`` and ``p2``. + points: Ideal two-dimensional normalized points before lens + distortion. Leading dimensions may represent a batch. + + Returns: + Distorted two-dimensional points in the same coordinate convention. + + Raises: + NotImplementedError: The Brown-Conrady transform interface is + declared here, but the concrete computation is not implemented. + """ + raise NotImplementedError + + def undistort(self, params: torch.Tensor, points: Vector2) -> Vector2: + """Remove Brown-Conrady lens distortion from observed points. + + Args: + params: Distortion parameter tensor matching the coefficients used + by :meth:`distort`. + points: Distorted two-dimensional points, usually measured in the + normalized image plane. + + Returns: + Undistorted two-dimensional points that approximate the ideal + pinhole projection. + + Raises: + NotImplementedError: The Brown-Conrady inverse transform interface + is declared here, but the concrete computation is not + implemented. + """ + raise NotImplementedError + + +class KannalaBrandtK3Transform: + """Apply the Kannala-Brandt (K3) distortion model. + + This model is specifically designed for fisheye lenses with significant + radial distortion, using a polynomial approximation for the projection. + """ + + def distort(self, params: torch.Tensor, points: Vector2) -> Vector2: + """Apply Kannala-Brandt K3 fisheye distortion to normalized points. + + Args: + params: Fisheye distortion coefficients for the K3 polynomial model. + points: Ideal two-dimensional normalized points before fisheye + distortion is applied. + + Returns: + Distorted two-dimensional points following the K3 fisheye model. + + Raises: + NotImplementedError: The K3 distortion interface is declared here, + but the concrete computation is not implemented. + """ + raise NotImplementedError + + def undistort(self, params: torch.Tensor, points: Vector2) -> Vector2: + """Remove Kannala-Brandt K3 fisheye distortion from observed points. + + Args: + params: Fisheye distortion coefficients matching the K3 model used + for distortion. + points: Distorted two-dimensional fisheye points. + + Returns: + Undistorted normalized points that approximate ideal pinhole + coordinates. + + Raises: + NotImplementedError: The K3 inverse distortion interface is + declared here, but the concrete computation is not implemented. + """ + raise NotImplementedError diff --git a/kornia/sensors/camera/projection_model.py b/kornia/sensors/camera/projection_model.py new file mode 100644 index 0000000..c8f1efa --- /dev/null +++ b/kornia/sensors/camera/projection_model.py @@ -0,0 +1,126 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch + +from kornia.geometry.vector import Vector2, Vector3 + + +class Z1Projection: + """Project 3D points from the camera frame into the canonical $z=1$ plane. + + This performs perspective division by dividing the $x$ and $y$ coordinates + by the depth $z$. + """ + + def project(self, points: Vector3) -> Vector2: + """Project one or more Vector3 from the camera frame into the canonical z=1 plane through perspective division. + + Args: + points: Vector3 representing the points to project. + + Returns: + Vector2 representing the projected points. + + Example: + >>> points = Vector3.from_coords(1., 2., 3.) + >>> Z1Projection().project(points) + x: 0.3333333432674408 + y: 0.6666666865348816 + + """ + xy = points.data[..., :2] + z = points.z + if len(z.shape): + uv = xy / z.unsqueeze(-1) + else: + # For scalar z, xy is 1-D, so no transpose needed + uv = xy * 1 / z + return Vector2(uv) + + def unproject(self, points: Vector2, depth: torch.Tensor | float) -> Vector3: + """Unproject one or more Vector2 from the canonical z=1 plane into the camera frame. + + Args: + points: Vector2 representing the points to unproject. + depth: torch.Tensor representing the depth of the points to unproject. + + Returns: + Vector3 representing the unprojected points. + + Example: + >>> points = Vector2.from_coords(1., 2.) + >>> Z1Projection().unproject(points, 3) + x: tensor([3.]) + y: tensor([6.]) + z: tensor([3.]) + + """ + if isinstance(depth, (float, int)): + depth = torch.Tensor([depth]) + return Vector3.from_coords(points.x * depth, points.y * depth, depth) + + +class OrthographicProjection: + """Project 3D points using an orthographic projection model. + + This model assumes parallel projection where the $z$ coordinate is + discarded and no perspective scaling is applied. + """ + + def project(self, points: Vector3) -> Vector2: + """Project 3D camera-frame points with an orthographic camera model. + + Orthographic projection keeps the horizontal and vertical coordinates + unchanged and discards depth. Unlike perspective projection, objects do + not shrink as their ``z`` value increases. + + Args: + points: Three-dimensional point container with coordinates + ``x``, ``y``, and ``z``. Leading dimensions may represent a + batch of points. + + Returns: + Two-dimensional point container containing the projected ``x`` and + ``y`` coordinates. + + Raises: + NotImplementedError: This projection model is declared as an + interface placeholder and is not implemented yet. + """ + raise NotImplementedError + + def unproject(self, points: Vector2, depth: torch.Tensor) -> Vector3: + """Lift orthographic image-plane points back into 3D using depth. + + Args: + points: Two-dimensional point container with image-plane + coordinates ``x`` and ``y``. + depth: Tensor containing the target ``z`` coordinate for each + unprojected point. + + Returns: + Three-dimensional point container with ``x`` and ``y`` copied from + ``points`` and ``z`` supplied by ``depth``. + + Raises: + NotImplementedError: This projection model is declared as an + interface placeholder and is not implemented yet. + """ + raise NotImplementedError diff --git a/kornia/tracking/__init__.py b/kornia/tracking/__init__.py new file mode 100644 index 0000000..5613e09 --- /dev/null +++ b/kornia/tracking/__init__.py @@ -0,0 +1,25 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Tracking submodule for Kornia. + +This package provides tracking algorithms and utilities, including planar homography tracking. +""" + +from .planar_tracker import HomographyTracker + +__all__ = ["HomographyTracker"] diff --git a/kornia/tracking/planar_tracker.py b/kornia/tracking/planar_tracker.py new file mode 100644 index 0000000..fc40d80 --- /dev/null +++ b/kornia/tracking/planar_tracker.py @@ -0,0 +1,228 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict, Optional, Tuple + +import torch +from torch import nn + +from kornia.feature import DescriptorMatcher, GFTTAffNetHardNet, LocalFeatureMatcher, LoFTR +from kornia.feature.integrated import LocalFeature +from kornia.geometry.linalg import transform_points +from kornia.geometry.ransac import RANSAC +from kornia.geometry.transform import warp_perspective + + +class HomographyTracker(nn.Module): + r"""Perform local-feature-based tracking of the target planar object in the sequence of the frames. + + Args: + initial_matcher: image matching module, e.g. :class:`~kornia.feature.LocalFeatureMatcher` + or :class:`~kornia.feature.LoFTR`. Default: :class:`~kornia.feature.GFTTAffNetHardNet`. + fast_matcher: fast image matching module, e.g. :class:`~kornia.feature.LocalFeatureMatcher` + or :class:`~kornia.feature.LoFTR`. Default: :class:`~kornia.feature.DescriptorMatcher`. + ransac: homography estimation module. Default: :class:`~kornia.geometry.RANSAC`. + minimum_inliers_num: threshold for number inliers for matching to be successful. + + """ + + def __init__( + self, + initial_matcher: Optional[LocalFeature] = None, + fast_matcher: Optional[nn.Module] = None, + ransac: Optional[nn.Module] = None, + minimum_inliers_num: int = 30, + ) -> None: + super().__init__() + self.initial_matcher = initial_matcher or ( + LocalFeatureMatcher(GFTTAffNetHardNet(3000), DescriptorMatcher("smnn", 0.95)) + ) + self.fast_matcher = fast_matcher or LoFTR("outdoor") + self.ransac = ransac or RANSAC("homography", inl_th=5.0, batch_size=4096, max_iter=10, max_lo_iters=10) + self.minimum_inliers_num = minimum_inliers_num + + # placeholders + self.target: torch.Tensor + self.target_initial_representation: Dict[str, torch.Tensor] = {} + self.target_fast_representation: Dict[str, torch.Tensor] = {} + self.previous_homography: Optional[torch.Tensor] = None + + self.inliers_num: int = 0 + self.keypoints0_num: int = 0 + self.keypoints1_num: int = 0 + + self.reset_tracking() + + @property + def device(self) -> torch.device: + """Return the device used by the current target image tensor. + + Returns: + The ``torch.device`` where ``self.target`` is allocated. + """ + return self.target.device + + @property + def dtype(self) -> torch.dtype: + """Return the data type used by the current target image tensor. + + Returns: + The ``torch.dtype`` of ``self.target``. + """ + return self.target.dtype + + @torch.no_grad() + def set_target(self, target: torch.Tensor) -> None: + """Register a new target image and refresh cached matcher features. + + Args: + target: Reference target image tensor used for subsequent matching. + + Returns: + None. + + The method clears previously cached features and precomputes new + feature representations when the configured matchers expose an + ``extract_features`` method. + """ + self.target = target + self.target_initial_representation = {} + self.target_fast_representation = {} + if hasattr(self.initial_matcher, "extract_features") and isinstance( + self.initial_matcher.extract_features, nn.Module + ): + self.target_initial_representation = self.initial_matcher.extract_features(target) + if hasattr(self.fast_matcher, "extract_features") and isinstance(self.fast_matcher.extract_features, nn.Module): + self.target_fast_representation = self.fast_matcher.extract_features(target) + + def reset_tracking(self) -> None: + """Reset temporal tracking state from previously processed frames. + + Returns: + None. + """ + self.previous_homography = None + + def no_match(self) -> Tuple[torch.Tensor, bool]: + """Return a failed-match response and clear current match statistics. + + Returns: + A tuple ``(H, is_valid)`` where ``H`` is an empty ``3 x 3`` tensor + on the tracker device and dtype, and ``is_valid`` is ``False``. + """ + self.inliers_num = 0 + self.keypoints0_num = 0 + self.keypoints1_num = 0 + return torch.empty(3, 3, device=self.device, dtype=self.dtype), False + + def match_initial(self, x: torch.Tensor) -> Tuple[torch.Tensor, bool]: + """Estimate a homography from the initial target frame to frame ``x``. + + Args: + x: Current frame tensor to match against the stored target image. + + Returns: + A tuple ``(H, is_valid)`` where ``H`` is the estimated homography + matrix and ``is_valid`` indicates whether enough inliers were found. + + The method updates keypoint counters, inlier statistics, and stores the + estimated homography as ``previous_homography`` on success. + """ + input_dict: Dict[str, torch.Tensor] = {"image0": self.target, "image1": x} + + for k, v in self.target_initial_representation.items(): + input_dict[f"{k}0"] = v + + match_dict: Dict[str, torch.Tensor] = self.initial_matcher(input_dict) + keypoints0 = match_dict["keypoints0"][match_dict["batch_indexes"] == 0] + keypoints1 = match_dict["keypoints1"][match_dict["batch_indexes"] == 0] + + self.keypoints0_num = len(keypoints0) + self.keypoints1_num = len(keypoints1) + + if self.keypoints0_num < self.minimum_inliers_num: + return self.no_match() + + H, inliers = self.ransac(keypoints0, keypoints1) + self.inliers_num = inliers.sum().item() + + if self.inliers_num < self.minimum_inliers_num: + return self.no_match() + self.previous_homography = H.clone() + + return H, True + + def track_next_frame(self, x: torch.Tensor) -> Tuple[torch.Tensor, bool]: + """Track the target in frame ``x`` using the previous homography prior. + + Args: + x: Current frame tensor to align with the target image. + + Returns: + A tuple ``(H, is_valid)`` where ``H`` is the updated homography and + ``is_valid`` indicates whether tracking remained reliable. + + The frame is first prewarped by the inverse of the previous homography, + then matched with ``fast_matcher`` and verified using RANSAC. + """ + if self.previous_homography is not None: # mypy, shut up + Hwarp = self.previous_homography.clone()[None] + # make a bit of border for safety + Hwarp[:, 0:2, 0:2] = Hwarp[:, 0:2, 0:2] / 0.8 + Hwarp[:, 0:2, 2] -= 10.0 + Hinv = torch.inverse(Hwarp) + h, w = self.target.shape[2:] + frame_warped = warp_perspective(x, Hinv, (h, w)) + input_dict: Dict[str, torch.Tensor] = {"image0": self.target, "image1": frame_warped} + for k, v in self.target_fast_representation.items(): + input_dict[f"{k}0"] = v + + match_dict = self.fast_matcher(input_dict) + keypoints0 = match_dict["keypoints0"][match_dict["batch_indexes"] == 0] + keypoints1 = match_dict["keypoints1"][match_dict["batch_indexes"] == 0] + keypoints1 = transform_points(Hwarp, keypoints1) + + self.keypoints0_num = len(keypoints0) + self.keypoints1_num = len(keypoints1) + + if self.keypoints0_num < self.minimum_inliers_num: + self.reset_tracking() + return self.no_match() + + H, inliers = self.ransac(keypoints0, keypoints1) + self.inliers_num = inliers.sum().item() + + if self.inliers_num < self.minimum_inliers_num: + self.reset_tracking() + return self.no_match() + + self.previous_homography = H.clone() + return H, True + + def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, bool]: + """Run one tracking step on frame ``x``. + + Args: + x: Current frame tensor. + + Returns: + A tuple ``(H, is_valid)`` from ``track_next_frame`` when previous + state exists, otherwise from ``match_initial``. + """ + if self.previous_homography is not None: + return self.track_next_frame(x) + return self.match_initial(x) diff --git a/kornia/transpiler/__init__.py b/kornia/transpiler/__init__.py new file mode 100644 index 0000000..9074294 --- /dev/null +++ b/kornia/transpiler/__init__.py @@ -0,0 +1,25 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Transpiler submodule for Kornia. + +This package provides utilities to transpile Kornia operations to other frameworks such as JAX, NumPy, and TensorFlow. +""" + +from .transpiler import to_jax, to_numpy, to_tensorflow + +__all__ = ["to_jax", "to_numpy", "to_tensorflow"] diff --git a/kornia/transpiler/transpiler.py b/kornia/transpiler/transpiler.py new file mode 100644 index 0000000..afec535 --- /dev/null +++ b/kornia/transpiler/transpiler.py @@ -0,0 +1,116 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Module for transpiling Kornia to other frameworks.""" + +from types import ModuleType + +import kornia +from kornia.core.external import ivy + + +def to_jax() -> ModuleType: + """Convert Kornia to JAX. + + Transpiles the Kornia library to JAX using [ivy](https://github.com/ivy-llc/ivy). The transpilation process + occurs lazily, so the transpilation on a given kornia function/class will only occur when it's called or + instantiated for the first time. This will make any functions/classes slow when being used for the first time, + but any subsequent uses should be as fast as expected. + + Return: + The Kornia library transpiled to JAX + + Example: + + .. highlight:: python + .. code-block:: python + + import kornia + jax_kornia = kornia.to_jax() + import jax + input = jax.random.normal(jax.random.key(42), shape=(2, 3, 4, 5)) + gray = jax_kornia.color.gray.rgb_to_grayscale(input) + + """ + return ivy.transpile( + kornia, + source="torch", + target="jax", + ) # type: ignore + + +def to_numpy() -> ModuleType: + """Convert Kornia to NumPy. + + Transpiles the Kornia library to NumPy using [ivy](https://github.com/ivy-llc/ivy). The transpilation process + occurs lazily, so the transpilation on a given kornia function/class will only occur when it's called or + instantiated for the first time. This will make any functions/classes slow when being used for the first time, + but any subsequent uses should be as fast as expected. + + Return: + The Kornia library transpiled to NumPy + + Example: + + .. highlight:: python + .. code-block:: python + + import kornia + np_kornia = kornia.to_numpy() + import numpy as np + input = np.random.normal(size=(2, 3, 4, 5)) + gray = np_kornia.color.gray.rgb_to_grayscale(input) + + Note: + Ivy does not currently support transpiling trainable modules to NumPy. + + """ + return ivy.transpile( + kornia, + source="torch", + target="numpy", + ) # type: ignore + + +def to_tensorflow() -> ModuleType: + """Convert Kornia to TensorFlow. + + Transpiles the Kornia library to TensorFlow using [ivy](https://github.com/ivy-llc/ivy). The transpilation process + occurs lazily, so the transpilation on a given kornia function/class will only occur when it's called or + instantiated for the first time. This will make any functions/classes slow when being used for the first time, + but any subsequent uses should be as fast as expected. + + Return: + The Kornia library transpiled to TensorFlow + + Example: + + .. highlight:: python + .. code-block:: python + + import kornia + tf_kornia = kornia.to_tensorflow() + import tensorflow as tf + input = tf.random.normal((2, 3, 4, 5)) + gray = tf_kornia.color.gray.rgb_to_grayscale(input) + + """ + return ivy.transpile( + kornia, + source="torch", + target="tensorflow", + ) # type: ignore diff --git a/kornia/utils/__init__.py b/kornia/utils/__init__.py new file mode 100644 index 0000000..653cf1f --- /dev/null +++ b/kornia/utils/__init__.py @@ -0,0 +1,211 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Utils submodule for Kornia. + +This module has been deprecated. Functions have been moved to their respective modules. +Import from the new locations instead (e.g., `kornia.image.draw_line` instead of `kornia.utils.draw_line`). +""" + +from typing import Any + +from kornia.core._compat import deprecated +from kornia.geometry import ( + create_meshgrid as _create_meshgrid, +) +from kornia.geometry import ( + create_meshgrid3d as _create_meshgrid3d, +) +from kornia.geometry import ( + load_pointcloud_ply as _load_pointcloud_ply, +) +from kornia.geometry import ( + save_pointcloud_ply as _save_pointcloud_ply, +) +from kornia.image import ( + draw_convex_polygon as _draw_convex_polygon, +) +from kornia.image import ( + draw_line as _draw_line, +) +from kornia.image import ( + draw_point2d as _draw_point2d, +) +from kornia.image import ( + draw_rectangle as _draw_rectangle, +) +from kornia.image import ( + image_to_string as _image_to_string, +) +from kornia.image import ( + image_to_tensor as _image_to_tensor, +) +from kornia.image import ( + print_image as _print_image, +) +from kornia.image import ( + tensor_to_image as _tensor_to_image, +) +from kornia.losses.one_hot import one_hot as _one_hot + + +# Re-export with deprecation warnings +@deprecated( + replace_with="kornia.geometry.create_meshgrid", + version="0.8.3", + extra_reason=" The `kornia.utils` module has been removed. Import from `kornia.geometry` instead.", +) +def create_meshgrid(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.geometry.create_meshgrid` instead.""" + return _create_meshgrid(*args, **kwargs) + + +@deprecated( + replace_with="kornia.geometry.create_meshgrid3d", + version="0.8.3", + extra_reason=" The `kornia.utils` module has been removed. Import from `kornia.geometry` instead.", +) +def create_meshgrid3d(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.geometry.create_meshgrid3d` instead.""" + return _create_meshgrid3d(*args, **kwargs) + + +@deprecated( + replace_with="kornia.geometry.load_pointcloud_ply", + version="0.8.3", + extra_reason=" The `kornia.utils` module has been removed. Import from `kornia.geometry` instead.", +) +def load_pointcloud_ply(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.geometry.load_pointcloud_ply` instead.""" + return _load_pointcloud_ply(*args, **kwargs) + + +@deprecated( + replace_with="kornia.geometry.save_pointcloud_ply", + version="0.8.3", + extra_reason=" The `kornia.utils` module has been removed. Import from `kornia.geometry` instead.", +) +def save_pointcloud_ply(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.geometry.save_pointcloud_ply` instead.""" + return _save_pointcloud_ply(*args, **kwargs) + + +@deprecated( + replace_with="kornia.image.draw_line", + version="0.8.3", + extra_reason=" The `kornia.utils` module has been removed. Import from `kornia.image` instead.", +) +def draw_line(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.image.draw_line` instead.""" + return _draw_line(*args, **kwargs) + + +@deprecated( + replace_with="kornia.image.draw_rectangle", + version="0.8.3", + extra_reason=" The `kornia.utils` module has been removed. Import from `kornia.image` instead.", +) +def draw_rectangle(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.image.draw_rectangle` instead.""" + return _draw_rectangle(*args, **kwargs) + + +@deprecated( + replace_with="kornia.image.draw_point2d", + version="0.8.3", + extra_reason=" The `kornia.utils` module has been removed. Import from `kornia.image` instead.", +) +def draw_point2d(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.image.draw_point2d` instead.""" + return _draw_point2d(*args, **kwargs) + + +@deprecated( + replace_with="kornia.image.draw_convex_polygon", + version="0.8.3", + extra_reason=" The `kornia.utils` module has been removed. Import from `kornia.image` instead.", +) +def draw_convex_polygon(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.image.draw_convex_polygon` instead.""" + return _draw_convex_polygon(*args, **kwargs) + + +@deprecated( + replace_with="kornia.image.image_to_string", + version="0.8.3", + extra_reason=" The `kornia.utils` module has been removed. Import from `kornia.image` instead.", +) +def image_to_string(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.image.image_to_string` instead.""" + return _image_to_string(*args, **kwargs) + + +@deprecated( + replace_with="kornia.image.print_image", + version="0.8.3", + extra_reason=" The `kornia.utils` module has been removed. Import from `kornia.image` instead.", +) +def print_image(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.image.print_image` instead.""" + return _print_image(*args, **kwargs) + + +@deprecated( + replace_with="kornia.image.image_to_tensor", + version="0.8.3", + extra_reason=" The `kornia.utils` module has been removed. Import from `kornia.image` instead.", +) +def image_to_tensor(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.image.image_to_tensor` instead.""" + return _image_to_tensor(*args, **kwargs) + + +@deprecated( + replace_with="kornia.image.tensor_to_image", + version="0.8.3", + extra_reason=" The `kornia.utils` module has been removed. Import from `kornia.image` instead.", +) +def tensor_to_image(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.image.tensor_to_image` instead.""" + return _tensor_to_image(*args, **kwargs) + + +@deprecated( + replace_with="kornia.losses.one_hot", + version="0.8.3", + extra_reason=" The `kornia.utils` module has been removed. Import from `kornia.losses` instead.", +) +def one_hot(*args: Any, **kwargs: Any) -> Any: + """Deprecated: Use `kornia.losses.one_hot` instead.""" + return _one_hot(*args, **kwargs) + + +__all__ = [ + "create_meshgrid", + "create_meshgrid3d", + "draw_convex_polygon", + "draw_line", + "draw_point2d", + "draw_rectangle", + "image_to_string", + "image_to_tensor", + "load_pointcloud_ply", + "one_hot", + "print_image", + "save_pointcloud_ply", + "tensor_to_image", +] diff --git a/pixi.lock b/pixi.lock new file mode 100644 index 0000000..c6a0df2 --- /dev/null +++ b/pixi.lock @@ -0,0 +1,2195 @@ +version: 6 +environments: + cuda: + channels: + - url: https://conda.anaconda.org/conda-forge/ + options: + pypi-prerelease-mode: if-necessary-or-explicit + packages: + linux-64: + - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.14-py311hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-78.1-h33c6efd_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.45-default_hbd61a6d_105.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.3-hecca717_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.5.2-h9ec8514_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.2.0-he0feb66_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.2.0-h69a702a_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-he0feb66_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hb9d3cd8_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.51.1-hf4e2dac_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.2.0-h934c35e_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.41.3-h5347b49_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.0-h26f9b46_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.11.14-hd63d673_2_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.11.14-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-h853b02a_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_ha0e22de_103.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tombi-0.7.11-hb17b654_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ty-0.0.9-h4e94fc0_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.9.18-h76e24b7_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda + linux-aarch64: + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h4777abc_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.14-py311hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/icu-78.1-hb1525cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.45-default_h1979696_105.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libexpat-2.7.3-hfae3067_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libffi-3.5.2-hd65408f_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-15.2.0-h8acb6b2_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-ng-15.2.0-he9431aa_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.2.0-h8acb6b2_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/liblzma-5.8.1-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libnsl-2.0.1-h86ecc28_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libsqlite-3.51.1-h10b116e_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-15.2.0-hef695bb_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libuuid-2.41.3-h1022ec0_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxcrypt-4.4.36-h31becfc_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libzlib-1.3.1-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ncurses-6.5-ha32ae93_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/openssl-3.6.0-h8e36d6e_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/python-3.11.14-h91f4b29_2_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.11.14-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/readline-8.3-hb682ff5_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h561c983_103.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tombi-0.7.11-h069e38c_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ty-0.0.9-h1339683_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/uv-0.9.18-haeed4ea_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.7-h85ac4a6_6.conda + osx-64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h500dc9f_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.14-py311hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/icu-78.1-h14c5de8_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-21.1.8-h3d58e20_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libexpat-2.7.3-heffb93a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.5.2-h750e83c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/liblzma-5.8.1-hd471939_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.51.1-hd09e2f1_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.1-hd23fc13_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.5-h0622a9a_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/openssl-3.6.0-h230baf5_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.11.14-h74c2667_2_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.11.14-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.3-h68b038d_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-hf689a15_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/tombi-0.7.11-ha9c3995_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ty-0.0.9-h3e8ffd6_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.9.18-h3315dae_0.conda + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.14-py311hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-21.1.8-hf598326_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.7.3-haf25636_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-he5f378a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.1-h39f12f2_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.51.1-h1b79a29_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.1-h8359307_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h5e97a16_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.0-h5503f6c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.11.14-h18782d2_2_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.11.14-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h892fb3f_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tombi-0.7.11-h6fdd925_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ty-0.0.9-ha73ee7d_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.9.18-h1bde295_0.conda + win-64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-h4c7d964_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.14-py311hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libexpat-2.7.3-hac47afa_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libffi-3.5.2-h52bdfb6_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/liblzma-5.8.1-h2466b09_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libsqlite-3.51.1-hf5d6505_1.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libzlib-1.3.1-h2466b09_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/openssl-3.6.0-h725018a_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/python-3.11.14-h0159041_2_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.11.14-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/tk-8.6.13-h2c6b04d_3.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/tombi-0.7.11-h18a1a76_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/ty-0.0.9-hc21aad4_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/ucrt-10.0.26100.0-h57928b3_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/uv-0.9.18-h3bd95fe_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/vc-14.3-h2b53caa_33.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/vc14_runtime-14.44.35208-h818238b_33.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/vcomp14-14.44.35208-h818238b_33.conda + default: + channels: + - url: https://conda.anaconda.org/conda-forge/ + options: + pypi-prerelease-mode: if-necessary-or-explicit + packages: + linux-64: + - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.14-py311hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-78.1-h33c6efd_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.45-default_hbd61a6d_105.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.3-hecca717_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.5.2-h9ec8514_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.2.0-he0feb66_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.2.0-h69a702a_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-he0feb66_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hb9d3cd8_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.51.1-hf4e2dac_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.2.0-h934c35e_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.41.3-h5347b49_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.0-h26f9b46_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.11.14-hd63d673_2_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.11.14-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-h853b02a_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_ha0e22de_103.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tombi-0.7.11-hb17b654_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ty-0.0.9-h4e94fc0_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.9.18-h76e24b7_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda + linux-aarch64: + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h4777abc_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.14-py311hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/icu-78.1-hb1525cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.45-default_h1979696_105.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libexpat-2.7.3-hfae3067_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libffi-3.5.2-hd65408f_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-15.2.0-h8acb6b2_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-ng-15.2.0-he9431aa_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.2.0-h8acb6b2_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/liblzma-5.8.1-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libnsl-2.0.1-h86ecc28_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libsqlite-3.51.1-h10b116e_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-15.2.0-hef695bb_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libuuid-2.41.3-h1022ec0_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxcrypt-4.4.36-h31becfc_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libzlib-1.3.1-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ncurses-6.5-ha32ae93_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/openssl-3.6.0-h8e36d6e_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/python-3.11.14-h91f4b29_2_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.11.14-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/readline-8.3-hb682ff5_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h561c983_103.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tombi-0.7.11-h069e38c_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ty-0.0.9-h1339683_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/uv-0.9.18-haeed4ea_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.7-h85ac4a6_6.conda + osx-64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h500dc9f_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.14-py311hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/icu-78.1-h14c5de8_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-21.1.8-h3d58e20_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libexpat-2.7.3-heffb93a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.5.2-h750e83c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/liblzma-5.8.1-hd471939_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.51.1-hd09e2f1_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.1-hd23fc13_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.5-h0622a9a_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/openssl-3.6.0-h230baf5_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.11.14-h74c2667_2_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.11.14-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.3-h68b038d_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-hf689a15_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/tombi-0.7.11-ha9c3995_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ty-0.0.9-h3e8ffd6_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.9.18-h3315dae_0.conda + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.14-py311hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-21.1.8-hf598326_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.7.3-haf25636_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-he5f378a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.1-h39f12f2_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.51.1-h1b79a29_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.1-h8359307_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h5e97a16_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.0-h5503f6c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.11.14-h18782d2_2_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.11.14-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h892fb3f_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tombi-0.7.11-h6fdd925_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ty-0.0.9-ha73ee7d_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.9.18-h1bde295_0.conda + win-64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-h4c7d964_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.14-py311hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libexpat-2.7.3-hac47afa_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libffi-3.5.2-h52bdfb6_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/liblzma-5.8.1-h2466b09_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libsqlite-3.51.1-hf5d6505_1.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libzlib-1.3.1-h2466b09_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/openssl-3.6.0-h725018a_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/python-3.11.14-h0159041_2_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.11.14-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/tk-8.6.13-h2c6b04d_3.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/tombi-0.7.11-h18a1a76_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/ty-0.0.9-hc21aad4_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/ucrt-10.0.26100.0-h57928b3_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/uv-0.9.18-h3bd95fe_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/vc-14.3-h2b53caa_33.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/vc14_runtime-14.44.35208-h818238b_33.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/vcomp14-14.44.35208-h818238b_33.conda + py311: + channels: + - url: https://conda.anaconda.org/conda-forge/ + options: + pypi-prerelease-mode: if-necessary-or-explicit + packages: + linux-64: + - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.14-py311hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-78.1-h33c6efd_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.45-default_hbd61a6d_105.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.3-hecca717_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.5.2-h9ec8514_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.2.0-he0feb66_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.2.0-h69a702a_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-he0feb66_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hb9d3cd8_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.51.1-hf4e2dac_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.2.0-h934c35e_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.41.3-h5347b49_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.0-h26f9b46_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.11.14-hd63d673_2_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.11.14-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-h853b02a_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_ha0e22de_103.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tombi-0.7.11-hb17b654_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ty-0.0.9-h4e94fc0_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.9.18-h76e24b7_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda + linux-aarch64: + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h4777abc_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.14-py311hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/icu-78.1-hb1525cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.45-default_h1979696_105.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libexpat-2.7.3-hfae3067_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libffi-3.5.2-hd65408f_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-15.2.0-h8acb6b2_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-ng-15.2.0-he9431aa_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.2.0-h8acb6b2_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/liblzma-5.8.1-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libnsl-2.0.1-h86ecc28_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libsqlite-3.51.1-h10b116e_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-15.2.0-hef695bb_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libuuid-2.41.3-h1022ec0_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxcrypt-4.4.36-h31becfc_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libzlib-1.3.1-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ncurses-6.5-ha32ae93_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/openssl-3.6.0-h8e36d6e_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/python-3.11.14-h91f4b29_2_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.11.14-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/readline-8.3-hb682ff5_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h561c983_103.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tombi-0.7.11-h069e38c_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ty-0.0.9-h1339683_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/uv-0.9.18-haeed4ea_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.7-h85ac4a6_6.conda + osx-64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h500dc9f_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.14-py311hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/icu-78.1-h14c5de8_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-21.1.8-h3d58e20_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libexpat-2.7.3-heffb93a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.5.2-h750e83c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/liblzma-5.8.1-hd471939_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.51.1-hd09e2f1_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.1-hd23fc13_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.5-h0622a9a_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/openssl-3.6.0-h230baf5_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.11.14-h74c2667_2_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.11.14-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.3-h68b038d_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-hf689a15_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/tombi-0.7.11-ha9c3995_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ty-0.0.9-h3e8ffd6_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.9.18-h3315dae_0.conda + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.14-py311hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-21.1.8-hf598326_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.7.3-haf25636_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-he5f378a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.1-h39f12f2_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.51.1-h1b79a29_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.1-h8359307_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h5e97a16_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.0-h5503f6c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.11.14-h18782d2_2_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.11.14-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h892fb3f_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tombi-0.7.11-h6fdd925_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ty-0.0.9-ha73ee7d_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.9.18-h1bde295_0.conda + win-64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-h4c7d964_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.14-py311hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libexpat-2.7.3-hac47afa_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libffi-3.5.2-h52bdfb6_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/liblzma-5.8.1-h2466b09_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libsqlite-3.51.1-hf5d6505_1.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libzlib-1.3.1-h2466b09_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/openssl-3.6.0-h725018a_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/python-3.11.14-h0159041_2_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.11.14-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/tk-8.6.13-h2c6b04d_3.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/tombi-0.7.11-h18a1a76_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/ty-0.0.9-hc21aad4_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/ucrt-10.0.26100.0-h57928b3_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/uv-0.9.18-h3bd95fe_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/vc-14.3-h2b53caa_33.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/vc14_runtime-14.44.35208-h818238b_33.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/vcomp14-14.44.35208-h818238b_33.conda + py312: + channels: + - url: https://conda.anaconda.org/conda-forge/ + options: + pypi-prerelease-mode: if-necessary-or-explicit + packages: + linux-64: + - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.12.12-py312hd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-78.1-h33c6efd_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.45-default_hbd61a6d_105.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.3-hecca717_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.5.2-h9ec8514_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.2.0-he0feb66_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.2.0-h69a702a_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-he0feb66_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hb9d3cd8_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.51.1-hf4e2dac_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.2.0-h934c35e_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.41.3-h5347b49_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.0-h26f9b46_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.12-hd63d673_1_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.12.12-hd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.12-8_cp312.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-h853b02a_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_ha0e22de_103.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tombi-0.7.11-hb17b654_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ty-0.0.9-h4e94fc0_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.9.18-h76e24b7_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda + linux-aarch64: + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h4777abc_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.12.12-py312hd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/icu-78.1-hb1525cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.45-default_h1979696_105.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libexpat-2.7.3-hfae3067_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libffi-3.5.2-hd65408f_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-15.2.0-h8acb6b2_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-ng-15.2.0-he9431aa_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.2.0-h8acb6b2_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/liblzma-5.8.1-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libnsl-2.0.1-h86ecc28_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libsqlite-3.51.1-h10b116e_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-15.2.0-hef695bb_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libuuid-2.41.3-h1022ec0_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxcrypt-4.4.36-h31becfc_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libzlib-1.3.1-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ncurses-6.5-ha32ae93_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/openssl-3.6.0-h8e36d6e_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/python-3.12.12-h91f4b29_1_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.12.12-hd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.12-8_cp312.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/readline-8.3-hb682ff5_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h561c983_103.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tombi-0.7.11-h069e38c_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ty-0.0.9-h1339683_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/uv-0.9.18-haeed4ea_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.7-h85ac4a6_6.conda + osx-64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h500dc9f_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.12.12-py312hd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/icu-78.1-h14c5de8_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-21.1.8-h3d58e20_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libexpat-2.7.3-heffb93a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.5.2-h750e83c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/liblzma-5.8.1-hd471939_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.51.1-hd09e2f1_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.1-hd23fc13_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.5-h0622a9a_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/openssl-3.6.0-h230baf5_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.12.12-h74c2667_1_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.12.12-hd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.12-8_cp312.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.3-h68b038d_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-hf689a15_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/tombi-0.7.11-ha9c3995_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ty-0.0.9-h3e8ffd6_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.9.18-h3315dae_0.conda + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.12.12-py312hd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-21.1.8-hf598326_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.7.3-haf25636_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-he5f378a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.1-h39f12f2_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.51.1-h1b79a29_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.1-h8359307_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h5e97a16_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.0-h5503f6c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.12.12-h18782d2_1_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.12.12-hd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.12-8_cp312.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h892fb3f_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tombi-0.7.11-h6fdd925_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ty-0.0.9-ha73ee7d_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.9.18-h1bde295_0.conda + win-64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-h4c7d964_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.12.12-py312hd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libexpat-2.7.3-hac47afa_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libffi-3.5.2-h52bdfb6_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/liblzma-5.8.1-h2466b09_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libsqlite-3.51.1-hf5d6505_1.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libzlib-1.3.1-h2466b09_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/openssl-3.6.0-h725018a_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/python-3.12.12-h0159041_1_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.12.12-hd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.12-8_cp312.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/tk-8.6.13-h2c6b04d_3.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/tombi-0.7.11-h18a1a76_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/ty-0.0.9-hc21aad4_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/ucrt-10.0.26100.0-h57928b3_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/uv-0.9.18-h3bd95fe_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/vc-14.3-h2b53caa_33.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/vc14_runtime-14.44.35208-h818238b_33.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/vcomp14-14.44.35208-h818238b_33.conda + py313: + channels: + - url: https://conda.anaconda.org/conda-forge/ + options: + pypi-prerelease-mode: if-necessary-or-explicit + packages: + linux-64: + - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.11-py313hd8ed1ab_100.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-78.1-h33c6efd_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.45-default_hbd61a6d_105.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.3-hecca717_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.5.2-h9ec8514_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.2.0-he0feb66_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-he0feb66_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.51.1-hf4e2dac_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.2.0-h934c35e_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.41.3-h5347b49_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.0-h26f9b46_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.13.11-hc97d973_100_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.13.11-h4df99d1_100.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-h853b02a_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_ha0e22de_103.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tombi-0.7.11-hb17b654_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ty-0.0.9-h4e94fc0_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.9.18-h76e24b7_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda + linux-aarch64: + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h4777abc_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.11-py313hd8ed1ab_100.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/icu-78.1-hb1525cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.45-default_h1979696_105.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libexpat-2.7.3-hfae3067_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libffi-3.5.2-hd65408f_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-15.2.0-h8acb6b2_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.2.0-h8acb6b2_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/liblzma-5.8.1-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libmpdec-4.0.0-h86ecc28_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libsqlite-3.51.1-h10b116e_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-15.2.0-hef695bb_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libuuid-2.41.3-h1022ec0_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libzlib-1.3.1-h86ecc28_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ncurses-6.5-ha32ae93_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/openssl-3.6.0-h8e36d6e_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/python-3.13.11-h4c0d347_100_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.13.11-h4df99d1_100.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/readline-8.3-hb682ff5_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h561c983_103.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tombi-0.7.11-h069e38c_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ty-0.0.9-h1339683_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/uv-0.9.18-haeed4ea_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.7-h85ac4a6_6.conda + osx-64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h500dc9f_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.11-py313hd8ed1ab_100.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/icu-78.1-h14c5de8_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-21.1.8-h3d58e20_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libexpat-2.7.3-heffb93a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.5.2-h750e83c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/liblzma-5.8.1-hd471939_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libmpdec-4.0.0-h6e16a3a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.51.1-hd09e2f1_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.1-hd23fc13_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.5-h0622a9a_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/openssl-3.6.0-h230baf5_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.13.11-h17c18a5_100_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.13.11-h4df99d1_100.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.3-h68b038d_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-hf689a15_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/tombi-0.7.11-ha9c3995_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ty-0.0.9-h3e8ffd6_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.9.18-h3315dae_0.conda + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.11-py313hd8ed1ab_100.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-21.1.8-hf598326_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.7.3-haf25636_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-he5f378a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.1-h39f12f2_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libmpdec-4.0.0-h5505292_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.51.1-h1b79a29_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.1-h8359307_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h5e97a16_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.0-h5503f6c_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.13.11-hfc2f54d_100_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.13.11-h4df99d1_100.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h892fb3f_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tombi-0.7.11-h6fdd925_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ty-0.0.9-ha73ee7d_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.9.18-h1bde295_0.conda + win-64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-h4c7d964_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.11-py313hd8ed1ab_100.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libexpat-2.7.3-hac47afa_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libffi-3.5.2-h52bdfb6_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/liblzma-5.8.1-h2466b09_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libmpdec-4.0.0-h2466b09_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libsqlite-3.51.1-hf5d6505_1.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libzlib-1.3.1-h2466b09_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/openssl-3.6.0-h725018a_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/python-3.13.11-h09917c8_100_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.13.11-h4df99d1_100.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/tk-8.6.13-h2c6b04d_3.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/tombi-0.7.11-h18a1a76_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/ty-0.0.9-hc21aad4_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/ucrt-10.0.26100.0-h57928b3_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/uv-0.9.18-h3bd95fe_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/vc-14.3-h2b53caa_33.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/vc14_runtime-14.44.35208-h818238b_33.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/vcomp14-14.44.35208-h818238b_33.conda +packages: +- conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 + sha256: fe51de6107f9edc7aa4f786a70f4a883943bc9d39b3bb7307c04c41410990726 + md5: d7c89558ba9fa0495403155b64376d81 + license: None + size: 2562 + timestamp: 1578324546067 +- conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + build_number: 16 + sha256: fbe2c5e56a653bebb982eda4876a9178aedfc2b545f25d0ce9c4c0b508253d22 + md5: 73aaf86a425cc6e73fcf236a5a46396d + depends: + - _libgcc_mutex 0.1 conda_forge + - libgomp >=7.5.0 + constrains: + - openmp_impl 9999 + license: BSD-3-Clause + license_family: BSD + size: 23621 + timestamp: 1650670423406 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-2_gnu.tar.bz2 + build_number: 16 + sha256: 3702bef2f0a4d38bd8288bbe54aace623602a1343c2cfbefd3fa188e015bebf0 + md5: 6168d71addc746e8f2b8d57dfd2edcea + depends: + - libgomp >=7.5.0 + constrains: + - openmp_impl 9999 + license: BSD-3-Clause + license_family: BSD + size: 23712 + timestamp: 1650670790230 +- conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda + sha256: a3967b937b9abf0f2a99f3173fa4630293979bd1644709d89580e7c62a544661 + md5: aaa2a381ccc56eac91d63b6c1240312f + depends: + - cpython + - python-gil + license: MIT + license_family: MIT + size: 8191 + timestamp: 1744137672556 +- conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_8.conda + sha256: c30daba32ddebbb7ded490f0e371eae90f51e72db620554089103b4a6934b0d5 + md5: 51a19bba1b8ebfb60df25cde030b7ebc + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: bzip2-1.0.6 + license_family: BSD + size: 260341 + timestamp: 1757437258798 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h4777abc_8.conda + sha256: d2a296aa0b5f38ed9c264def6cf775c0ccb0f110ae156fcde322f3eccebf2e01 + md5: 2921ac0b541bf37c69e66bd6d9a43bca + depends: + - libgcc >=14 + license: bzip2-1.0.6 + license_family: BSD + size: 192536 + timestamp: 1757437302703 +- conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h500dc9f_8.conda + sha256: 8f50b58efb29c710f3cecf2027a8d7325ba769ab10c746eff75cea3ac050b10c + md5: 97c4b3bd8a90722104798175a1bdddbf + depends: + - __osx >=10.13 + license: bzip2-1.0.6 + license_family: BSD + size: 132607 + timestamp: 1757437730085 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_8.conda + sha256: b456200636bd5fecb2bec63f7e0985ad2097cf1b83d60ce0b6968dffa6d02aa1 + md5: 58fd217444c2a5701a44244faf518206 + depends: + - __osx >=11.0 + license: bzip2-1.0.6 + license_family: BSD + size: 125061 + timestamp: 1757437486465 +- conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda + sha256: d882712855624641f48aa9dc3f5feea2ed6b4e6004585d3616386a18186fe692 + md5: 1077e9333c41ff0be8edd1a5ec0ddace + depends: + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: bzip2-1.0.6 + license_family: BSD + size: 55977 + timestamp: 1757437738856 +- conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-h4c7d964_0.conda + sha256: 686a13bd2d4024fc99a22c1e0e68a7356af3ed3304a8d3ff6bb56249ad4e82f0 + md5: f98fb7db808b94bc1ec5b0e62f9f1069 + depends: + - __win + license: ISC + size: 152827 + timestamp: 1762967310929 +- conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.11.12-hbd8a1cb_0.conda + sha256: b986ba796d42c9d3265602bc038f6f5264095702dd546c14bc684e60c385e773 + md5: f0991f0f84902f6b6009b4d2350a83aa + depends: + - __unix + license: ISC + size: 152432 + timestamp: 1762967197890 +- conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.14-py311hd8ed1ab_2.conda + noarch: generic + sha256: c871fe68dcc6b79b322e4fcf9f2b131162094c32a68d48c87aa8582995948a01 + md5: 43ed151bed1a0eb7181d305fed7cf051 + depends: + - python >=3.11,<3.12.0a0 + - python_abi * *_cp311 + license: Python-2.0 + size: 47257 + timestamp: 1761172995774 +- conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.12.12-py312hd8ed1ab_1.conda + noarch: generic + sha256: b88c76a6d6b45378552ccfd9e88b2a073161fe83fd1294c8fa103ffd32f7934a + md5: 99d689ccc1a360639eec979fd7805be9 + depends: + - python >=3.12,<3.13.0a0 + - python_abi * *_cp312 + license: Python-2.0 + size: 45767 + timestamp: 1761175217281 +- conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.11-py313hd8ed1ab_100.conda + noarch: generic + sha256: 63f677762304e6f8dc55e11dff6aafe71129cbbd0a77d176b99ba1f6a5053b77 + md5: 5bf347916a543bcb290c780fa449bf73 + depends: + - python >=3.13,<3.14.0a0 + - python_abi * *_cp313 + license: Python-2.0 + size: 48369 + timestamp: 1765019689213 +- conda: https://conda.anaconda.org/conda-forge/linux-64/icu-78.1-h33c6efd_0.conda + sha256: 7d6463d0be5092b2ae8f2fad34dc84de83eab8bd44cc0d4be8931881c973c48f + md5: 518e9bbbc3e3486d6a4519192ba690f8 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - libstdcxx >=14 + license: MIT + size: 12722920 + timestamp: 1766299101259 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/icu-78.1-hb1525cb_0.conda + sha256: 550c581d08eefe420f9ed14148f1c1d59a3e33de78806a1b8d610d207d06374c + md5: 5eba836ceb0cccf969d9518ca884de2a + depends: + - libgcc >=14 + - libstdcxx >=14 + license: MIT + size: 12835377 + timestamp: 1766304007889 +- conda: https://conda.anaconda.org/conda-forge/osx-64/icu-78.1-h14c5de8_0.conda + sha256: 256df2229f930d7c83d8e2d36fdfce1f78980272558095ce741a9fccc5ed8998 + md5: 1e648e0c6657a29dc44102d6e3b10ebc + depends: + - __osx >=10.13 + license: MIT + size: 12273114 + timestamp: 1766299263503 +- conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.45-default_hbd61a6d_105.conda + sha256: 1027bd8aa0d5144e954e426ab6218fd5c14e54a98f571985675468b339c808ca + md5: 3ec0aa5037d39b06554109a01e6fb0c6 + depends: + - __glibc >=2.17,<3.0.a0 + - zstd >=1.5.7,<1.6.0a0 + constrains: + - binutils_impl_linux-64 2.45 + license: GPL-3.0-only + size: 730831 + timestamp: 1766513089214 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.45-default_h1979696_105.conda + sha256: 12e7341b89e9ea319a3b4de03d02cd988fa02b8a678f4e46779515009b5e475c + md5: 849c4cbbf8dd1d71e66c13afed1d2f12 + depends: + - zstd >=1.5.7,<1.6.0a0 + constrains: + - binutils_impl_linux-aarch64 2.45 + license: GPL-3.0-only + size: 876257 + timestamp: 1766513180236 +- conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-21.1.8-h3d58e20_0.conda + sha256: cbd8e821e97436d8fc126c24b50df838b05ba4c80494fbb93ccaf2e3b2d109fb + md5: 9f8a60a77ecafb7966ca961c94f33bd1 + depends: + - __osx >=10.13 + license: Apache-2.0 WITH LLVM-exception + license_family: Apache + size: 569777 + timestamp: 1765919624323 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-21.1.8-hf598326_0.conda + sha256: 82e228975fd491bcf1071ecd0a6ec2a0fcc5f57eb0bd1d52cb13a18d57c67786 + md5: 780f0251b757564e062187044232c2b7 + depends: + - __osx >=11.0 + license: Apache-2.0 WITH LLVM-exception + license_family: Apache + size: 569118 + timestamp: 1765919724254 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.3-hecca717_0.conda + sha256: 1e1b08f6211629cbc2efe7a5bca5953f8f6b3cae0eeb04ca4dacee1bd4e2db2f + md5: 8b09ae86839581147ef2e5c5e229d164 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + constrains: + - expat 2.7.3.* + license: MIT + license_family: MIT + size: 76643 + timestamp: 1763549731408 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libexpat-2.7.3-hfae3067_0.conda + sha256: cc2581a78315418cc2e0bb2a273d37363203e79cefe78ba6d282fed546262239 + md5: b414e36fbb7ca122030276c75fa9c34a + depends: + - libgcc >=14 + constrains: + - expat 2.7.3.* + license: MIT + license_family: MIT + size: 76201 + timestamp: 1763549910086 +- conda: https://conda.anaconda.org/conda-forge/osx-64/libexpat-2.7.3-heffb93a_0.conda + sha256: d11b3a6ce5b2e832f430fd112084533a01220597221bee16d6c7dc3947dffba6 + md5: 222e0732a1d0780a622926265bee14ef + depends: + - __osx >=10.13 + constrains: + - expat 2.7.3.* + license: MIT + license_family: MIT + size: 74058 + timestamp: 1763549886493 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.7.3-haf25636_0.conda + sha256: fce22610ecc95e6d149e42a42fbc3cc9d9179bd4eb6232639a60f06e080eec98 + md5: b79875dbb5b1db9a4a22a4520f918e1a + depends: + - __osx >=11.0 + constrains: + - expat 2.7.3.* + license: MIT + license_family: MIT + size: 67800 + timestamp: 1763549994166 +- conda: https://conda.anaconda.org/conda-forge/win-64/libexpat-2.7.3-hac47afa_0.conda + sha256: 844ab708594bdfbd7b35e1a67c379861bcd180d6efe57b654f482ae2f7f5c21e + md5: 8c9e4f1a0e688eef2e95711178061a0f + depends: + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + constrains: + - expat 2.7.3.* + license: MIT + license_family: MIT + size: 70137 + timestamp: 1763550049107 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.5.2-h9ec8514_0.conda + sha256: 25cbdfa65580cfab1b8d15ee90b4c9f1e0d72128f1661449c9a999d341377d54 + md5: 35f29eec58405aaf55e01cb470d8c26a + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: MIT + license_family: MIT + size: 57821 + timestamp: 1760295480630 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libffi-3.5.2-hd65408f_0.conda + sha256: 6c3332e78a975e092e54f87771611db81dcb5515a3847a3641021621de76caea + md5: 0c5ad486dcfb188885e3cf8ba209b97b + depends: + - libgcc >=14 + license: MIT + license_family: MIT + size: 55586 + timestamp: 1760295405021 +- conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.5.2-h750e83c_0.conda + sha256: 277dc89950f5d97f1683f26e362d6dca3c2efa16cb2f6fdb73d109effa1cd3d0 + md5: d214916b24c625bcc459b245d509f22e + depends: + - __osx >=10.13 + license: MIT + license_family: MIT + size: 52573 + timestamp: 1760295626449 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-he5f378a_0.conda + sha256: 9b8acdf42df61b7bfe8bdc545c016c29e61985e79748c64ad66df47dbc2e295f + md5: 411ff7cd5d1472bba0f55c0faf04453b + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + size: 40251 + timestamp: 1760295839166 +- conda: https://conda.anaconda.org/conda-forge/win-64/libffi-3.5.2-h52bdfb6_0.conda + sha256: ddff25aaa4f0aa535413f5d831b04073789522890a4d8626366e43ecde1534a3 + md5: ba4ad812d2afc22b9a34ce8327a0930f + depends: + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: MIT + license_family: MIT + size: 44866 + timestamp: 1760295760649 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.2.0-he0feb66_16.conda + sha256: 6eed58051c2e12b804d53ceff5994a350c61baf117ec83f5f10c953a3f311451 + md5: 6d0363467e6ed84f11435eb309f2ff06 + depends: + - __glibc >=2.17,<3.0.a0 + - _openmp_mutex >=4.5 + constrains: + - libgcc-ng ==15.2.0=*_16 + - libgomp 15.2.0 he0feb66_16 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + size: 1042798 + timestamp: 1765256792743 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-15.2.0-h8acb6b2_16.conda + sha256: 44bfc6fe16236babb271e0c693fe7fd978f336542e23c9c30e700483796ed30b + md5: cf9cd6739a3b694dcf551d898e112331 + depends: + - _openmp_mutex >=4.5 + constrains: + - libgomp 15.2.0 h8acb6b2_16 + - libgcc-ng ==15.2.0=*_16 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + size: 620637 + timestamp: 1765256938043 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.2.0-h69a702a_16.conda + sha256: 5f07f9317f596a201cc6e095e5fc92621afca64829785e483738d935f8cab361 + md5: 5a68259fac2da8f2ee6f7bfe49c9eb8b + depends: + - libgcc 15.2.0 he0feb66_16 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + size: 27256 + timestamp: 1765256804124 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-ng-15.2.0-he9431aa_16.conda + sha256: 22d7e63a00c880bd14fbbc514ec6f553b9325d705f08582e9076c7e73c93a2e1 + md5: 3e54a6d0f2ff0172903c0acfda9efc0e + depends: + - libgcc 15.2.0 h8acb6b2_16 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + size: 27356 + timestamp: 1765256948637 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-he0feb66_16.conda + sha256: 5b3e5e4e9270ecfcd48f47e3a68f037f5ab0f529ccb223e8e5d5ac75a58fc687 + md5: 26c46f90d0e727e95c6c9498a33a09f3 + depends: + - __glibc >=2.17,<3.0.a0 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + size: 603284 + timestamp: 1765256703881 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.2.0-h8acb6b2_16.conda + sha256: 0a9d77c920db691eb42b78c734d70c5a1d00b3110c0867cfff18e9dd69bc3c29 + md5: 4d2f224e8186e7881d53e3aead912f6c + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + size: 587924 + timestamp: 1765256821307 +- conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.1-hb9d3cd8_2.conda + sha256: f2591c0069447bbe28d4d696b7fcb0c5bd0b4ac582769b89addbcf26fb3430d8 + md5: 1a580f7796c7bf6393fddb8bbbde58dc + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + constrains: + - xz 5.8.1.* + license: 0BSD + size: 112894 + timestamp: 1749230047870 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/liblzma-5.8.1-h86ecc28_2.conda + sha256: 498ea4b29155df69d7f20990a7028d75d91dbea24d04b2eb8a3d6ef328806849 + md5: 7d362346a479256857ab338588190da0 + depends: + - libgcc >=13 + constrains: + - xz 5.8.1.* + license: 0BSD + size: 125103 + timestamp: 1749232230009 +- conda: https://conda.anaconda.org/conda-forge/osx-64/liblzma-5.8.1-hd471939_2.conda + sha256: 7e22fd1bdb8bf4c2be93de2d4e718db5c548aa082af47a7430eb23192de6bb36 + md5: 8468beea04b9065b9807fc8b9cdc5894 + depends: + - __osx >=10.13 + constrains: + - xz 5.8.1.* + license: 0BSD + size: 104826 + timestamp: 1749230155443 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.1-h39f12f2_2.conda + sha256: 0cb92a9e026e7bd4842f410a5c5c665c89b2eb97794ffddba519a626b8ce7285 + md5: d6df911d4564d77c4374b02552cb17d1 + depends: + - __osx >=11.0 + constrains: + - xz 5.8.1.* + license: 0BSD + size: 92286 + timestamp: 1749230283517 +- conda: https://conda.anaconda.org/conda-forge/win-64/liblzma-5.8.1-h2466b09_2.conda + sha256: 55764956eb9179b98de7cc0e55696f2eff8f7b83fc3ebff5e696ca358bca28cc + md5: c15148b2e18da456f5108ccb5e411446 + depends: + - ucrt >=10.0.20348.0 + - vc >=14.2,<15 + - vc14_runtime >=14.29.30139 + constrains: + - xz 5.8.1.* + license: 0BSD + size: 104935 + timestamp: 1749230611612 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-hb9d3cd8_0.conda + sha256: 3aa92d4074d4063f2a162cd8ecb45dccac93e543e565c01a787e16a43501f7ee + md5: c7e925f37e3b40d893459e625f6a53f1 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + license: BSD-2-Clause + license_family: BSD + size: 91183 + timestamp: 1748393666725 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libmpdec-4.0.0-h86ecc28_0.conda + sha256: ef8697f934c80b347bf9d7ed45650928079e303bad01bd064995b0e3166d6e7a + md5: 78cfed3f76d6f3f279736789d319af76 + depends: + - libgcc >=13 + license: BSD-2-Clause + license_family: BSD + size: 114064 + timestamp: 1748393729243 +- conda: https://conda.anaconda.org/conda-forge/osx-64/libmpdec-4.0.0-h6e16a3a_0.conda + sha256: 98299c73c7a93cd4f5ff8bb7f43cd80389f08b5a27a296d806bdef7841cc9b9e + md5: 18b81186a6adb43f000ad19ed7b70381 + depends: + - __osx >=10.13 + license: BSD-2-Clause + license_family: BSD + size: 77667 + timestamp: 1748393757154 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libmpdec-4.0.0-h5505292_0.conda + sha256: 0a1875fc1642324ebd6c4ac864604f3f18f57fbcf558a8264f6ced028a3c75b2 + md5: 85ccccb47823dd9f7a99d2c7f530342f + depends: + - __osx >=11.0 + license: BSD-2-Clause + license_family: BSD + size: 71829 + timestamp: 1748393749336 +- conda: https://conda.anaconda.org/conda-forge/win-64/libmpdec-4.0.0-h2466b09_0.conda + sha256: fc529fc82c7caf51202cc5cec5bb1c2e8d90edbac6d0a4602c966366efe3c7bf + md5: 74860100b2029e2523cf480804c76b9b + depends: + - ucrt >=10.0.20348.0 + - vc >=14.2,<15 + - vc14_runtime >=14.29.30139 + license: BSD-2-Clause + license_family: BSD + size: 88657 + timestamp: 1723861474602 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hb9d3cd8_1.conda + sha256: 927fe72b054277cde6cb82597d0fcf6baf127dcbce2e0a9d8925a68f1265eef5 + md5: d864d34357c3b65a4b731f78c0801dc4 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + license: LGPL-2.1-only + license_family: GPL + size: 33731 + timestamp: 1750274110928 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libnsl-2.0.1-h86ecc28_1.conda + sha256: c0dc4d84198e3eef1f37321299e48e2754ca83fd12e6284754e3cb231357c3a5 + md5: d5d58b2dc3e57073fe22303f5fed4db7 + depends: + - libgcc >=13 + license: LGPL-2.1-only + license_family: GPL + size: 34831 + timestamp: 1750274211 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.51.1-hf4e2dac_1.conda + sha256: d614540c55f22ad555633f75e174089018ddfc65c49f447f7bbdbc3c3013bec1 + md5: b1f35e70f047918b49fb4b181e40300e + depends: + - __glibc >=2.17,<3.0.a0 + - icu >=78.1,<79.0a0 + - libgcc >=14 + - libzlib >=1.3.1,<2.0a0 + license: blessing + size: 943451 + timestamp: 1766319676469 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libsqlite-3.51.1-h10b116e_1.conda + sha256: f80893874d5ba5ac754b2d65ec392c46841bfe57bd89499aa0e1965c720babbd + md5: 9fd37e702b4e7c85462fe79baf13974d + depends: + - icu >=78.1,<79.0a0 + - libgcc >=14 + - libzlib >=1.3.1,<2.0a0 + license: blessing + size: 943924 + timestamp: 1766319577347 +- conda: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.51.1-hd09e2f1_1.conda + sha256: 497b0a698ae87e024d24e242f93c56303731844d10861e1448f6d0a3d69c9ea7 + md5: 75ba9aba95c277f12e23cdb0856fd9cd + depends: + - __osx >=10.13 + - icu >=78.1,<79.0a0 + - libzlib >=1.3.1,<2.0a0 + license: blessing + size: 991497 + timestamp: 1766319979749 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.51.1-h1b79a29_1.conda + sha256: f2c3cbf2ca7d697098964a748fbf19d6e4adcefa23844ec49f0166f1d36af83c + md5: 8c3951797658e10b610929c3e57e9ad9 + depends: + - __osx >=11.0 + - libzlib >=1.3.1,<2.0a0 + license: blessing + size: 905861 + timestamp: 1766319901587 +- conda: https://conda.anaconda.org/conda-forge/win-64/libsqlite-3.51.1-hf5d6505_1.conda + sha256: d6d86715a1afe11f626b7509935e9d2e14a4946632c0ac474526e20fc6c55f99 + md5: be65be5f758709fc01b01626152e96b0 + depends: + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: blessing + size: 1292859 + timestamp: 1766319616777 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.2.0-h934c35e_16.conda + sha256: 813427918316a00c904723f1dfc3da1bbc1974c5cfe1ed1e704c6f4e0798cbc6 + md5: 68f68355000ec3f1d6f26ea13e8f525f + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc 15.2.0 he0feb66_16 + constrains: + - libstdcxx-ng ==15.2.0=*_16 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + size: 5856456 + timestamp: 1765256838573 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-15.2.0-hef695bb_16.conda + sha256: 4db11a903707068ae37aa6909511c68e9af6a2e97890d1b73b0a8d87cb74aba9 + md5: 52d9df8055af3f1665ba471cce77da48 + depends: + - libgcc 15.2.0 h8acb6b2_16 + constrains: + - libstdcxx-ng ==15.2.0=*_16 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + size: 5541149 + timestamp: 1765256980783 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.41.3-h5347b49_0.conda + sha256: 1a7539cfa7df00714e8943e18de0b06cceef6778e420a5ee3a2a145773758aee + md5: db409b7c1720428638e7c0d509d3e1b5 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: BSD-3-Clause + size: 40311 + timestamp: 1766271528534 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libuuid-2.41.3-h1022ec0_0.conda + sha256: c37a8e89b700646f3252608f8368e7eb8e2a44886b92776e57ad7601fc402a11 + md5: cf2861212053d05f27ec49c3784ff8bb + depends: + - libgcc >=14 + license: BSD-3-Clause + size: 43453 + timestamp: 1766271546875 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda + sha256: 6ae68e0b86423ef188196fff6207ed0c8195dd84273cb5623b85aa08033a410c + md5: 5aa797f8787fe7a17d1b0821485b5adc + depends: + - libgcc-ng >=12 + license: LGPL-2.1-or-later + size: 100393 + timestamp: 1702724383534 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libxcrypt-4.4.36-h31becfc_1.conda + sha256: 6b46c397644091b8a26a3048636d10b989b1bf266d4be5e9474bf763f828f41f + md5: b4df5d7d4b63579d081fd3a4cf99740e + depends: + - libgcc-ng >=12 + license: LGPL-2.1-or-later + size: 114269 + timestamp: 1702724369203 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda + sha256: d4bfe88d7cb447768e31650f06257995601f89076080e76df55e3112d4e47dc4 + md5: edb0dca6bc32e4f4789199455a1dbeb8 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + constrains: + - zlib 1.3.1 *_2 + license: Zlib + license_family: Other + size: 60963 + timestamp: 1727963148474 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libzlib-1.3.1-h86ecc28_2.conda + sha256: 5a2c1eeef69342e88a98d1d95bff1603727ab1ff4ee0e421522acd8813439b84 + md5: 08aad7cbe9f5a6b460d0976076b6ae64 + depends: + - libgcc >=13 + constrains: + - zlib 1.3.1 *_2 + license: Zlib + license_family: Other + size: 66657 + timestamp: 1727963199518 +- conda: https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.1-hd23fc13_2.conda + sha256: 8412f96504fc5993a63edf1e211d042a1fd5b1d51dedec755d2058948fcced09 + md5: 003a54a4e32b02f7355b50a837e699da + depends: + - __osx >=10.13 + constrains: + - zlib 1.3.1 *_2 + license: Zlib + license_family: Other + size: 57133 + timestamp: 1727963183990 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.1-h8359307_2.conda + sha256: ce34669eadaba351cd54910743e6a2261b67009624dbc7daeeafdef93616711b + md5: 369964e85dc26bfe78f41399b366c435 + depends: + - __osx >=11.0 + constrains: + - zlib 1.3.1 *_2 + license: Zlib + license_family: Other + size: 46438 + timestamp: 1727963202283 +- conda: https://conda.anaconda.org/conda-forge/win-64/libzlib-1.3.1-h2466b09_2.conda + sha256: ba945c6493449bed0e6e29883c4943817f7c79cbff52b83360f7b341277c6402 + md5: 41fbfac52c601159df6c01f875de31b9 + depends: + - ucrt >=10.0.20348.0 + - vc >=14.2,<15 + - vc14_runtime >=14.29.30139 + constrains: + - zlib 1.3.1 *_2 + license: Zlib + license_family: Other + size: 55476 + timestamp: 1727963768015 +- conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda + sha256: 3fde293232fa3fca98635e1167de6b7c7fda83caf24b9d6c91ec9eefb4f4d586 + md5: 47e340acb35de30501a76c7c799c41d7 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + license: X11 AND BSD-3-Clause + size: 891641 + timestamp: 1738195959188 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ncurses-6.5-ha32ae93_3.conda + sha256: 91cfb655a68b0353b2833521dc919188db3d8a7f4c64bea2c6a7557b24747468 + md5: 182afabe009dc78d8b73100255ee6868 + depends: + - libgcc >=13 + license: X11 AND BSD-3-Clause + size: 926034 + timestamp: 1738196018799 +- conda: https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.5-h0622a9a_3.conda + sha256: ea4a5d27ded18443749aefa49dc79f6356da8506d508b5296f60b8d51e0c4bd9 + md5: ced34dd9929f491ca6dab6a2927aff25 + depends: + - __osx >=10.13 + license: X11 AND BSD-3-Clause + size: 822259 + timestamp: 1738196181298 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h5e97a16_3.conda + sha256: 2827ada40e8d9ca69a153a45f7fd14f32b2ead7045d3bbb5d10964898fe65733 + md5: 068d497125e4bf8a66bf707254fff5ae + depends: + - __osx >=11.0 + license: X11 AND BSD-3-Clause + size: 797030 + timestamp: 1738196177597 +- conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.0-h26f9b46_0.conda + sha256: a47271202f4518a484956968335b2521409c8173e123ab381e775c358c67fe6d + md5: 9ee58d5c534af06558933af3c845a780 + depends: + - __glibc >=2.17,<3.0.a0 + - ca-certificates + - libgcc >=14 + license: Apache-2.0 + license_family: Apache + size: 3165399 + timestamp: 1762839186699 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/openssl-3.6.0-h8e36d6e_0.conda + sha256: 8dd3b4c31fe176a3e51c5729b2c7f4c836a2ce3bd5c82082dc2a503ba9ee0af3 + md5: 7624c6e01aecba942e9115e0f5a2af9d + depends: + - ca-certificates + - libgcc >=14 + license: Apache-2.0 + license_family: Apache + size: 3705625 + timestamp: 1762841024958 +- conda: https://conda.anaconda.org/conda-forge/osx-64/openssl-3.6.0-h230baf5_0.conda + sha256: 36fe9fb316be22fcfb46d5fa3e2e85eec5ef84f908b7745f68f768917235b2d5 + md5: 3f50cdf9a97d0280655758b735781096 + depends: + - __osx >=10.13 + - ca-certificates + license: Apache-2.0 + license_family: Apache + size: 2778996 + timestamp: 1762840724922 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.0-h5503f6c_0.conda + sha256: ebe93dafcc09e099782fe3907485d4e1671296bc14f8c383cb6f3dfebb773988 + md5: b34dc4172653c13dcf453862f251af2b + depends: + - __osx >=11.0 + - ca-certificates + license: Apache-2.0 + license_family: Apache + size: 3108371 + timestamp: 1762839712322 +- conda: https://conda.anaconda.org/conda-forge/win-64/openssl-3.6.0-h725018a_0.conda + sha256: 6d72d6f766293d4f2aa60c28c244c8efed6946c430814175f959ffe8cab899b3 + md5: 84f8fb4afd1157f59098f618cd2437e4 + depends: + - ca-certificates + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: Apache-2.0 + license_family: Apache + size: 9440812 + timestamp: 1762841722179 +- conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.11.14-hd63d673_2_cpython.conda + build_number: 2 + sha256: 5b872f7747891e50e990a96d2b235236a5c66cc9f8c9dcb7149aee674ea8145a + md5: c4202a55b4486314fbb8c11bc43a29a0 + depends: + - __glibc >=2.17,<3.0.a0 + - bzip2 >=1.0.8,<2.0a0 + - ld_impl_linux-64 >=2.36.1 + - libexpat >=2.7.1,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - libgcc >=14 + - liblzma >=5.8.1,<6.0a0 + - libnsl >=2.0.1,<2.1.0a0 + - libsqlite >=3.50.4,<4.0a0 + - libuuid >=2.41.2,<3.0a0 + - libxcrypt >=4.4.36 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.4,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + constrains: + - python_abi 3.11.* *_cp311 + license: Python-2.0 + size: 30874708 + timestamp: 1761174520369 +- conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.12-hd63d673_1_cpython.conda + build_number: 1 + sha256: 39898d24769a848c057ab861052e50bdc266310a7509efa3514b840e85a2ae98 + md5: 5c00c8cea14ee8d02941cab9121dce41 + depends: + - __glibc >=2.17,<3.0.a0 + - bzip2 >=1.0.8,<2.0a0 + - ld_impl_linux-64 >=2.36.1 + - libexpat >=2.7.1,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - libgcc >=14 + - liblzma >=5.8.1,<6.0a0 + - libnsl >=2.0.1,<2.1.0a0 + - libsqlite >=3.50.4,<4.0a0 + - libuuid >=2.41.2,<3.0a0 + - libxcrypt >=4.4.36 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.4,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + constrains: + - python_abi 3.12.* *_cp312 + license: Python-2.0 + size: 31537229 + timestamp: 1761176876216 +- conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.13.11-hc97d973_100_cp313.conda + build_number: 100 + sha256: 9cf014cf28e93ee242bacfbf664e8b45ae06e50b04291e640abeaeb0cba0364c + md5: 0cbb0010f1d8ecb64a428a8d4214609e + depends: + - __glibc >=2.17,<3.0.a0 + - bzip2 >=1.0.8,<2.0a0 + - ld_impl_linux-64 >=2.36.1 + - libexpat >=2.7.3,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - libgcc >=14 + - liblzma >=5.8.1,<6.0a0 + - libmpdec >=4.0.0,<5.0a0 + - libsqlite >=3.51.1,<4.0a0 + - libuuid >=2.41.2,<3.0a0 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.4,<4.0a0 + - python_abi 3.13.* *_cp313 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + license: Python-2.0 + size: 37226336 + timestamp: 1765021889577 + python_site_packages_path: lib/python3.13/site-packages +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/python-3.11.14-h91f4b29_2_cpython.conda + build_number: 2 + sha256: c920bcd33f20f9fb671d0e816e9df88515e6618c8a5835276af4b4f7b70b0db9 + md5: 622ae39bb186be3eeeaa564a9c7e1eec + depends: + - bzip2 >=1.0.8,<2.0a0 + - ld_impl_linux-aarch64 >=2.36.1 + - libexpat >=2.7.1,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - libgcc >=14 + - liblzma >=5.8.1,<6.0a0 + - libnsl >=2.0.1,<2.1.0a0 + - libsqlite >=3.50.4,<4.0a0 + - libuuid >=2.41.2,<3.0a0 + - libxcrypt >=4.4.36 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.4,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + constrains: + - python_abi 3.11.* *_cp311 + license: Python-2.0 + size: 15534042 + timestamp: 1761172955688 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/python-3.12.12-h91f4b29_1_cpython.conda + build_number: 1 + sha256: a635a01f696d4c62b739073eb241e83a35894f1aabb0f590957a05a23aa3ad28 + md5: 823e8543dd3abc98b2982fea10f3daea + depends: + - bzip2 >=1.0.8,<2.0a0 + - ld_impl_linux-aarch64 >=2.36.1 + - libexpat >=2.7.1,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - libgcc >=14 + - liblzma >=5.8.1,<6.0a0 + - libnsl >=2.0.1,<2.1.0a0 + - libsqlite >=3.50.4,<4.0a0 + - libuuid >=2.41.2,<3.0a0 + - libxcrypt >=4.4.36 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.4,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + constrains: + - python_abi 3.12.* *_cp312 + license: Python-2.0 + size: 13683458 + timestamp: 1761175192478 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/python-3.13.11-h4c0d347_100_cp313.conda + build_number: 100 + sha256: bbb0b341c3ce460d02087e1c5e0d3bb814c270f4ae61f82c0e2996ec3902d301 + md5: a6e2b5b263090516ec36efdd51dcc35b + depends: + - bzip2 >=1.0.8,<2.0a0 + - ld_impl_linux-aarch64 >=2.36.1 + - libexpat >=2.7.3,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - libgcc >=14 + - liblzma >=5.8.1,<6.0a0 + - libmpdec >=4.0.0,<5.0a0 + - libsqlite >=3.51.1,<4.0a0 + - libuuid >=2.41.2,<3.0a0 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.4,<4.0a0 + - python_abi 3.13.* *_cp313 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + license: Python-2.0 + size: 33896215 + timestamp: 1765020450426 + python_site_packages_path: lib/python3.13/site-packages +- conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.11.14-h74c2667_2_cpython.conda + build_number: 2 + sha256: 0a17479efb8df514c3777c015ffe430d38a3a59c01dc46358e87d7ff459c9aeb + md5: 37ac5f13a245f08746e0d658b245d670 + depends: + - __osx >=10.13 + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.7.1,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - liblzma >=5.8.1,<6.0a0 + - libsqlite >=3.50.4,<4.0a0 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.4,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + constrains: + - python_abi 3.11.* *_cp311 + license: Python-2.0 + size: 15697126 + timestamp: 1761174493171 +- conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.12.12-h74c2667_1_cpython.conda + build_number: 1 + sha256: 7d711e7a5085c05d186e1dbc86b8f10fb3d88fb3ce3034944ededef39173ff32 + md5: 902046b662c35d8d644514df0d9c7109 + depends: + - __osx >=10.13 + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.7.1,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - liblzma >=5.8.1,<6.0a0 + - libsqlite >=3.50.4,<4.0a0 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.4,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + constrains: + - python_abi 3.12.* *_cp312 + license: Python-2.0 + size: 13779792 + timestamp: 1761176993883 +- conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.13.11-h17c18a5_100_cp313.conda + build_number: 100 + sha256: 58e23beaf3174a809c785900477c37df9f88993b5a3ccd0d76d57d6688a1be37 + md5: 6ffffd784fe1126b73329e29c80ddf53 + depends: + - __osx >=10.13 + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.7.3,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - liblzma >=5.8.1,<6.0a0 + - libmpdec >=4.0.0,<5.0a0 + - libsqlite >=3.51.1,<4.0a0 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.4,<4.0a0 + - python_abi 3.13.* *_cp313 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + license: Python-2.0 + size: 17360881 + timestamp: 1765022591905 + python_site_packages_path: lib/python3.13/site-packages +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.11.14-h18782d2_2_cpython.conda + build_number: 2 + sha256: 64a2bc6be8582fae75f1f2da7bdc49afd81c2793f65bb843fc37f53c99734063 + md5: da948e6cd735249ab4cfbb3fdede785e + depends: + - __osx >=11.0 + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.7.1,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - liblzma >=5.8.1,<6.0a0 + - libsqlite >=3.50.4,<4.0a0 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.4,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + constrains: + - python_abi 3.11.* *_cp311 + license: Python-2.0 + size: 14788204 + timestamp: 1761174033541 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.12.12-h18782d2_1_cpython.conda + build_number: 1 + sha256: 626da9bb78459ce541407327d1e22ee673fd74e9103f1a0e0f4e3967ad0a23a7 + md5: 0322f2ddca2cafbf34ef3ddbea100f73 + depends: + - __osx >=11.0 + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.7.1,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - liblzma >=5.8.1,<6.0a0 + - libsqlite >=3.50.4,<4.0a0 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.4,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + constrains: + - python_abi 3.12.* *_cp312 + license: Python-2.0 + size: 12062421 + timestamp: 1761176476561 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.13.11-hfc2f54d_100_cp313.conda + build_number: 100 + sha256: c476f4e9b6d97c46b496b442878924868a54e5727251549ebfc82027aa52af68 + md5: 18a8c69608151098a8fb75eea64cc266 + depends: + - __osx >=11.0 + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.7.3,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - liblzma >=5.8.1,<6.0a0 + - libmpdec >=4.0.0,<5.0a0 + - libsqlite >=3.51.1,<4.0a0 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.4,<4.0a0 + - python_abi 3.13.* *_cp313 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + license: Python-2.0 + size: 12920650 + timestamp: 1765020887340 + python_site_packages_path: lib/python3.13/site-packages +- conda: https://conda.anaconda.org/conda-forge/win-64/python-3.11.14-h0159041_2_cpython.conda + build_number: 2 + sha256: d5f455472597aefcdde1bc39bca313fcb40bf084f3ad987da0441f2a2ec242e4 + md5: 02a9ba5950d8b78e6c9862d6ba7a5045 + depends: + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.7.1,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - liblzma >=5.8.1,<6.0a0 + - libsqlite >=3.50.4,<4.0a0 + - libzlib >=1.3.1,<2.0a0 + - openssl >=3.5.4,<4.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + constrains: + - python_abi 3.11.* *_cp311 + license: Python-2.0 + size: 18514691 + timestamp: 1761172844103 +- conda: https://conda.anaconda.org/conda-forge/win-64/python-3.12.12-h0159041_1_cpython.conda + build_number: 1 + sha256: 9b163b0426c92eee1881d5c838e230a750a3fa372092db494772886ab91c2548 + md5: 42ae551e4c15837a582bea63412dc0b4 + depends: + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.7.1,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - liblzma >=5.8.1,<6.0a0 + - libsqlite >=3.50.4,<4.0a0 + - libzlib >=1.3.1,<2.0a0 + - openssl >=3.5.4,<4.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + constrains: + - python_abi 3.12.* *_cp312 + license: Python-2.0 + size: 15883484 + timestamp: 1761175152489 +- conda: https://conda.anaconda.org/conda-forge/win-64/python-3.13.11-h09917c8_100_cp313.conda + build_number: 100 + sha256: 0ee0402368783e1fad10025719530499c517a3dbbdfbe18351841d9b7aef1d6a + md5: 9e4c9a7ee9c4ab5b3778ab73e583283e + depends: + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.7.3,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - liblzma >=5.8.1,<6.0a0 + - libmpdec >=4.0.0,<5.0a0 + - libsqlite >=3.51.1,<4.0a0 + - libzlib >=1.3.1,<2.0a0 + - openssl >=3.5.4,<4.0a0 + - python_abi 3.13.* *_cp313 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: Python-2.0 + size: 16617922 + timestamp: 1765019627175 + python_site_packages_path: Lib/site-packages +- conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.11.14-hd8ed1ab_2.conda + sha256: 9261923f0dc8a3c8c517c29d8f3b7eea80f2577b094198239895bd7a88b534c5 + md5: a4effc7e6eb335d0e1080a5554590425 + depends: + - cpython 3.11.14.* + - python_abi * *_cp311 + license: Python-2.0 + size: 47569 + timestamp: 1761172935811 +- conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.12.12-hd8ed1ab_1.conda + sha256: 59f17182813f8b23709b7d4cfda82c33b72dd007cb729efa0033c609fbd92122 + md5: c20172b4c59fbe288fa50cdc1b693d73 + depends: + - cpython 3.12.12.* + - python_abi * *_cp312 + license: Python-2.0 + size: 45888 + timestamp: 1761175248278 +- conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.13.11-h4df99d1_100.conda + sha256: 4b08d4c2c4b956d306b4868d3faf724eebb5d6e6b170fad2eb0f2d4eb227f1af + md5: d1461b2e63b1909f4f5b41c823bd90ae + depends: + - cpython 3.13.11.* + - python_abi * *_cp313 + license: Python-2.0 + size: 48352 + timestamp: 1765019767640 +- conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + build_number: 8 + sha256: fddf123692aa4b1fc48f0471e346400d9852d96eeed77dbfdd746fa50a8ff894 + md5: 8fcb6b0e2161850556231336dae58358 + constrains: + - python 3.11.* *_cpython + license: BSD-3-Clause + license_family: BSD + size: 7003 + timestamp: 1752805919375 +- conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.12-8_cp312.conda + build_number: 8 + sha256: 80677180dd3c22deb7426ca89d6203f1c7f1f256f2d5a94dc210f6e758229809 + md5: c3efd25ac4d74b1584d2f7a57195ddf1 + constrains: + - python 3.12.* *_cpython + license: BSD-3-Clause + license_family: BSD + size: 6958 + timestamp: 1752805918820 +- conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda + build_number: 8 + sha256: 210bffe7b121e651419cb196a2a63687b087497595c9be9d20ebe97dd06060a7 + md5: 94305520c52a4aa3f6c2b1ff6008d9f8 + constrains: + - python 3.13.* *_cp313 + license: BSD-3-Clause + license_family: BSD + size: 7002 + timestamp: 1752805902938 +- conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-h853b02a_0.conda + sha256: 12ffde5a6f958e285aa22c191ca01bbd3d6e710aa852e00618fa6ddc59149002 + md5: d7d95fc8287ea7bf33e0e7116d2b95ec + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - ncurses >=6.5,<7.0a0 + license: GPL-3.0-only + license_family: GPL + size: 345073 + timestamp: 1765813471974 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/readline-8.3-hb682ff5_0.conda + sha256: fe695f9d215e9a2e3dd0ca7f56435ab4df24f5504b83865e3d295df36e88d216 + md5: 3d49cad61f829f4f0e0611547a9cda12 + depends: + - libgcc >=14 + - ncurses >=6.5,<7.0a0 + license: GPL-3.0-only + license_family: GPL + size: 357597 + timestamp: 1765815673644 +- conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.3-h68b038d_0.conda + sha256: 4614af680aa0920e82b953fece85a03007e0719c3399f13d7de64176874b80d5 + md5: eefd65452dfe7cce476a519bece46704 + depends: + - __osx >=10.13 + - ncurses >=6.5,<7.0a0 + license: GPL-3.0-only + license_family: GPL + size: 317819 + timestamp: 1765813692798 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda + sha256: a77010528efb4b548ac2a4484eaf7e1c3907f2aec86123ed9c5212ae44502477 + md5: f8381319127120ce51e081dce4865cf4 + depends: + - __osx >=11.0 + - ncurses >=6.5,<7.0a0 + license: GPL-3.0-only + license_family: GPL + size: 313930 + timestamp: 1765813902568 +- conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_ha0e22de_103.conda + sha256: 1544760538a40bcd8ace2b1d8ebe3eb5807ac268641f8acdc18c69c5ebfeaf64 + md5: 86bc20552bf46075e3d92b67f089172d + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - libzlib >=1.3.1,<2.0a0 + constrains: + - xorg-libx11 >=1.8.12,<2.0a0 + license: TCL + license_family: BSD + size: 3284905 + timestamp: 1763054914403 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h561c983_103.conda + sha256: 154e73f6269f92ad5257aa2039278b083998fd19d371e150f307483fb93c07ae + md5: 631db4799bc2bfe4daccf80bb3cbc433 + depends: + - libgcc >=13 + - libzlib >=1.3.1,<2.0a0 + constrains: + - xorg-libx11 >=1.8.12,<2.0a0 + license: TCL + license_family: BSD + size: 3333495 + timestamp: 1763059192223 +- conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-hf689a15_3.conda + sha256: 0d0b6cef83fec41bc0eb4f3b761c4621b7adfb14378051a8177bd9bb73d26779 + md5: bd9f1de651dbd80b51281c694827f78f + depends: + - __osx >=10.13 + - libzlib >=1.3.1,<2.0a0 + license: TCL + license_family: BSD + size: 3262702 + timestamp: 1763055085507 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h892fb3f_3.conda + sha256: ad0c67cb03c163a109820dc9ecf77faf6ec7150e942d1e8bb13e5d39dc058ab7 + md5: a73d54a5abba6543cb2f0af1bfbd6851 + depends: + - __osx >=11.0 + - libzlib >=1.3.1,<2.0a0 + license: TCL + license_family: BSD + size: 3125484 + timestamp: 1763055028377 +- conda: https://conda.anaconda.org/conda-forge/win-64/tk-8.6.13-h2c6b04d_3.conda + sha256: 4581f4ffb432fefa1ac4f85c5682cc27014bcd66e7beaa0ee330e927a7858790 + md5: 7cb36e506a7dba4817970f8adb6396f9 + depends: + - ucrt >=10.0.20348.0 + - vc >=14.2,<15 + - vc14_runtime >=14.29.30139 + license: TCL + license_family: BSD + size: 3472313 + timestamp: 1763055164278 +- conda: https://conda.anaconda.org/conda-forge/linux-64/tombi-0.7.11-hb17b654_0.conda + sha256: d1d49b17bcbb92186a1c950720ee4a2682c7d17babebd91adec47c81a92d23db + md5: 32c738a2366e3e5b5b6a5d5516bb0fa1 + depends: + - libgcc >=14 + - __glibc >=2.17,<3.0.a0 + constrains: + - __glibc >=2.17 + license: MIT + size: 6003044 + timestamp: 1766686466709 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tombi-0.7.11-h069e38c_0.conda + sha256: e8ce67dd48fdac7d21a28d0dffa8e6ad9dd903ab7c886cccdee739a8957e872e + md5: 64078b5720a40ea52f97219647beb28d + depends: + - libgcc >=14 + constrains: + - __glibc >=2.17 + license: MIT + size: 5658247 + timestamp: 1766686477352 +- conda: https://conda.anaconda.org/conda-forge/osx-64/tombi-0.7.11-ha9c3995_0.conda + sha256: 7ab45c7554640a765aea2bd3362f16cd5625c87b6345a3a81505d2ae13c9e548 + md5: 5887e000279710aa1f43d32de20d4939 + depends: + - __osx >=10.13 + constrains: + - __osx >=10.13 + license: MIT + size: 5921351 + timestamp: 1766686473435 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/tombi-0.7.11-h6fdd925_0.conda + sha256: 91871ac8a162cdfa32907b7655ea616c188086849aea1506b5f108f9bf13e165 + md5: 62620b4ac92c26f5f7201c8c94d9e043 + depends: + - __osx >=11.0 + constrains: + - __osx >=11.0 + license: MIT + size: 5454830 + timestamp: 1766686492003 +- conda: https://conda.anaconda.org/conda-forge/win-64/tombi-0.7.11-h18a1a76_0.conda + sha256: c3ca5aa5a9b41e9570cc4adc165680a170efd7c485862a547867a73d60b9d8f5 + md5: 254f4ffca123f028fd150ce3a9be3bcc + depends: + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + - ucrt >=10.0.20348.0 + license: MIT + size: 6535303 + timestamp: 1766686560871 +- conda: https://conda.anaconda.org/conda-forge/linux-64/ty-0.0.9-h4e94fc0_0.conda + noarch: python + sha256: d4bd1e56b87e369841bdec38ea917ccfc078568ed46418936c490b74303d96e3 + md5: 56db074973271f308793a97069d0c1c1 + depends: + - python + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - _python_abi3_support 1.* + - cpython >=3.10 + constrains: + - __glibc >=2.17 + license: MIT + license_family: MIT + size: 8607307 + timestamp: 1767627165150 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ty-0.0.9-h1339683_0.conda + noarch: python + sha256: 648eac1131af9992d35d3ff0423623541bf1c0852c05540fa412625ddccaa199 + md5: 64191db770949076221c6f97674721c2 + depends: + - python + - libgcc >=14 + - _python_abi3_support 1.* + - cpython >=3.10 + constrains: + - __glibc >=2.17 + license: MIT + license_family: MIT + size: 8269481 + timestamp: 1767627183203 +- conda: https://conda.anaconda.org/conda-forge/osx-64/ty-0.0.9-h3e8ffd6_0.conda + noarch: python + sha256: c2c3ef88ff823b3202a287c9e4f8b2a6fa358e70df1eb4a582d7288cbe2daf34 + md5: 1ffea72c30cf48b434c39413d25768c6 + depends: + - python + - __osx >=10.13 + - _python_abi3_support 1.* + - cpython >=3.10 + constrains: + - __osx >=10.13 + license: MIT + license_family: MIT + size: 8295791 + timestamp: 1767627181604 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/ty-0.0.9-ha73ee7d_0.conda + noarch: python + sha256: 695eb5986559da4b86b5d555130b9573fd4f415c86f753c21d3d0a56dc73a937 + md5: a59a708a5f9a65b2d20266b9cf2853ec + depends: + - python + - __osx >=11.0 + - _python_abi3_support 1.* + - cpython >=3.10 + constrains: + - __osx >=11.0 + license: MIT + license_family: MIT + size: 7750797 + timestamp: 1767627188492 +- conda: https://conda.anaconda.org/conda-forge/win-64/ty-0.0.9-hc21aad4_0.conda + noarch: python + sha256: c74f8c1db8b66ef7570d13365cc8b41d96461c54942458e7de0b987fa70e468f + md5: b5a7e604632446440a56569782c7053d + depends: + - python + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + - ucrt >=10.0.20348.0 + - _python_abi3_support 1.* + - cpython >=3.10 + license: MIT + license_family: MIT + size: 8576443 + timestamp: 1767627193217 +- conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-h8577fbf_0.conda + sha256: 50fad5db6734d1bb73df1cf5db73215e326413d4b2137933f70708aa1840e25b + md5: 338201218b54cadff2e774ac27733990 + license: LicenseRef-Public-Domain + size: 119204 + timestamp: 1765745742795 +- conda: https://conda.anaconda.org/conda-forge/win-64/ucrt-10.0.26100.0-h57928b3_0.conda + sha256: 3005729dce6f3d3f5ec91dfc49fc75a0095f9cd23bab49efb899657297ac91a5 + md5: 71b24316859acd00bdb8b38f5e2ce328 + constrains: + - vc14_runtime >=14.29.30037 + - vs2015_runtime >=14.29.30037 + license: LicenseRef-MicrosoftWindowsSDK10 + size: 694692 + timestamp: 1756385147981 +- conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.9.18-h76e24b7_0.conda + sha256: ae73274a70fbec55ef064b8cd1c16fa31770ad2825fcccf2ecba1465da87801c + md5: 66a5d1348be63f1874f5a0dd0add29c2 + depends: + - libstdcxx >=14 + - libgcc >=14 + - __glibc >=2.17,<3.0.a0 + constrains: + - __glibc >=2.17 + license: Apache-2.0 OR MIT + size: 17782803 + timestamp: 1765925067699 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/uv-0.9.18-haeed4ea_0.conda + sha256: e33672e1cbf15f434135232aed1f4d71d805f1ddbcd4631d6b60a130384edaeb + md5: 795d48acc0a5ce4625b2ab911fcd75ae + depends: + - libstdcxx >=14 + - libgcc >=14 + constrains: + - __glibc >=2.17 + license: Apache-2.0 OR MIT + size: 17261454 + timestamp: 1765925375807 +- conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.9.18-h3315dae_0.conda + sha256: 4d0e487f936b34d13678038c8e405251f7cdcc2f024d0cea47972da937a1f3ea + md5: 1e29d8deef75dbab7518b3b06c3a1336 + depends: + - __osx >=10.13 + - libcxx >=19 + constrains: + - __osx >=10.13 + license: Apache-2.0 OR MIT + size: 16685712 + timestamp: 1765925009765 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.9.18-h1bde295_0.conda + sha256: b38769e558e7fbbeabd6d3d76e623840b705a0d59fb313ee5d1cebb39624214b + md5: 2b426bd923b5a6a7fc85c37304a05a35 + depends: + - __osx >=11.0 + - libcxx >=19 + constrains: + - __osx >=11.0 + license: Apache-2.0 OR MIT + size: 15391764 + timestamp: 1765925027508 +- conda: https://conda.anaconda.org/conda-forge/win-64/uv-0.9.18-h3bd95fe_0.conda + sha256: 3e66475d65054a70c0a0121c3746ef84f3a18d88f20ee01f4dc41989e95f497c + md5: 7ac52346d9108544895ce451d4586034 + depends: + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + - ucrt >=10.0.20348.0 + license: Apache-2.0 OR MIT + size: 18314146 + timestamp: 1765925001916 +- conda: https://conda.anaconda.org/conda-forge/win-64/vc-14.3-h2b53caa_33.conda + sha256: 7036945b5fff304064108c22cbc1bb30e7536363782b0456681ee6cf209138bd + md5: 2d1c042360c09498891809a3765261be + depends: + - vc14_runtime >=14.42.34433 + track_features: + - vc14 + license: BSD-3-Clause + license_family: BSD + size: 19070 + timestamp: 1765216452130 +- conda: https://conda.anaconda.org/conda-forge/win-64/vc14_runtime-14.44.35208-h818238b_33.conda + sha256: 7e8f7da25d7ce975bbe7d7e6d6e899bf1f253e524a3427cc135a79f3a79c457c + md5: fb8e4914c5ad1c71b3c519621e1df7b8 + depends: + - ucrt >=10.0.20348.0 + - vcomp14 14.44.35208 h818238b_33 + constrains: + - vs2015_runtime 14.44.35208.* *_33 + license: LicenseRef-MicrosoftVisualCpp2015-2022Runtime + license_family: Proprietary + size: 684323 + timestamp: 1765216366832 +- conda: https://conda.anaconda.org/conda-forge/win-64/vcomp14-14.44.35208-h818238b_33.conda + sha256: f79edd878094e86af2b2bc1455b0a81e02839a784fb093d5996ad4cf7b810101 + md5: 4cb6942b4bd846e51b4849f4a93c7e6d + depends: + - ucrt >=10.0.20348.0 + constrains: + - vs2015_runtime 14.44.35208.* *_33 + license: LicenseRef-MicrosoftVisualCpp2015-2022Runtime + license_family: Proprietary + size: 115073 + timestamp: 1765216325898 +- conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda + sha256: 68f0206ca6e98fea941e5717cec780ed2873ffabc0e1ed34428c061e2c6268c7 + md5: 4a13eeac0b5c8e5b8ab496e6c4ddd829 + depends: + - __glibc >=2.17,<3.0.a0 + - libzlib >=1.3.1,<2.0a0 + license: BSD-3-Clause + license_family: BSD + size: 601375 + timestamp: 1764777111296 +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.7-h85ac4a6_6.conda + sha256: 569990cf12e46f9df540275146da567d9c618c1e9c7a0bc9d9cfefadaed20b75 + md5: c3655f82dcea2aa179b291e7099c1fcc + depends: + - libzlib >=1.3.1,<2.0a0 + license: BSD-3-Clause + license_family: BSD + size: 614429 + timestamp: 1764777145593 diff --git a/pixi.toml b/pixi.toml new file mode 100644 index 0000000..4bb331c --- /dev/null +++ b/pixi.toml @@ -0,0 +1,86 @@ +[workspace] +name = "kornia" +description = "Open Source Differentiable Computer Vision Library for PyTorch" +authors = ["Edgar Riba "] +channels = ["conda-forge"] +platforms = ["linux-64", "linux-aarch64", "osx-arm64", "osx-64", "win-64"] + +# ------------------------------------------------------------------------------ +# Base: System dependencies +# ------------------------------------------------------------------------------ + +[dependencies] +python = ">=3.11,<3.14" +uv = "*" +tombi = "*" +ty = ">=0.0.9,<0.0.10" + +# ------------------------------------------------------------------------------ +# Environments +# ------------------------------------------------------------------------------ + +[environments] +# Default environment with Python 3.11 (best compatibility) +default = { features = ["py311"], solve-group = "default" } +# CUDA environment for GPU development +cuda = { features = ["cuda", "py311"], solve-group = "cuda" } +# CI environments with specific Python versions +py311 = { features = ["py311"], solve-group = "py311" } +py312 = { features = ["py312"], solve-group = "py312" } +py313 = { features = ["py313"], solve-group = "py313" } + +# ------------------------------------------------------------------------------ +# Tasks (shared across environments) +# ------------------------------------------------------------------------------ + +[tasks] +# Installation +install = { cmd = "uv sync --extra dev", description = "Install dev dependencies" } +install-docs = { cmd = "uv sync --extra dev --extra docs", description = "Install dev + docs dependencies" } +# Tests - use env vars (KORNIA_TEST_DEVICE, KORNIA_TEST_DTYPE, KORNIA_TEST_RUNSLOW) +test = { cmd = "uv run pytest -v tests/", description = "Run tests (configure via KORNIA_TEST_* env vars)" } +test-f32 = { cmd = "uv run pytest -v tests/", env = { KORNIA_TEST_DTYPE = "float32" } } +test-f64 = { cmd = "uv run pytest -v tests/", env = { KORNIA_TEST_DTYPE = "float64" } } +test-slow = { cmd = "uv run pytest -v tests/", env = { KORNIA_TEST_RUNSLOW = "true" } } +test-quick = { cmd = "uv run pytest -v -m 'not (jit or grad or nn)' tests/" } +test-module = { cmd = "uv run pytest -v", description = "Run tests for a specific module (usage: pixi run test-module tests/, e.g., pixi run test-module tests/contrib/)" } +# Quality +lint = { cmd = "uv run pre-commit run ruff-check --all-files && uv run pre-commit run ruff-format --all-files" } +typecheck = { cmd = "ty check kornia" } +doctest = { cmd = "uv run pytest -v --doctest-modules kornia/" } +toml-fmt = { cmd = "tombi format", description = "Format TOML files" } +toml-fmt-check = { cmd = "tombi format --check", description = "Check TOML formatting" } +# Docs +build-docs = { cmd = "uv run python -m sphinx -W -b html docs/source docs/build/html", depends-on = [ + "install-docs" +] } +# Utility +clean = { cmd = "find . -type f -name '*.pyc' -delete && find . -type d -name '__pycache__' -delete && rm -rf .venv" } + +# ------------------------------------------------------------------------------ +# Feature: CUDA (PyTorch CUDA wheels) - for local GPU development +# Requires reinstall because uv doesn't support per-environment sources +# ------------------------------------------------------------------------------ + +[feature.cuda.tasks] +install = { cmd = "uv sync --extra dev && uv pip install --reinstall torch torchvision --index pytorch-cu121", description = "Install dev dependencies (CUDA PyTorch)" } +install-docs = { cmd = "uv sync --extra dev --extra docs --extra x && uv pip install --reinstall torch torchvision --index pytorch-cu121", description = "Install dev + docs (CUDA PyTorch)" } +# CUDA test tasks +test-cuda = { cmd = "uv run pytest -v tests/", env = { KORNIA_TEST_DEVICE = "cuda" }, description = "Run tests on CUDA" } +test-cuda-f32 = { cmd = "uv run pytest -v tests/", env = { KORNIA_TEST_DEVICE = "cuda", KORNIA_TEST_DTYPE = "float32" } } +test-cuda-f64 = { cmd = "uv run pytest -v tests/", env = { KORNIA_TEST_DEVICE = "cuda", KORNIA_TEST_DTYPE = "float64" } } +test-half = { cmd = "uv run pytest -v tests/", env = { KORNIA_TEST_DTYPE = "float16,bfloat16" }, description = "Run half-precision tests (CPU; use test-cuda-half for CUDA isolation)" } +test-cuda-half = { cmd = "uv run pytest -v tests/ --isolate-half-precision", env = { KORNIA_TEST_DEVICE = "cuda", KORNIA_TEST_DTYPE = "float16,bfloat16" }, description = "Run half-precision CUDA tests with per-test subprocess isolation" } + +# ------------------------------------------------------------------------------ +# Python version features (for CI matrix) +# ------------------------------------------------------------------------------ + +[feature.py311.dependencies] +python = "3.11.*" + +[feature.py312.dependencies] +python = "3.12.*" + +[feature.py313.dependencies] +python = "3.13.*" diff --git a/pr_body.txt b/pr_body.txt new file mode 100644 index 0000000..8d47aaa --- /dev/null +++ b/pr_body.txt @@ -0,0 +1,5 @@ +> Description: This PR introduces Google-style structural docstrings to the kornia.losses.dice module, resolving the missing D101, D102, and D103 Ruff violations. + +Related Issue: Fixes #3083 +Verification: 12 passed, 43 warnings (local Windows test-bed). +AI Disclosure: ?? AI-assisted via GitHub Copilot CLI; verified by Siddharth Jaspal. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..bc20473 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,392 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "kornia" +description = "Open Source Differentiable Computer Vision Library for PyTorch" +readme = "README.md" +requires-python = ">=3.11" +license = { text = "Apache-2.0" } +authors = [{ name = "Edgar Riba", email = "edgar@kornia.org" }] +keywords = ["computer vision", "deep learning", "pytorch"] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Environment :: GPU", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Scientific/Engineering :: Image Processing", + "Topic :: Software Development :: Libraries", +] +dependencies = ["kornia_rs>=0.1.9", "numpy>=1.21", "packaging", "torch>=2.0.0"] +dynamic = ["version"] + +[project.urls] +"Bug Tracker" = "https://github.com/kornia/kornia/issues" +Documentation = "https://kornia.readthedocs.io/en/latest" +Download = "https://github.com/kornia/kornia" +Homepage = "https://kornia.github.io/" +Issues = "https://github.com/kornia/kornia/issues" +"Source Code" = "https://github.com/kornia/kornia" + +[project.optional-dependencies] +dev = [ + "coverage", + "diffusers", + "ivy>=1.0.0.0", + "numpy<3", + "onnx", + "onnxruntime", + "onnxscript", + "pillow", + "pre-commit", + "pytest", + "pytest-cov", + "pytest-timeout", + "requests", + "ruff", + "setuptools", # For torch.compile + "transformers<5.6", +] +docs = [ + "furo", + "ivy>=1.0.0.0", + "kornia_moons", + "matplotlib", + "onnx", + "onnxruntime", + "opencv-python", + "PyYAML>=5.1", + "sphinx", + "sphinx-autodoc-defaultargs", + "sphinx-autodoc-typehints", + "sphinx-copybutton>=0.3", + "sphinx-design", + "sphinx-notfound-page", + "sphinxcontrib-bibtex", + "sphinxcontrib-gtagjs", + "sphinxcontrib-youtube", +] + +[tool.hatch.version] +path = "kornia/__init__.py" + +[tool.hatch.build.targets.wheel] +packages = ["kornia"] +exclude = ["**/*.md"] # Exclude markdown files from wheel + +[tool.hatch.build.targets.sdist] +include = ["kornia/", "LICENSE", "README.md"] + +[tool.codespell] +ignore-words-list = "ans,hist,laf,cofusion" +skip = "*.bib,*.ipynb" + +[tool.ruff] +target-version = "py311" +line-length = 120 + +[tool.ruff.format] +skip-magic-trailing-comma = false + +[tool.ruff.lint] +select = [ + "AIR", # Airflow + "ASYNC", # flake8-async + "B", # flake8-bugbear + "BLE", # flake8-blind-except + "C4", # flake8-comprehensions + "C90", # McCabe cyclomatic complexity + # "CPY", # Copyright-related rules + "DTZ", # flake8-datetimez + "E", # pycodestyle + "F", # Pyflakes + "FLY", # flynt + "I", # isort + "ICN", # flake8-import-conventions + "INT", # flake8-gettext + "NPY", # NumPy-specific rules + "PL", # Pylint + "PYI", # flake8-pyi + "RSE", # flake8-raise + "RUF", # Ruff-specific rules + "S", # flake8-bandit + "SLOT", # flake8-slots + "T10", # flake8-debugger + "TID", # flake8-tidy-imports + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 + # "A", # flake8-builtins + # "ANN", # flake8-annotations + # "ARG", # flake8-unused-arguments + # "COM", # flake8-commas + "D", # pydocstyle + "D400", # missing-trailing-period + # "DJ", # flake8-django + # "EM", # flake8-errmsg + # "ERA", # eradicate + # "EXE", # flake8-executable + # "FA", # flake8-future-annotations + # "FBT", # flake8-boolean-trap + # "FIX", # flake8-fixme + # "G", # flake8-logging-format + # "INP", # flake8-no-pep420 + # "ISC", # flake8-implicit-str-concat + # "N", # pep8-naming + # "PD", # pandas-vet + # "PERF", # Perflint + # "PGH", # pygrep-hooks + # "PIE", # flake8-pie + # "PT", # flake8-pytest-style + # "PTH", # flake8-use-pathlib + # "Q", # flake8-quotes + # "RET", # flake8-return + # "SIM", # flake8-simplify + # "SLF", # flake8-self + # "T20", # flake8-print + # "TCH", # flake8-type-checking + # "TD", # flake8-todos + # "TRY", # tryceratops +] +ignore = [ + "B905", # `zip()` without explicit `strict=` - too many to fix, not a bug + "RUF007", # Prefer `itertools.pairwise()` over `zip()` - style preference + "PLR0915", # Allow condition check in list comprehension + "PLC0415", # `import` should be at the top-level of a file + "PLW2901", # Allow overwritten values on loops + "PLW1641", # Object does not implement `__hash__` method + "UP007", # Prefer Optional[], Union[] over | due to torch jit scripting + "UP006", # Prefer List[], over list due to torch jit scripting + "UP035", # Ignore deprecated typing because of jit scripting + "UP045", # Use `X | None` for type annotations + "RUF005", # Consider `(*points_in_cam_canonical.shape[:-1], 1)` instead of concatenation. Note: breaks JIT. + "D100", # Allow Undocumented public module + "D101", # TODO: Undocumented public class + "D102", # TODO: Undocumented public method + "D104", # TODO: Undocumented public package + "D105", # Allow Undocumented magic method + "D107", # TODO: Undocumented public init +] + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.lint.isort] +forced-separate = ["testing", "tests"] +known-first-party = ["kornia"] +split-on-trailing-comma = true + +[tool.ruff.lint.mccabe] +max-complexity = 20 + +[tool.ruff.lint.pylint] +allow-magic-value-types = ["bytes", "float", "int", "str"] +max-args = 30 # Recommended: 5 +max-branches = 21 # Recommended: 12 +max-returns = 13 # Recommended: 6 +max-statements = 64 # Recommended: 50 + +[tool.ruff.lint.per-file-ignores] +"*/__init__.py" = ["F401", "F403"] # Allow unused imports and star imports +"benchmarks/*" = [ + "BLE", + "RUF005", + "RUF012", + "S101", + "S311", + "S603", + "D", +] # allow assert, random, subprocess calls, ignore BLE, mutable class attr +"docs/*" = [ + "PLR0912", + "PLR0915", + "S101", + "D", +] # allow assert, ignore max branches and statements +"docs/generate_examples.py" = ["C901"] # Allow too complex function +"kornia/__init__.py" = ["I001"] # Allow unsorted imports +"kornia/feature/dedode/*" = [ + "C408", + "F401", + "F841", + "FLY002", + "PLR1714", +] # allow DINOv2 things +"testing/*" = [ + "S101", # allow assert + "D", # Don't enforce documentation rules +] +"tests/*" = [ + "BLE", + "RUF005", + "RUF012", + "S101", + "S311", + "B017", # Check for Exception since KORNIA_CHECK raises it + "D", # Don't enforce documentation rules +] # allow assert, random, ignore BLE, mutable class attr + +[tool.pytest.ini_options] +addopts = "--color=yes -v" +testpaths = ["tests"] +markers = [ + "grad: mark a test as gradcheck test", + "jit: mark a test as torchscript test", + "nn: mark a test as module test", + "slow: mark test as slow to run", + "tf32: mark a test as sensitive to TF32 (TensorFloat-32) CUDA matmul precision; xfail by default, run with --tf32 to activate", +] +filterwarnings = [ + "ignore::DeprecationWarning:onnxscript.converter", +] +# Test configuration via environment variables: +# KORNIA_TEST_DEVICE: cpu, cuda, mps (default: cpu) +# KORNIA_TEST_DTYPE: float32, float64, float16, bfloat16 (default: float32) +# KORNIA_TEST_OPTIMIZER: inductor, jit, etc. (default: inductor) +# KORNIA_TEST_RUNSLOW: true/false (default: false) + +[tool.coverage.run] +branch = true +source = ["kornia/"] +omit = [ + "*/__main__.py", + "*/setup.py", + "kornia/models/sam/*", + "kornia/models/sam3/*", + "kornia/models/siglip2/*", + "kornia/models/paligemma/*", + "kornia/models/kimi_vl/*", + "kornia/models/qwen25/*", + "kornia/models/smolvlm2/*", + "kornia/models/efficient_vit/*", + "kornia/models/rt_detr/*", + "kornia/models/depth_estimation/*", + "kornia/models/_hf_models/*", + "kornia/models/processors/*", + "kornia/models/vit.py", + "kornia/models/vit_mobile.py", + "kornia/models/tiny_vit.py", + # Feature model building blocks (vendored architectures) + "kornia/feature/adalam/*", + "kornia/feature/dedode/transformer/*", + "kornia/feature/dedode/dedode_models.py", + "kornia/feature/dedode/vgg.py", + "kornia/feature/loftr/loftr_module/*", + "kornia/feature/loftr/backbone/*", + "kornia/feature/loftr/utils/*", + "kornia/feature/sold2/backbones.py", + "kornia/feature/sold2/sold2_detector.py", + "kornia/feature/lightglue.py", + "kornia/feature/lightglue_onnx/*", + "kornia/feature/aliked/deform_conv2d.py", + "kornia/feature/disk/_unets/*", + "kornia/feature/defmo.py", + # DexiNed vendored architecture (building blocks only; high-level API in kornia/contrib/edge_detection.py is kept) + "kornia/filters/dexined.py", + "kornia/models/dexined.py", + # HuggingFace diffusers wrapper — requires diffusers as optional dep, not unit-testable as a filter + "kornia/filters/dissolving.py", +] + +[tool.coverage.report] +show_missing = true +skip_covered = true +fail_under = 84 +exclude_lines = [ + # Based into the covdefaults plugin config + # a more strict default pragma + '\# pragma: no cover\b', + # allow defensive code + '^\s*raise AssertionError\b', + '^\s*raise NotImplementedError\b', + '^\s*return NotImplemented\b', + '^\s*raise$', + # typing-related code + '^\s*if (False|TYPE_CHECKING):', + ': \.\.\.(\s*#.*)?$', + '^ +\.\.\.$', + # ---------------------------- + "def __repr__", + "if __name__ == .__main__.:", + "if 0:", + "if self.debug:", +] +partial_branches = [ + # a more strict default pragma + '\# pragma: no cover\b', +] + +[tool.pydocstyle] +match = '.*\.py' + +[tool.codeflash] +module-root = "kornia" +tests-root = "tests" +test-framework = "pytest" +ignore-paths = [] +formatter-cmds = ["disabled"] + +# ============================================================================= +# uv configuration +# ============================================================================= +# PyTorch: Use standard PyPI version (simpler, more compatible) +# For CUDA: pixi run -e cuda install (uses reinstall workaround) +# Note: uv doesn't support per-environment sources + +# torch is installed from standard PyPI (no custom index needed) + +[[tool.uv.index]] +name = "pytorch-cu121" +url = "https://download.pytorch.org/whl/cu121" +explicit = true + +[[tool.uv.index]] +name = "pytorch-cu124" +url = "https://download.pytorch.org/whl/cu124" +explicit = true + +[tool.ty.src] +include = ["kornia"] +exclude = ["tests", "docs", "conftest.py"] + +[[tool.ty.overrides]] +include = ["kornia"] + +[tool.ty.overrides.rules] +unresolved-import = "ignore" +unresolved-attribute = "ignore" +call-non-callable = "ignore" +invalid-argument-type = "ignore" +invalid-method-override = "ignore" +invalid-type-form = "ignore" +no-matching-overload = "ignore" +unsupported-operator = "ignore" +possibly-missing-attribute = "ignore" +not-subscriptable = "ignore" +# Silenced because ty has issues with: +# - Iterator[Enum] from metaclass __iter__ returning Iterator[object] +# - Scalar wrapper return types in geometry/plane.py +invalid-return-type = "ignore" +# Silenced because ty doesn't understand star unpacking with union types +# e.g., `[-i for i in unpadding]` where unpadding is int | tuple +# or `*images.shape, images.device` unpacking +not-iterable = "ignore" +# Silenced because ty struggles with: +# - Star unpacking assignments like `b, c, h, w, device = *tensor.shape, tensor.device` +# - dict containing partial() alongside type[nn.Module] +# - Attribute assignments with getattr fallback to None +invalid-assignment = "ignore" +# Silenced because KORNIA_UNWRAP intentionally uses cast for type narrowing +redundant-cast = "ignore" diff --git a/readthedocs.yml b/readthedocs.yml new file mode 100644 index 0000000..e9666e3 --- /dev/null +++ b/readthedocs.yml @@ -0,0 +1,25 @@ +# Read the Docs configuration file for Sphinx projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/source/conf.py + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/testing/__init__.py b/testing/__init__.py new file mode 100644 index 0000000..b851fe0 --- /dev/null +++ b/testing/__init__.py @@ -0,0 +1,20 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .parametrized_tester import parametrized_test + +__all__ = ["parametrized_test"] diff --git a/testing/augmentation/__init__.py b/testing/augmentation/__init__.py new file mode 100644 index 0000000..4d273d5 --- /dev/null +++ b/testing/augmentation/__init__.py @@ -0,0 +1,16 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/testing/augmentation/datasets.py b/testing/augmentation/datasets.py new file mode 100644 index 0000000..c76218e --- /dev/null +++ b/testing/augmentation/datasets.py @@ -0,0 +1,102 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch.nn +from torch.utils.data import Dataset + +from kornia.augmentation import ( + ColorJiggle, + ColorJitter, + RandomAffine, + RandomAffine3D, + RandomCrop, + RandomCrop3D, + RandomCutMixV2, + RandomErasing, + RandomGaussianBlur, + RandomJigsaw, + RandomMixUpV2, + RandomMosaic, + RandomMotionBlur, + RandomMotionBlur3D, + RandomPerspective, + RandomPerspective3D, + RandomPlanckianJitter, + RandomPosterize, + RandomRain, + RandomRotation3D, + RandomShear, + RandomTranslate, + Resize, +) + + +class DummyMPDataset(Dataset): + def __init__(self, context: str): + super().__init__() + # we add all transforms that could potentially fail in + # multiprocessing with a spawn context below, that is all the + # transforms that define a RNG + transforms = [ + RandomTranslate(), + RandomShear(0.1), + RandomPosterize(), + RandomErasing(), + RandomMotionBlur(kernel_size=3, angle=(0, 360), direction=(-1, 1)), + RandomGaussianBlur(3, (0.1, 2.0)), + RandomPerspective(), + ColorJitter(), + ColorJiggle(), + RandomJigsaw(), + RandomAffine(degrees=15), + RandomMotionBlur3D(kernel_size=3, angle=(0, 360), direction=(-1, 1)), + RandomPerspective3D(), + RandomAffine3D(degrees=15), + RandomRotation3D(degrees=15), + ] + + if context != "fork": + # random planckian jitter auto selects a GPU. But it is not possible + # to init a CUDA context in a forked process. + # So we skip it in this case. + transforms.append(RandomPlanckianJitter()) + + self._transform = torch.nn.Sequential() + + self._resize = Resize((10, 10)) + self._mosaic = RandomMosaic((2, 2)) + self._crop = RandomCrop((5, 5)) + self._crop3d = RandomCrop3D((5, 5, 5)) + self._mixup = RandomMixUpV2() + self._cutmix = RandomCutMixV2() + self._rain = RandomRain(p=1, drop_height=(1, 2), drop_width=(1, 2), number_of_drops=(1, 1)) + + def __len__(self): + return 10 + + def __getitem__(self, _): + mosaic = self._mosaic(torch.rand(1, 3, 64, 64)) + rain = self._rain(torch.rand(1, 1, 5, 5)) + rain = self._resize(rain) + cropped = self._crop(torch.rand(3, 3, 64, 64)) + cropped3d = self._crop3d(torch.rand(3, 64, 64, 64)) + mixed = self._mixup(torch.rand(3, 3, 64, 64), torch.rand(3, 3, 64, 64)) + mixed = self._cutmix(torch.rand(3, 3, 64, 64), mixed) + + return (self._transform(mixed), cropped, cropped3d, mixed, mosaic, rain) diff --git a/testing/augmentation/utils.py b/testing/augmentation/utils.py new file mode 100644 index 0000000..e9e6417 --- /dev/null +++ b/testing/augmentation/utils.py @@ -0,0 +1,45 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch + +from testing.base import assert_close + + +def reproducibility_test(input, seq): + """Any tests failed here indicate the output cannot be reproduced by the same params.""" + if isinstance(input, (tuple, list)): + output_1 = seq(*input) + output_2 = seq(*input, params=seq._params) + else: + output_1 = seq(input) + output_2 = seq(input, params=seq._params) + + if isinstance(output_1, (tuple, list)) and isinstance(output_2, (tuple, list)): + [ + assert_close(o1, o2) + for o1, o2 in zip(output_1, output_2) + if isinstance(o1, (torch.Tensor,)) and isinstance(o2, (torch.Tensor,)) + ] + elif isinstance(output_1, (tuple, list)) and isinstance(output_2, (torch.Tensor,)): + assert_close(output_1[0], output_2) + elif isinstance(output_2, (tuple, list)) and isinstance(output_1, (torch.Tensor,)): + assert_close(output_1, output_2[0]) + elif isinstance(output_2, (torch.Tensor,)) and isinstance(output_1, (torch.Tensor,)): + assert_close(output_1, output_2, msg=f"{seq._params}") + else: + raise AssertionError(f"cannot compare {type(output_1)} and {type(output_2)}") diff --git a/testing/base.py b/testing/base.py new file mode 100644 index 0000000..91bb412 --- /dev/null +++ b/testing/base.py @@ -0,0 +1,172 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import math +from typing import Any, Callable, Optional, Sequence, Union + +import pytest +import torch +from torch.autograd import gradcheck +from torch.testing import assert_close as _assert_close + +Dtype = Union[torch.dtype, None] +Tensor = torch.Tensor + +# {dtype: (rtol, atol)} +_DTYPE_PRECISIONS = { + torch.bfloat16: (7.8e-3, 7.8e-3), + torch.float16: (1e-3, 1e-3), + torch.float32: (1e-4, 1e-5), # TODO: Update to ~1.2e-7 + # TODO: Update to ~2.3e-16 for fp64 + torch.float64: (1e-5, 1e-5), # TODO: BaseTester used (1.3e-6, 1e-5), but it fails for general cases +} + + +def _default_tolerances(*inputs: Any) -> tuple[float, float]: + rtols, atols = zip(*[_DTYPE_PRECISIONS.get(torch.as_tensor(input_).dtype, (0.0, 0.0)) for input_ in inputs]) + return max(rtols), max(atols) + + +def assert_close( + actual: Tensor, expected: Tensor, *, rtol: Optional[float] = None, atol: Optional[float] = None, **kwargs: Any +) -> None: + if rtol is None and atol is None: + # `torch.testing.assert_close` used different default tolerances than `torch.testing.assert_allclose`. + # TODO: remove this special handling as soon as https://github.com/kornia/kornia/issues/1134 is resolved + # Basically, this whole wrapper function can be removed and `torch.testing.assert_close` can be used + # directly. + rtol, atol = _default_tolerances(actual, expected) + + return _assert_close( + actual, + expected, + rtol=rtol, + atol=atol, + # this is the default value for torch>=1.10, but not for torch==1.9 + # TODO: remove this if kornia relies on torch>=1.10 + check_stride=False, + equal_nan=False, + **kwargs, + ) + + +def tensor_to_gradcheck_var( + tensor: Tensor, dtype: Dtype = torch.float64, requires_grad: bool = True +) -> Union[Tensor, str]: + """Convert the input tensor to a valid variable to check the gradient. + + `gradcheck` needs 64-bit floating point and requires gradient. + """ + if not torch.is_tensor(tensor): + raise AssertionError(type(tensor)) + t = tensor.type(dtype) + + if t.is_floating_point(): + return t.requires_grad_(requires_grad) + + return t + + +class BaseTester: + @staticmethod + def assert_close( + actual: Tensor | float, + expected: Tensor | float, + rtol: Optional[float] = None, + atol: Optional[float] = None, + low_tolerance: bool = False, + ) -> None: + """Asserts that `actual` and `expected` are close. + + Args: + actual: Actual input. + expected: Expected input. + rtol: Relative tolerance. + atol: Absolute tolerance. + low_tolerance: + This parameter allows to reduce tolerance. Half the decimal places. + Example, 1e-4 -> 1e-2 or 1e-6 -> 1e-3 + + """ + if hasattr(actual, "data"): + actual = actual.data + if hasattr(expected, "data"): + expected = expected.data + + if (isinstance(actual, Tensor) and "xla" in actual.device.type) or ( + isinstance(expected, Tensor) and "xla" in expected.device.type + ): + rtol, atol = 1e-2, 1e-2 + + if (isinstance(actual, Tensor) and isinstance(expected, Tensor)) and rtol is None and atol is None: + actual_rtol, actual_atol = _DTYPE_PRECISIONS.get(actual.dtype, (0.0, 0.0)) + expected_rtol, expected_atol = _DTYPE_PRECISIONS.get(expected.dtype, (0.0, 0.0)) + rtol, atol = max(actual_rtol, expected_rtol), max(actual_atol, expected_atol) + + # halve the tolerance if `low_tolerance` is true + rtol = math.sqrt(rtol) if low_tolerance else rtol + atol = math.sqrt(atol) if low_tolerance else atol + + return assert_close(actual, expected, rtol=rtol, atol=atol) + + @staticmethod + def gradcheck( + func: Callable[..., Union[torch.Tensor, Sequence[torch.Tensor]]], + inputs: Union[torch.Tensor, Sequence[Any]], + *, + raise_exception: bool = True, + fast_mode: bool = True, + requires_grad: Sequence[bool] = [], + dtypes: Sequence[Dtype] = [], + **kwargs: Any, + ) -> bool: + """It will gradcheck the function using the `torch.autograd.gradcheck` method. + + By default this method will pass all tensor to `tensor_to_gradcheck_var` which casts the tensor + to be float64 dtype, and requires grad as True. You can overwrite which tensors should have requires grad + equals True, by using a Sequence of the same length of the sequence of inputs, within the requires_grad + per item. You also, can overwrite with the same mechanics the dtype using the `dtypes` + parameter. + """ + requires_grad = requires_grad if len(requires_grad) > 0 else [True] * len(inputs) + dtypes = dtypes if len(dtypes) > 0 else [torch.float64] * len(inputs) + + # MPS does not support float64; gradcheck requires float64, so skip on MPS + _all_inputs = ( + [inputs] + if isinstance(inputs, torch.Tensor) + else list(inputs.values() if isinstance(inputs, dict) else inputs) + ) + if any(isinstance(t, torch.Tensor) and t.device.type == "mps" for t in _all_inputs): + pytest.skip("gradcheck requires float64 which is not supported on MPS") + + if isinstance(inputs, torch.Tensor): + inputs = tensor_to_gradcheck_var(inputs) + elif isinstance(inputs, dict): + inputs = { + k: tensor_to_gradcheck_var(v, d, r) if isinstance(v, torch.Tensor) else v + for (k, v), d, r in zip(inputs.items(), dtypes, requires_grad) + } + else: + inputs = [ + tensor_to_gradcheck_var(i, d, r) if isinstance(i, torch.Tensor) else i + for i, r, d in zip(inputs, requires_grad, dtypes) + ] + + return gradcheck(func, inputs, raise_exception=raise_exception, fast_mode=fast_mode, **kwargs) diff --git a/testing/casts.py b/testing/casts.py new file mode 100644 index 0000000..0aa4495 --- /dev/null +++ b/testing/casts.py @@ -0,0 +1,31 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Any, TypeVar + +import torch + +T = TypeVar("T") + + +def dict_to(data: dict[T, Any], device: torch.device, dtype: torch.dtype) -> dict[T, Any]: + out: dict[T, Any] = {} + for key, val in data.items(): + out[key] = val.to(device, dtype) if isinstance(val, torch.Tensor) else val + return out diff --git a/testing/error.py b/testing/error.py new file mode 100644 index 0000000..0f27949 --- /dev/null +++ b/testing/error.py @@ -0,0 +1,25 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch + + +def compute_patch_abs_error(x: torch.Tensor, y: torch.Tensor, h: int, w: int) -> torch.Tensor: + """Compute the absolute error between patches.""" + return torch.abs(x - y)[..., h // 4 : -h // 4, w // 4 : -w // 4].mean() diff --git a/testing/geometry/__init__.py b/testing/geometry/__init__.py new file mode 100644 index 0000000..4d273d5 --- /dev/null +++ b/testing/geometry/__init__.py @@ -0,0 +1,16 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/testing/geometry/create.py b/testing/geometry/create.py new file mode 100644 index 0000000..c70e132 --- /dev/null +++ b/testing/geometry/create.py @@ -0,0 +1,110 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Optional, Union + +import torch + +import kornia.geometry.epipolar as epi +from kornia.core.ops import eye_like + + +def create_random_homography(data: torch.Tensor, eye_size: int, std_val: float = 1e-3) -> torch.Tensor: + """Create a batch of random homographies of shape Bx3x3.""" + std = torch.zeros(data.shape[0], eye_size, eye_size, device=data.device, dtype=data.dtype) + eye = eye_like(eye_size, data) + return eye + std.uniform_(-std_val, std_val) + + +def create_rectified_fundamental_matrix( + batch_size: int, dtype: Optional[torch.dtype] = None, device: Optional[Union[str, torch.device]] = None +) -> torch.Tensor: + """Create a batch of rectified fundamental matrices of shape Bx3x3.""" + F_rect = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, -1.0], [0.0, 1.0, 0.0]], device=device, dtype=dtype).view( + 1, 3, 3 + ) + F_repeat = F_rect.expand(batch_size, 3, 3) + return F_repeat + + +def create_random_fundamental_matrix( + batch_size: int, + std_val: float = 1e-3, + dtype: Optional[torch.dtype] = None, + device: Optional[Union[str, torch.device]] = None, +) -> torch.Tensor: + """Create a batch of random fundamental matrices of shape Bx3x3.""" + F_rect = create_rectified_fundamental_matrix(batch_size, dtype, device) + H_left = create_random_homography(F_rect, 3, std_val) + H_right = create_random_homography(F_rect, 3, std_val) + return H_left.permute(0, 2, 1) @ F_rect @ H_right + + +def generate_two_view_random_scene( + device: Optional[torch.device] = None, dtype: torch.dtype = torch.float32 +) -> dict[str, torch.Tensor]: + if device is None: + device = torch.device("cpu") + num_views: int = 2 + num_points: int = 30 + + scene: dict[str, torch.Tensor] = epi.generate_scene(num_views, num_points) + + # internal parameters (same K) + K1 = scene["K"].to(device, dtype) + K2 = K1.clone() + + # rotation + R1 = scene["R"][0:1].to(device, dtype) + R2 = scene["R"][1:2].to(device, dtype) + + # translation + t1 = scene["t"][0:1].to(device, dtype) + t2 = scene["t"][1:2].to(device, dtype) + + # projection matrix, P = K(R|t) + P1 = scene["P"][0:1].to(device, dtype) + P2 = scene["P"][1:2].to(device, dtype) + + # fundamental matrix + F_mat = epi.fundamental_from_projections(P1[..., :3, :], P2[..., :3, :]) + + F_mat = epi.normalize_transformation(F_mat) + + # points 3d + X = scene["points3d"].to(device, dtype) + + # projected points + x1 = scene["points2d"][0:1].to(device, dtype) + x2 = scene["points2d"][1:2].to(device, dtype) + + return { + "K1": K1, + "K2": K2, + "R1": R1, + "R2": R2, + "t1": t1, + "t2": t2, + "P1": P1, + "P2": P2, + "F": F_mat, + "X": X, + "x1": x1, + "x2": x2, + } diff --git a/testing/geometry/linalg.py b/testing/geometry/linalg.py new file mode 100644 index 0000000..82ac12a --- /dev/null +++ b/testing/geometry/linalg.py @@ -0,0 +1,91 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch + + +def identity_matrix(batch_size, device, dtype): + r"""Create a batched homogeneous identity matrix.""" + return torch.eye(4, device=device, dtype=dtype).repeat(batch_size, 1, 1) # Nx4x4 + + +def euler_angles_to_rotation_matrix(x, y, z): + r"""Create a rotation matrix from x, y, z angles.""" + assert x.dim() == 1, x.shape + assert x.shape == y.shape == z.shape + ones, zeros = torch.ones_like(x), torch.zeros_like(x) + # the rotation matrix for the x-axis + rx_tmp = [ + ones, + zeros, + zeros, + zeros, + zeros, + torch.cos(x), + -torch.sin(x), + zeros, + zeros, + torch.sin(x), + torch.cos(x), + zeros, + zeros, + zeros, + zeros, + ones, + ] + rx = torch.stack(rx_tmp, dim=-1).view(-1, 4, 4) + # the rotation matrix for the y-axis + ry_tmp = [ + torch.cos(y), + zeros, + torch.sin(y), + zeros, + zeros, + ones, + zeros, + zeros, + -torch.sin(y), + zeros, + torch.cos(y), + zeros, + zeros, + zeros, + zeros, + ones, + ] + ry = torch.stack(ry_tmp, dim=-1).view(-1, 4, 4) + # the rotation matrix for the z-axis + rz_tmp = [ + torch.cos(z), + -torch.sin(z), + zeros, + zeros, + torch.sin(z), + torch.cos(z), + zeros, + zeros, + zeros, + zeros, + ones, + zeros, + zeros, + zeros, + zeros, + ones, + ] + rz = torch.stack(rz_tmp, dim=-1).view(-1, 4, 4) + return torch.matmul(rz, torch.matmul(ry, rx)) # Bx4x4 diff --git a/testing/nerf.py b/testing/nerf.py new file mode 100644 index 0000000..6ef7aea --- /dev/null +++ b/testing/nerf.py @@ -0,0 +1,26 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +# Shim imports for backward compatibility - implementations moved to kornia.geometry.camera.utils +from kornia.geometry.camera.utils import ( + create_pinhole_camera, +) + +# Deprecated alias for backward compatibility - use create_pinhole_camera instead +create_one_camera = create_pinhole_camera diff --git a/testing/overwrite.py b/testing/overwrite.py new file mode 100644 index 0000000..ff9c497 --- /dev/null +++ b/testing/overwrite.py @@ -0,0 +1,34 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from copy import deepcopy +from typing import Any, Optional + + +def default_with_one_parameter_changed(*, default: Optional[dict[str, Any]] = None, **possible_parameters: Any) -> Any: + if default is None: + default = {} + if not isinstance(default, dict): + raise AssertionError(f"default should be a dict not a {type(default)}") + + for parameter_name, possible_values in possible_parameters.items(): + for v in possible_values: + param_set = deepcopy(default) + param_set[parameter_name] = v + yield param_set diff --git a/testing/parametrized_tester.py b/testing/parametrized_tester.py new file mode 100644 index 0000000..67b6564 --- /dev/null +++ b/testing/parametrized_tester.py @@ -0,0 +1,231 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Utilities for automating parametrized test generation across devices and dtypes. + +This module provides decorators and utilities to automatically generate common test methods +(test_smoke, test_cardinality, test_gradcheck) parametrized across devices and dtypes. + +Example: + >>> @parametrized_test( + ... smoke_inputs=lambda device, dtype: (tensor([1.0], device=device, dtype=dtype),), + ... cardinality_tests=[ + ... {"inputs": lambda device, dtype: (tensor([1.0], device=device, dtype=dtype),), + ... "expected_shape": (1,)} + ... ], + ... ) + ... class TestMyFunction(BaseTester): + ... def setup_method(self): + ... self.func = my_function +""" + +from __future__ import annotations + +import warnings +from typing import Any, Callable, Optional, Union + +import torch + +Dtype = Union[torch.dtype, None] +Tensor = torch.Tensor + + +def parametrized_test( + smoke_inputs: Optional[Callable[[torch.device, Dtype], tuple[Any, ...]]] = None, + cardinality_tests: Optional[list[dict[str, Any]]] = None, + gradcheck_inputs: Optional[Callable[[torch.device], tuple[Any, ...]]] = None, +) -> Callable[[type], type]: + """Decorator to automatically generate parametrized test methods. + + Generates test_smoke, test_cardinality, and test_gradcheck methods that are + automatically parametrized across devices and dtypes. + + Args: + smoke_inputs: Callable that takes (device, dtype) and returns input arguments tuple + for smoke testing. If provided, generates test_smoke method. + cardinality_tests: List of dicts with 'inputs' (callable) and 'expected_shape' keys. + 'inputs' callable takes (device, dtype) and returns inputs tuple. + If provided, generates test_cardinality method. + gradcheck_inputs: Callable that takes (device,) and returns input arguments tuple + for gradcheck. If provided, generates test_gradcheck method. + + Returns: + Decorator function that adds parametrized test methods to the test class. + + Example: + >>> @parametrized_test( + ... smoke_inputs=lambda dev, dtype: (torch.randn(2, 3, device=dev, dtype=dtype),), + ... cardinality_tests=[ + ... {"inputs": lambda dev, dtype: (torch.randn(2, 3, device=dev, dtype=dtype),), + ... "expected_shape": torch.Size([2, 3])} + ... ], + ... gradcheck_inputs=lambda dev: (torch.randn(2, 3, device=dev, requires_grad=True, dtype=torch.float64),), + ... ) + ... class MyTestClass(BaseTester): + ... def setup_method(self): + ... self.func = some_function + """ + + def decorator(cls: type) -> type: + # Check if class has a func or function_under_test attribute + if not hasattr(cls, "func") and not hasattr(cls, "function_under_test"): + + def setup_method_wrapper(self): + """Default setup that expects subclass to define func or function_under_test.""" + if not hasattr(self, "func") and not hasattr(self, "function_under_test"): + raise NotImplementedError( + f"{cls.__name__} must define either 'func' or 'function_under_test' " + "attribute or override setup_method()" + ) + + if not hasattr(cls, "setup_method"): + cls.setup_method = setup_method_wrapper + + _generate_test_smoke(cls, smoke_inputs) + _generate_test_cardinality(cls, cardinality_tests) + _generate_test_gradcheck(cls, gradcheck_inputs) + + return cls + + return decorator + + +def _generate_test_smoke( + cls: type, + smoke_inputs: Optional[Callable[[torch.device, Dtype], tuple[Any, ...]]] = None, +) -> None: + """Generate test_smoke method if smoke_inputs provided.""" + if smoke_inputs is None: + return + + # Warn if the method already exists (but don't fail, as parent classes may define it) + if "test_smoke" in cls.__dict__: + warnings.warn( + f"{cls.__name__} already defines 'test_smoke' method. " + "The @parametrized_test decorator will overwrite it. " + "Remove the existing method or the decorator parameter.", + UserWarning, + stacklevel=2, + ) + + def test_smoke(self, device: torch.device, dtype: Dtype) -> None: + """Smoke test: verify function runs with provided inputs.""" + func = getattr(self, "func", None) or getattr(self, "function_under_test", None) + if func is None: + raise NotImplementedError(f"{cls.__name__} must define 'func' or 'function_under_test'") + + inputs = smoke_inputs(device, dtype) + try: + func(*inputs) + except Exception as e: + raise AssertionError(f"Smoke test failed: {e}") from e + + cls.test_smoke = test_smoke + + +def _generate_test_cardinality( + cls: type, + cardinality_tests: Optional[list[dict[str, Any]]] = None, +) -> None: + """Generate test_cardinality method if cardinality_tests provided.""" + if cardinality_tests is None: + return + + # Warn if the method already exists (but don't fail, as parent classes may define it) + if "test_cardinality" in cls.__dict__: + warnings.warn( + f"{cls.__name__} already defines 'test_cardinality' method. " + "The @parametrized_test decorator will overwrite it. " + "Remove the existing method or the decorator parameter.", + UserWarning, + stacklevel=2, + ) + + def test_cardinality(self, device: torch.device, dtype: Dtype) -> None: + """Cardinality test: verify output shape matches expected shape.""" + func = getattr(self, "func", None) or getattr(self, "function_under_test", None) + if func is None: + raise NotImplementedError(f"{cls.__name__} must define 'func' or 'function_under_test'") + + for i, test_case in enumerate(cardinality_tests): + inputs = test_case["inputs"](device, dtype) + expected_shape = test_case["expected_shape"] + + try: + output = func(*inputs) + except Exception as e: + raise AssertionError(f"Cardinality test {i} failed to execute: {e}") from e + + _check_output_shape(cls, output, expected_shape, i) + + cls.test_cardinality = test_cardinality + + +def _check_output_shape( + cls: type, + output: Any, + expected_shape: Any, + test_case_idx: int, +) -> None: + """Check if output shape matches expected shape.""" + if isinstance(output, Tensor): + actual_shape = output.shape + assert actual_shape == expected_shape, ( + f"Test case {test_case_idx}: Expected shape {expected_shape}, got {actual_shape}" + ) + elif isinstance(output, (tuple, list)): + for j, out in enumerate(output): + if isinstance(out, Tensor): + actual_shape = out.shape + expected = expected_shape[j] if isinstance(expected_shape, (tuple, list)) else expected_shape + assert actual_shape == expected, ( + f"Test case {test_case_idx}, output {j}: Expected shape {expected}, got {actual_shape}" + ) + + +def _generate_test_gradcheck( + cls: type, + gradcheck_inputs: Optional[Callable[[torch.device], tuple[Any, ...]]] = None, +) -> None: + """Generate test_gradcheck method if gradcheck_inputs provided.""" + if gradcheck_inputs is None: + return + + # Warn if the method already exists (but don't fail, as parent classes may define it) + if "test_gradcheck" in cls.__dict__: + warnings.warn( + f"{cls.__name__} already defines 'test_gradcheck' method. " + "The @parametrized_test decorator will overwrite it. " + "Remove the existing method or the decorator parameter.", + UserWarning, + stacklevel=2, + ) + + def test_gradcheck(self, device: torch.device) -> None: + """Gradcheck test: verify gradient computation.""" + func = getattr(self, "func", None) or getattr(self, "function_under_test", None) + if func is None: + raise NotImplementedError(f"{cls.__name__} must define 'func' or 'function_under_test'") + + inputs = gradcheck_inputs(device) + try: + result = self.gradcheck(func, inputs) + assert result, "Gradcheck failed" + except Exception as e: + raise AssertionError(f"Gradcheck test failed: {e}") from e + + cls.test_gradcheck = test_gradcheck diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..4d273d5 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,16 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/augmentation/_2d/mix/test_patchmix.py b/tests/augmentation/_2d/mix/test_patchmix.py new file mode 100644 index 0000000..83c4498 --- /dev/null +++ b/tests/augmentation/_2d/mix/test_patchmix.py @@ -0,0 +1,58 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.augmentation import PatchMix + + +@pytest.mark.parametrize( + "batch_size,channels,height,width,patch_size", + [ + (4, 3, 32, 32, 8), + (2, 1, 16, 16, 4), + ], +) +def test_patchmix_shape_and_type(batch_size, channels, height, width, patch_size): + aug = PatchMix(alpha=1.0, patch_size=patch_size) + x = torch.rand(batch_size, channels, height, width) + params = aug.generate_parameters(x.shape) + out = aug.apply_transform(x, params, {}) + assert out.shape == x.shape + assert params["mix_pairs"].shape[0] == batch_size + assert params["lam"].shape[0] == batch_size + assert out.dtype == x.dtype + + +def test_patchmix_different_inputs(): + aug = PatchMix(alpha=1.0, patch_size=8) + x = torch.rand(4, 3, 32, 32) + y = torch.rand(4, 3, 32, 32) + params = aug.generate_parameters(x.shape) + out1 = aug.apply_transform(x, params, {}) + out2 = aug.apply_transform(y, params, {}) + assert not torch.allclose(out1, out2) + + +def test_patchmix_grad(): + aug = PatchMix(alpha=1.0, patch_size=8) + x = torch.rand(2, 3, 16, 16, requires_grad=True) + params = aug.generate_parameters(x.shape) + out = aug.apply_transform(x, params, {}) + out.sum().backward() + assert x.grad is not None diff --git a/tests/augmentation/_3d/test_center_crop_3d.py b/tests/augmentation/_3d/test_center_crop_3d.py new file mode 100644 index 0000000..63dc6a4 --- /dev/null +++ b/tests/augmentation/_3d/test_center_crop_3d.py @@ -0,0 +1,45 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch + +from kornia.augmentation import CenterCrop3D + +from testing.base import BaseTester + + +class TestCenterCrop3D(BaseTester): + def test_no_transform(self, device, dtype): + inp = torch.rand(1, 2, 4, 4, 4, device=device, dtype=dtype) + out = CenterCrop3D(2)(inp) + assert out.shape == (1, 2, 2, 2, 2) + + def test_transform(self, device, dtype): + inp = torch.rand(1, 2, 5, 4, 8, device=device, dtype=dtype) + aug = CenterCrop3D(2) + out = aug(inp) + assert out.shape == (1, 2, 2, 2, 2) + assert aug.transform_matrix.shape == (1, 4, 4) + + def test_no_transform_tuple(self, device, dtype): + inp = torch.rand(1, 2, 5, 4, 8, device=device, dtype=dtype) + out = CenterCrop3D((3, 4, 5))(inp) + assert out.shape == (1, 2, 3, 4, 5) + + def test_gradcheck(self, device): + input_tensor = torch.rand(1, 2, 3, 4, 5, device=device, dtype=torch.float64) + self.gradcheck(CenterCrop3D(3), (input_tensor,)) diff --git a/tests/augmentation/_3d/test_random_affine_3d.py b/tests/augmentation/_3d/test_random_affine_3d.py new file mode 100644 index 0000000..41e1a42 --- /dev/null +++ b/tests/augmentation/_3d/test_random_affine_3d.py @@ -0,0 +1,52 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.augmentation import RandomAffine3D + +from testing.base import BaseTester + + +class TestRandomAffine3D(BaseTester): + def test_batch_random_affine_3d(self, device, dtype): + # TODO(jian): cuda and fp64 + if "cuda" in str(device) and dtype == torch.float64: + pytest.skip("AssertionError: assert tensor(False, device='cuda:0')") + + f = RandomAffine3D((0, 0, 0), p=1.0) # No rotation + tensor = torch.tensor( + [[[[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]]]]], device=device, dtype=dtype + ) # 1 x 1 x 1 x 3 x 3 + + expected = torch.tensor( + [[[[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]]]]], device=device, dtype=dtype + ) # 1 x 1 x 1 x 3 x 3 + + expected_transform = torch.tensor( + [[[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) # 1 x 4 x 4 + + tensor = tensor.repeat(5, 3, 1, 1, 1) # 5 x 3 x 3 x 3 x 3 + expected = expected.repeat(5, 3, 1, 1, 1) # 5 x 3 x 3 x 3 x 3 + expected_transform = expected_transform.repeat(5, 1, 1) # 5 x 4 x 4 + + self.assert_close(f(tensor), expected) + self.assert_close(f.transform_matrix, expected_transform) diff --git a/tests/augmentation/_3d/test_random_crop_3d.py b/tests/augmentation/_3d/test_random_crop_3d.py new file mode 100644 index 0000000..9b20d30 --- /dev/null +++ b/tests/augmentation/_3d/test_random_crop_3d.py @@ -0,0 +1,198 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia +from kornia.augmentation import RandomCrop, RandomCrop3D + +from testing.base import BaseTester + + +class TestRandomCrop3D(BaseTester): + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a torch.Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self): + f = RandomCrop3D(size=(2, 3, 4), padding=(0, 1, 2), fill=10, pad_if_needed=False, p=1.0) + repr = ( + "RandomCrop3D(crop_size=(2, 3, 4), padding=(0, 1, 2), fill=10, pad_if_needed=False, " + "padding_mode=constant, resample=BILINEAR, p=1.0, p_batch=1.0, same_on_batch=False, " + "return_transform=None)" + ) + assert str(f) == repr + + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_no_padding(self, batch_size, device, dtype): + torch.manual_seed(42) + input_tensor = torch.tensor( + [ + [ + [ + [ + [0.0, 1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0, 9.0], + [10, 11, 12, 13, 14], + [15, 16, 17, 18, 19], + [20, 21, 22, 23, 24], + ] + ] + ] + ], + device=device, + dtype=dtype, + ).repeat(batch_size, 1, 5, 1, 1) + f = RandomCrop3D(size=(2, 3, 4), padding=None, align_corners=True, p=1.0) + out = f(input_tensor) + if batch_size == 1: + expected = torch.tensor( + [[[[[11, 12, 13, 14], [16, 17, 18, 19], [21, 22, 23, 24]]]]], device=device, dtype=dtype + ).repeat(batch_size, 1, 2, 1, 1) + if batch_size == 2: + expected = torch.tensor( + [ + [ + [ + [ + [6.0000, 7.0000, 8.0000, 9.0000], + [11.0000, 12.0000, 13.0000, 14.0000], + [16.0000, 17.0000, 18.0000, 19.0000], + ], + [ + [6.0000, 7.0000, 8.0000, 9.0000], + [11.0000, 12.0000, 13.0000, 14.0000], + [16.0000, 17.0000, 18.0000, 19.0000], + ], + ] + ], + [ + [ + [ + [11.0000, 12.0000, 13.0000, 14.0000], + [16.0000, 17.0000, 18.0000, 19.0000], + [21.0000, 22.0000, 23.0000, 24.0000], + ], + [ + [11.0000, 12.0000, 13.0000, 14.0000], + [16.0000, 17.0000, 18.0000, 19.0000], + [21.0000, 22.0000, 23.0000, 24.0000], + ], + ] + ], + ], + device=device, + dtype=dtype, + ) + + self.assert_close(out, expected, atol=1e-4, rtol=1e-4) + + def test_same_on_batch(self, device, dtype): + f = RandomCrop3D(size=(2, 3, 4), padding=None, align_corners=True, p=1.0, same_on_batch=True) + input_tensor = ( + torch.eye(6, device=device, dtype=dtype) + .unsqueeze(dim=0) + .unsqueeze(dim=0) + .unsqueeze(dim=0) + .repeat(2, 3, 5, 1, 1) + ) + res = f(input_tensor) + self.assert_close(res[0], res[1]) + + @pytest.mark.parametrize("padding", [1, (1, 1, 1), (1, 1, 1, 1, 1, 1)]) + def test_padding_batch(self, padding, device, dtype): + torch.manual_seed(42) + batch_size = 2 + input_tensor = torch.tensor( + [[[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]]], device=device, dtype=dtype + ).repeat(batch_size, 1, 3, 1, 1) + expected = torch.tensor( + [ + [ + [ + [[0.0, 1.0, 2.0, 10.0], [3.0, 4.0, 5.0, 10.0], [6.0, 7.0, 8.0, 10.0]], + [[0.0, 1.0, 2.0, 10.0], [3.0, 4.0, 5.0, 10.0], [6.0, 7.0, 8.0, 10.0]], + ] + ], + [ + [ + [[3.0, 4.0, 5.0, 10.0], [6.0, 7.0, 8.0, 10.0], [10, 10, 10, 10.0]], + [[3.0, 4.0, 5.0, 10.0], [6.0, 7.0, 8.0, 10.0], [10, 10, 10, 10.0]], + ] + ], + ], + device=device, + dtype=dtype, + ) + f = RandomCrop3D(size=(2, 3, 4), fill=10.0, padding=padding, align_corners=True, p=1.0) + out = f(input_tensor) + + self.assert_close(out, expected, atol=1e-4, rtol=1e-4) + + def test_pad_if_needed(self, device, dtype): + torch.manual_seed(42) + input_tensor = torch.tensor([[[0.0, 1.0, 2.0]]], device=device, dtype=dtype) + expected = torch.tensor( + [ + [ + [ + [[9.0, 9.0, 9.0, 9.0], [9.0, 9.0, 9.0, 9.0], [9.0, 9.0, 9.0, 9.0]], + [[0.0, 1.0, 2.0, 9.0], [9.0, 9.0, 9.0, 9.0], [9.0, 9.0, 9.0, 9.0]], + ] + ] + ], + device=device, + dtype=dtype, + ) + rc = RandomCrop3D(size=(2, 3, 4), pad_if_needed=True, fill=9, align_corners=True, p=1.0) + out = rc(input_tensor) + + self.assert_close(out, expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + torch.manual_seed(0) # for random reproductibility + input_tensor = torch.rand((3, 3, 3), device=device, dtype=torch.float64) # 3 x 3 + self.gradcheck(RandomCrop3D(size=(3, 3, 3), p=1.0), (input_tensor,)) + + @pytest.mark.skip("Need to fix Union type") + def test_jit(self, device, dtype): + # Define script + op = RandomCrop(size=(3, 3), p=1.0).forward + op_script = torch.jit.script(op) + img = torch.ones(1, 1, 5, 6, device=device, dtype=dtype) + + actual = op_script(img) + expected = kornia.geometry.transform.center_crop3d(img) + self.assert_close(actual, expected) + + @pytest.mark.skip("Need to fix Union type") + def test_jit_trace(self, device, dtype): + # Define script + op = RandomCrop(size=(3, 3), p=1.0).forward + op_script = torch.jit.script(op) + # 1. Trace op + img = torch.ones(1, 1, 5, 6, device=device, dtype=dtype) + + op_trace = torch.jit.trace(op_script, (img,)) + + # 2. Generate new input + img = torch.ones(1, 1, 5, 6, device=device, dtype=dtype) + + # 3. Evaluate + actual = op_trace(img) + expected = op(img) + self.assert_close(actual, expected) diff --git a/tests/augmentation/_3d/test_random_depthical_flip_3d.py b/tests/augmentation/_3d/test_random_depthical_flip_3d.py new file mode 100644 index 0000000..d56f430 --- /dev/null +++ b/tests/augmentation/_3d/test_random_depthical_flip_3d.py @@ -0,0 +1,157 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.augmentation import RandomDepthicalFlip3D +from kornia.augmentation.container.augment import AugmentationSequential + +from testing.base import BaseTester + + +class TestRandomDepthicalFlip3D(BaseTester): + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a torch.Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self): + f = RandomDepthicalFlip3D(0.5) + repr = "RandomDepthicalFlip3D(p=0.5, p_batch=1.0, same_on_batch=False, return_transform=None)" + assert str(f) == repr + + def test_random_dflip(self, device, dtype): + f = RandomDepthicalFlip3D(p=1.0) + f1 = RandomDepthicalFlip3D(p=0.0) + + input_tensor = torch.tensor( + [ + [ + [ + [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0]], + [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 2.0]], + ] + ] + ], + device=device, + dtype=dtype, + ) # 2 x 3 x 4 + + expected = torch.tensor( + [ + [ + [ + [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 2.0]], + [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0]], + ] + ] + ], + device=device, + dtype=dtype, + ) # 2 x 3 x 4 + + expected_transform = torch.tensor( + [[[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, -1.0, 1.0], [0.0, 0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) # 4 x 4 + + identity = torch.tensor( + [[[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) # 4 x 4 + + self.assert_close(f(input_tensor), expected) + self.assert_close(f.transform_matrix, expected_transform) + self.assert_close(f1(input_tensor), input_tensor) + self.assert_close(f1.transform_matrix, identity) + + def test_batch_random_dflip(self, device): + f = RandomDepthicalFlip3D(p=1.0) + f1 = RandomDepthicalFlip3D(p=0.0) + + input_tensor = torch.tensor( + [ + [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0]], + [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 2.0]], + ] + ) # 2 x 3 x 4 + + input_tensor = input_tensor.to(device) + + expected = torch.tensor( + [ + [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 2.0]], + [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0]], + ] + ) # 2 x 3 x 4 + expected = expected.to(device) + + expected_transform = torch.tensor( + [[[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, -1.0, 1.0], [0.0, 0.0, 0.0, 1.0]]] + ) # 1 x 4 x 4 + expected_transform = expected_transform.to(device) + + identity = torch.tensor( + [[[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]] + ) # 1 x 4 x 4 + identity = identity.to(device) + + input_tensor = input_tensor.repeat(5, 3, 1, 1, 1) # 5 x 3 x 3 x 3 x 3 + expected = expected.repeat(5, 3, 1, 1, 1) # 5 x 3 x 3 x 3 x 3 + expected_transform = expected_transform.repeat(5, 1, 1) # 5 x 4 x 4 + identity = identity.repeat(5, 1, 1) # 5 x 4 x 4 + + self.assert_close(f(input_tensor), expected) + self.assert_close(f.transform_matrix, expected_transform) + self.assert_close(f1(input_tensor), input_tensor) + self.assert_close(f1.transform_matrix, identity) + + def test_same_on_batch(self, device): + f = RandomDepthicalFlip3D(p=0.5, same_on_batch=True) + input_tensor = torch.eye(3).unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 1, 2, 1, 1) + res = f(input_tensor) + self.assert_close(res[0], res[1]) + + def test_sequential(self, device): + f = AugmentationSequential(RandomDepthicalFlip3D(p=1.0), RandomDepthicalFlip3D(p=1.0)) + + input_tensor = torch.tensor( + [ + [ + [ + [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0]], + [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 2.0]], + ] + ] + ] + ) # 2 x 3 x 4 + input_tensor = input_tensor.to(device) + + expected_transform = torch.tensor( + [[[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, -1.0, 1.0], [0.0, 0.0, 0.0, 1.0]]] + ) # 1 x 4 x 4 + expected_transform = expected_transform.to(device) + + expected_transform_1 = expected_transform @ expected_transform + + self.assert_close(f(input_tensor), input_tensor) + self.assert_close(f.transform_matrix, expected_transform_1) + + def test_gradcheck(self, device): + input_tensor = torch.rand((1, 3, 3)).to(device) # 4 x 4 + self.gradcheck(RandomDepthicalFlip3D(p=1.0), (input_tensor,)) diff --git a/tests/augmentation/_3d/test_random_equalize_3d.py b/tests/augmentation/_3d/test_random_equalize_3d.py new file mode 100644 index 0000000..a62a947 --- /dev/null +++ b/tests/augmentation/_3d/test_random_equalize_3d.py @@ -0,0 +1,99 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia +from kornia.augmentation import RandomEqualize3D + +from testing.base import BaseTester + + +class TestRandomEqualize3D(BaseTester): + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a torch.Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self, device, dtype): + f = RandomEqualize3D(p=0.5) + repr = "RandomEqualize3D(p=0.5, p_batch=1.0, same_on_batch=False, return_transform=None)" + assert str(f) == repr + + def test_random_equalize(self, device, dtype): + f = RandomEqualize3D(p=1.0) + f1 = RandomEqualize3D(p=0.0) + + bs, channels, depth, height, width = 1, 3, 6, 10, 10 + + inputs3d = self.build_input(channels, depth, height, width, bs, device=device, dtype=dtype) + + row_expected = torch.tensor( + [0.0000, 0.11764, 0.2353, 0.3529, 0.4706, 0.5882, 0.7059, 0.8235, 0.9412, 1.0000], + device=device, + dtype=dtype, + ) + expected = self.build_input(channels, depth, height, width, bs=1, row=row_expected, device=device, dtype=dtype) + + identity = kornia.core.ops.eye_like(4, expected) + + self.assert_close(f(inputs3d), expected, rtol=1e-4, atol=1e-4) + self.assert_close(f.transform_matrix, identity, rtol=1e-4, atol=1e-4) + self.assert_close(f1(inputs3d), inputs3d, rtol=1e-4, atol=1e-4) + self.assert_close(f1.transform_matrix, identity, rtol=1e-4, atol=1e-4) + + def test_batch_random_equalize(self, device, dtype): + f = RandomEqualize3D(p=1.0) + f1 = RandomEqualize3D(p=0.0) + + bs, channels, depth, height, width = 2, 3, 6, 10, 10 + + inputs3d = self.build_input(channels, depth, height, width, bs, device=device, dtype=dtype) + + row_expected = torch.tensor([0.0000, 0.11764, 0.2353, 0.3529, 0.4706, 0.5882, 0.7059, 0.8235, 0.9412, 1.0000]) + expected = self.build_input(channels, depth, height, width, bs, row=row_expected, device=device, dtype=dtype) + + identity = kornia.core.ops.eye_like(4, expected) # 2 x 4 x 4 + + self.assert_close(f(inputs3d), expected, rtol=1e-4, atol=1e-4) + self.assert_close(f.transform_matrix, identity, rtol=1e-4, atol=1e-4) + self.assert_close(f1(inputs3d), inputs3d, rtol=1e-4, atol=1e-4) + self.assert_close(f1.transform_matrix, identity, rtol=1e-4, atol=1e-4) + + def test_same_on_batch(self, device, dtype): + f = RandomEqualize3D(p=0.5, same_on_batch=True) + input_tensor = torch.eye(4, device=device, dtype=dtype) + input_tensor = input_tensor.unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 1, 2, 1, 1) + res = f(input_tensor) + self.assert_close(res[0], res[1]) + + def test_gradcheck(self, device, dtype): + torch.manual_seed(0) # for random reproductibility + + inputs3d = torch.rand((3, 3, 3), device=device, dtype=dtype) # 3 x 3 x 3 + self.gradcheck(RandomEqualize3D(p=0.5), (inputs3d,)) + + @staticmethod + def build_input(channels, depth, height, width, bs=1, row=None, device="cpu", dtype=torch.float32): + if row is None: + row = torch.arange(width, device=device, dtype=dtype) / float(width) + + channel = torch.stack([row] * height) + image = torch.stack([channel] * channels) + image3d = torch.stack([image] * depth).transpose(0, 1) + batch = torch.stack([image3d] * bs) + + return batch.to(device, dtype) diff --git a/tests/augmentation/_3d/test_random_horizontal_flip_3d.py b/tests/augmentation/_3d/test_random_horizontal_flip_3d.py new file mode 100644 index 0000000..fd2dae5 --- /dev/null +++ b/tests/augmentation/_3d/test_random_horizontal_flip_3d.py @@ -0,0 +1,127 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.augmentation import ( + RandomHorizontalFlip3D, +) +from kornia.augmentation.container.augment import AugmentationSequential + +from testing.base import BaseTester + + +class TestRandomHorizontalFlip3D(BaseTester): + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a torch.Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self, device): + f = RandomHorizontalFlip3D(0.5) + repr = "RandomHorizontalFlip3D(p=0.5, p_batch=1.0, same_on_batch=False, return_transform=None)" + assert str(f) == repr + + def test_random_hflip(self, device): + f = RandomHorizontalFlip3D(p=1.0, keepdim=True) + f1 = RandomHorizontalFlip3D(p=0.0, keepdim=True) + + input_tensor = torch.tensor( + [ + [ + [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 1.0, 2.0]], + [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 1.0, 2.0]], + ] + ], + device=device, + ) # 1 x 2 x 3 x 4 + + expected = torch.tensor( + [ + [ + [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [2.0, 1.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [2.0, 1.0, 0.0, 0.0]], + ] + ], + device=device, + ) # 1 x 2 x 3 x 4 + + expected_transform = torch.tensor( + [[[-1.0, 0.0, 0.0, 3.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]], device=device + ) # 1 x 4 x 4 + + identity = torch.tensor( + [[[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]], device=device + ) # 1 x 4 x 4 + + self.assert_close(f(input_tensor), expected) + self.assert_close(f.transform_matrix, expected_transform) + self.assert_close(f1(input_tensor), input_tensor) + self.assert_close(f1.transform_matrix, identity) + + def test_batch_random_hflip(self, device): + f = RandomHorizontalFlip3D(p=1.0) + + input_tensor = torch.tensor([[[[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]]]]]) # 1 x 1 x 1 x 3 x 3 + input_tensor = input_tensor.to(device) + + expected = torch.tensor([[[[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [1.0, 1.0, 0.0]]]]]) # 1 x 1 x 1 x 3 x 3 + expected = expected.to(device) + + expected_transform = torch.tensor( + [[[-1.0, 0.0, 0.0, 2.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]] + ) # 1 x 4 x 4 + expected_transform = expected_transform.to(device) + + identity = torch.tensor( + [[[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]] + ) # 1 x 4 x 4 + identity = identity.to(device) + + input_tensor = input_tensor.repeat(5, 3, 1, 1, 1) # 5 x 3 x 3 x 3 x 3 + expected = expected.repeat(5, 3, 1, 1, 1) # 5 x 3 x 3 x 3 x 3 + expected_transform = expected_transform.repeat(5, 1, 1) # 5 x 4 x 4 + identity = identity.repeat(5, 1, 1) # 5 x 4 x 4 + + self.assert_close(f(input_tensor), expected) + self.assert_close(f.transform_matrix, expected_transform) + + def test_same_on_batch(self, device): + f = RandomHorizontalFlip3D(p=0.5, same_on_batch=True) + input_tensor = torch.eye(3, device=device).unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 1, 1, 1, 1) + res = f(input_tensor) + self.assert_close(res[0], res[1]) + + def test_sequential(self, device): + f = AugmentationSequential(RandomHorizontalFlip3D(p=1.0), RandomHorizontalFlip3D(p=1.0)) + + input_tensor = torch.tensor([[[[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]]]]]) # 1 x 1 x 1 x 3 x 3 + input_tensor = input_tensor.to(device) + + expected_transform = torch.tensor( + [[[-1.0, 0.0, 0.0, 2.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]] + ) # 1 x 4 x 4 + expected_transform = expected_transform.to(device) + + expected_transform_1 = expected_transform @ expected_transform + expected_transform_1 = expected_transform_1.to(device) + + self.assert_close(f(input_tensor), input_tensor) + self.assert_close(f.transform_matrix, expected_transform_1) + + def test_gradcheck(self, device): + input_tensor = torch.rand((1, 3, 3), dtype=torch.float64, device=device) # 3 x 3 + self.gradcheck(RandomHorizontalFlip3D(p=1.0), (input_tensor,)) diff --git a/tests/augmentation/_3d/test_random_rotation_3d.py b/tests/augmentation/_3d/test_random_rotation_3d.py new file mode 100644 index 0000000..9927a4d --- /dev/null +++ b/tests/augmentation/_3d/test_random_rotation_3d.py @@ -0,0 +1,201 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.augmentation import RandomRotation3D +from kornia.augmentation.container.augment import AugmentationSequential + +from testing.base import BaseTester + + +class TestRandomRotation3D(BaseTester): + torch.manual_seed(0) # for random reproductibility + + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a torch.Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self): + f = RandomRotation3D(degrees=45.5) + repr = ( + """RandomRotation3D(degrees=tensor([[-45.5000, 45.5000], + [-45.5000, 45.5000], + [-45.5000, 45.5000]]), resample=BILINEAR, align_corners=False, p=0.5, """ + """p_batch=1.0, same_on_batch=False, return_transform=None)""" + ) + assert str(f) == repr + + def test_random_rotation(self, device, dtype): + # This is included in doctest + torch.manual_seed(0) # for random reproductibility + + f = RandomRotation3D(degrees=45.0) + + input_tensor = torch.tensor( + [ + [[1.0, 0.0, 0.0, 2.0], [0.0, 0.0, 0.0, 0.0], [0.0, 1.0, 2.0, 0.0], [0.0, 0.0, 1.0, 2.0]], + [[1.0, 0.0, 0.0, 2.0], [0.0, 0.0, 0.0, 0.0], [0.0, 1.0, 2.0, 0.0], [0.0, 0.0, 1.0, 2.0]], + [[1.0, 0.0, 0.0, 2.0], [0.0, 0.0, 0.0, 0.0], [0.0, 1.0, 2.0, 0.0], [0.0, 0.0, 1.0, 2.0]], + ], + device=device, + dtype=dtype, + ) # 3 x 4 x 4 + + expected = torch.tensor( + [ + [ + [ + [ + [0.0000, 0.0000, 0.6810, 0.5250], + [0.5052, 0.0000, 0.0000, 0.0613], + [0.1159, 0.1072, 0.5324, 0.0870], + [0.0000, 0.0000, 0.1927, 0.0000], + ], + [ + [0.0000, 0.1683, 0.6963, 0.1131], + [0.0566, 0.0000, 0.5215, 0.2796], + [0.0694, 0.6039, 1.4519, 1.1240], + [0.0000, 0.1325, 0.1542, 0.2510], + ], + [ + [0.0000, 0.2054, 0.0000, 0.0000], + [0.0026, 0.6088, 0.7358, 0.2319], + [0.1261, 1.0830, 1.3687, 1.4940], + [0.0000, 0.0416, 0.2012, 0.3124], + ], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected_transform = torch.tensor( + [ + [ + [0.6523, 0.3666, -0.6635, 0.6352], + [-0.6185, 0.7634, -0.1862, 1.4689], + [0.4382, 0.5318, 0.7247, -1.1797], + [0.0000, 0.0000, 0.0000, 1.0000], + ] + ], + device=device, + dtype=dtype, + ) + + out = f(input_tensor) + atol = 5e-3 if (device.type == "cuda" and dtype == torch.float32) else 1e-4 + rtol = 1e-3 if (device.type == "cuda" and dtype == torch.float32) else 1e-6 + self.assert_close(out, expected, rtol=rtol, atol=atol) + self.assert_close(f.transform_matrix, expected_transform, rtol=rtol, atol=atol) + + def test_batch_random_rotation(self, device, dtype): + # Verifies per-element random rotation invariants on a batch input: + # p=1.0 forces every element to be transformed so the assertions don't depend on + # how the underlying RNG happens to roll the per-element apply mask. + torch.manual_seed(24) + f = RandomRotation3D(degrees=45.0, p=1.0, same_on_batch=False) + + input_tensor = torch.tensor( + [ + [ + [[1.0, 0.0, 0.0, 2.0], [0.0, 0.0, 0.0, 0.0], [0.0, 1.0, 2.0, 0.0], [0.0, 0.0, 1.0, 2.0]], + [[1.0, 0.0, 0.0, 2.0], [0.0, 0.0, 0.0, 0.0], [0.0, 1.0, 2.0, 0.0], [0.0, 0.0, 1.0, 2.0]], + [[1.0, 0.0, 0.0, 2.0], [0.0, 0.0, 0.0, 0.0], [0.0, 1.0, 2.0, 0.0], [0.0, 0.0, 1.0, 2.0]], + ] + ], + device=device, + dtype=dtype, + ).repeat(2, 1, 1, 1, 1) # (2, 1, 3, 4, 4) + + atol = 5e-3 if (device.type == "cuda" and dtype == torch.float32) else 1e-4 + rtol = 1e-3 if (device.type == "cuda" and dtype == torch.float32) else 1e-6 + + torch.manual_seed(24) + out = f(input_tensor) + transform = f.transform_matrix + + # 1. Shape preservation + assert out.shape == input_tensor.shape + assert transform.shape == (2, 4, 4) + + # 2. Per-element variation: the two batch elements were rotated by *different* angles + assert not torch.allclose(transform[0], transform[1], atol=1e-3) + + # 3. Valid rigid 3D transforms: rotation block is orthogonal, bottom row is [0,0,0,1] + bottom_row = torch.tensor([0.0, 0.0, 0.0, 1.0], device=device, dtype=dtype).expand(2, 4) + self.assert_close(transform[:, 3, :], bottom_row, rtol=rtol, atol=atol) + rot = transform[:, :3, :3] + rot_rt = rot @ rot.transpose(-1, -2) + eye = torch.eye(3, device=device, dtype=dtype).expand(2, 3, 3) + self.assert_close(rot_rt, eye, rtol=rtol, atol=atol) + + # 4. Reproducibility under fixed seed + torch.manual_seed(24) + out2 = f(input_tensor) + self.assert_close(out, out2, rtol=rtol, atol=atol) + self.assert_close(transform, f.transform_matrix, rtol=rtol, atol=atol) + + def test_same_on_batch(self, device, dtype): + f = RandomRotation3D(degrees=40, same_on_batch=True) + input_tensor = torch.eye(6, device=device, dtype=dtype).unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 3, 6, 1, 1) + res = f(input_tensor) + self.assert_close(res[0], res[1]) + + def test_sequential(self, device, dtype): + # Verifies AugmentationSequential composition for 3D rotations: + # Sequential(aug_a, aug_b)(x) == aug_b(aug_a(x)) + # and the composed transform matrix equals aug_b.transform_matrix @ aug_a.transform_matrix. + # p=1.0 forces both rotations to fire so the assertion is independent of the per-element + # apply mask. Two distinct instances are required because nn.Module dedupes children + # passed by the same instance. + atol = 5e-3 if (device.type == "cuda" and dtype == torch.float32) else 1e-4 + rtol = 1e-3 if (device.type == "cuda" and dtype == torch.float32) else 1e-6 + + aug_a = RandomRotation3D(degrees=torch.tensor([-45.0, 90.0]), p=1.0) + aug_b = RandomRotation3D(degrees=10.4, p=1.0) + f = AugmentationSequential(aug_a, aug_b) + + input_tensor = torch.tensor( + [ + [[1.0, 0.0, 0.0, 2.0], [0.0, 0.0, 0.0, 0.0], [0.0, 1.0, 2.0, 0.0], [0.0, 0.0, 1.0, 2.0]], + [[1.0, 0.0, 0.0, 2.0], [0.0, 0.0, 0.0, 0.0], [0.0, 1.0, 2.0, 0.0], [0.0, 0.0, 1.0, 2.0]], + [[1.0, 0.0, 0.0, 2.0], [0.0, 0.0, 0.0, 0.0], [0.0, 1.0, 2.0, 0.0], [0.0, 0.0, 1.0, 2.0]], + ], + device=device, + dtype=dtype, + ) # 3 x 4 x 4 + + torch.manual_seed(24) + out_seq = f(input_tensor) + transform_seq = f.transform_matrix + + torch.manual_seed(24) + out_a = aug_a(input_tensor) + transform_a = aug_a.transform_matrix + out_manual = aug_b(out_a) + transform_manual = aug_b.transform_matrix @ transform_a + + self.assert_close(out_seq, out_manual, rtol=rtol, atol=atol) + self.assert_close(transform_seq, transform_manual, rtol=rtol, atol=atol) + + def test_gradcheck(self, device): + torch.manual_seed(0) # for random reproductibility + + input_tensor = torch.rand((3, 3, 3), device=device, dtype=torch.float64) # 3 x 3 x 3 + self.gradcheck(RandomRotation3D(degrees=(15.0, 15.0), p=1.0), (input_tensor,)) diff --git a/tests/augmentation/_3d/test_random_vertical_flip_3d.py b/tests/augmentation/_3d/test_random_vertical_flip_3d.py new file mode 100644 index 0000000..ab4a003 --- /dev/null +++ b/tests/augmentation/_3d/test_random_vertical_flip_3d.py @@ -0,0 +1,139 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.augmentation import ( + RandomVerticalFlip3D, +) +from kornia.augmentation.container.augment import AugmentationSequential + +from testing.base import BaseTester + + +class TestRandomVerticalFlip3D(BaseTester): + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a torch.Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self): + f = RandomVerticalFlip3D(0.5) + repr = "RandomVerticalFlip3D(p=0.5, p_batch=1.0, same_on_batch=False, return_transform=None)" + assert str(f) == repr + + def test_random_vflip(self, device, dtype): + f = RandomVerticalFlip3D(p=1.0) + f1 = RandomVerticalFlip3D(p=0.0) + + input_tensor = torch.tensor( + [ + [ + [ + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]], + ] + ] + ], + device=device, + dtype=dtype, + ) # 1 x 1 x 2 x 3 x 3 + + expected = torch.tensor( + [ + [ + [ + [[0.0, 1.0, 1.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 1.0, 1.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + ] + ] + ], + device=device, + dtype=dtype, + ) # 1 x 1 x 2 x 3 x 3 + + expected_transform = torch.tensor( + [[[1.0, 0.0, 0.0, 0.0], [0.0, -1.0, 0.0, 2.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) # 4 x 4 + + identity = torch.tensor( + [[[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) # 1 x 4 x 4 + + self.assert_close(f(input_tensor), expected) + self.assert_close(f.transform_matrix, expected_transform) + self.assert_close(f1(input_tensor), input_tensor) + self.assert_close(f1.transform_matrix, identity) + + def test_batch_random_vflip(self, device): + f = RandomVerticalFlip3D(p=1.0) + f1 = RandomVerticalFlip3D(p=0.0) + + input_tensor = torch.tensor([[[[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]]]]]) # 1 x 1 x 1 x 3 x 3 + input_tensor = input_tensor.to(device) + + expected = torch.tensor([[[[[0.0, 1.0, 1.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]]]) # 1 x 1 x 1 x 3 x 3 + expected = expected.to(device) + + expected_transform = torch.tensor( + [[[1.0, 0.0, 0.0, 0.0], [0.0, -1.0, 0.0, 2.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]] + ) # 1 x 4 x 4 + expected_transform = expected_transform.to(device) + + identity = torch.tensor( + [[[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]] + ) # 1 x 4 x 4 + identity = identity.to(device) + + input_tensor = input_tensor.repeat(5, 3, 1, 1, 1) # 5 x 3 x 3 x 3 x 3 + expected = expected.repeat(5, 3, 1, 1, 1) # 5 x 3 x 3 x 3 x 3 + expected_transform = expected_transform.repeat(5, 1, 1) # 5 x 4 x 4 + identity = identity.repeat(5, 1, 1) # 5 x 4 x 4 + + self.assert_close(f(input_tensor), expected) + self.assert_close(f.transform_matrix, expected_transform) + self.assert_close(f1(input_tensor), input_tensor) + self.assert_close(f1.transform_matrix, identity) + + def test_same_on_batch(self, device): + f = RandomVerticalFlip3D(p=0.5, same_on_batch=True) + input_tensor = torch.eye(3).unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 1, 1, 1, 1) + res = f(input_tensor) + self.assert_close(res[0], res[1]) + + def test_sequential(self, device): + f = AugmentationSequential(RandomVerticalFlip3D(p=1.0), RandomVerticalFlip3D(p=1.0)) + + input_tensor = torch.tensor([[[[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]]]]]) # 1 x 1 x 1 x 4 x 4 + input_tensor = input_tensor.to(device) + + expected_transform = torch.tensor( + [[[1.0, 0.0, 0.0, 0.0], [0.0, -1.0, 0.0, 2.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]] + ) # 1 x 4 x 4 + expected_transform = expected_transform.to(device) + + expected_transform_1 = expected_transform @ expected_transform + + self.assert_close(f(input_tensor), input_tensor) + self.assert_close(f.transform_matrix, expected_transform_1) + + def test_gradcheck(self, device): + input_tensor = torch.rand((1, 3, 3)).to(device) # 4 x 4 + self.gradcheck(RandomVerticalFlip3D(p=1.0), (input_tensor,)) diff --git a/tests/augmentation/container/test_augmentation_sequential.py b/tests/augmentation/container/test_augmentation_sequential.py new file mode 100644 index 0000000..d6205b4 --- /dev/null +++ b/tests/augmentation/container/test_augmentation_sequential.py @@ -0,0 +1,573 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from functools import partial +from unittest.mock import patch + +import pytest +import torch + +import kornia +import kornia.augmentation as K +from kornia.augmentation.container.base import ParamItem +from kornia.constants import BorderType +from kornia.geometry.bbox import bbox_to_mask + +from testing.augmentation.utils import reproducibility_test +from testing.base import assert_close + + +class TestAugmentationSequential: + @pytest.mark.parametrize( + "data_keys", ["input", "image", ["mask", "input"], ["input", "bbox_yxyx"], [0, 10], [BorderType.REFLECT]] + ) + @pytest.mark.parametrize("augmentation_list", [K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0)]) + def test_exception(self, augmentation_list, data_keys, device, dtype): + with pytest.raises(Exception): # AssertError and NotImplementedError + K.AugmentationSequential(augmentation_list, data_keys=data_keys) + + @pytest.mark.slow + @pytest.mark.parametrize("same_on_batch", [True, False]) + @pytest.mark.parametrize("random_apply", [1, (2, 2), (1, 2), (2,), 10, True, False]) + @pytest.mark.parametrize("inp", [torch.randn(1, 3, 1000, 500), torch.randn(3, 1000, 500)]) + def test_mixup(self, inp, random_apply, same_on_batch, device, dtype): + inp = torch.as_tensor(inp, device=device, dtype=dtype) + aug = K.AugmentationSequential( + K.ImageSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0)), + K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + K.RandomAffine(360, p=1.0), + K.RandomMixUpV2(p=1.0), + data_keys=["input"], + random_apply=random_apply, + same_on_batch=same_on_batch, + ) + out = aug(inp) + assert out.shape[-3:] == inp.shape[-3:] + reproducibility_test(inp, aug) + + def test_mixup_cutmix_only(self, device, dtype): + mixup = K.RandomMixUpV2(p=1.0, data_keys=["input"]) + cutmix = K.RandomCutMixV2(p=1.0, data_keys=["input"]) + aug = K.AugmentationSequential( + mixup, + cutmix, + data_keys=["input"], + random_apply=1, + ) + + input = torch.randn(2, 3, 224, 224, device=device, dtype=dtype) + + out_input = aug(input) + + assert out_input.shape == input.shape + + def test_video(self, device, dtype): + input = torch.randn(2, 3, 5, 6, device=device, dtype=dtype)[None] + bbox = torch.tensor([[[1.0, 1.0], [2.0, 1.0], [2.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype).expand( + 2, 1, -1, -1 + )[None] + points = torch.tensor([[[1.0, 1.0]]], device=device, dtype=dtype).expand(2, -1, -1)[None] + aug_list = K.AugmentationSequential( + K.VideoSequential( + kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), kornia.augmentation.RandomAffine(360, p=1.0) + ), + data_keys=["input", "mask", "bbox", "keypoints"], + ) + out = aug_list(input, input, bbox, points) + assert out[0].shape == input.shape + assert out[1].shape == input.shape + assert out[2].shape == bbox.shape + assert out[3].shape == points.shape + + out_inv = aug_list.inverse(*out) + assert out_inv[0].shape == input.shape + assert out_inv[1].shape == input.shape + assert out_inv[2].shape == bbox.shape + assert out_inv[3].shape == points.shape + + def test_3d_augmentations(self, device, dtype): + input = torch.randn(2, 2, 3, 5, 6, device=device, dtype=dtype) + aug_list = K.AugmentationSequential( + K.RandomAffine3D(360.0, p=1.0), K.RandomHorizontalFlip3D(p=1.0), data_keys=["input"] + ) + out = aug_list(input) + assert out.shape == input.shape + + @pytest.mark.parametrize("image_dtype", [torch.float16, torch.float32, torch.float64, torch.bfloat16]) + def test_mixed_image_bbox_dtypes(self, device, image_dtype): + # Regression test for https://github.com/kornia/kornia/issues/3705 and #3706: + # bbox stays in fp32 while the image uses a half/double compute dtype. + torch.manual_seed(0) + img = torch.rand(2, 3, 32, 32, device=device, dtype=image_dtype) + bb = torch.tensor( + [[[[4.0, 4.0], [12.0, 4.0], [12.0, 12.0], [4.0, 12.0]]]] * 2, + device=device, + dtype=torch.float32, + ) + aug = K.AugmentationSequential(K.RandomAffine(degrees=10, p=1.0), data_keys=["image", "bbox"]) + out_img, out_bb = aug(img, bb) + assert out_img.dtype == image_dtype + assert out_bb.dtype == torch.float32 + assert out_img.shape == img.shape + assert out_bb.shape == bb.shape + + def test_random_flips(self, device, dtype): + inp = torch.randn(1, 3, 510, 1020, device=device, dtype=dtype) + bbox = torch.tensor([[[355, 10], [660, 10], [660, 250], [355, 250]]], device=device, dtype=dtype) + + expected_bbox_vertical_flip = torch.tensor( + [[[355, 259], [660, 259], [660, 499], [355, 499]]], device=device, dtype=dtype + ) + expected_bbox_horizontal_flip = torch.tensor( + [[[359, 10], [664, 10], [664, 250], [359, 250]]], device=device, dtype=dtype + ) + + aug_ver = K.AugmentationSequential( + K.RandomVerticalFlip(p=1.0), data_keys=["input", "bbox"], same_on_batch=False + ) + + aug_hor = K.AugmentationSequential( + K.RandomHorizontalFlip(p=1.0), data_keys=["image", "bbox"], same_on_batch=False + ) + + out_ver = aug_ver(inp.clone(), bbox.clone()) + out_hor = aug_hor(inp.clone(), bbox.clone()) + + assert_close(out_ver[1], expected_bbox_vertical_flip) + assert_close(out_hor[1], expected_bbox_horizontal_flip) + + def test_with_mosaic(self, device, dtype): + width, height = 100, 100 + crop_width, crop_height = 3, 3 + input = torch.randn(3, 3, width, height, device=device, dtype=dtype) + bbox = torch.tensor( + [[[1.0, 1.0, 2.0, 2.0], [0.0, 0.0, 1.0, 2.0], [0.0, 0.0, 2.0, 1.0]]], device=device, dtype=dtype + ).expand(3, -1, -1) + aug = K.AugmentationSequential( + K.RandomCrop((crop_width, crop_height), padding=1, cropping_mode="resample", fill=0), + K.RandomHorizontalFlip(p=1.0), + K.RandomMosaic(p=1.0), + data_keys=["input", "bbox_xyxy"], + ) + + reproducibility_test((input, bbox), aug) + + def test_random_crops_and_flips(self, device, dtype): + width, height = 100, 100 + crop_width, crop_height = 3, 3 + input = torch.randn(3, 3, width, height, device=device, dtype=dtype) + bbox = torch.tensor( + [[[1.0, 1.0, 2.0, 2.0], [0.0, 0.0, 1.0, 2.0], [0.0, 0.0, 2.0, 1.0]]], device=device, dtype=dtype + ).expand(3, -1, -1) + aug = K.AugmentationSequential( + K.RandomCrop((crop_width, crop_height), padding=1, cropping_mode="resample", fill=0), + K.RandomHorizontalFlip(p=1.0), + data_keys=["input", "bbox_xyxy"], + ) + + reproducibility_test((input, bbox), aug) + + _params = aug.forward_parameters(input.shape) + # specifying the crop locations allows us to compute by hand the expected outputs + crop_locations = torch.tensor( + [[1.0, 2.0], [1.0, 1.0], [2.0, 0.0]], + device=_params[0].data["src"].device, + dtype=_params[0].data["src"].dtype, + ) + crops = crop_locations.expand(4, -1, -1).permute(1, 0, 2).clone() + crops[:, 1:3, 0] += crop_width - 1 + crops[:, 2:4, 1] += crop_height - 1 + _params[0].data["src"] = crops + + # expected output bboxes after crop for specified crop locations and crop size (3,3) + expected_out_bbox = torch.tensor( + [ + [[1.0, 0.0, 2.0, 1.0], [0.0, -1.0, 1.0, 1.0], [0.0, -1.0, 2.0, 0.0]], + [[1.0, 1.0, 2.0, 2.0], [0.0, 0.0, 1.0, 2.0], [0.0, 0.0, 2.0, 1.0]], + [[0.0, 2.0, 1.0, 3.0], [-1.0, 1.0, 0.0, 3.0], [-1.0, 1.0, 1.0, 2.0]], + ], + device=device, + dtype=dtype, + ) + # horizontally flip boxes based on crop width + xmins = expected_out_bbox[..., 0].clone() + xmaxs = expected_out_bbox[..., 2].clone() + expected_out_bbox[..., 0] = crop_width - xmaxs - 1 + expected_out_bbox[..., 2] = crop_width - xmins - 1 + + out = aug(input, bbox, params=_params) + assert out[1].shape == bbox.shape + assert_close(out[1], expected_out_bbox, atol=1e-4, rtol=1e-4) + + out_inv = aug.inverse(*out) + assert out_inv[1].shape == bbox.shape + assert_close(out_inv[1], bbox, atol=1e-4, rtol=1e-4) + + def test_random_erasing(self, device, dtype): + fill_value = 0.5 + input = torch.randn(3, 3, 100, 100, device=device, dtype=dtype) + aug = K.AugmentationSequential(K.RandomErasing(p=1.0, value=fill_value), data_keys=["image", "mask"]) + + reproducibility_test((input, input), aug) + + out = aug(input, input) + assert torch.all(out[1][out[0] == fill_value] == 0.0) + + def test_resize(self, device, dtype): + size = 50 + input = torch.randn(3, 3, 100, 100, device=device, dtype=dtype) + mask = torch.randn(3, 1, 100, 100, device=device, dtype=dtype) + aug = K.AugmentationSequential(K.Resize((size, size), p=1.0), data_keys=["input", "mask"]) + + reproducibility_test((input, mask), aug) + + out = aug(input, mask) + assert out[0].shape == (3, 3, size, size) + assert out[1].shape == (3, 1, size, size) + + def test_random_crops(self, device, dtype): + # Test with relaxed tolerance for platform-specific numerical precision + torch.manual_seed(233) + input = torch.randn(3, 3, 3, 3, device=device, dtype=dtype) + bbox = torch.tensor( + [[[1.0, 1.0, 2.0, 2.0], [0.0, 0.0, 1.0, 2.0], [0.0, 0.0, 2.0, 1.0]]], device=device, dtype=dtype + ).expand(3, -1, -1) + points = torch.tensor([[[0.0, 0.0], [1.0, 1.0]]], device=device, dtype=dtype).expand(3, -1, -1) + aug = K.AugmentationSequential( + K.RandomCrop((3, 3), padding=1, cropping_mode="resample", fill=0), + K.RandomAffine((360.0, 360.0), p=1.0), + data_keys=["input", "mask", "bbox_xyxy", "keypoints"], + extra_args={}, + ) + + reproducibility_test((input, input, bbox, points), aug) + + _params = aug.forward_parameters(input.shape) + # specifying the crops allows us to compute by hand the expected outputs + _params[0].data["src"] = torch.tensor( + [ + [[1.0, 2.0], [3.0, 2.0], [3.0, 4.0], [1.0, 4.0]], + [[1.0, 1.0], [3.0, 1.0], [3.0, 3.0], [1.0, 3.0]], + [[2.0, 0.0], [4.0, 0.0], [4.0, 2.0], [2.0, 2.0]], + ], + device=_params[0].data["src"].device, + dtype=_params[0].data["src"].dtype, + ) + + expected_out_bbox = torch.tensor( + [ + [[1.0, 0.0, 2.0, 1.0], [0.0, -1.0, 1.0, 1.0], [0.0, -1.0, 2.0, 0.0]], + [[1.0, 1.0, 2.0, 2.0], [0.0, 0.0, 1.0, 2.0], [0.0, 0.0, 2.0, 1.0]], + [[0.0, 2.0, 1.0, 3.0], [-1.0, 1.0, 0.0, 3.0], [-1.0, 1.0, 1.0, 2.0]], + ], + device=device, + dtype=dtype, + ) + expected_out_points = torch.tensor( + [[[0.0, -1.0], [1.0, 0.0]], [[0.0, 0.0], [1.0, 1.0]], [[-1.0, 1.0], [0.0, 2.0]]], device=device, dtype=dtype + ) + + out = aug(input, input, bbox, points, params=_params) + assert out[0].shape == (3, 3, 3, 3) + assert_close(out[0], out[1], atol=1e-4, rtol=1e-4) + assert out[2].shape == bbox.shape + assert_close(out[2], expected_out_bbox, atol=1e-3, rtol=1e-3) + assert out[3].shape == points.shape + assert_close(out[3], expected_out_points, atol=1e-4, rtol=1e-4) + + out_inv = aug.inverse(*out) + assert out_inv[0].shape == input.shape + assert_close(out_inv[0], out_inv[1], atol=1e-4, rtol=1e-4) + assert out_inv[2].shape == bbox.shape + assert_close(out_inv[2], bbox, atol=1e-3, rtol=1e-3) + assert out_inv[3].shape == points.shape + assert_close(out_inv[3], points, atol=1e-4, rtol=1e-4) + + def test_random_resized_crop(self, device, dtype): + size = 50 + input = torch.randn(3, 3, 100, 100, device=device, dtype=dtype) + mask = torch.randn(3, 1, 100, 100, device=device, dtype=dtype) + aug = K.AugmentationSequential(K.RandomResizedCrop((size, size), p=1.0), data_keys=["input", "mask"]) + + reproducibility_test((input, mask), aug) + + out = aug(input, mask) + assert out[0].shape == (3, 3, size, size) + assert out[1].shape == (3, 1, size, size) + + @pytest.mark.parametrize( + "bbox", + [ + [ + torch.tensor([[1, 5, 2, 7], [0, 3, 9, 9]]), + torch.tensor([[1, 5, 2, 7], [0, 3, 9, 9], [0, 5, 8, 7]]), + torch.empty((0, 4)), + ], + torch.empty((3, 0, 4)), + torch.tensor([[[1, 5, 2, 7], [0, 3, 9, 9]], [[1, 5, 2, 7], [0, 3, 9, 9]], [[0, 5, 8, 7], [0, 2, 5, 5]]]), + ], + ) + @pytest.mark.parametrize( + "augmentation", [K.RandomCrop((30, 30), padding=1, cropping_mode="resample", fill=0), K.Resize((30, 30))] + ) + def test_bbox(self, bbox, augmentation, device, dtype): + img = torch.rand((3, 3, 10, 10), device=device, dtype=dtype) + if isinstance(bbox, list): + for i, b in enumerate(bbox): + bbox[i] = b.to(device=device, dtype=dtype) + else: + bbox = bbox.to(device=device, dtype=dtype) + + inputs = [img, bbox] + + aug = K.AugmentationSequential(augmentation, data_keys=["input", "bbox_xyxy"]) + + transformed = aug(*inputs) + + assert len(transformed) == len(inputs) + bboxes_transformed = transformed[-1] + assert len(bboxes_transformed) == len(bbox) + assert bboxes_transformed.__class__ == bbox.__class__ + for i in range(len(bbox)): + assert len(bboxes_transformed[i]) == len(bbox[i]) + + def test_class(self, device, dtype): + img = torch.zeros((5, 1, 5, 5)) + labels = torch.randint(0, 10, size=(5, 1)) + aug = K.AugmentationSequential(K.RandomCrop((3, 3), pad_if_needed=True), data_keys=["input", "class"]) + + _, out_labels = aug(img, labels) + assert labels is out_labels + + @pytest.mark.slow + @pytest.mark.parametrize("random_apply", [1, (2, 2), (1, 2), (2,), 10, True, False]) + def test_forward_and_inverse(self, random_apply, device, dtype): + inp = torch.randn(1, 3, 1000, 500, device=device, dtype=dtype) + bbox = torch.tensor([[[355, 10], [660, 10], [660, 250], [355, 250]]], device=device, dtype=dtype) + keypoints = torch.tensor([[[465, 115], [545, 116]]], device=device, dtype=dtype) + mask = bbox_to_mask( + torch.tensor([[[155, 0], [900, 0], [900, 400], [155, 400]]], device=device, dtype=dtype), 1000, 500 + )[:, None] + aug = K.AugmentationSequential( + K.ImageSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0)), + K.AugmentationSequential( + K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + K.RandomAffine(360, p=1.0), + K.RandomAffine(360, p=1.0), + data_keys=["input", "mask", "bbox", "keypoints"], + ), + K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + K.RandomAffine(360, p=1.0), + data_keys=["input", "mask", "bbox", "keypoints"], + random_apply=random_apply, + ) + out = aug(inp, mask, bbox, keypoints) + assert out[0].shape == inp.shape + assert out[1].shape == mask.shape + assert out[2].shape == bbox.shape + assert out[3].shape == keypoints.shape + assert set(out[1].unique().tolist()).issubset(set(mask.unique().tolist())) + + out_inv = aug.inverse(*out) + assert out_inv[0].shape == inp.shape + assert out_inv[1].shape == mask.shape + assert out_inv[2].shape == bbox.shape + assert out_inv[3].shape == keypoints.shape + assert set(out_inv[1].unique().tolist()).issubset(set(mask.unique().tolist())) + + if random_apply is False: + reproducibility_test((inp, mask, bbox, keypoints), aug) + + @pytest.mark.slow + def test_individual_forward_and_inverse(self, device, dtype): + inp = torch.randn(1, 3, 1000, 500, device=device, dtype=dtype) + bbox = torch.tensor([[[[355, 10], [660, 10], [660, 250], [355, 250]]]], device=device, dtype=dtype) + keypoints = torch.tensor([[[465, 115], [545, 116]]], device=device, dtype=dtype) + mask = bbox_to_mask( + torch.tensor([[[155, 0], [900, 0], [900, 400], [155, 400]]], device=device, dtype=dtype), 500, 1000 + )[:, None] + crop_size = (200, 200) + + aug = K.AugmentationSequential( + K.ImageSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0)), + K.AugmentationSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0)), + K.RandomAffine(360, p=1.0), + K.RandomCrop(crop_size, padding=1, cropping_mode="resample", fill=0), + data_keys=["input", "mask", "bbox", "keypoints"], + extra_args={}, + ) + # NOTE: Mask data with nearest not passing reproducibility check under float64. + reproducibility_test((inp, mask, bbox, keypoints), aug) + + out = aug(inp, mask, bbox, keypoints) + assert out[0].shape == (*inp.shape[:2], *crop_size) + assert out[1].shape == (*mask.shape[:2], *crop_size) + assert out[2].shape == bbox.shape + assert out[3].shape == keypoints.shape + + out_inv = aug.inverse(*out) + assert out_inv[0].shape == inp.shape + assert out_inv[1].shape == mask.shape + assert out_inv[2].shape == bbox.shape + assert out_inv[3].shape == keypoints.shape + + aug = K.AugmentationSequential(K.RandomAffine(360, p=1.0)) + assert aug(inp, data_keys=["input"]).shape == inp.shape + aug = K.AugmentationSequential(K.RandomAffine(360, p=1.0)) + assert aug(inp, data_keys=["input"]).shape == inp.shape + assert aug(mask, data_keys=["mask"], params=aug._params).shape == mask.shape + + assert aug.inverse(inp, data_keys=["input"]).shape == inp.shape + assert aug.inverse(bbox, data_keys=["bbox"]).shape == bbox.shape + assert aug.inverse(keypoints, data_keys=["keypoints"]).shape == keypoints.shape + assert aug.inverse(mask, data_keys=["mask"]).shape == mask.shape + + @pytest.mark.slow + @pytest.mark.parametrize("random_apply", [2, (1, 1), (2,), 10, True, False]) + def test_forward_and_inverse_return_transform(self, random_apply, device, dtype): + inp = torch.randn(1, 3, 1000, 500, device=device, dtype=dtype) + bbox = torch.tensor([[[355, 10], [660, 10], [660, 250], [355, 250]]], device=device, dtype=dtype) + keypoints = torch.tensor([[[465, 115], [545, 116]]], device=device, dtype=dtype) + mask = bbox_to_mask( + torch.tensor([[[155, 0], [900, 0], [900, 400], [155, 400]]], device=device, dtype=dtype), 1000, 500 + )[:, None] + aug = K.AugmentationSequential( + K.ImageSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0)), + K.AugmentationSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0)), + K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + K.RandomAffine(360, p=1.0), + data_keys=["input", "mask", "bbox", "keypoints"], + random_apply=random_apply, + extra_args={}, + ) + out = aug(inp, mask, bbox, keypoints) + assert out[0].shape == inp.shape + assert out[1].shape == mask.shape + assert out[2].shape == bbox.shape + assert out[3].shape == keypoints.shape + + reproducibility_test((inp, mask, bbox, keypoints), aug) + + out_inv = aug.inverse(*out) + assert out_inv[0].shape == inp.shape + assert out_inv[1].shape == mask.shape + assert out_inv[2].shape == bbox.shape + assert out_inv[3].shape == keypoints.shape + + @pytest.mark.slow + @pytest.mark.parametrize("random_apply", [1, (2, 2), (1, 2), (2,), 10, True, False]) + def test_inverse_and_forward_return_transform(self, random_apply, device, dtype): + inp = torch.randn(1, 3, 1000, 500, device=device, dtype=dtype) + bbox = torch.tensor([[[355, 10], [660, 10], [660, 250], [355, 250]]], device=device, dtype=dtype) + bbox_2 = [ + # torch.tensor([[[355, 10], [660, 10], [660, 250], [355, 250]]], device=device, dtype=dtype), + torch.tensor( + [[[355, 10], [660, 10], [660, 250], [355, 250]], [[355, 10], [660, 10], [660, 250], [355, 250]]], + device=device, + dtype=dtype, + ) + ] + bbox_wh = torch.tensor([[[30, 40, 100, 100]]], device=device, dtype=dtype) + bbox_wh_2 = [ + # torch.tensor([[30, 40, 100, 100]], device=device, dtype=dtype), + torch.tensor([[30, 40, 100, 100], [30, 40, 100, 100]], device=device, dtype=dtype) + ] + keypoints = torch.tensor([[[465, 115], [545, 116]]], device=device, dtype=dtype) + mask = bbox_to_mask( + torch.tensor([[[155, 0], [900, 0], [900, 400], [155, 400]]], device=device, dtype=dtype), 1000, 500 + )[:, None] + aug = K.AugmentationSequential( + K.ImageSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0)), + K.AugmentationSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0)), + K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + K.RandomAffine(360, p=1.0), + data_keys=["input", "mask", "bbox", "keypoints", "bbox", "BBOX_XYWH", "BBOX_XYWH"], + random_apply=random_apply, + ) + with pytest.raises(Exception): # No parameters available for inversing. + aug.inverse(inp, mask, bbox, keypoints, bbox_2, bbox_wh, bbox_wh_2) + + out = aug(inp, mask, bbox, keypoints, bbox_2, bbox_wh, bbox_wh_2) + assert out[0].shape == inp.shape + assert out[1].shape == mask.shape + assert out[2].shape == bbox.shape + assert out[3].shape == keypoints.shape + + if random_apply is False: + reproducibility_test((inp, mask, bbox, keypoints, bbox_2, bbox_wh, bbox_wh_2), aug) + + @pytest.mark.jit() + @pytest.mark.skip(reason="turn off due to Union Type") + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = K.AugmentationSequential( + K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0), same_on_batch=True + ) + op_jit = torch.jit.script(op) + assert_close(op(img), op_jit(img)) + + @pytest.mark.parametrize("batch_prob", [[True, True], [False, True], [False, False]]) + @pytest.mark.parametrize("box", ["bbox", "bbox_xyxy", "bbox_xywh"]) + def test_autocast(self, batch_prob, box, device, dtype): + if not hasattr(torch, "autocast"): + pytest.skip("PyTorch version without autocast support") + + def mock_forward_parameters_sequential(batch_shape, cls, batch_prob): + named_modules = cls.get_forward_sequence() + params = [] + for name, module in named_modules: + if isinstance(module, (K.base._AugmentationBase, K.MixAugmentationBaseV2, K.ImageSequential)): + with patch.object(module, "__batch_prob_generator__", return_value=batch_prob): + mod_param = module.forward_parameters(batch_shape) + + param = ParamItem(name, mod_param) + else: + param = ParamItem(name, None) + batch_shape = K.container.image._get_new_batch_shape(param, batch_shape) + params.append(param) + return params + + tfs = (K.RandomAffine(0.5, (0.1, 0.5), (0.5, 1.5), 1.2, p=1.0), K.RandomGaussianBlur((3, 3), (0.1, 3), p=1)) + data_keys = ["input", "mask", box, "keypoints"] + aug = K.AugmentationSequential(*tfs, data_keys=data_keys, random_apply=True) + bs = len(batch_prob) + imgs = torch.rand(bs, 3, 7, 4, dtype=dtype, device=device) + if box == "bbox": + bb = torch.tensor([[[1.0, 1.0], [2.0, 1.0], [2.0, 2.0], [1.0, 2.0]]], dtype=dtype, device=device).expand( + bs, 1, -1, -1 + ) + else: + bb = torch.rand(bs, 1, 4, dtype=dtype, device=device) + + msk = torch.zeros_like(imgs) + msk[..., 3:, 2] = 1.0 + points = torch.rand(bs, 1, 2, dtype=dtype, device=device) + + to_apply = torch.tensor(batch_prob, device=device) + + fwd_params = partial(mock_forward_parameters_sequential, cls=aug, batch_prob=to_apply) + with patch.object(aug, "forward_parameters", fwd_params): + params = aug.forward_parameters(imgs.shape) + + with torch.autocast(device.type): + outputs = aug(imgs, msk, bb, points, params=params) + + assert outputs[0].dtype == dtype, "Output image dtype should match the input dtype" + assert outputs[1].dtype == dtype, "Output mask dtype should match the input dtype" + assert outputs[2].dtype == dtype, "Output box dtype should match the input dtype" + assert outputs[3].dtype == dtype, "Output keypoints dtype should match the input dtype" diff --git a/tests/augmentation/container/test_dispatcher.py b/tests/augmentation/container/test_dispatcher.py new file mode 100644 index 0000000..1133c68 --- /dev/null +++ b/tests/augmentation/container/test_dispatcher.py @@ -0,0 +1,95 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia +import kornia.augmentation as K + + +class TestDispatcher: + def test_many_to_many(self, device, dtype): + input_1 = torch.randn(2, 3, 5, 6, device=device, dtype=dtype) + input_2 = torch.randn(2, 3, 5, 6, device=device, dtype=dtype) + mask_1 = torch.ones(2, 1, 5, 6, device=device, dtype=dtype) + mask_2 = torch.ones(2, 1, 5, 6, device=device, dtype=dtype) + aug_list = K.ManyToManyAugmentationDispather( + K.AugmentationSequential( + kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + kornia.augmentation.RandomAffine(360, p=1.0), + data_keys=["input", "mask"], + ), + K.AugmentationSequential( + kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + kornia.augmentation.RandomAffine(360, p=1.0), + data_keys=["input", "mask"], + ), + ) + output = aug_list((input_1, mask_1), (input_2, mask_2)) + + assert output[0][0].shape == input_1.shape + assert output[1][0].shape == input_2.shape + assert output[0][1].shape == mask_1.shape + assert output[1][1].shape == mask_2.shape + + @pytest.mark.parametrize("strict", [True, False]) + def test_many_to_one(self, strict, device, dtype): + input = torch.randn(2, 3, 5, 6, device=device, dtype=dtype) + mask = torch.ones(2, 1, 5, 6, device=device, dtype=dtype) + aug_list = K.ManyToOneAugmentationDispather( + K.AugmentationSequential( + kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + kornia.augmentation.RandomAffine(360, p=1.0), + data_keys=["input", "mask"], + ), + K.AugmentationSequential( + kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + kornia.augmentation.RandomAffine(360, p=1.0), + data_keys=["input", "mask"], + ), + strict=strict, + ) + output = aug_list(input, mask) + + assert output[0][0].shape == input.shape + assert output[1][0].shape == input.shape + assert output[0][1].shape == mask.shape + assert output[1][1].shape == mask.shape + + @pytest.mark.parametrize("strict", [True, False]) + def test_many_to_one_strict_mode(self, strict): + def _init_many_to_one(strict): + K.ManyToOneAugmentationDispather( + K.AugmentationSequential( + kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + kornia.augmentation.RandomAffine(360, p=1.0), + data_keys=["input"], + ), + K.AugmentationSequential( + kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + kornia.augmentation.RandomAffine(360, p=1.0), + data_keys=["input", "mask"], + ), + strict=strict, + ) + + if strict: + with pytest.raises(RuntimeError): + _init_many_to_one(strict) # fails + else: + _init_many_to_one(strict) # passes diff --git a/tests/augmentation/container/test_patch_sequential.py b/tests/augmentation/container/test_patch_sequential.py new file mode 100644 index 0000000..411cf17 --- /dev/null +++ b/tests/augmentation/container/test_patch_sequential.py @@ -0,0 +1,126 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia.augmentation as K + +from testing.augmentation.utils import reproducibility_test + + +class TestPatchSequential: + @pytest.mark.parametrize( + "error_param", + [ + {"random_apply": False, "patchwise_apply": True, "grid_size": (2, 3)}, + {"random_apply": 2, "patchwise_apply": True}, + {"random_apply": (2, 3), "patchwise_apply": True}, + ], + ) + def test_exception(self, error_param): + with pytest.raises(Exception): # AssertError and NotImplementedError + K.PatchSequential( + K.ImageSequential( + K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=0.5), + K.RandomPerspective(0.2, p=0.5), + K.RandomSolarize(0.1, 0.1, p=0.5), + ), + K.ColorJiggle(0.1, 0.1, 0.1, 0.1), + K.ImageSequential( + K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=0.5), + K.RandomPerspective(0.2, p=0.5), + K.RandomSolarize(0.1, 0.1, p=0.5), + ), + K.ColorJiggle(0.1, 0.1, 0.1, 0.1), + **error_param, + ) + + @pytest.mark.parametrize("shape", [(2, 3, 24, 24)]) + @pytest.mark.parametrize("padding", ["same", "valid"]) + @pytest.mark.parametrize("patchwise_apply", [True, False]) + @pytest.mark.parametrize("same_on_batch", [True, False, None]) + @pytest.mark.parametrize("keepdim", [True, False, None]) + @pytest.mark.parametrize("random_apply", [1, (2, 2), (1, 2), (2,), 10, True, False]) + def test_forward(self, shape, padding, patchwise_apply, same_on_batch, keepdim, random_apply, device, dtype): + torch.manual_seed(11) + try: # skip wrong param settings. + seq = K.PatchSequential( + K.color.RgbToBgr(), + K.ColorJiggle(0.1, 0.1, 0.1, 0.1), + K.ImageSequential( + K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=0.5), + K.RandomPerspective(0.2, p=0.5), + K.RandomSolarize(0.1, 0.1, p=0.5), + ), + K.RandomMixUpV2(p=1.0), + grid_size=(2, 2), + padding=padding, + patchwise_apply=patchwise_apply, + same_on_batch=same_on_batch, + keepdim=keepdim, + random_apply=random_apply, + ) + # TODO: improve me and remove the exception. + except Exception: + return + + input = torch.randn(*shape, device=device, dtype=dtype) + out = seq(input) + assert out.shape[-3:] == input.shape[-3:] + + reproducibility_test(input, seq) + + def test_intensity_only(self): + seq = K.PatchSequential( + K.ImageSequential( + K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=0.5), + K.RandomPerspective(0.2, p=0.5), + K.RandomSolarize(0.1, 0.1, p=0.5), + ), + K.ColorJiggle(0.1, 0.1, 0.1, 0.1), + K.ImageSequential( + K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=0.5), + K.RandomPerspective(0.2, p=0.5), + K.RandomSolarize(0.1, 0.1, p=0.5), + ), + K.ColorJiggle(0.1, 0.1, 0.1, 0.1), + grid_size=(2, 2), + ) + assert not seq.is_intensity_only() + + seq = K.PatchSequential( + K.ImageSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=0.5)), + K.ColorJiggle(0.1, 0.1, 0.1, 0.1), + K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=0.5), + K.ColorJiggle(0.1, 0.1, 0.1, 0.1), + grid_size=(2, 2), + ) + assert seq.is_intensity_only() + + def test_autocast(self, device, dtype): + if not hasattr(torch, "autocast"): + pytest.skip("PyTorch version without autocast support") + + tfs = (K.RandomAffine(0.5, (0.1, 0.5), (0.5, 1.5), 1.2, p=1.0), K.RandomGaussianBlur((3, 3), (0.1, 3), p=1)) + aug = K.PatchSequential(*tfs, grid_size=(2, 2), random_apply=True) + imgs = torch.rand(2, 3, 7, 4, dtype=dtype, device=device) + + with torch.autocast(device.type): + output = aug(imgs) + + assert output.dtype == dtype, "Output image dtype should match the input dtype" diff --git a/tests/augmentation/container/test_sequential_aug.py b/tests/augmentation/container/test_sequential_aug.py new file mode 100644 index 0000000..2e34d3e --- /dev/null +++ b/tests/augmentation/container/test_sequential_aug.py @@ -0,0 +1,70 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia +import kornia.augmentation as K + +from testing.augmentation.utils import reproducibility_test + + +class TestSequential: + @pytest.mark.parametrize("random_apply_weights", [None, [0.8, 0.9]]) + def test_exception(self, random_apply_weights, device, dtype): + inp = torch.randn(1, 3, 30, 30, device=device, dtype=dtype) + with pytest.raises(Exception): # AssertError and NotImplementedError + K.ImageSequential( + K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), random_apply_weights=random_apply_weights + ).inverse(inp) + + @pytest.mark.parametrize("same_on_batch", [True, False, None]) + @pytest.mark.parametrize("keepdim", [True, False, None]) + @pytest.mark.parametrize("random_apply", [1, (2, 2), (1, 2), (2,), 20, True, False]) + def test_construction(self, same_on_batch, keepdim, random_apply): + aug = K.ImageSequential( + K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + K.RandomAffine(360, p=1.0), + K.RandomMixUpV2(p=1.0), + same_on_batch=same_on_batch, + keepdim=keepdim, + random_apply=random_apply, + ) + aug.same_on_batch = True + aug.keepdim = True + for m in aug.children(): + assert m.same_on_batch is True, m.same_on_batch + assert m.keepdim is True, m.keepdim + + @pytest.mark.parametrize("random_apply", [1, (2, 2), (1, 2), (2,), 10, True, False]) + def test_forward(self, random_apply, device, dtype): + inp = torch.randn(1, 3, 30, 30, device=device, dtype=dtype) + aug = K.ImageSequential( + K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + kornia.filters.MedianBlur((3, 3)), + K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + K.ImageSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0)), + K.ImageSequential(K.RandomAffine(360, p=1.0)), + K.RandomAffine(360, p=1.0), + K.RandomMixUpV2(p=1.0), + random_apply=random_apply, + ) + out = aug(inp) + assert out.shape == inp.shape + aug.inverse(inp) + reproducibility_test(inp, aug) diff --git a/tests/augmentation/container/test_video_sequential.py b/tests/augmentation/container/test_video_sequential.py new file mode 100644 index 0000000..a8736b2 --- /dev/null +++ b/tests/augmentation/container/test_video_sequential.py @@ -0,0 +1,214 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia +import kornia.augmentation as K + +from testing.augmentation.utils import assert_close, reproducibility_test + + +class TestVideoSequential: + def test_smoke(self, device, dtype): + input_1 = torch.randn(2, 3, 1, 5, 6, device=device, dtype=dtype).repeat(1, 1, 3, 1, 1) + input_2 = torch.randn(4, 3, 1, 5, 6, device=device, dtype=dtype).repeat(1, 1, 3, 1, 1) + aug_list = K.VideoSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1)) + aug_list(input_1) + aug_list(input_2) + + @pytest.mark.parametrize("shape", [(3, 4), (2, 3, 4), (2, 3, 5, 6), (2, 3, 4, 5, 6, 7)]) + @pytest.mark.parametrize("data_format", ["BCTHW", "BTCHW"]) + def test_exception(self, shape, data_format, device, dtype): + aug_list = K.VideoSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1), data_format=data_format, same_on_frame=True) + with pytest.raises(AssertionError): + img = torch.randn(*shape, device=device, dtype=dtype) + aug_list(img) + + @pytest.mark.parametrize( + "augmentation", + [ + K.RandomAffine(360, p=1.0), + K.CenterCrop((3, 3), p=1.0), + K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), + K.RandomCrop((5, 5), p=1.0), + K.RandomErasing(p=1.0), + K.RandomGrayscale(p=1.0), + K.RandomHorizontalFlip(p=1.0), + K.RandomVerticalFlip(p=1.0), + K.RandomPerspective(p=1.0), + K.RandomResizedCrop((5, 5), p=1.0), + K.RandomRotation(360.0, p=1.0), + K.RandomSolarize(p=1.0), + K.RandomPosterize(p=1.0), + K.RandomSharpness(p=1.0), + K.RandomEqualize(p=1.0), + K.RandomMotionBlur(3, 35.0, 0.5, p=1.0), + K.Normalize(torch.tensor([0.5, 0.5, 0.5]), torch.tensor([0.5, 0.5, 0.5]), p=1.0), + K.Denormalize(torch.tensor([0.5, 0.5, 0.5]), torch.tensor([0.5, 0.5, 0.5]), p=1.0), + K.RandomAutoContrast(p=1.0), + K.RandomShear((10.0, 10.0), p=1.0), + K.RandomTranslate((0.5, 0.5), p=1.0), + ], + ) + @pytest.mark.parametrize("data_format", ["BCTHW", "BTCHW"]) + def test_augmentation(self, augmentation, data_format, device, dtype): + input = torch.randint(255, (1, 3, 3, 5, 6), device=device, dtype=dtype).repeat(2, 1, 1, 1, 1) / 255.0 + torch.manual_seed(21) + aug_list = K.VideoSequential(augmentation, data_format=data_format, same_on_frame=True) + reproducibility_test(input, aug_list) + + @pytest.mark.parametrize( + "augmentations", + [ + [K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0)], + [K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0)], + [K.RandomAffine(360, p=1.0), kornia.color.BgrToRgb()], + [K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=0.0), K.RandomAffine(360, p=0.0)], + [K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=0.0)], + [K.RandomAffine(360, p=0.0)], + # NOTE: RandomMixUpV2 failed occasionally but always passed in the debugger. Unable to debug now. + # [K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0), K.RandomMixUpV2(p=1.0)], + ], + ) + @pytest.mark.parametrize("data_format", ["BCTHW", "BTCHW"]) + @pytest.mark.parametrize("random_apply", [1, (1, 1), (1,), 10, True, False]) + def test_same_on_frame(self, augmentations, data_format, random_apply, device, dtype): + aug_list = K.VideoSequential( + *augmentations, data_format=data_format, same_on_frame=True, random_apply=random_apply + ) + + if data_format == "BCTHW": + input = torch.randn(2, 3, 1, 5, 6, device=device, dtype=dtype).repeat(1, 1, 4, 1, 1) + output = aug_list(input) + assert_close(output[:, :, 0], output[:, :, 1]) + assert_close(output[:, :, 1], output[:, :, 2]) + assert_close(output[:, :, 2], output[:, :, 3]) + if data_format == "BTCHW": + input = torch.randn(2, 1, 3, 5, 6, device=device, dtype=dtype).repeat(1, 4, 1, 1, 1) + output = aug_list(input) + assert_close(output[:, 0], output[:, 1]) + assert_close(output[:, 1], output[:, 2]) + assert_close(output[:, 2], output[:, 3]) + reproducibility_test(input, aug_list) + + @pytest.mark.parametrize( + "augmentations", + [ + [K.RandomAffine(360, p=1.0)], + [K.RandomCrop((2, 2), padding=2)], + [K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0)], + [K.RandomAffine(360, p=0.0), K.ImageSequential(K.RandomAffine(360, p=0.0))], + ], + ) + @pytest.mark.parametrize("data_format", ["BCTHW", "BTCHW"]) + def test_against_sequential(self, augmentations, data_format, device, dtype): + aug_list_1 = K.VideoSequential(*augmentations, data_format=data_format, same_on_frame=False) + aug_list_2 = torch.nn.Sequential(*augmentations) + + if data_format == "BCTHW": + input = torch.randn(2, 3, 1, 5, 6, device=device, dtype=dtype).repeat(1, 1, 4, 1, 1) + if data_format == "BTCHW": + input = torch.randn(2, 1, 3, 5, 6, device=device, dtype=dtype).repeat(1, 4, 1, 1, 1) + + torch.manual_seed(0) + output_1 = aug_list_1(input) + + torch.manual_seed(0) + if data_format == "BCTHW": + input = input.transpose(1, 2) + output_2 = aug_list_2(input.reshape(-1, 3, 5, 6)) + if any(isinstance(a, K.RandomCrop) for a in augmentations): + output_2 = output_2.view(2, 4, 3, 2, 2) + else: + output_2 = output_2.view(2, 4, 3, 5, 6) + if data_format == "BCTHW": + output_2 = output_2.transpose(1, 2) + assert_close(output_1, output_2) + + @pytest.mark.jit() + @pytest.mark.skip(reason="turn off due to Union Type") + def test_jit(self, device, dtype): + B, C, D, H, W = 2, 3, 5, 4, 4 + img = torch.ones(B, C, D, H, W, device=device, dtype=dtype) + op = K.VideoSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1), same_on_frame=True) + op_jit = torch.jit.script(op) + assert_close(op(img), op_jit(img)) + + @pytest.mark.parametrize("data_format", ["BCTHW", "BTCHW"]) + def test_autocast(self, data_format, device, dtype): + if not hasattr(torch, "autocast"): + pytest.skip("PyTorch version without autocast support") + + tfs = (K.RandomAffine(0.5, (0.1, 0.5), (0.5, 1.5), 1.2, p=1.0), K.RandomGaussianBlur((3, 3), (0.1, 3), p=1)) + aug = K.VideoSequential(*tfs, data_format=data_format, random_apply=True) + if data_format == "BCTHW": + imgs = torch.randn(2, 3, 1, 5, 6, device=device, dtype=dtype).repeat(1, 1, 4, 1, 1) + elif data_format == "BTCHW": + imgs = torch.randn(2, 1, 3, 5, 6, device=device, dtype=dtype).repeat(1, 4, 1, 1, 1) + + with torch.autocast(device.type): + output = aug(imgs) + + assert output.dtype == dtype, "Output image dtype should match the input dtype" + + @pytest.mark.parametrize("data_format", ["BCTHW", "BTCHW"]) + @pytest.mark.parametrize( + "same_on_frame,same_on_batch", + [ + (True, True), + (True, False), + (False, True), + (False, False), + ], + ) + def test_all_frame_and_batch(self, data_format, same_on_frame, same_on_batch, device, dtype): + aug = K.VideoSequential( + K.RandomCrop(size=(4, 4), p=1.0, same_on_batch=same_on_batch), + data_format=data_format, + same_on_frame=same_on_frame, + ) + + if data_format == "BCTHW": + if same_on_frame: + start = torch.randn(1, 3, 1, 5, 6, device=device, dtype=dtype).repeat(1, 1, 4, 1, 1) + x = start.repeat(2, 1, 1, 1, 1) + else: + start = torch.randn(1, 3, 4, 5, 6, device=device, dtype=dtype) + x = start.repeat(2, 1, 1, 1, 1) + elif same_on_frame: + start = torch.randn(1, 1, 3, 5, 6, device=device, dtype=dtype).repeat(1, 4, 1, 1, 1) + x = start.repeat(2, 1, 1, 1, 1) + else: + start = torch.randn(1, 4, 3, 5, 6, device=device, dtype=dtype) + x = start.repeat(2, 1, 1, 1, 1) + + torch.manual_seed(0) + y = aug(x) + + if data_format == "BCTHW": + same_frame = torch.allclose(y[:, :, 0], y[:, :, 1]) + same_batch = torch.allclose(y[0, :, 0], y[1, :, 0]) + else: + same_frame = torch.allclose(y[:, 0], y[:, 1]) + same_batch = torch.allclose(y[0, 0], y[1, 0]) + + assert same_frame == same_on_frame + assert same_batch == same_on_batch + + reproducibility_test(x, aug) diff --git a/tests/augmentation/test_augmentation.py b/tests/augmentation/test_augmentation.py new file mode 100644 index 0000000..fa38b5e --- /dev/null +++ b/tests/augmentation/test_augmentation.py @@ -0,0 +1,5345 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import sys +from typing import Any, Dict, Optional, Tuple, Type +from unittest.mock import patch + +import pytest +import torch + +import kornia +from kornia.augmentation import ( + AugmentationSequential, + CenterCrop, + ColorJiggle, + ColorJitter, + Denormalize, + LongestMaxSize, + Normalize, + PadTo, + RandomBoxBlur, + RandomBrightness, + RandomChannelDropout, + RandomChannelShuffle, + RandomClahe, + RandomContrast, + RandomCrop, + RandomDissolving, + RandomElasticTransform, + RandomEqualize, + RandomErasing, + RandomFisheye, + RandomGamma, + RandomGaussianBlur, + RandomGaussianIllumination, + RandomGaussianNoise, + RandomGrayscale, + RandomHorizontalFlip, + RandomHue, + RandomInvert, + RandomJPEG, + RandomLinearCornerIllumination, + RandomLinearIllumination, + RandomMedianBlur, + RandomPlanckianJitter, + RandomPlasmaBrightness, + RandomPlasmaContrast, + RandomPlasmaShadow, + RandomPosterize, + RandomRain, + RandomResizedCrop, + RandomRGBShift, + RandomRotation, + RandomRotation90, + RandomSaltAndPepperNoise, + RandomSaturation, + RandomSnow, + RandomThinPlateSpline, + RandomVerticalFlip, + Resize, + SmallestMaxSize, +) +from kornia.augmentation._2d.base import AugmentationBase2D +from kornia.constants import Resample, pi +from kornia.core._compat import torch_version, torch_version_le +from kornia.core.utils import _torch_inverse_cast +from kornia.geometry import create_meshgrid, transform_points + +from testing.augmentation.datasets import DummyMPDataset +from testing.base import BaseTester +from testing.overwrite import default_with_one_parameter_changed + + +@pytest.mark.usefixtures("device", "dtype") +class CommonTests(BaseTester): + # TODO same_on_batch tests? + + fixture_names = ("device", "dtype") + + ############################################################################################################ + # Attribute variables to set + ############################################################################################################ + _augmentation_cls: Optional[Type[AugmentationBase2D]] = None + _default_param_set: Dict["str", Any] = {} + + ############################################################################################################ + # Fixtures + ############################################################################################################ + + @pytest.fixture(autouse=True) + def auto_injector_fixture(self, request): + for fixture_name in self.fixture_names: + setattr(self, fixture_name, request.getfixturevalue(fixture_name)) + + @pytest.fixture(scope="class") + def param_set(self, request): + raise NotImplementedError("param_set must be overridden in subclasses") + + ############################################################################################################ + # Test cases + ############################################################################################################ + def test_smoke(self, param_set): + self._test_smoke_implementation(params=param_set) + self._test_smoke_call_implementation(params=param_set) + + @pytest.mark.parametrize( + "input_shape", + [(4, 5), (3, 4, 5), (2, 3, 4, 5)], + ) + @pytest.mark.parametrize( + "keepdim", + [True, False], + ) + def test_cardinality(self, input_shape, keepdim): + self._test_cardinality_implementation(input_shape=input_shape, keepdim=keepdim, params=self._default_param_set) + + def test_random_p_0(self): + self._test_random_p_0_implementation(params=self._default_param_set) + + @pytest.mark.skip(reason="Not implemented in base class") + def test_random_p_1(self): ... + + def test_inverse_coordinate_check(self): + self._test_inverse_coordinate_check_implementation(params=self._default_param_set) + + @pytest.mark.skip(reason="Not implemented in base class") + def test_exception(self): ... + + @pytest.mark.skip(reason="Not implemented in base class") + def test_batch(self): ... + + @pytest.mark.skip(reason="turn off all jit for a while") + def test_jit(self): ... + + def test_module(self): + self._test_module_implementation(params=self._default_param_set) + + @pytest.mark.slow + def test_gradcheck(self): + self._test_gradcheck_implementation(params=self._default_param_set) + + # TODO Implement + # test_batch + # test_batch_return_transform + # test_coordinate check + # test_jit + # test_gradcheck + + def _create_augmentation_from_params(self, **params): + return self._augmentation_cls(**params) + + ############################################################################################################ + # Test case implementations + ############################################################################################################ + + def _test_smoke_implementation(self, params): + assert issubclass(self._augmentation_cls, AugmentationBase2D), ( + f"{self._augmentation_cls} is not a subclass of AugmentationBase2D" + ) + + # Can be instantiated + augmentation = self._create_augmentation_from_params(**params) + assert issubclass(type(augmentation), AugmentationBase2D), ( + f"{type(augmentation)} is not a subclass of AugmentationBase2D" + ) + + # generate_parameters can be called and returns the correct amount of parameters + batch_shape = (4, 3, 5, 6) + generated_params = augmentation.forward_parameters(batch_shape) + assert isinstance(generated_params, dict) + + # compute_transformation now operates on the full batch (ONNX-friendly contract) + # and returns a (B, 3, 3) transform matrix. + expected_transformation_shape = torch.Size((batch_shape[0], 3, 3)) + test_input = torch.ones(batch_shape, device=self.device, dtype=self.dtype) + transformation = augmentation.compute_transformation(test_input, generated_params, augmentation.flags) + assert transformation.shape == expected_transformation_shape + + # apply_transform can be called on the full batch and returns full-batch output + output = augmentation.apply_transform( + test_input, + generated_params, + augmentation.flags, + transformation, + ) + assert output.shape[0] == batch_shape[0] + + def _test_smoke_call_implementation(self, params): + batch_shape = (4, 3, 5, 6) + expected_transformation_shape = torch.Size((batch_shape[0], 3, 3)) + test_input = torch.ones(batch_shape, device=self.device, dtype=self.dtype) + augmentation = self._create_augmentation_from_params(**params) + generated_params = augmentation.forward_parameters(batch_shape) + + output = augmentation(test_input, params=generated_params) + assert output.shape[0] == batch_shape[0] + assert augmentation.transform_matrix.shape == expected_transformation_shape + + def _test_cardinality_implementation(self, input_shape, params, keepdim=False, expected_output_shape=None): + # p==0.0 + augmentation = self._create_augmentation_from_params(**params, p=0.0, keepdim=keepdim) + test_input = torch.rand(input_shape, device=self.device, dtype=self.dtype) + output = augmentation(test_input) + + if keepdim: + assert output.shape == input_shape + else: + assert len(output.shape) == 4 + assert output.shape == torch.Size((1,) * (4 - len(input_shape)) + tuple(input_shape)) + + # p==1.0 + augmentation = self._create_augmentation_from_params(**params, p=1.0, keepdim=keepdim) + test_input = torch.rand(input_shape, device=self.device, dtype=self.dtype) + output = augmentation(test_input) + + if expected_output_shape: + assert len(output.shape) == 4 + assert output.shape == expected_output_shape + elif keepdim: + assert output.shape == input_shape + else: + assert (*(1,) * (4 - len(input_shape)), *input_shape) == output.shape + + def _test_random_p_0_implementation(self, params): + augmentation = self._create_augmentation_from_params(**params, p=0.0) + test_input = torch.rand((2, 3, 4, 5), device=self.device, dtype=self.dtype) + output = augmentation(test_input) + self.assert_close(output, test_input) + + def _test_random_p_1_implementation(self, input_tensor, expected_output, params, expected_transformation=None): + augmentation = self._create_augmentation_from_params(**params, p=1.0) + output = augmentation(input_tensor.to(self.device).to(self.dtype)) + + # Output should match + assert output.shape == expected_output.shape + self.assert_close( + output, + expected_output.to(device=self.device, dtype=self.dtype), + low_tolerance=True, + ) + if expected_transformation is not None: + transform = augmentation.transform_matrix + self.assert_close(transform, expected_transformation, low_tolerance=True) + + def _test_module_implementation(self, params): + # Verifies the composition invariant of AugmentationSequential: + # Sequential(aug_a, aug_b)(x) == aug_b(aug_a(x)) + # and the transform matrix is the product of the individual matrices. + # + # Two distinct instances are required because `AugmentationSequential(aug, aug)` + # registers a single child (nn.Module dedupes same-instance children). + # We force p=1.0 so both augmentations always fire. + augmentation_a = self._create_augmentation_from_params(**params, p=1.0) + augmentation_b = self._create_augmentation_from_params(**params, p=1.0) + + augmentation_sequence = AugmentationSequential(augmentation_a, augmentation_b) + + input_tensor = torch.rand(2, 3, 5, 5, device=self.device, dtype=self.dtype) + + torch.manual_seed(42) + out_manual_a = augmentation_a(input_tensor) + transform_a = augmentation_a.transform_matrix + out_manual = augmentation_b(out_manual_a) + transform_manual = augmentation_b.transform_matrix @ transform_a + + torch.manual_seed(42) + out_sequence = augmentation_sequence(input_tensor) + transform_sequence = augmentation_sequence.transform_matrix + + assert out_manual.shape == out_sequence.shape + assert transform_manual.shape == transform_sequence.shape + self.assert_close(out_manual, out_sequence, low_tolerance=True) + self.assert_close(transform_manual, transform_sequence, low_tolerance=True) + + def _test_inverse_coordinate_check_implementation(self, params): + torch.manual_seed(42) + + input_tensor = torch.zeros((1, 3, 50, 100), device=self.device, dtype=self.dtype) + input_tensor[:, :, 20:30, 40:60] = 1.0 + + augmentation = self._create_augmentation_from_params(**params, p=1.0) + output = augmentation(input_tensor) + transform = augmentation.transform_matrix + + if (transform == kornia.core.ops.eye_like(3, transform)).all(): + pytest.skip("Test not relevant for intensity augmentations.") + + indices = create_meshgrid( + height=output.shape[-2], + width=output.shape[-1], + normalized_coordinates=False, + device=self.device, + ) + output_indices = indices.reshape((1, -1, 2)).to(dtype=self.dtype) + input_indices = transform_points(_torch_inverse_cast(transform.to(self.dtype)), output_indices) + + output_indices = output_indices.round().long().squeeze(0) + input_indices = input_indices.round().long().squeeze(0) + output_values = output[0, 0, output_indices[:, 1], output_indices[:, 0]] + value_mask = output_values > 0.9999 + + output_values = output[0, :, output_indices[:, 1][value_mask], output_indices[:, 0][value_mask]] + input_values = input_tensor[0, :, input_indices[:, 1][value_mask], input_indices[:, 0][value_mask]] + + self.assert_close(output_values, input_values, low_tolerance=True) + + @pytest.mark.slow + def _test_gradcheck_implementation(self, params): + input_tensor = torch.rand((3, 5, 5), device=self.device, dtype=self.dtype) # 3 x 3 + self.gradcheck(self._create_augmentation_from_params(**params, p=1.0), (input_tensor,)) + + +class TestRandomEqualizeAlternative(CommonTests): + possible_params: Dict["str", Tuple] = {} + + _augmentation_cls = RandomEqualize + _default_param_set: Dict["str", Any] = {} + + @pytest.fixture(params=[_default_param_set], scope="class") + def param_set(self, request): + return request.param + + def test_random_p_1(self): + input_tensor = torch.arange(20.0, device=self.device, dtype=self.dtype) / 20 + input_tensor = input_tensor.repeat(1, 2, 20, 1) + + expected_output = torch.tensor( + [ + 0.0000, + 0.07843, + 0.15686, + 0.2353, + 0.3137, + 0.3922, + 0.4706, + 0.5490, + 0.6275, + 0.7059, + 0.7843, + 0.8627, + 0.9412, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + ], + device=self.device, + dtype=self.dtype, + ) + expected_output = expected_output.repeat(1, 2, 20, 1) + + parameters = {} + self._test_random_p_1_implementation( + input_tensor=input_tensor, + expected_output=expected_output, + params=parameters, + ) + + def test_batch(self): + input_tensor = torch.arange(20.0, device=self.device, dtype=self.dtype) / 20 + input_tensor = input_tensor.repeat(2, 3, 20, 1) + + expected_output = torch.tensor( + [ + 0.0000, + 0.07843, + 0.15686, + 0.2353, + 0.3137, + 0.3922, + 0.4706, + 0.5490, + 0.6275, + 0.7059, + 0.7843, + 0.8627, + 0.9412, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + ], + device=self.device, + dtype=self.dtype, + ) + expected_output = expected_output.repeat(2, 3, 20, 1) + + expected_transformation = kornia.core.ops.eye_like(3, input_tensor) + parameters = {} + self._test_random_p_1_implementation( + input_tensor=input_tensor, + expected_output=expected_output, + expected_transformation=expected_transformation, + params=parameters, + ) + + def test_exception(self): + with pytest.raises(ValueError): + self._create_augmentation_from_params(p=1.0)( + torch.ones((1, 3, 4, 5) * 3, device=self.device, dtype=self.dtype) + ) + + +class TestCenterCropAlternative(CommonTests): + possible_params: Dict["str", Tuple] = { + "size": (2, (2, 2)), + "resample": (0, Resample.BILINEAR.name, Resample.BILINEAR), + "align_corners": (False, True), + } + _augmentation_cls = CenterCrop + _default_param_set: Dict["str", Any] = {"size": (2, 2), "align_corners": True} + + @pytest.fixture( + params=default_with_one_parameter_changed(default=_default_param_set, **possible_params), + scope="class", + ) + def param_set(self, request): + return request.param + + @pytest.mark.parametrize( + "input_shape,expected_output_shape", + [ + ((4, 5), (1, 1, 2, 3)), + ((3, 4, 5), (1, 3, 2, 3)), + ((2, 3, 4, 5), (2, 3, 2, 3)), + ], + ) + def test_cardinality(self, input_shape, expected_output_shape): + self._test_cardinality_implementation( + input_shape=input_shape, + expected_output_shape=expected_output_shape, + params={"size": (2, 3), "align_corners": True}, + ) + + @pytest.mark.xfail(reason="size=(1,2) results in RuntimeError: solve_cpu: For batch 0: U(3,3) is zero, singular U.") + def test_random_p_1(self): + torch.manual_seed(42) + + input_tensor = torch.tensor( + [[[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 0.0, 0.1, 0.2]]], + device=self.device, + dtype=self.dtype, + ) + expected_output = torch.tensor([[[[0.6, 0.7]]]], device=self.device, dtype=self.dtype) + + parameters = {"size": (1, 2), "align_corners": True, "resample": 0} + self._test_random_p_1_implementation( + input_tensor=input_tensor, + expected_output=expected_output, + params=parameters, + ) + + def test_batch(self): + torch.manual_seed(42) + + input_tensor = torch.rand((2, 3, 4, 4), device=self.device, dtype=self.dtype) + expected_output = input_tensor[:, :, 1:3, 1:3] + expected_transformation = torch.tensor( + [[[1.0, 0.0, -1.0], [0.0, 1.0, -1.0], [0.0, 0.0, 1.0]]], + device=self.device, + dtype=self.dtype, + ).repeat(2, 1, 1) + parameters = { + "size": (2, 2), + "align_corners": True, + "resample": 0, + "cropping_mode": "resample", + } + self._test_random_p_1_implementation( + input_tensor=input_tensor, + expected_output=expected_output, + expected_transformation=expected_transformation, + params=parameters, + ) + + @pytest.mark.xfail(reason="No input validation is implemented yet.") + def test_exception(self): + # Wrong type + with pytest.raises(TypeError): + self._create_augmentation_from_params(size=0.0) + with pytest.raises(TypeError): + self._create_augmentation_from_params(size=2, align_corners=0) + with pytest.raises(TypeError): + self._create_augmentation_from_params(size=2, resample=True) + + # Bound check + with pytest.raises(ValueError): + self._create_augmentation_from_params(size=-1) + with pytest.raises(ValueError): + self._create_augmentation_from_params(size=(-1, 2)) + with pytest.raises(ValueError): + self._create_augmentation_from_params(size=(2, -1)) + + +class TestRandomHorizontalFlipAlternative(CommonTests): + possible_params: Dict["str", Tuple] = {} + _augmentation_cls = RandomHorizontalFlip + _default_param_set: Dict["str", Any] = {} + + @pytest.fixture(params=[_default_param_set], scope="class") + def param_set(self, request): + return request.param + + def test_random_p_1(self): + torch.manual_seed(42) + + input_tensor = torch.tensor( + [[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]]], + device=self.device, + dtype=self.dtype, + ) + expected_output = torch.tensor( + [[[[0.3, 0.2, 0.1], [0.6, 0.5, 0.4], [0.9, 0.8, 0.7]]]], + device=self.device, + dtype=self.dtype, + ) + + parameters = {} + self._test_random_p_1_implementation( + input_tensor=input_tensor, + expected_output=expected_output, + params=parameters, + ) + + def test_batch(self): + torch.manual_seed(12) + + input_tensor = torch.tensor( + [[[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]]]], + device=self.device, + dtype=self.dtype, + ).repeat((2, 1, 1, 1)) + expected_output = torch.tensor( + [[[[0.3, 0.2, 0.1], [0.6, 0.5, 0.4], [0.9, 0.8, 0.7]]]], + device=self.device, + dtype=self.dtype, + ).repeat((2, 1, 1, 1)) + expected_transformation = torch.tensor( + [[[-1.0, 0.0, 2.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], + device=self.device, + dtype=self.dtype, + ).repeat((2, 1, 1)) + parameters = {} + self._test_random_p_1_implementation( + input_tensor=input_tensor, + expected_output=expected_output, + expected_transformation=expected_transformation, + params=parameters, + ) + + @pytest.mark.skip(reason="No special parameters to validate.") + def test_exception(self): + pass + + +class TestRandomVerticalFlipAlternative(CommonTests): + possible_params: Dict["str", Tuple] = {} + _augmentation_cls = RandomVerticalFlip + _default_param_set: Dict["str", Any] = {} + + @pytest.fixture(params=[_default_param_set], scope="class") + def param_set(self, request): + return request.param + + def test_random_p_1(self): + torch.manual_seed(42) + + input_tensor = torch.tensor( + [[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]]], + device=self.device, + dtype=self.dtype, + ) + expected_output = torch.tensor( + [[[[0.7, 0.8, 0.9], [0.4, 0.5, 0.6], [0.1, 0.2, 0.3]]]], + device=self.device, + dtype=self.dtype, + ) + + parameters = {} + self._test_random_p_1_implementation( + input_tensor=input_tensor, + expected_output=expected_output, + params=parameters, + ) + + def test_batch(self): + torch.manual_seed(12) + + input_tensor = torch.tensor( + [[[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]]]], + device=self.device, + dtype=self.dtype, + ).repeat((2, 1, 1, 1)) + expected_output = torch.tensor( + [[[[0.7, 0.8, 0.9], [0.4, 0.5, 0.6], [0.1, 0.2, 0.3]]]], + device=self.device, + dtype=self.dtype, + ).repeat((2, 1, 1, 1)) + expected_transformation = torch.tensor( + [[[1.0, 0.0, 0.0], [0.0, -1.0, 2.0], [0.0, 0.0, 1.0]]], + device=self.device, + dtype=self.dtype, + ).repeat((2, 1, 1)) + parameters = {} + self._test_random_p_1_implementation( + input_tensor=input_tensor, + expected_output=expected_output, + expected_transformation=expected_transformation, + params=parameters, + ) + + @pytest.mark.skip(reason="No special parameters to validate.") + def test_exception(self): + pass + + +class TestRandomRotationAlternative(CommonTests): + possible_params: Dict["str", Tuple] = { + "degrees": (0.0, (-360.0, 360.0), [0.0, 0.0], torch.tensor((-180.0, 180))), + "resample": (0, Resample.BILINEAR.name, Resample.BILINEAR), + "align_corners": (False, True), + } + _augmentation_cls = RandomRotation + _default_param_set: Dict["str", Any] = { + "degrees": (30.0, 30.0), + "align_corners": True, + } + + @pytest.fixture( + params=default_with_one_parameter_changed(default=_default_param_set, **possible_params), + scope="class", + ) + def param_set(self, request): + return request.param + + def test_random_p_1(self): + torch.manual_seed(42) + + input_tensor = torch.tensor( + [[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]]], + device=self.device, + dtype=self.dtype, + ) + expected_output = torch.tensor( + [[[[0.3, 0.6, 0.9], [0.2, 0.5, 0.8], [0.1, 0.4, 0.7]]]], + device=self.device, + dtype=self.dtype, + ) + + parameters = {"degrees": (90.0, 90.0), "align_corners": True} + self._test_random_p_1_implementation( + input_tensor=input_tensor, + expected_output=expected_output, + params=parameters, + ) + + def test_batch(self): + if self.dtype == torch.float16: + pytest.skip("not work for half-precision") + + torch.manual_seed(12) + + input_tensor = torch.tensor( + [[[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]]]], + device=self.device, + dtype=self.dtype, + ).repeat((2, 1, 1, 1)) + expected_output = input_tensor + expected_transformation = kornia.core.ops.eye_like(3, input_tensor) + parameters = {"degrees": (-360.0, -360.0), "align_corners": True} + self._test_random_p_1_implementation( + input_tensor=input_tensor, + expected_output=expected_output, + expected_transformation=expected_transformation, + params=parameters, + ) + + @pytest.mark.xfail(reason="No input validation is implemented yet.") + def test_exception(self): + # Wrong type + with pytest.raises(TypeError): + self._create_augmentation_from_params(degrees="") + with pytest.raises(TypeError): + self._create_augmentation_from_params(degrees=(3, 3), align_corners=0) + with pytest.raises(TypeError): + self._create_augmentation_from_params(degrees=(3, 3), resample=True) + + # Bound check + with pytest.raises(ValueError): + self._create_augmentation_from_params(degrees=-361.0) + with pytest.raises(ValueError): + self._create_augmentation_from_params(degrees=(-361.0, 360.0)) + with pytest.raises(ValueError): + self._create_augmentation_from_params(degrees=(-360.0, 361.0)) + with pytest.raises(ValueError): + self._create_augmentation_from_params(degrees=(360.0, -360.0)) + + +class TestRandomRotation90(CommonTests): + possible_params: dict["str", tuple] = { + "times": ((-3, 3), (1, 1)), + "resample": (0, Resample.BILINEAR.name, Resample.BILINEAR), + "align_corners": (False, True), + } + _augmentation_cls = RandomRotation90 + _default_param_set: dict["str", Any] = { + "times": (-3, 3), + "align_corners": True, + } + + @pytest.mark.skip(reason="not working for all torch versions") + def test_gradcheck(): ... + + @pytest.fixture(params=[_default_param_set], scope="class") + def param_set(self, request): + return request.param + + @pytest.mark.parametrize( + "input_shape,expected_output_shape", + [((3, 4, 5), (1, 3, 4, 5)), ((2, 3, 4, 5), (2, 3, 4, 5))], + ) + def test_cardinality(self, input_shape, expected_output_shape): + self._test_cardinality_implementation( + input_shape=input_shape, + expected_output_shape=expected_output_shape, + params=self._default_param_set, + ) + + def test_random_p_1(self): + torch.manual_seed(42) + + input_tensor = torch.tensor( + [[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]]], + device=self.device, + dtype=self.dtype, + ) + expected_output = torch.tensor( + [[[[0.3, 0.6, 0.9], [0.2, 0.5, 0.8], [0.1, 0.4, 0.7]]]], + device=self.device, + dtype=self.dtype, + ) + + parameters = {"times": (1, 1), "align_corners": True} + self._test_random_p_1_implementation( + input_tensor=input_tensor, + expected_output=expected_output, + params=parameters, + ) + + def test_batch(self): + if self.dtype == torch.float16: + pytest.skip("not work for half-precision") + + torch.manual_seed(12) + + input_tensor = torch.tensor( + [[[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]]]], + device=self.device, + dtype=self.dtype, + ).repeat((2, 1, 1, 1)) + expected_output = input_tensor + expected_transformation = kornia.core.ops.eye_like(3, input_tensor) + parameters = {"times": (0, 0), "align_corners": True} + self._test_random_p_1_implementation( + input_tensor=input_tensor, + expected_output=expected_output, + expected_transformation=expected_transformation, + params=parameters, + ) + + def test_exception(self): + # Wrong type + with pytest.raises(TypeError): + self._create_augmentation_from_params(times="") + with pytest.raises(ValueError): + self._create_augmentation_from_params(times=(30, 60), align_corners=0) + + +class TestRandomGrayscaleAlternative(CommonTests): + possible_params: Dict["str", Tuple] = {} + + _augmentation_cls = RandomGrayscale + _default_param_set: Dict["str", Any] = {} + + @pytest.fixture(params=[_default_param_set], scope="class") + def param_set(self, request): + return request.param + + @pytest.mark.parametrize( + "input_shape,expected_output_shape", + [((3, 4, 5), (1, 3, 4, 5)), ((2, 3, 4, 5), (2, 3, 4, 5))], + ) + def test_cardinality(self, input_shape, expected_output_shape): + self._test_cardinality_implementation( + input_shape=input_shape, + expected_output_shape=expected_output_shape, + params=self._default_param_set, + ) + + def test_random_p_1(self): + torch.manual_seed(42) + + input_tensor = torch.tensor( + [[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 0.0, 0.1, 0.2]], + device=self.device, + dtype=self.dtype, + ).repeat(1, 3, 1, 1) + expected_output = ( + (input_tensor * torch.tensor([0.299, 0.587, 0.114], device=self.device, dtype=self.dtype).view(1, 3, 1, 1)) + .sum(dim=1, keepdim=True) + .repeat(1, 3, 1, 1) + ) + + parameters = {} + self._test_random_p_1_implementation( + input_tensor=input_tensor, + expected_output=expected_output, + params=parameters, + ) + + def test_batch(self): + torch.manual_seed(42) + + input_tensor = torch.tensor( + [[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 0.0, 0.1, 0.2]], + device=self.device, + dtype=self.dtype, + ).repeat(2, 3, 1, 1) + expected_output = ( + (input_tensor * torch.tensor([0.299, 0.587, 0.114], device=self.device, dtype=self.dtype).view(1, 3, 1, 1)) + .sum(dim=1, keepdim=True) + .repeat(1, 3, 1, 1) + ) + + expected_transformation = kornia.core.ops.eye_like(3, input_tensor) + parameters = {} + self._test_random_p_1_implementation( + input_tensor=input_tensor, + expected_output=expected_output, + expected_transformation=expected_transformation, + params=parameters, + ) + + @pytest.mark.xfail(reason="No input validation is implemented yet when p=0.") + def test_exception(self): + torch.manual_seed(42) + + with pytest.raises(ValueError): + self._create_augmentation_from_params(p=0.0)(torch.rand((1, 1, 4, 5), device=self.device, dtype=self.dtype)) + with pytest.raises(ValueError): + self._create_augmentation_from_params(p=1.0)(torch.rand((1, 4, 4, 5), device=self.device, dtype=self.dtype)) + + +class TestRandomHorizontalFlip(BaseTester): + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self): + f = RandomHorizontalFlip(p=0.5) + repr = "RandomHorizontalFlip(p=0.5, p_batch=1.0, same_on_batch=False)" + assert str(f) == repr + + def test_random_hflip(self, device, dtype): + f = RandomHorizontalFlip(p=1.0, keepdim=True) + f1 = RandomHorizontalFlip(p=0.0, keepdim=True) + + input = torch.tensor( + [[[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 1.0, 2.0]]], + device=device, + dtype=dtype, + ) # 1 x 3 x 4 + + expected = torch.tensor( + [[[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [2.0, 1.0, 0.0, 0.0]]], + device=device, + dtype=dtype, + ) # 1 x 3 x 4 + + expected = expected.to(device) + + expected_transform = torch.tensor( + [[[-1.0, 0.0, 3.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) # 1 x 3 x 3 + + identity = torch.tensor( + [[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) # 1 x 3 x 3 + + self.assert_close(f(input), expected) + self.assert_close(f.transform_matrix, expected_transform) + self.assert_close(f1(input), input) + self.assert_close(f1.transform_matrix, identity) + self.assert_close(f.inverse(expected), input) + self.assert_close(f1.inverse(expected), expected) + + def test_batch_random_hflip(self, device, dtype): + f = RandomHorizontalFlip(p=1.0) + f1 = RandomHorizontalFlip(p=0.0) + + input = torch.tensor( + [[[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + + expected = torch.tensor( + [[[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [1.0, 1.0, 0.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + + expected_transform = torch.tensor( + [[[-1.0, 0.0, 2.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) # 1 x 3 x 3 + + identity = torch.tensor( + [[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) # 1 x 3 x 3 + + input = input.repeat(5, 3, 1, 1) # 5 x 3 x 3 x 3 + expected = expected.repeat(5, 3, 1, 1) # 5 x 3 x 3 x 3 + expected_transform = expected_transform.repeat(5, 1, 1) # 5 x 3 x 3 + identity = identity.repeat(5, 1, 1) # 5 x 3 x 3 + + self.assert_close(f(input), expected) + self.assert_close(f.transform_matrix, expected_transform) + self.assert_close(f1(input), input) + self.assert_close(f1.transform_matrix, identity) + self.assert_close(f.inverse(expected), input) + self.assert_close(f1.inverse(expected), expected) + + def test_same_on_batch(self, device, dtype): + f = RandomHorizontalFlip(p=0.5, same_on_batch=True) + input = torch.eye(3, device=device, dtype=dtype).unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 1, 1, 1) + res = f(input) + self.assert_close(res[0], res[1]) + self.assert_close(f.inverse(res), input) + + def test_sequential(self, device, dtype): + f = AugmentationSequential(RandomHorizontalFlip(p=1.0), RandomHorizontalFlip(p=1.0)) + + input = torch.tensor( + [[[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + + expected_transform = torch.tensor( + [[[-1.0, 0.0, 2.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) # 1 x 3 x 3 + + expected_transform_1 = expected_transform @ expected_transform + + out = f(input) + self.assert_close(out, input) + self.assert_close(f.transform_matrix, expected_transform_1) + self.assert_close(f.inverse(out), input) + + def test_random_hflip_coord_check(self, device, dtype): + f = RandomHorizontalFlip(p=1.0) + + input = torch.tensor( + [[[[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [9.0, 10.0, 11.0, 12.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 4 + + input_coordinates = torch.tensor( + [ + [ + [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3], # x coord + [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2], # y coord + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + ] + ], + device=device, + dtype=dtype, + ) # 1 x 3 x 3 + + expected_output = torch.tensor( + [[[[4.0, 3.0, 2.0, 1.0], [8.0, 7.0, 6.0, 5.0], [12.0, 11.0, 10.0, 9.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 4 + + output = f(input) + transform = f.transform_matrix + result_coordinates = transform @ input_coordinates + # NOTE: without rounding it might produce unexpected results + input_coordinates = input_coordinates.round().long() + result_coordinates = result_coordinates.round().long() + + # Tensors must have the same shapes and values + assert output.shape == expected_output.shape + self.assert_close(output, expected_output) + # Transformed indices must not be out of bound + assert ( + torch.torch.logical_and( + result_coordinates[0, 0, :] >= 0, + result_coordinates[0, 0, :] < input.shape[-1], + ) + ).all() + assert ( + torch.torch.logical_and( + result_coordinates[0, 1, :] >= 0, + result_coordinates[0, 1, :] < input.shape[-2], + ) + ).all() + # Values in the output tensor at the places of transformed indices must + # have the same value as the input tensor has at the corresponding + # positions + self.assert_close( + output[..., result_coordinates[0, 1, :], result_coordinates[0, 0, :]], + input[..., input_coordinates[0, 1, :], input_coordinates[0, 0, :]], + ) + + @pytest.mark.slow + def test_gradcheck(self, device): + input = torch.rand((3, 3), device=device, dtype=torch.float64) # 3 x 3 + self.gradcheck(RandomHorizontalFlip(p=1.0), (input,)) + + +class TestRandomVerticalFlip(BaseTester): + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self): + f = RandomVerticalFlip(p=0.5) + repr = "RandomVerticalFlip(p=0.5, p_batch=1.0, same_on_batch=False)" + assert str(f) == repr + + def test_random_vflip(self, device, dtype): + f = RandomVerticalFlip(p=1.0) + f1 = RandomVerticalFlip(p=0.0) + + input = torch.tensor( + [[[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + + expected = torch.tensor( + [[[[0.0, 1.0, 1.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + + expected_transform = torch.tensor( + [[[1.0, 0.0, 0.0], [0.0, -1.0, 2.0], [0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) # 3 x 3 + + identity = torch.tensor( + [[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) # 3 x 3 + + self.assert_close(f(input), expected, low_tolerance=True) + self.assert_close(f.transform_matrix, expected_transform, low_tolerance=True) + self.assert_close(f1(input), input, low_tolerance=True) + self.assert_close(f1.transform_matrix, identity, low_tolerance=True) + + self.assert_close(f.inverse(expected), input, low_tolerance=True) + self.assert_close(f1.inverse(input), input, low_tolerance=True) + + def test_batch_random_vflip(self, device, dtype): + f = RandomVerticalFlip(p=1.0) + + input = torch.tensor( + [[[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + + expected = torch.tensor( + [[[[0.0, 1.0, 1.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + + expected_transform = torch.tensor( + [[[1.0, 0.0, 0.0], [0.0, -1.0, 2.0], [0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) # 1 x 3 x 3 + + identity = torch.tensor( + [[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) # 1 x 3 x 3 + + input = input.repeat(5, 3, 1, 1) # 5 x 3 x 3 x 3 + expected = expected.repeat(5, 3, 1, 1) # 5 x 3 x 3 x 3 + expected_transform = expected_transform.repeat(5, 1, 1) # 5 x 3 x 3 + identity = identity.repeat(5, 1, 1) # 5 x 3 x 3 + + self.assert_close(f(input), expected, low_tolerance=True) + self.assert_close(f.transform_matrix, expected_transform, low_tolerance=True) + self.assert_close(f.inverse(expected), input, low_tolerance=True) + + def test_same_on_batch(self, device, dtype): + f = RandomVerticalFlip(p=0.5, same_on_batch=True) + input = torch.eye(3, device=device, dtype=dtype).unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 1, 1, 1) + res = f(input) + self.assert_close(res[0], res[1]) + self.assert_close(f.inverse(res), input) + + def test_sequential(self, device, dtype): + f = AugmentationSequential(RandomVerticalFlip(p=1.0), RandomVerticalFlip(p=1.0)) + + input = torch.tensor( + [[[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + + expected_transform = torch.tensor( + [[[1.0, 0.0, 0.0], [0.0, -1.0, 2.0], [0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) # 1 x 3 x 3 + + expected_transform_1 = expected_transform @ expected_transform + + self.assert_close(f(input), input, low_tolerance=True) + self.assert_close(f.transform_matrix, expected_transform_1, low_tolerance=True) + + def test_random_vflip_coord_check(self, device, dtype): + f = RandomVerticalFlip(p=1.0) + + input = torch.tensor( + [[[[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [9.0, 10.0, 11.0, 12.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 4 + + input_coordinates = torch.tensor( + [ + [ + [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3], # x coord + [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2], # y coord + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + ] + ], + device=device, + dtype=dtype, + ) # 1 x 3 x 3 + + expected_output = torch.tensor( + [[[[9.0, 10.0, 11.0, 12.0], [5.0, 6.0, 7.0, 8.0], [1.0, 2.0, 3.0, 4.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 4 + + output = f(input) + transform = f.transform_matrix + result_coordinates = transform @ input_coordinates + # NOTE: without rounding it might produce unexpected results + input_coordinates = input_coordinates.round().long() + result_coordinates = result_coordinates.round().long() + + # Tensors must have the same shapes and values + assert output.shape == expected_output.shape + self.assert_close(output, expected_output) + # Transformed indices must not be out of bound + assert ( + torch.torch.logical_and( + result_coordinates[0, 0, :] >= 0, + result_coordinates[0, 0, :] < input.shape[-1], + ) + ).all() + assert ( + torch.torch.logical_and( + result_coordinates[0, 1, :] >= 0, + result_coordinates[0, 1, :] < input.shape[-2], + ) + ).all() + # Values in the output tensor at the places of transformed indices must + # have the same value as the input tensor has at the corresponding + # positions + self.assert_close( + output[..., result_coordinates[0, 1, :], result_coordinates[0, 0, :]], + input[..., input_coordinates[0, 1, :], input_coordinates[0, 0, :]], + ) + + @pytest.mark.slow + def test_gradcheck(self, device): + input = torch.rand((3, 3), device=device, dtype=torch.float64) + self.gradcheck(RandomVerticalFlip(p=1.0), (input,)) + + +class TestColorJiggle(BaseTester): + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self): + f = ColorJiggle(brightness=0.5, contrast=0.3, saturation=[0.2, 1.2], hue=0.1) + repr = ( + "ColorJiggle(brightness=tensor([0.5000, 1.5000]), contrast=tensor([0.7000, 1.3000]), " + "saturation=tensor([0.2000, 1.2000]), hue=tensor([-0.1000, 0.1000]), " + "p=1.0, p_batch=1.0, same_on_batch=False)" + ) + assert str(f) == repr + + @pytest.mark.parametrize("C", [1, 3]) + def test_color_jiggle(self, device, dtype, C): + f = ColorJiggle() + + input = torch.rand(C, 5, 5, device=device, dtype=dtype).unsqueeze(0) # 3 x 5 x 5 + expected = input + + expected_transform = torch.eye(3, device=device, dtype=dtype).unsqueeze(0) # 3 x 3 + + self.assert_close(f(input), expected, low_tolerance=True) + self.assert_close(f.transform_matrix, expected_transform, low_tolerance=True) + + def test_color_jiggle_batch(self, device, dtype): + f = ColorJiggle() + + input = torch.rand(2, 3, 5, 5, device=device, dtype=dtype) # 2 x 3 x 5 x 5 + expected = input + + expected_transform = torch.eye(3, device=device, dtype=dtype).unsqueeze(0).expand((2, 3, 3)) # 2 x 3 x 3 + + self.assert_close(f(input), expected, low_tolerance=True) + self.assert_close(f.transform_matrix, expected_transform, low_tolerance=True) + + def test_same_on_batch(self, device, dtype): + f = ColorJiggle(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.1, same_on_batch=True) + input = torch.eye(3).unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 3, 1, 1) + res = f(input) + self.assert_close(res[0], res[1]) + + def _get_expected_brightness(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.2529, 0.3529, 0.4529], + [0.7529, 0.6529, 0.5529], + [0.8529, 0.9529, 1.0000], + ], + [ + [0.2529, 0.3529, 0.4529], + [0.7529, 0.6529, 0.5529], + [0.8529, 0.9529, 1.0000], + ], + [ + [0.2529, 0.3529, 0.4529], + [0.7529, 0.6529, 0.5529], + [0.8529, 0.9529, 1.0000], + ], + ], + [ + [ + [0.2660, 0.3660, 0.4660], + [0.7660, 0.6660, 0.5660], + [0.8660, 0.9660, 1.0000], + ], + [ + [0.2660, 0.3660, 0.4660], + [0.7660, 0.6660, 0.5660], + [0.8660, 0.9660, 1.0000], + ], + [ + [0.2660, 0.3660, 0.4660], + [0.7660, 0.6660, 0.5660], + [0.8660, 0.9660, 1.0000], + ], + ], + ], + device=device, + dtype=dtype, + ) + + def test_random_brightness(self, device, dtype): + torch.manual_seed(42) + f = ColorJiggle(brightness=0.2) + + input = torch.tensor( + [[[[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 3, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_brightness(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def test_random_brightness_tuple(self, device, dtype): + torch.manual_seed(42) + f = ColorJiggle(brightness=(0.8, 1.2)) + + input = torch.tensor( + [[[[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 3, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_brightness(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def _get_expected_contrast(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.0953, 0.1906, 0.2859], + [0.5719, 0.4766, 0.3813], + [0.6672, 0.7625, 0.9531], + ], + [ + [0.0953, 0.1906, 0.2859], + [0.5719, 0.4766, 0.3813], + [0.6672, 0.7625, 0.9531], + ], + [ + [0.0953, 0.1906, 0.2859], + [0.5719, 0.4766, 0.3813], + [0.6672, 0.7625, 0.9531], + ], + ], + [ + [ + [0.1184, 0.2367, 0.3551], + [0.7102, 0.5919, 0.4735], + [0.8286, 0.9470, 1.0000], + ], + [ + [0.1184, 0.2367, 0.3551], + [0.7102, 0.5919, 0.4735], + [0.8286, 0.9470, 1.0000], + ], + [ + [0.1184, 0.2367, 0.3551], + [0.7102, 0.5919, 0.4735], + [0.8286, 0.9470, 1.0000], + ], + ], + ], + device=device, + dtype=dtype, + ) + + def test_random_contrast(self, device, dtype): + torch.manual_seed(42) + f = ColorJiggle(contrast=0.2) + + input = torch.tensor( + [[[[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 3, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_contrast(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def test_random_contrast_list(self, device, dtype): + torch.manual_seed(42) + f = ColorJiggle(contrast=[0.8, 1.2]) + + input = torch.tensor( + [[[[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 3, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_contrast(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def _get_expected_saturation(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.1876, 0.2584, 0.3389], + [0.6292, 0.5000, 0.4000], + [0.7097, 0.8000, 1.0000], + ], + [ + [1.0000, 0.5292, 0.6097], + [0.6292, 0.3195, 0.2195], + [0.8000, 0.1682, 0.2779], + ], + [ + [0.6389, 0.8000, 0.7000], + [0.9000, 0.3195, 0.2195], + [0.8000, 0.4389, 0.5487], + ], + ], + [ + [ + [0.0000, 0.1295, 0.2530], + [0.5648, 0.5000, 0.4000], + [0.6883, 0.8000, 1.0000], + ], + [ + [1.0000, 0.4648, 0.5883], + [0.5648, 0.2765, 0.1765], + [0.8000, 0.0178, 0.1060], + ], + [ + [0.5556, 0.8000, 0.7000], + [0.9000, 0.2765, 0.1765], + [0.8000, 0.3530, 0.4413], + ], + ], + ], + device=device, + dtype=dtype, + ) + + def test_random_saturation(self, device, dtype): + torch.manual_seed(42) + f = ColorJiggle(saturation=0.2) + + input = torch.tensor( + [ + [ + [[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]], + [[1.0, 0.5, 0.6], [0.6, 0.3, 0.2], [0.8, 0.1, 0.2]], + [[0.6, 0.8, 0.7], [0.9, 0.3, 0.2], [0.8, 0.4, 0.5]], + ] + ], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 1, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_saturation(device, dtype) + self.assert_close(f(input), expected, low_tolerance=True) + + def test_random_saturation_tensor(self, device, dtype): + torch.manual_seed(42) + f = ColorJiggle(saturation=torch.tensor(0.2)) + + input = torch.tensor( + [ + [ + [[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]], + [[1.0, 0.5, 0.6], [0.6, 0.3, 0.2], [0.8, 0.1, 0.2]], + [[0.6, 0.8, 0.7], [0.9, 0.3, 0.2], [0.8, 0.4, 0.5]], + ] + ], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 1, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_saturation(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def test_random_saturation_tuple(self, device, dtype): + torch.manual_seed(42) + f = ColorJiggle(saturation=(0.8, 1.2)) + + input = torch.tensor( + [ + [ + [[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]], + [[1.0, 0.5, 0.6], [0.6, 0.3, 0.2], [0.8, 0.1, 0.2]], + [[0.6, 0.8, 0.7], [0.9, 0.3, 0.2], [0.8, 0.4, 0.5]], + ] + ], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 1, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_saturation(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def _get_expected_hue(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.1000, 0.2000, 0.3000], + [0.6000, 0.5000, 0.4000], + [0.7000, 0.8000, 1.0000], + ], + [ + [1.0000, 0.5251, 0.6167], + [0.6126, 0.3000, 0.2000], + [0.8000, 0.1000, 0.2000], + ], + [ + [0.5623, 0.8000, 0.7000], + [0.9000, 0.3084, 0.2084], + [0.7958, 0.4293, 0.5335], + ], + ], + [ + [ + [0.1000, 0.2000, 0.3000], + [0.6116, 0.5000, 0.4000], + [0.7000, 0.8000, 1.0000], + ], + [ + [1.0000, 0.4769, 0.5846], + [0.6000, 0.3077, 0.2077], + [0.7961, 0.1000, 0.2000], + ], + [ + [0.6347, 0.8000, 0.7000], + [0.9000, 0.3000, 0.2000], + [0.8000, 0.3730, 0.4692], + ], + ], + ], + device=device, + dtype=dtype, + ) + + def test_random_hue(self, device, dtype): + torch.manual_seed(42) + f = ColorJiggle(hue=0.1 / pi.item()) + + input = torch.tensor( + [ + [ + [[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]], + [[1.0, 0.5, 0.6], [0.6, 0.3, 0.2], [0.8, 0.1, 0.2]], + [[0.6, 0.8, 0.7], [0.9, 0.3, 0.2], [0.8, 0.4, 0.5]], + ] + ], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 1, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_hue(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def test_random_hue_list(self, device, dtype): + torch.manual_seed(42) + f = ColorJiggle(hue=[-0.1 / pi, 0.1 / pi]) + + input = torch.tensor( + [ + [ + [[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]], + [[1.0, 0.5, 0.6], [0.6, 0.3, 0.2], [0.8, 0.1, 0.2]], + [[0.6, 0.8, 0.7], [0.9, 0.3, 0.2], [0.8, 0.4, 0.5]], + ] + ], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 1, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_hue(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def test_sequential(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + f = AugmentationSequential(ColorJiggle(), ColorJiggle()) + + input = torch.rand(3, 5, 5, device=device, dtype=dtype).unsqueeze(0) # 1 x 3 x 5 x 5 + + expected = input + + expected_transform = torch.eye(3, device=device, dtype=dtype).unsqueeze(0) # 3 x 3 + + self.assert_close(f(input), expected) + self.assert_close(f.transform_matrix, expected_transform) + + def test_color_jitter_batch_sequential(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + f = AugmentationSequential(ColorJiggle(), ColorJiggle()) + + input = torch.rand(2, 3, 5, 5, device=device, dtype=dtype) # 2 x 3 x 5 x 5 + expected = input + + expected_transform = torch.eye(3, device=device, dtype=dtype).unsqueeze(0).expand((2, 3, 3)) # 2 x 3 x 3 + + self.assert_close(f(input), expected, low_tolerance=True) + self.assert_close(f(input), expected, low_tolerance=True) + self.assert_close(f.transform_matrix, expected_transform, low_tolerance=True) + + @pytest.mark.slow + def test_gradcheck(self, device): + input = torch.rand((3, 5, 5), device=device, dtype=torch.float64).unsqueeze(0) # 3 x 3 + self.gradcheck(ColorJiggle(p=1.0), (input,)) + + +class TestColorJitter(BaseTester): + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self): + f = ColorJitter(brightness=0.5, contrast=0.3, saturation=[0.2, 1.2], hue=0.1) + repr = ( + "ColorJitter(brightness=tensor([0.5000, 1.5000]), contrast=tensor([0.7000, 1.3000]), " + "saturation=tensor([0.2000, 1.2000]), hue=tensor([-0.1000, 0.1000]), " + "p=1.0, p_batch=1.0, same_on_batch=False)" + ) + assert str(f) == repr + + def test_color_jitter(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + f = ColorJitter() + + input = torch.rand(3, 5, 5, device=device, dtype=dtype).unsqueeze(0) # 3 x 5 x 5 + expected = input + + expected_transform = torch.eye(3, device=device, dtype=dtype).unsqueeze(0) # 3 x 3 + + self.assert_close(f(input), expected) + self.assert_close(f.transform_matrix, expected_transform) + + def test_color_jitter_batch(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + f = ColorJitter() + + input = torch.rand(2, 3, 5, 5, device=device, dtype=dtype) # 2 x 3 x 5 x 5 + expected = input + + expected_transform = torch.eye(3, device=device, dtype=dtype).unsqueeze(0).expand((2, 3, 3)) # 2 x 3 x 3 + + self.assert_close(f(input), expected) + self.assert_close(f.transform_matrix, expected_transform) + + def test_same_on_batch(self, device, dtype): + f = ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.1, same_on_batch=True) + input = torch.eye(3).unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 3, 1, 1) + res = f(input) + self.assert_close(res[0], res[1]) + + def _get_expected_brightness(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.1153, 0.2306, 0.3459], + [0.6917, 0.5764, 0.4612], + [0.8070, 0.9223, 1.0000], + ], + [ + [0.1153, 0.2306, 0.3459], + [0.6917, 0.5764, 0.4612], + [0.8070, 0.9223, 1.0000], + ], + [ + [0.1153, 0.2306, 0.3459], + [0.6917, 0.5764, 0.4612], + [0.8070, 0.9223, 1.0000], + ], + ], + [ + [ + [0.1166, 0.2332, 0.3498], + [0.6996, 0.5830, 0.4664], + [0.8162, 0.9328, 1.0000], + ], + [ + [0.1166, 0.2332, 0.3498], + [0.6996, 0.5830, 0.4664], + [0.8162, 0.9328, 1.0000], + ], + [ + [0.1166, 0.2332, 0.3498], + [0.6996, 0.5830, 0.4664], + [0.8162, 0.9328, 1.0000], + ], + ], + ], + device=device, + dtype=dtype, + ) + + def test_random_brightness(self, device, dtype): + torch.manual_seed(42) + f = ColorJitter(brightness=0.2) + + input = torch.tensor( + [[[[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 3, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_brightness(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def test_random_brightness_tuple(self, device, dtype): + torch.manual_seed(42) + f = ColorJitter(brightness=(0.8, 1.2)) + + input = torch.tensor( + [[[[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 3, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_brightness(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def _get_expected_contrast(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.1193, 0.2146, 0.3099], + [0.5958, 0.5005, 0.4052], + [0.6911, 0.7865, 0.9771], + ], + [ + [0.1193, 0.2146, 0.3099], + [0.5958, 0.5005, 0.4052], + [0.6911, 0.7865, 0.9771], + ], + [ + [0.1193, 0.2146, 0.3099], + [0.5958, 0.5005, 0.4052], + [0.6911, 0.7865, 0.9771], + ], + ], + [ + [ + [0.0245, 0.1428, 0.2612], + [0.6163, 0.4980, 0.3796], + [0.7347, 0.8531, 1.0000], + ], + [ + [0.0245, 0.1428, 0.2612], + [0.6163, 0.4980, 0.3796], + [0.7347, 0.8531, 1.0000], + ], + [ + [0.0245, 0.1428, 0.2612], + [0.6163, 0.4980, 0.3796], + [0.7347, 0.8531, 1.0000], + ], + ], + ], + device=device, + dtype=dtype, + ) + + def test_random_contrast(self, device, dtype): + torch.manual_seed(42) + f = ColorJitter(contrast=0.2) + + input = torch.tensor( + [[[[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 3, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_contrast(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def test_random_contrast_list(self, device, dtype): + torch.manual_seed(42) + f = ColorJitter(contrast=[0.8, 1.2]) + + input = torch.tensor( + [[[[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 3, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_contrast(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def _get_expected_saturation(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.1570, 0.2238, 0.3216], + [0.6033, 0.4863, 0.3863], + [0.7068, 0.7555, 0.9487], + ], + [ + [0.9693, 0.4946, 0.5924], + [0.6033, 0.3058, 0.2058], + [0.7971, 0.1237, 0.2266], + ], + [ + [0.6083, 0.7654, 0.6826], + [0.8741, 0.3058, 0.2058], + [0.7971, 0.3945, 0.4974], + ], + ], + [ + [ + [0.0312, 0.1713, 0.2740], + [0.5960, 0.5165, 0.4165], + [0.6918, 0.8536, 1.0000], + ], + [ + [1.0000, 0.5065, 0.6092], + [0.5960, 0.2930, 0.1930], + [0.8035, 0.0714, 0.1679], + ], + [ + [0.5900, 0.8418, 0.7210], + [0.9312, 0.2930, 0.1930], + [0.8035, 0.4066, 0.5031], + ], + ], + ], + device=device, + dtype=dtype, + ) + + def test_random_saturation(self, device, dtype): + torch.manual_seed(42) + f = ColorJitter(saturation=0.2) + + input = torch.tensor( + [ + [ + [[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]], + [[1.0, 0.5, 0.6], [0.6, 0.3, 0.2], [0.8, 0.1, 0.2]], + [[0.6, 0.8, 0.7], [0.9, 0.3, 0.2], [0.8, 0.4, 0.5]], + ] + ], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 1, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_saturation(device, dtype) + self.assert_close(f(input), expected, low_tolerance=True) + + def test_random_saturation_tensor(self, device, dtype): + torch.manual_seed(42) + f = ColorJitter(saturation=torch.tensor(0.2)) + + input = torch.tensor( + [ + [ + [[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]], + [[1.0, 0.5, 0.6], [0.6, 0.3, 0.2], [0.8, 0.1, 0.2]], + [[0.6, 0.8, 0.7], [0.9, 0.3, 0.2], [0.8, 0.4, 0.5]], + ] + ], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 1, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_saturation(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def test_random_saturation_tuple(self, device, dtype): + torch.manual_seed(42) + f = ColorJitter(saturation=(0.8, 1.2)) + + input = torch.tensor( + [ + [ + [[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]], + [[1.0, 0.5, 0.6], [0.6, 0.3, 0.2], [0.8, 0.1, 0.2]], + [[0.6, 0.8, 0.7], [0.9, 0.3, 0.2], [0.8, 0.4, 0.5]], + ] + ], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 1, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_saturation(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def _get_expected_hue(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.1000, 0.2000, 0.3000], + [0.6000, 0.5000, 0.4000], + [0.7000, 0.8000, 1.0000], + ], + [ + [1.0000, 0.5251, 0.6167], + [0.6126, 0.3000, 0.2000], + [0.8000, 0.1000, 0.2000], + ], + [ + [0.5623, 0.8000, 0.7000], + [0.9000, 0.3084, 0.2084], + [0.7958, 0.4293, 0.5335], + ], + ], + [ + [ + [0.1000, 0.2000, 0.3000], + [0.6116, 0.5000, 0.4000], + [0.7000, 0.8000, 1.0000], + ], + [ + [1.0000, 0.4769, 0.5846], + [0.6000, 0.3077, 0.2077], + [0.7961, 0.1000, 0.2000], + ], + [ + [0.6347, 0.8000, 0.7000], + [0.9000, 0.3000, 0.2000], + [0.8000, 0.3730, 0.4692], + ], + ], + ], + device=device, + dtype=dtype, + ) + + def test_random_hue(self, device, dtype): + torch.manual_seed(42) + f = ColorJiggle(hue=0.1 / pi.item()) + + input = torch.tensor( + [ + [ + [[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]], + [[1.0, 0.5, 0.6], [0.6, 0.3, 0.2], [0.8, 0.1, 0.2]], + [[0.6, 0.8, 0.7], [0.9, 0.3, 0.2], [0.8, 0.4, 0.5]], + ] + ], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 1, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_hue(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def test_random_hue_list(self, device, dtype): + torch.manual_seed(42) + f = ColorJiggle(hue=[-0.1 / pi, 0.1 / pi]) + + input = torch.tensor( + [ + [ + [[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]], + [[1.0, 0.5, 0.6], [0.6, 0.3, 0.2], [0.8, 0.1, 0.2]], + [[0.6, 0.8, 0.7], [0.9, 0.3, 0.2], [0.8, 0.4, 0.5]], + ] + ], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 1, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_hue(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def test_sequential(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + f = AugmentationSequential(ColorJiggle(), ColorJiggle()) + + input = torch.rand(3, 5, 5, device=device, dtype=dtype).unsqueeze(0).repeat(4, 1, 1, 1) # 4 x 3 x 5 x 5 + + expected = input + + expected_transform = torch.eye(3, device=device, dtype=dtype).unsqueeze(0).repeat(4, 1, 1) # 4 x 3 x 3 + + self.assert_close(f(input), expected) + self.assert_close(f.transform_matrix, expected_transform) + + def test_color_jitter_batch_sequential(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + f = AugmentationSequential(ColorJitter(), ColorJitter()) + + input = torch.rand(2, 3, 5, 5, device=device, dtype=dtype) # 2 x 3 x 5 x 5 + expected = input + + expected_transform = torch.eye(3, device=device, dtype=dtype).unsqueeze(0).expand((2, 3, 3)) # 2 x 3 x 3 + + self.assert_close(f(input), expected) + self.assert_close(f(input), expected) + self.assert_close(f.transform_matrix, expected_transform) + + @pytest.mark.slow + def test_dynamo(self, device, dtype, torch_optimizer): + input = torch.rand((1, 3, 5, 5), device=device, dtype=dtype) + f = ColorJitter(p=1.0).compile(fullgraph=True) + out = f(input) + assert out.shape == input.shape + + +class TestRandomBrightness(BaseTester): + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self): + f = RandomBrightness(brightness=(0.5, 1.5)) + repr = "RandomBrightness(brightness_factor=tensor([0.5000, 1.5000]), p=1.0, p_batch=1.0, same_on_batch=False))" + assert str(f.__repr__) == repr + + def test_random_brighness_identity(self, device, dtype): + f = RandomBrightness() + + input = torch.rand(3, 5, 5, device=device, dtype=dtype).unsqueeze(0) # 3 x 5 x 5 + expected = input + + expected_transform = torch.eye(3, device=device, dtype=dtype).unsqueeze(0) # 3 x 3 + + self.assert_close(f(input), expected, low_tolerance=True) + self.assert_close(f.transform_matrix, expected_transform, low_tolerance=True) + + def test_same_on_batch(self, device, dtype): + f = RandomBrightness(brightness=(0.5, 1.5), same_on_batch=True) + input = torch.eye(3).unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 3, 1, 1) + res = f(input) + self.assert_close(res[0], res[1]) + + def _get_expected_brightness(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.2529, 0.3529, 0.4529], + [0.7529, 0.6529, 0.5529], + [0.8529, 0.9529, 1.0000], + ], + [ + [0.2529, 0.3529, 0.4529], + [0.7529, 0.6529, 0.5529], + [0.8529, 0.9529, 1.0000], + ], + [ + [0.2529, 0.3529, 0.4529], + [0.7529, 0.6529, 0.5529], + [0.8529, 0.9529, 1.0000], + ], + ], + [ + [ + [0.2660, 0.3660, 0.4660], + [0.7660, 0.6660, 0.5660], + [0.8660, 0.9660, 1.0000], + ], + [ + [0.2660, 0.3660, 0.4660], + [0.7660, 0.6660, 0.5660], + [0.8660, 0.9660, 1.0000], + ], + [ + [0.2660, 0.3660, 0.4660], + [0.7660, 0.6660, 0.5660], + [0.8660, 0.9660, 1.0000], + ], + ], + ], + device=device, + dtype=dtype, + ) + + def test_random_brightness(self, device, dtype): + torch.manual_seed(42) + f = RandomBrightness(brightness=(0.8, 1.2)) + + input = torch.tensor( + [[[[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 3, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_brightness(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def test_sequential(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + torch.manual_seed(27) + + f = AugmentationSequential(RandomBrightness()) + + input = torch.rand(3, 5, 5, device=device, dtype=dtype).unsqueeze(0) # 1 x 3 x 5 x 5 + + expected = input + + self.assert_close(f(input), expected) + + def test_random_brightness_batch_sequential(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + f = AugmentationSequential(RandomBrightness(), RandomBrightness()) + + input = torch.rand(2, 3, 5, 5, device=device, dtype=dtype) # 2 x 3 x 5 x 5 + expected = input + + self.assert_close(f(input), expected, low_tolerance=True) + + @pytest.mark.slow + def test_gradcheck(self, device): + input = torch.rand((3, 5, 5), device=device, dtype=torch.float64).unsqueeze(0) # 3 x 3 + self.gradcheck(RandomBrightness(p=1.0), (input,)) + + +class TestRandomContrast(BaseTester): + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self): + f = RandomContrast(contrast=(0.7, 1.3)) + repr = "RandomContrast(contrast=tensor([0.7000, 1.3000]), p=1.0, p_batch=1.0, same_on_batch=False)" + assert str(f) == repr + + def test_random_contrast_identity(self, device, dtype): + f = RandomContrast() + + input = torch.rand(3, 5, 5, device=device, dtype=dtype).unsqueeze(0) # 3 x 5 x 5 + expected = input + + expected_transform = torch.eye(3, device=device, dtype=dtype).unsqueeze(0) # 3 x 3 + + self.assert_close(f(input), expected, low_tolerance=True) + self.assert_close(f.transform_matrix, expected_transform, low_tolerance=True) + + def test_same_on_batch(self, device, dtype): + f = RandomContrast(contrast=(0.5, 1.5), same_on_batch=True) + input = torch.eye(3).unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 3, 1, 1) + res = f(input) + self.assert_close(res[0], res[1]) + + def _get_expected_contrast(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.1153, 0.2306, 0.3459], + [0.6917, 0.5765, 0.4612], + [0.8070, 0.9225, 1.0000], + ], + [ + [0.1153, 0.2306, 0.3459], + [0.6917, 0.5765, 0.4612], + [0.8070, 0.9225, 1.0000], + ], + [ + [0.1153, 0.2306, 0.3459], + [0.6917, 0.5765, 0.4612], + [0.8070, 0.9225, 1.0000], + ], + ], + [ + [ + [0.1166, 0.2332, 0.3498], + [0.6996, 0.5830, 0.4664], + [0.8162, 0.9328, 1.0000], + ], + [ + [0.1166, 0.2332, 0.3498], + [0.6996, 0.5830, 0.4664], + [0.8162, 0.9328, 1.0000], + ], + [ + [0.1166, 0.2332, 0.3498], + [0.6996, 0.5830, 0.4664], + [0.8162, 0.9328, 1.0000], + ], + ], + ], + device=device, + dtype=dtype, + ) + + def test_random_contrast(self, device, dtype): + torch.manual_seed(42) + f = RandomContrast(contrast=(0.8, 1.2)) + + input = torch.tensor( + [[[[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]]]], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 3, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_contrast(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def test_sequential(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + torch.manual_seed(27) + + f = AugmentationSequential(RandomContrast()) + + input = torch.rand(3, 5, 5, device=device, dtype=dtype).unsqueeze(0) # 1 x 3 x 5 x 5 + + expected = input + + self.assert_close(f(input), expected) + + def test_random_contrast_batch_sequential(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + f = AugmentationSequential(RandomContrast(), RandomContrast()) + + input = torch.rand(2, 3, 5, 5, device=device, dtype=dtype) # 2 x 3 x 5 x 5 + expected = input + + self.assert_close(f(input), expected, low_tolerance=True) + + @pytest.mark.slow + def test_gradcheck(self, device): + input = torch.rand((3, 5, 5), device=device, dtype=torch.float64).unsqueeze(0) # 3 x 3 + self.gradcheck(RandomContrast(p=1.0), (input,)) + + +class TestRandomHue(BaseTester): + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self): + f = RandomHue(hue=(-0.2, 0.3)) + repr = "RandomHue(hue=tensor([-0.2000, 0.3000]), p=1.0, p_batch=1.0, same_on_batch=False)" + assert str(f) == repr + + def test_random_hue_identity(self, device, dtype): + f = RandomHue(hue=(0.0, 0.0)) + + input = torch.rand(3, 5, 5, device=device, dtype=dtype).unsqueeze(0) # 3 x 5 x 5 + expected = input + + expected_transform = torch.eye(3, device=device, dtype=dtype).unsqueeze(0) # 3 x 3 + + self.assert_close(f(input), expected, low_tolerance=True) + self.assert_close(f.transform_matrix, expected_transform, low_tolerance=True) + + def test_same_on_batch(self, device, dtype): + f = RandomHue(hue=(-0.5, 0.5), same_on_batch=True) + input = torch.eye(3).unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 3, 1, 1) + res = f(input) + self.assert_close(res[0], res[1]) + + def _get_expected_hue(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.1000, 0.2000, 0.3000], + [0.6438, 0.5000, 0.4000], + [0.7000, 0.8000, 1.0000], + ], + [ + [1.0000, 0.4124, 0.5416], + [0.6000, 0.3292, 0.2292], + [0.7854, 0.1000, 0.2000], + ], + [ + [0.7314, 0.8000, 0.7000], + [0.9000, 0.3000, 0.2000], + [0.8000, 0.2978, 0.3832], + ], + ], + [ + [ + [0.1000, 0.2000, 0.3000], + [0.6476, 0.5000, 0.4000], + [0.7000, 0.8000, 1.0000], + ], + [ + [1.0000, 0.4049, 0.5366], + [0.6000, 0.3317, 0.2317], + [0.7841, 0.1000, 0.2000], + ], + [ + [0.7427, 0.8000, 0.7000], + [0.9000, 0.3000, 0.2000], + [0.8000, 0.2890, 0.3732], + ], + ], + ], + device=device, + dtype=dtype, + ) + + def test_random_hue(self, device, dtype): + torch.manual_seed(42) + f = RandomHue(hue=(-0.1 / pi, 0.1 / pi)) + + input = torch.tensor( + [ + [ + [[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]], + [[1.0, 0.5, 0.6], [0.6, 0.3, 0.2], [0.8, 0.1, 0.2]], + [[0.6, 0.8, 0.7], [0.9, 0.3, 0.2], [0.8, 0.4, 0.5]], + ] + ], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 1, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_hue(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def test_sequential(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + torch.manual_seed(27) + + f = AugmentationSequential(RandomHue()) + + input = torch.rand(3, 5, 5, device=device, dtype=dtype).unsqueeze(0) # 1 x 3 x 5 x 5 + + expected = input + + self.assert_close(f(input), expected) + + def test_random_hue_batch_sequential(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + f = AugmentationSequential(RandomHue(), RandomHue()) + + input = torch.rand(2, 3, 5, 5, device=device, dtype=dtype) # 2 x 3 x 5 x 5 + expected = input + + self.assert_close(f(input), expected, low_tolerance=True) + + @pytest.mark.slow + def test_gradcheck(self, device): + input = torch.rand((3, 5, 5), device=device, dtype=torch.float64).unsqueeze(0) # 3 x 3 + self.gradcheck(RandomHue(p=1.0), (input,)) + + +class TestRandomSaturation(BaseTester): + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self): + f = RandomSaturation(saturation=(0.5, 1.5)) + repr = "RandomSaturation(hue=tensor([0.5000, 1.5000]), p=1.0, p_batch=1.0, same_on_batch=False)" + assert str(f) == repr + + def test_random_saturation_identity(self, device, dtype): + f = RandomSaturation(saturation=(1.0, 1.0)) + + input = torch.rand(3, 5, 5, device=device, dtype=dtype).unsqueeze(0) # 3 x 5 x 5 + expected = input + + expected_transform = torch.eye(3, device=device, dtype=dtype).unsqueeze(0) # 3 x 3 + + self.assert_close(f(input), expected, low_tolerance=True) + self.assert_close(f.transform_matrix, expected_transform, low_tolerance=True) + + def test_same_on_batch(self, device, dtype): + f = RandomSaturation(saturation=(0.5, 1.5), same_on_batch=True) + input = torch.eye(3).unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 3, 1, 1) + res = f(input) + self.assert_close(res[0], res[1]) + + def _get_expected_saturation(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.0000, 0.0000, 0.1471], + [0.4853, 0.5000, 0.4000], + [0.6618, 0.8000, 1.0000], + ], + [ + [1.0000, 0.4000, 0.5618], + [0.4853, 0.2235, 0.1235], + [0.8000, 0.0000, 0.0000], + ], + [ + [0.5556, 0.8000, 0.7000], + [0.9000, 0.2235, 0.1235], + [0.8000, 0.3429, 0.3750], + ], + ], + [ + [ + [0.0000, 0.0000, 0.1340], + [0.4755, 0.5000, 0.4000], + [0.6585, 0.8000, 1.0000], + ], + [ + [1.0000, 0.4000, 0.5585], + [0.4755, 0.2170, 0.1170], + [0.8000, 0.0000, 0.0000], + ], + [ + [0.5556, 0.8000, 0.7000], + [0.9000, 0.2170, 0.1170], + [0.8000, 0.3429, 0.3750], + ], + ], + ], + device=device, + dtype=dtype, + ) + + def test_random_saturation(self, device, dtype): + torch.manual_seed(42) + f = RandomSaturation(saturation=(0.5, 1.5)) + + input = torch.tensor( + [ + [ + [[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]], + [[1.0, 0.5, 0.6], [0.6, 0.3, 0.2], [0.8, 0.1, 0.2]], + [[0.6, 0.8, 0.7], [0.9, 0.3, 0.2], [0.8, 0.4, 0.5]], + ] + ], + device=device, + dtype=dtype, + ) # 1 x 1 x 3 x 3 + input = input.repeat(2, 1, 1, 1) # 2 x 3 x 3 + + expected = self._get_expected_saturation(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def test_sequential(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + torch.manual_seed(27) + + f = AugmentationSequential(RandomSaturation()) + + input = torch.rand(3, 5, 5, device=device, dtype=dtype).unsqueeze(0) # 1 x 3 x 5 x 5 + + expected = input + + self.assert_close(f(input), expected) + + def test_random_saturation_batch_sequential(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + f = AugmentationSequential(RandomSaturation(), RandomSaturation()) + + input = torch.rand(2, 3, 5, 5, device=device, dtype=dtype) # 2 x 3 x 5 x 5 + expected = input + + self.assert_close(f(input), expected, low_tolerance=True) + + @pytest.mark.slow + def test_gradcheck(self, device): + input = torch.rand((3, 5, 5), device=device, dtype=torch.float64).unsqueeze(0) # 3 x 3 + self.gradcheck(RandomSaturation(p=1.0), (input,)) + + +class TestRectangleRandomErasing(BaseTester): + @pytest.mark.parametrize("erase_scale_range", [(0.001, 0.001), (1.0, 1.0)]) + @pytest.mark.parametrize("aspect_ratio_range", [(0.1, 0.1), (10.0, 10.0)]) + @pytest.mark.parametrize("batch_shape", [(1, 4, 8, 15), (2, 3, 11, 7)]) + def test_random_rectangle_erasing_shape(self, batch_shape, erase_scale_range, aspect_ratio_range): + input = torch.rand(batch_shape) + rand_rec = RandomErasing(erase_scale_range, aspect_ratio_range, p=1.0) + assert rand_rec(input).shape == batch_shape + + @pytest.mark.parametrize("erase_scale_range", [(0.001, 0.001), (1.0, 1.0)]) + @pytest.mark.parametrize("aspect_ratio_range", [(0.1, 0.1), (10.0, 10.0)]) + @pytest.mark.parametrize("batch_shape", [(1, 4, 8, 15), (2, 3, 11, 7)]) + def test_no_rectangle_erasing_shape(self, batch_shape, erase_scale_range, aspect_ratio_range): + input = torch.rand(batch_shape) + rand_rec = RandomErasing(erase_scale_range, aspect_ratio_range, p=0.0) + assert rand_rec(input).equal(input) + + @pytest.mark.parametrize("erase_scale_range", [(0.001, 0.001), (1.0, 1.0)]) + @pytest.mark.parametrize("aspect_ratio_range", [(0.1, 0.1), (10.0, 10.0)]) + @pytest.mark.parametrize("shape", [(3, 11, 7)]) + def test_same_on_batch(self, shape, erase_scale_range, aspect_ratio_range): + f = RandomErasing(erase_scale_range, aspect_ratio_range, same_on_batch=True, p=0.5) + input = torch.rand(shape).unsqueeze(dim=0).repeat(2, 1, 1, 1) + res = f(input) + self.assert_close(res[0], res[1]) + + @pytest.mark.slow + def test_gradcheck(self, device): + # test parameters + batch_shape = (2, 3, 11, 7) + erase_scale_range = (0.2, 0.4) + aspect_ratio_range = (0.3, 0.5) + + rand_rec = RandomErasing(erase_scale_range, aspect_ratio_range, p=1.0) + rect_params = rand_rec.forward_parameters(batch_shape) + + # evaluate function gradient + input = torch.rand(batch_shape, device=device, dtype=torch.float64) + self.gradcheck(rand_rec, (input, rect_params)) + + +class TestRandomGamma(BaseTester): + # return values such a Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self): + f = RandomGamma(gamma=(0.5, 2.0), gain=(0.5, 0.5)) + repr = ( + "RandomGamma(gamma=tensor([0.5000, 2.5000]), gain=tensor([0.5000, 1.5000]), " + "p=1.0, p_batch=1.0, same_on_batch=False)" + ) + assert str(f) == repr + + def test_same_on_batch(self, device, dtype): + f = RandomGamma(gamma=(0.5, 2.0), gain=(0.5, 0.5), same_on_batch=True) + input = torch.eye(3).unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 3, 1, 1) + res = f(input) + self.assert_close(res[0], res[1]) + + def _get_expected_gamma(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.0703, 0.1564, 0.2496], + [0.5549, 0.4497, 0.3477], + [0.6628, 0.7732, 1.0000], + ], + [ + [1.0000, 0.4497, 0.5549], + [0.5549, 0.2496, 0.1564], + [0.7732, 0.0703, 0.1564], + ], + [ + [0.5549, 0.7732, 0.6628], + [0.8856, 0.2496, 0.1564], + [0.7732, 0.3477, 0.4497], + ], + ], + [ + [ + [0.0682, 0.1531, 0.2457], + [0.5512, 0.4457, 0.3436], + [0.6598, 0.7709, 1.0000], + ], + [ + [1.0000, 0.4457, 0.5512], + [0.5512, 0.2457, 0.1531], + [0.7709, 0.0682, 0.1531], + ], + [ + [0.5512, 0.7709, 0.6598], + [0.8844, 0.2457, 0.1531], + [0.7709, 0.3436, 0.4457], + ], + ], + ], + device=device, + dtype=dtype, + ) + + def test_random_gamma(self, device, dtype): + torch.manual_seed(42) + f = RandomGamma(gamma=(0.8, 1.2)) + input = torch.tensor( + [ + [ + [[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]], + [[1.0, 0.5, 0.6], [0.6, 0.3, 0.2], [0.8, 0.1, 0.2]], + [[0.6, 0.8, 0.7], [0.9, 0.3, 0.2], [0.8, 0.4, 0.5]], + ] + ], + device=device, + dtype=dtype, + ) # 1 x 3 x 3 x 3 + + input = input.repeat(2, 1, 1, 1) # 2 x 3 x 3 x 3 + + expected = self._get_expected_gamma(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def test_random_gamma_tuple(self, device, dtype): + torch.manual_seed(42) + + f = RandomGamma(gamma=(0.8, 1.2)) + + input = torch.tensor( + [ + [ + [[0.1, 0.2, 0.3], [0.6, 0.5, 0.4], [0.7, 0.8, 1.0]], + [[1.0, 0.5, 0.6], [0.6, 0.3, 0.2], [0.8, 0.1, 0.2]], + [[0.6, 0.8, 0.7], [0.9, 0.3, 0.2], [0.8, 0.4, 0.5]], + ] + ], + device=device, + dtype=dtype, + ) # 1 x 3 x 3 x 3 + + input = input.repeat(2, 1, 1, 1) # 2 x 3 x 3 x 3 + + expected = self._get_expected_gamma(device, dtype) + + self.assert_close(f(input), expected, low_tolerance=True) + + def test_sequential(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + torch.manual_seed(27) + + f = AugmentationSequential(RandomGamma(gamma=(0.5, 0.5)), RandomGamma(gamma=(2.0, 2.0))) + + input = torch.rand(3, 5, 5, device=device, dtype=dtype).unsqueeze(0) # 1 x 3 x 5 x 5 + + expected = input + + self.assert_close(f(input), expected) + + def test_random_gamma_batch_sequential(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + f = AugmentationSequential(RandomGamma(gamma=(0.5, 0.5)), RandomGamma(gamma=(2.0, 2.0))) + + input = torch.rand(2, 3, 5, 5, device=device, dtype=dtype) # 2 x 3 x 5 x 5 + expected = input + + self.assert_close(f(input), expected, low_tolerance=True) + + @pytest.mark.slow + def test_gradcheck(self, device): + input = torch.rand((3, 5, 5), device=device, dtype=torch.float64).unsqueeze(0) # 3 x 3 + self.gradcheck(RandomGamma(p=1.0), (input,)) + + +class TestRandomGrayscale(BaseTester): + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self): + f = RandomGrayscale() + repr = "RandomGrayscale(p=0.1, p_batch=1.0, same_on_batch=False)" + assert str(f) == repr + + def test_random_grayscale(self, device, dtype): + f = RandomGrayscale() + + input = torch.rand(3, 5, 5, device=device, dtype=dtype) # 3 x 5 x 5 + + expected_transform = torch.eye(3, device=device, dtype=dtype).unsqueeze(0) # 3 x 3 + f(input) + self.assert_close(f.transform_matrix, expected_transform) + + def test_same_on_batch(self, device, dtype): + f = RandomGrayscale(p=0.5, same_on_batch=True) + input = torch.eye(3, device=device, dtype=dtype).unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 3, 1, 1) + res = f(input) + self.assert_close(res[0], res[1]) + + def test_opencv_true(self, device, dtype): + data = torch.tensor( + [ + [ + [ + [0.3944633, 0.8597369, 0.1670904, 0.2825457, 0.0953912], + [0.1251704, 0.8020709, 0.8933256, 0.9170977, 0.1497008], + [0.2711633, 0.1111478, 0.0783281, 0.2771807, 0.5487481], + [0.0086008, 0.8288748, 0.9647092, 0.8922020, 0.7614344], + [0.2898048, 0.1282895, 0.7621747, 0.5657831, 0.9918593], + ], + [ + [0.5414237, 0.9962701, 0.8947155, 0.5900949, 0.9483274], + [0.0468036, 0.3933847, 0.8046577, 0.3640994, 0.0632100], + [0.6171775, 0.8624780, 0.4126036, 0.7600935, 0.7279997], + [0.4237089, 0.5365476, 0.5591233, 0.1523191, 0.1382165], + [0.8932794, 0.8517839, 0.7152701, 0.8983801, 0.5905426], + ], + [ + [0.2869580, 0.4700376, 0.2743714, 0.8135023, 0.2229074], + [0.9306560, 0.3734594, 0.4566821, 0.7599275, 0.7557513], + [0.7415742, 0.6115875, 0.3317572, 0.0379378, 0.1315770], + [0.8692724, 0.0809556, 0.7767404, 0.8742208, 0.1522012], + [0.7708948, 0.4509611, 0.0481175, 0.2358997, 0.6900532], + ], + ] + ], + device=device, + dtype=dtype, + ) + + # Output data generated with OpenCV 4.1.1: cv2.cvtColor(img_np, cv2.COLOR_RGB2GRAY) + expected = torch.tensor( + [ + [ + [ + [0.4684734, 0.8954562, 0.6064363, 0.5236061, 0.6106016], + [0.1709944, 0.5133104, 0.7915002, 0.5745703, 0.1680204], + [0.5279005, 0.6092287, 0.3034387, 0.5333768, 0.6064113], + [0.3503858, 0.5720159, 0.7052018, 0.4558409, 0.3261529], + [0.6988886, 0.5897652, 0.6532392, 0.7234108, 0.7218805], + ], + [ + [0.4684734, 0.8954562, 0.6064363, 0.5236061, 0.6106016], + [0.1709944, 0.5133104, 0.7915002, 0.5745703, 0.1680204], + [0.5279005, 0.6092287, 0.3034387, 0.5333768, 0.6064113], + [0.3503858, 0.5720159, 0.7052018, 0.4558409, 0.3261529], + [0.6988886, 0.5897652, 0.6532392, 0.7234108, 0.7218805], + ], + [ + [0.4684734, 0.8954562, 0.6064363, 0.5236061, 0.6106016], + [0.1709944, 0.5133104, 0.7915002, 0.5745703, 0.1680204], + [0.5279005, 0.6092287, 0.3034387, 0.5333768, 0.6064113], + [0.3503858, 0.5720159, 0.7052018, 0.4558409, 0.3261529], + [0.6988886, 0.5897652, 0.6532392, 0.7234108, 0.7218805], + ], + ] + ], + device=device, + dtype=dtype, + ) + + img_gray = RandomGrayscale(p=1.0)(data) + self.assert_close(img_gray, expected) + + def test_opencv_false(self, device, dtype): + data = torch.tensor( + [ + [ + [ + [0.3944633, 0.8597369, 0.1670904, 0.2825457, 0.0953912], + [0.1251704, 0.8020709, 0.8933256, 0.9170977, 0.1497008], + [0.2711633, 0.1111478, 0.0783281, 0.2771807, 0.5487481], + [0.0086008, 0.8288748, 0.9647092, 0.8922020, 0.7614344], + [0.2898048, 0.1282895, 0.7621747, 0.5657831, 0.9918593], + ], + [ + [0.5414237, 0.9962701, 0.8947155, 0.5900949, 0.9483274], + [0.0468036, 0.3933847, 0.8046577, 0.3640994, 0.0632100], + [0.6171775, 0.8624780, 0.4126036, 0.7600935, 0.7279997], + [0.4237089, 0.5365476, 0.5591233, 0.1523191, 0.1382165], + [0.8932794, 0.8517839, 0.7152701, 0.8983801, 0.5905426], + ], + [ + [0.2869580, 0.4700376, 0.2743714, 0.8135023, 0.2229074], + [0.9306560, 0.3734594, 0.4566821, 0.7599275, 0.7557513], + [0.7415742, 0.6115875, 0.3317572, 0.0379378, 0.1315770], + [0.8692724, 0.0809556, 0.7767404, 0.8742208, 0.1522012], + [0.7708948, 0.4509611, 0.0481175, 0.2358997, 0.6900532], + ], + ] + ], + device=device, + dtype=dtype, + ) + + expected = data + + img_gray = RandomGrayscale(p=0.0)(data) + self.assert_close(img_gray, expected) + + def test_opencv_true_batch(self, device, dtype): + data = torch.tensor( + [ + [ + [0.3944633, 0.8597369, 0.1670904, 0.2825457, 0.0953912], + [0.1251704, 0.8020709, 0.8933256, 0.9170977, 0.1497008], + [0.2711633, 0.1111478, 0.0783281, 0.2771807, 0.5487481], + [0.0086008, 0.8288748, 0.9647092, 0.8922020, 0.7614344], + [0.2898048, 0.1282895, 0.7621747, 0.5657831, 0.9918593], + ], + [ + [0.5414237, 0.9962701, 0.8947155, 0.5900949, 0.9483274], + [0.0468036, 0.3933847, 0.8046577, 0.3640994, 0.0632100], + [0.6171775, 0.8624780, 0.4126036, 0.7600935, 0.7279997], + [0.4237089, 0.5365476, 0.5591233, 0.1523191, 0.1382165], + [0.8932794, 0.8517839, 0.7152701, 0.8983801, 0.5905426], + ], + [ + [0.2869580, 0.4700376, 0.2743714, 0.8135023, 0.2229074], + [0.9306560, 0.3734594, 0.4566821, 0.7599275, 0.7557513], + [0.7415742, 0.6115875, 0.3317572, 0.0379378, 0.1315770], + [0.8692724, 0.0809556, 0.7767404, 0.8742208, 0.1522012], + [0.7708948, 0.4509611, 0.0481175, 0.2358997, 0.6900532], + ], + ], + device=device, + dtype=dtype, + ) + data = data.unsqueeze(0).repeat(4, 1, 1, 1) + + # Output data generated with OpenCV 4.1.1: cv2.cvtColor(img_np, cv2.COLOR_RGB2GRAY) + expected = torch.tensor( + [ + [ + [0.4684734, 0.8954562, 0.6064363, 0.5236061, 0.6106016], + [0.1709944, 0.5133104, 0.7915002, 0.5745703, 0.1680204], + [0.5279005, 0.6092287, 0.3034387, 0.5333768, 0.6064113], + [0.3503858, 0.5720159, 0.7052018, 0.4558409, 0.3261529], + [0.6988886, 0.5897652, 0.6532392, 0.7234108, 0.7218805], + ], + [ + [0.4684734, 0.8954562, 0.6064363, 0.5236061, 0.6106016], + [0.1709944, 0.5133104, 0.7915002, 0.5745703, 0.1680204], + [0.5279005, 0.6092287, 0.3034387, 0.5333768, 0.6064113], + [0.3503858, 0.5720159, 0.7052018, 0.4558409, 0.3261529], + [0.6988886, 0.5897652, 0.6532392, 0.7234108, 0.7218805], + ], + [ + [0.4684734, 0.8954562, 0.6064363, 0.5236061, 0.6106016], + [0.1709944, 0.5133104, 0.7915002, 0.5745703, 0.1680204], + [0.5279005, 0.6092287, 0.3034387, 0.5333768, 0.6064113], + [0.3503858, 0.5720159, 0.7052018, 0.4558409, 0.3261529], + [0.6988886, 0.5897652, 0.6532392, 0.7234108, 0.7218805], + ], + ], + device=device, + dtype=dtype, + ) + expected = expected.unsqueeze(0).repeat(4, 1, 1, 1) + + img_gray = RandomGrayscale(p=1.0)(data) + self.assert_close(img_gray, expected) + + def test_opencv_false_batch(self, device, dtype): + data = torch.tensor( + [ + [ + [0.3944633, 0.8597369, 0.1670904, 0.2825457, 0.0953912], + [0.1251704, 0.8020709, 0.8933256, 0.9170977, 0.1497008], + [0.2711633, 0.1111478, 0.0783281, 0.2771807, 0.5487481], + [0.0086008, 0.8288748, 0.9647092, 0.8922020, 0.7614344], + [0.2898048, 0.1282895, 0.7621747, 0.5657831, 0.9918593], + ], + [ + [0.5414237, 0.9962701, 0.8947155, 0.5900949, 0.9483274], + [0.0468036, 0.3933847, 0.8046577, 0.3640994, 0.0632100], + [0.6171775, 0.8624780, 0.4126036, 0.7600935, 0.7279997], + [0.4237089, 0.5365476, 0.5591233, 0.1523191, 0.1382165], + [0.8932794, 0.8517839, 0.7152701, 0.8983801, 0.5905426], + ], + [ + [0.2869580, 0.4700376, 0.2743714, 0.8135023, 0.2229074], + [0.9306560, 0.3734594, 0.4566821, 0.7599275, 0.7557513], + [0.7415742, 0.6115875, 0.3317572, 0.0379378, 0.1315770], + [0.8692724, 0.0809556, 0.7767404, 0.8742208, 0.1522012], + [0.7708948, 0.4509611, 0.0481175, 0.2358997, 0.6900532], + ], + ], + device=device, + dtype=dtype, + ) + data = data.unsqueeze(0).repeat(4, 1, 1, 1) + + expected = data + + img_gray = RandomGrayscale(p=0.0)(data) + self.assert_close(img_gray, expected) + + def test_random_grayscale_sequential_batch(self, device, dtype): + f = AugmentationSequential(RandomGrayscale(p=0.0), RandomGrayscale(p=0.0)) + + input = torch.rand(2, 3, 5, 5, device=device, dtype=dtype) # 2 x 3 x 5 x 5 + expected = input + + expected_transform = torch.eye(3, device=device, dtype=dtype).unsqueeze(0).expand((2, 3, 3)) # 2 x 3 x 3 + expected_transform = expected_transform.to(device) + + self.assert_close(f(input), expected) + self.assert_close(f.transform_matrix, expected_transform) + + @pytest.mark.slow + @pytest.mark.parametrize("p", [0.0, 1.0]) + def test_gradcheck(self, device, p): + input = torch.rand((3, 5, 5), device=device, dtype=torch.float64) # 3 x 3 + self.gradcheck(RandomGrayscale(p=p), (input,)) + + +class TestCenterCrop(BaseTester): + def test_no_transform(self, device, dtype): + inp = torch.rand(1, 2, 4, 4, device=device, dtype=dtype) + out = CenterCrop(2)(inp) + assert out.shape == (1, 2, 2, 2) + aug = CenterCrop(2, cropping_mode="resample") + out = aug(inp) + assert out.shape == (1, 2, 2, 2) + assert aug.inverse(out).shape == (1, 2, 4, 4) + + def test_transform(self, device, dtype): + inp = torch.rand(1, 2, 5, 4, device=device, dtype=dtype) + aug = CenterCrop(2) + out = aug(inp) + assert out.shape == (1, 2, 2, 2) + assert aug.transform_matrix.shape == (1, 3, 3) + aug = CenterCrop(2, cropping_mode="resample") + out = aug(inp) + assert out.shape == (1, 2, 2, 2) + assert aug.transform_matrix.shape == (1, 3, 3) + assert aug.inverse(out).shape == (1, 2, 5, 4) + + def test_no_transform_tuple(self, device, dtype): + inp = torch.rand(1, 2, 5, 4, device=device, dtype=dtype) + out = CenterCrop((3, 4))(inp) + assert out.shape == (1, 2, 3, 4) + aug = CenterCrop((3, 4), cropping_mode="resample") + out = aug(inp) + assert out.shape == (1, 2, 3, 4) + assert aug.inverse(out).shape == (1, 2, 5, 4) + + def test_crop_modes(self, device, dtype): + torch.manual_seed(0) + img = torch.rand(1, 3, 5, 5, device=device, dtype=dtype) + + op1 = CenterCrop(size=(2, 2), cropping_mode="resample") + out = op1(img) + + op2 = CenterCrop(size=(2, 2), cropping_mode="slice") + + self.assert_close(out, op2(img, op1._params)) + + @pytest.mark.slow + def test_gradcheck(self, device): + input = torch.rand(1, 2, 3, 4, device=device, dtype=torch.float64) + self.gradcheck(CenterCrop(3), (input,)) + + +class TestRandomRotation(BaseTester): + torch.manual_seed(0) # for random reproductibility + + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self): + f = RandomRotation(degrees=45.5) + repr = ( + "RandomRotation(degrees=tensor([-45.5000, 45.5000]), interpolation=BILINEAR, p=0.5, " + "p_batch=1.0, same_on_batch=False)" + ) + assert str(f) == repr + + def test_random_rotation(self, device, dtype): + # This is included in doctest + torch.manual_seed(0) # for random reproductibility + + f = RandomRotation(degrees=45.0, p=1.0) + + input = torch.tensor( + [ + [1.0, 0.0, 0.0, 2.0], + [0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 2.0, 0.0], + [0.0, 0.0, 1.0, 2.0], + ], + device=device, + dtype=dtype, + ) # 4 x 4 + + expected = torch.tensor( + [ + [ + [ + [0.9824, 0.0088, 0.0000, 1.9649], + [0.0000, 0.0029, 0.0000, 0.0176], + [0.0029, 1.0000, 1.9883, 0.0000], + [0.0000, 0.0088, 1.0117, 1.9649], + ] + ] + ], + device=device, + dtype=dtype, + ) # 1 x 4 x 4 + + expected_transform = torch.tensor( + [ + [ + [1.0000, -0.0059, 0.0088], + [0.0059, 1.0000, -0.0088], + [0.0000, 0.0000, 1.0000], + ] + ], + device=device, + dtype=dtype, + ) # 1 x 3 x 3 + + out = f(input) + self.assert_close(out, expected, low_tolerance=True) + self.assert_close(f.transform_matrix, expected_transform, low_tolerance=True) + + def test_batch_random_rotation(self, device, dtype): + torch.manual_seed(0) # for random reproductibility + + f = RandomRotation(degrees=45.0, p=1.0) + + input = torch.tensor( + [ + [ + [ + [1.0, 0.0, 0.0, 2.0], + [0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 2.0, 0.0], + [0.0, 0.0, 1.0, 2.0], + ] + ] + ], + device=device, + dtype=dtype, + ) # 1 x 1 x 4 x 4 + + expected = torch.tensor( + [ + [ + [ + [0.9824, 0.0088, 0.0000, 1.9649], + [0.0000, 0.0029, 0.0000, 0.0176], + [0.0029, 1.0000, 1.9883, 0.0000], + [0.0000, 0.0088, 1.0117, 1.9649], + ] + ], + [ + [ + [0.1322, 0.0000, 0.7570, 0.2644], + [0.3785, 0.0000, 0.4166, 0.0000], + [0.0000, 0.6309, 1.5910, 1.2371], + [0.0000, 0.1444, 0.3177, 0.6499], + ] + ], + ], + device=device, + dtype=dtype, + ) # 2 x 1 x 4 x 4 + + expected_transform = torch.tensor( + [ + [ + [1.0000, -0.0059, 0.0088], + [0.0059, 1.0000, -0.0088], + [0.0000, 0.0000, 1.0000], + ], + [ + [0.9125, 0.4090, -0.4823], + [-0.4090, 0.9125, 0.7446], + [0.0000, 0.0000, 1.0000], + ], + ], + device=device, + dtype=dtype, + ) # 2 x 3 x 3 + + input = input.repeat(2, 1, 1, 1) # 5 x 3 x 3 x 3 + + out = f(input) + self.assert_close(out, expected, low_tolerance=True) + self.assert_close(f.transform_matrix, expected_transform, low_tolerance=True) + + def test_same_on_batch(self, device, dtype): + f = RandomRotation(degrees=40, same_on_batch=True) + input = torch.eye(6, device=device, dtype=dtype).unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 3, 1, 1) + res = f(input) + self.assert_close(res[0], res[1]) + + def test_sequential(self, device, dtype): + torch.manual_seed(0) # for random reproductibility + + f = AugmentationSequential( + RandomRotation(torch.tensor([-45.0, 90]), p=1.0), + RandomRotation(10.4, p=1.0), + ) + + input = torch.tensor( + [ + [1.0, 0.0, 0.0, 2.0], + [0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 2.0, 0.0], + [0.0, 0.0, 1.0, 2.0], + ], + device=device, + dtype=dtype, + ) # 4 x 4 + + expected = torch.tensor( + [ + [ + [ + [0.1314, 0.1050, 0.6649, 0.2628], + [0.3234, 0.0202, 0.4256, 0.1671], + [0.0525, 0.5976, 1.5199, 1.1306], + [0.0000, 0.1453, 0.3224, 0.5796], + ] + ] + ], + device=device, + dtype=dtype, + ) # 1 x 4 x 4 + + expected_transform = torch.tensor( + [ + [ + [0.8864, 0.4629, -0.5240], + [-0.4629, 0.8864, 0.8647], + [0.0000, 0.0000, 1.0000], + ] + ], + device=device, + dtype=dtype, + ) # 1 x 3 x 3 + + out = f(input) + self.assert_close(out, expected, low_tolerance=True) + self.assert_close(f.transform_matrix, expected_transform, low_tolerance=True) + + @pytest.mark.slow + def test_gradcheck(self, device): + torch.manual_seed(0) # for random reproductibility + + input = torch.rand((3, 3), device=device, dtype=torch.float64) # 3 x 3 + self.gradcheck(RandomRotation(degrees=(15.0, 15.0), p=1.0), (input,)) + + +class TestRandomCrop(BaseTester): + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self): + f = RandomCrop(size=(2, 3), padding=(0, 1), fill=10, pad_if_needed=False, p=1.0) + repr = ( + "RandomCrop(crop_size=(2, 3), padding=(0, 1), fill=10, pad_if_needed=False, padding_mode=constant, " + "resample=BILINEAR, p=1.0, p_batch=1.0, same_on_batch=False)" + ) + assert str(f) == repr + + def test_no_padding(self, device, dtype): + torch.manual_seed(0) + inp = torch.tensor( + [[[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]]], + device=device, + dtype=dtype, + ) + expected = torch.tensor([[[[3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]]], device=device, dtype=dtype) + rc = RandomCrop(size=(2, 3), padding=None, align_corners=True, p=1.0) + out = rc(inp) + + torch.manual_seed(0) + out2 = rc(inp.squeeze()) + + self.assert_close(out, expected) + self.assert_close(out2, expected) + torch.manual_seed(0) + inversed = torch.tensor( + [[[[0.0, 0.0, 0.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]]], + device=device, + dtype=dtype, + ) + aug = RandomCrop( + size=(2, 3), + padding=None, + align_corners=True, + p=1.0, + cropping_mode="resample", + ) + out = aug(inp) + self.assert_close(out, expected) + self.assert_close(aug.inverse(out), inversed) + + def test_no_padding_batch(self, device, dtype): + torch.manual_seed(42) + batch_size = 2 + inp = torch.tensor( + [[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]], + device=device, + dtype=dtype, + ).repeat(batch_size, 1, 1, 1) + expected = torch.tensor( + [ + [[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]], + [[[3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]], + ], + device=device, + dtype=dtype, + ) + rc = RandomCrop(size=(2, 3), padding=None, align_corners=True, p=1.0) + out = rc(inp) + self.assert_close(out, expected) + + torch.manual_seed(42) + inversed = torch.tensor( + [ + [[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [0.0, 0.0, 0.0]]], + [[[0.0, 0.0, 0.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]], + ], + device=device, + dtype=dtype, + ) + aug = RandomCrop( + size=(2, 3), + padding=None, + align_corners=True, + p=1.0, + cropping_mode="resample", + ) + out = aug(inp) + self.assert_close(out, expected) + self.assert_close(aug.inverse(out), inversed) + + def test_same_on_batch(self, device, dtype): + f = RandomCrop(size=(2, 3), padding=1, same_on_batch=True, align_corners=True, p=1.0) + input = torch.eye(3, device=device, dtype=dtype).unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 3, 1, 1) + res = f(input) + self.assert_close(res[0], res[1]) + + def test_padding(self, device, dtype): + torch.manual_seed(42) + inp = torch.tensor( + [[[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]]], + device=device, + dtype=dtype, + ) + expected = torch.tensor([[[[7.0, 8.0, 7.0], [4.0, 5.0, 4.0]]]], device=device, dtype=dtype) + rc = RandomCrop(size=(2, 3), padding=1, padding_mode="reflect", align_corners=True, p=1.0) + out = rc(inp) + + torch.manual_seed(42) + out2 = rc(inp.squeeze()) + + self.assert_close(out, expected) + self.assert_close(out2, expected) + torch.manual_seed(42) + inversed = torch.tensor( + [[[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 7.0, 8.0]]]], + device=device, + dtype=dtype, + ) + aug = RandomCrop( + size=(2, 3), + padding=1, + padding_mode="reflect", + align_corners=True, + p=1.0, + cropping_mode="resample", + ) + out = aug(inp) + self.assert_close(out, expected) + self.assert_close(aug.inverse(out, padding_mode="constant"), inversed) + + def test_padding_batch_1(self, device, dtype): + torch.manual_seed(42) + batch_size = 2 + inp = torch.tensor( + [[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]], + device=device, + dtype=dtype, + ).repeat(batch_size, 1, 1, 1) + expected = torch.tensor( + [ + [[[1.0, 2.0, 0.0], [4.0, 5.0, 0.0]]], + [[[7.0, 8.0, 0.0], [0.0, 0.0, 0.0]]], + ], + device=device, + dtype=dtype, + ) + rc = RandomCrop(size=(2, 3), padding=1, align_corners=True, p=1.0) + out = rc(inp) + + self.assert_close(out, expected) + + torch.manual_seed(42) + inversed = torch.tensor( + [ + [[[0.0, 1.0, 2.0], [0.0, 4.0, 5.0], [0.0, 0.0, 0.0]]], + [[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 7.0, 8.0]]], + ], + device=device, + dtype=dtype, + ) + aug = RandomCrop(size=(2, 3), padding=1, align_corners=True, p=1.0, cropping_mode="resample") + out = aug(inp) + self.assert_close(out, expected) + self.assert_close(aug.inverse(out), inversed) + + def test_padding_batch_2(self, device, dtype): + torch.manual_seed(42) + batch_size = 2 + padding = (0, 1) # order: left-right, top-bottom + inp = torch.tensor( + [[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]], + device=device, + dtype=dtype, + ).repeat(batch_size, 1, 1, 1) + expected = torch.tensor( + [ + [[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]], + [[[6.0, 7.0, 8.0], [10.0, 10.0, 10.0]]], + ], + device=device, + dtype=dtype, + ) + rc = RandomCrop(size=(2, 3), padding=padding, fill=10, align_corners=True, p=1.0) + out = rc(inp) + + assert rc._params["input_size"][0][0] == (inp.shape[-2] + 2 * padding[1]) # height + top + bottom + assert rc._params["input_size"][0][1] == (inp.shape[-1] + 2 * padding[0]) # height + left + right + self.assert_close(out, expected) + torch.manual_seed(42) + inversed = torch.tensor( + [ + [[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [0.0, 0.0, 0.0]]], + [[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [6.0, 7.0, 8.0]]], + ], + device=device, + dtype=dtype, + ) + aug = RandomCrop( + size=(2, 3), + padding=padding, + fill=10, + align_corners=True, + p=1.0, + cropping_mode="resample", + ) + out = aug(inp) + self.assert_close(out, expected) + self.assert_close(aug.inverse(out), inversed) + + def test_padding_batch_3(self, device, dtype): + torch.manual_seed(0) + batch_size = 2 + padding = (0, 1, 2, 3) # order: left, top, right, bottom + inp = torch.tensor( + [[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]], + device=device, + dtype=dtype, + ).repeat(batch_size, 1, 1, 1) + expected = torch.tensor( + [ + [[[8.0, 8.0, 8.0], [1.0, 2.0, 8.0]]], + [[[8.0, 8.0, 8.0], [2.0, 8.0, 8.0]]], + ], + device=device, + dtype=dtype, + ) + rc = RandomCrop(size=(2, 3), padding=padding, fill=8, align_corners=True, p=1.0) + out = rc(inp) + + assert rc._params["input_size"][0][0] == (inp.shape[-2] + padding[1] + padding[3]) # height + top + bottom + assert rc._params["input_size"][0][1] == (inp.shape[-1] + padding[0] + padding[2]) # height + left + right + self.assert_close(out, expected, low_tolerance=True) + + torch.manual_seed(0) + inversed = torch.tensor( + [ + [[[0.0, 1.0, 2.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]], + [[[0.0, 0.0, 2.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]], + ], + device=device, + dtype=dtype, + ) + aug = RandomCrop( + size=(2, 3), + padding=(0, 1, 2, 3), + fill=8, + align_corners=True, + p=1.0, + cropping_mode="resample", + ) + out = aug(inp) + self.assert_close(out, expected, low_tolerance=True) + self.assert_close(aug.inverse(out), inversed, low_tolerance=True) + + def test_padding_no_forward(self, device, dtype): + torch.manual_seed(0) + inp = torch.tensor([[[[3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]]], device=device, dtype=dtype) + trans = torch.eye(3, device=device, dtype=dtype)[None] + # Not return transform + rc = RandomCrop( + size=(2, 3), + padding=(0, 1, 2, 3), + fill=9, + align_corners=True, + p=0.0, + cropping_mode="resample", + ) + + out = rc(inp) + self.assert_close(out, inp) + self.assert_close(rc.transform_matrix, trans) + + def test_pad_if_needed_width(self, device, dtype): + torch.manual_seed(0) + batch_size = 2 + inp = torch.tensor([[[0.0], [1.0], [2.0]]], device=device, dtype=dtype).repeat(batch_size, 1, 1, 1) + expected = torch.tensor( + [ + [[[9.0, 0.0], [9.0, 1.0], [9.0, 2.0]]], + [[[0.0, 9.0], [1.0, 9.0], [2.0, 9.0]]], + ], + device=device, + dtype=dtype, + ) + rc = RandomCrop(size=(3, 2), pad_if_needed=True, fill=9, align_corners=True, p=1.0) + out = rc(inp) + + self.assert_close(out, expected) + + torch.manual_seed(0) + aug = RandomCrop( + size=(3, 2), + pad_if_needed=True, + fill=9, + align_corners=True, + p=1.0, + cropping_mode="resample", + ) + out = aug(inp) + self.assert_close(out, expected) + self.assert_close(aug.inverse(out), inp) + + def test_pad_if_needed_height(self, device, dtype): + torch.manual_seed(0) + batch_size = 2 + inp = torch.tensor([[[0.0, 1.0, 2.0]]], device=device, dtype=dtype).repeat(batch_size, 1, 1, 1) + expected = torch.tensor( + [ + [[[9.0, 9.0, 9.0], [0.0, 1.0, 2.0]]], + [[[9.0, 9.0, 9.0], [0.0, 1.0, 2.0]]], + ], + device=device, + dtype=dtype, + ) + rc = RandomCrop(size=(2, 3), pad_if_needed=True, fill=9, align_corners=True, p=1.0) + out = rc(inp) + + self.assert_close(out, expected) + + torch.manual_seed(0) + aug = RandomCrop( + size=(2, 3), + pad_if_needed=True, + fill=9, + align_corners=True, + p=1.0, + cropping_mode="resample", + ) + out = aug(inp) + self.assert_close(out, expected) + self.assert_close(aug.inverse(out), inp) + + def test_pad_if_needed_both(self, device, dtype): + torch.manual_seed(0) + batch_size = 2 + inp = torch.tensor([[[0.0], [1.0]]], device=device, dtype=dtype).repeat(batch_size, 1, 1, 1) + expected = torch.tensor( + [ + [[[9.0, 9.0], [9.0, 0.0], [9.0, 1.0]]], + [[[9.0, 9.0], [0.0, 9.0], [1.0, 9.0]]], + ], + device=device, + dtype=dtype, + ) + rc = RandomCrop(size=(3, 2), pad_if_needed=True, fill=9, align_corners=True, p=1.0) + out = rc(inp) + + self.assert_close(out, expected) + + torch.manual_seed(0) + aug = RandomCrop( + size=(3, 2), + pad_if_needed=True, + fill=9, + align_corners=True, + p=1.0, + cropping_mode="resample", + ) + out = aug(inp) + self.assert_close(out, expected) + self.assert_close(aug.inverse(out), inp) + + def test_crop_modes(self, device, dtype): + torch.manual_seed(0) + img = torch.tensor( + [[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]], + device=device, + dtype=dtype, + ) + + op1 = RandomCrop(size=(2, 2), cropping_mode="resample") + out = op1(img) + + op2 = RandomCrop(size=(2, 2), cropping_mode="slice") + + self.assert_close(out, op2(img, op1._params)) + + @pytest.mark.slow + def test_gradcheck(self, device): + torch.manual_seed(0) # for random reproductibility + inp = torch.rand((3, 3, 3), device=device, dtype=torch.float64) # 3 x 3 + self.gradcheck(RandomCrop(size=(3, 3), p=1.0), (inp,)) + + @pytest.mark.skip("Need to fix Union type") + def test_jit(self, device, dtype): + # Define script + op = RandomCrop(size=(3, 3), p=1.0).forward + op_script = torch.jit.script(op) + img = torch.ones(1, 1, 5, 6, device=device, dtype=dtype) + + actual = op_script(img) + expected = kornia.geometry.transform.center_crop3d(img) + self.assert_close(actual, expected) + + @pytest.mark.skip("Need to fix Union type") + def test_jit_trace(self, device, dtype): + # Define script + op = RandomCrop(size=(3, 3), p=1.0).forward + op_script = torch.jit.script(op) + # 1. Trace op + img = torch.ones(1, 1, 5, 6, device=device, dtype=dtype) + + op_trace = torch.jit.trace(op_script, (img,)) + + # 2. Generate new input + img = torch.ones(1, 1, 5, 6, device=device, dtype=dtype) + + # 3. Evaluate + actual = op_trace(img) + expected = op(img) + self.assert_close(actual, expected) + + +class TestRandomResizedCrop(BaseTester): + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self): + f = RandomResizedCrop(size=(2, 3), scale=(1.0, 1.0), ratio=(1.0, 1.0)) + repr = ( + "RandomResizedCrop(size=(2, 3), scale=tensor([1., 1.]), ratio=tensor([1., 1.]), " + "interpolation=BILINEAR, p=1.0, p_batch=1.0, same_on_batch=False)" + ) + assert str(f) == repr + + def test_no_resize(self, device, dtype): + torch.manual_seed(0) + inp = torch.tensor( + [[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [[[[0.0000, 1.0000, 2.0000], [6.0000, 7.0000, 8.0000]]]], + device=device, + dtype=dtype, + ) + + rrc = RandomResizedCrop(size=(2, 3), scale=(1.0, 1.0), ratio=(1.0, 1.0)) + # It will crop a size of (2, 3) from the aspect ratio implementation of torch + out = rrc(inp) + self.assert_close(out, expected) + + torch.manual_seed(0) + aug = RandomResizedCrop(size=(2, 3), scale=(1.0, 1.0), ratio=(1.0, 1.0), cropping_mode="resample") + out = aug(inp) + self.assert_close(out, expected) + self.assert_close(aug.inverse(out), inp[None]) + + def test_same_on_batch(self, device, dtype): + f = RandomResizedCrop(size=(2, 3), scale=(1.0, 1.0), ratio=(1.0, 1.0), same_on_batch=True) + input = torch.tensor( + [[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]], + device=device, + dtype=dtype, + ).repeat(2, 1, 1, 1) + res = f(input) + self.assert_close(res[0], res[1]) + + torch.manual_seed(0) + aug = RandomResizedCrop( + size=(2, 3), + scale=(1.0, 1.0), + ratio=(1.0, 1.0), + same_on_batch=True, + cropping_mode="resample", + ) + out = aug(input) + inversed = aug.inverse(out) + self.assert_close(inversed[0], inversed[1]) + + def test_crop_scale_ratio(self, device, dtype): + # This is included in doctest + torch.manual_seed(0) + inp = torch.tensor( + [[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [ + [ + [ + [1.0000, 1.5000, 2.0000], + [4.0000, 4.5000, 5.0000], + [7.0000, 7.5000, 8.0000], + ] + ] + ], + device=device, + dtype=dtype, + ) + rrc = RandomResizedCrop(size=(3, 3), scale=(3.0, 3.0), ratio=(2.0, 2.0)) + # It will crop a size of (3, 3) from the aspect ratio implementation of torch + out = rrc(inp) + self.assert_close(out, expected) + + torch.manual_seed(0) + inversed = torch.tensor( + [[[[0.0, 1.0, 2.0], [0.0, 4.0, 5.0], [0.0, 7.0, 8.0]]]], + device=device, + dtype=dtype, + ) + aug = RandomResizedCrop(size=(3, 3), scale=(3.0, 3.0), ratio=(2.0, 2.0), cropping_mode="resample") + out = aug(inp) + self.assert_close(out, expected) + self.assert_close(aug.inverse(out), inversed) + + def test_crop_size_greater_than_input(self, device, dtype): + # This is included in doctest + torch.manual_seed(0) + inp = torch.tensor( + [[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]], + device=device, + dtype=dtype, + ) + + exp = torch.tensor( + [ + [ + [ + [1.0000, 1.3333, 1.6667, 2.0000], + [3.0000, 3.3333, 3.6667, 4.0000], + [5.0000, 5.3333, 5.6667, 6.0000], + [7.0000, 7.3333, 7.6667, 8.0000], + ] + ] + ], + device=device, + dtype=dtype, + ) + + rrc = RandomResizedCrop(size=(4, 4), scale=(3.0, 3.0), ratio=(2.0, 2.0)) + # It will crop a size of (3, 3) from the aspect ratio implementation of torch + out = rrc(inp) + assert out.shape == torch.Size([1, 1, 4, 4]) + self.assert_close(out, exp, low_tolerance=True) + + torch.manual_seed(0) + inversed = torch.tensor( + [[[[0.0, 1.0, 2.0], [0.0, 4.0, 5.0], [0.0, 7.0, 8.0]]]], + device=device, + dtype=dtype, + ) + aug = RandomResizedCrop(size=(4, 4), scale=(3.0, 3.0), ratio=(2.0, 2.0), cropping_mode="resample") + out = aug(inp) + self.assert_close(out, exp, low_tolerance=True) + self.assert_close(aug.inverse(out), inversed, low_tolerance=True) + + def test_crop_scale_ratio_batch(self, device, dtype): + torch.manual_seed(0) + batch_size = 2 + inp = torch.tensor( + [[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]], + device=device, + dtype=dtype, + ).repeat(batch_size, 1, 1, 1) + + expected = torch.tensor( + [ + [ + [ + [1.0000, 1.5000, 2.0000], + [4.0000, 4.5000, 5.0000], + [7.0000, 7.5000, 8.0000], + ] + ], + [ + [ + [0.0000, 0.5000, 1.0000], + [3.0000, 3.5000, 4.0000], + [6.0000, 6.5000, 7.0000], + ] + ], + ], + device=device, + dtype=dtype, + ) + rrc = RandomResizedCrop(size=(3, 3), scale=(3.0, 3.0), ratio=(2.0, 2.0)) + # It will crop a size of (2, 2) from the aspect ratio implementation of torch + out = rrc(inp) + self.assert_close(out, expected) + + torch.manual_seed(0) + inversed = torch.tensor( + [ + [[[0.0, 1.0, 2.0], [0.0, 4.0, 5.0], [0.0, 7.0, 8.0]]], + [[[0.0, 1.0, 0.0], [3.0, 4.0, 0.0], [6.0, 7.0, 0.0]]], + ], + device=device, + dtype=dtype, + ) + aug = RandomResizedCrop(size=(3, 3), scale=(3.0, 3.0), ratio=(2.0, 2.0), cropping_mode="resample") + out = aug(inp) + self.assert_close(out, expected) + self.assert_close(aug.inverse(out), inversed) + + def test_crop_modes(self, device, dtype): + torch.manual_seed(0) + img = torch.tensor( + [[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]], + device=device, + dtype=dtype, + ) + + op1 = RandomResizedCrop(size=(4, 4), cropping_mode="resample") + out = op1(img) + + op2 = RandomResizedCrop(size=(4, 4), cropping_mode="slice") + + self.assert_close(out, op2(img, op1._params)) + + @pytest.mark.slow + def test_gradcheck(self, device): + torch.manual_seed(0) # for random reproductibility + inp = torch.rand((1, 3, 3), device=device, dtype=torch.float64) # 3 x 3 + self.gradcheck(RandomResizedCrop(size=(3, 3), scale=(1.0, 1.0), ratio=(1.0, 1.0)), (inp,)) + + +class TestRandomEqualize(BaseTester): + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing precision.") + def test_smoke(self, device, dtype): + f = RandomEqualize(p=0.5) + repr = "RandomEqualize(p=0.5, p_batch=1.0, same_on_batch=False)" + assert str(f) == repr + + def test_random_equalize(self, device, dtype): + f = RandomEqualize(p=1.0) + f1 = RandomEqualize(p=0.0) + + bs, channels, height, width = 1, 3, 20, 20 + + inputs = self.build_input(channels, height, width, bs, device=device, dtype=dtype) + + row_expected = torch.tensor( + [ + 0.0000, + 0.07843, + 0.15686, + 0.2353, + 0.3137, + 0.3922, + 0.4706, + 0.5490, + 0.6275, + 0.7059, + 0.7843, + 0.8627, + 0.9412, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + ] + ) + expected = self.build_input(channels, height, width, bs=1, row=row_expected, device=device, dtype=dtype) + identity = kornia.core.ops.eye_like(3, expected) # 3 x 3 + + self.assert_close(f(inputs), expected, low_tolerance=True) + self.assert_close(f.transform_matrix, identity, low_tolerance=True) + self.assert_close(f1(inputs), inputs, low_tolerance=True) + self.assert_close(f1.transform_matrix, identity, low_tolerance=True) + + def test_batch_random_equalize(self, device, dtype): + f = RandomEqualize(p=1.0) + f1 = RandomEqualize(p=0.0) + + bs, channels, height, width = 2, 3, 20, 20 + + inputs = self.build_input(channels, height, width, bs, device=device, dtype=dtype) + + row_expected = torch.tensor( + [ + 0.0000, + 0.07843, + 0.15686, + 0.2353, + 0.3137, + 0.3922, + 0.4706, + 0.5490, + 0.6275, + 0.7059, + 0.7843, + 0.8627, + 0.9412, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + ] + ) + expected = self.build_input(channels, height, width, bs, row=row_expected, device=device, dtype=dtype) + + identity = kornia.core.ops.eye_like(3, expected) # 2 x 3 x 3 + + self.assert_close(f(inputs), expected, low_tolerance=True) + self.assert_close(f.transform_matrix, identity, low_tolerance=True) + self.assert_close(f1(inputs), inputs, low_tolerance=True) + self.assert_close(f1.transform_matrix, identity, low_tolerance=True) + + def test_same_on_batch(self, device, dtype): + f = RandomEqualize(p=0.5, same_on_batch=True) + input = torch.eye(4, device=device, dtype=dtype) + input = input.unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 1, 1, 1) + res = f(input) + self.assert_close(res[0], res[1]) + + @pytest.mark.slow + def test_gradcheck(self, device): + torch.manual_seed(0) # for random reproductibility + + input = torch.rand((3, 3, 3), device=device, dtype=torch.float64) # 3 x 3 x 3 + self.gradcheck(RandomEqualize(p=0.5), (input,)) + + @staticmethod + def build_input(channels, height, width, bs=1, row=None, device="cpu", dtype=torch.float32): + if row is None: + row = torch.arange(width, device=device, dtype=dtype) / float(width) + + channel = torch.stack([row] * height) + image = torch.stack([channel] * channels) + batch = torch.stack([image] * bs) + + return batch.to(device, dtype) + + +class TestRandomGaussianBlur(BaseTester): + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self): + f = RandomGaussianBlur((3, 3), (0.1, 2.0), p=1.0) + repr = "RandomGaussianBlur(sigma=(0.1, 2.0), p=1.0, p_batch=1.0, same_on_batch=False)" + assert str(f) == repr + + @pytest.mark.parametrize("batch_shape", [(1, 4, 8, 15), (2, 3, 11, 7)]) + def test_cardinality(self, batch_shape, device, dtype): + kernel_size = (5, 7) + sigma = (1.5, 2.1) + input = torch.rand(batch_shape, device=device, dtype=dtype) + aug = RandomGaussianBlur(kernel_size, sigma, "replicate") + actual = aug(input) + assert actual.shape == batch_shape + + def test_noncontiguous(self, device, dtype): + batch_size = 3 + input = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + + kernel_size = (3, 3) + sigma = (1.5, 2.1) + aug = RandomGaussianBlur(kernel_size, sigma, "replicate") + actual = aug(input) + self.assert_close(actual, actual) + + def test_module(self, device, dtype): + func_params = [(3, 3), torch.tensor([1.5, 1.5]).view(1, -1)] + params = [(3, 3), (1.5, 1.5)] + op = kornia.filters.gaussian_blur2d + op_module = RandomGaussianBlur(*params) + + img = torch.ones(1, 3, 5, 5, device=device, dtype=dtype) + self.assert_close(op(img, *func_params), op_module(img)) + + def test_module_kernel_int(self, device, dtype): + func_params = [3, torch.tensor([1.5, 1.5]).view(1, -1)] + params = [3, (1.5, 1.5)] + op = kornia.filters.gaussian_blur2d + op_module = RandomGaussianBlur(*params) + + img = torch.ones(1, 3, 5, 5, device=device, dtype=dtype) + self.assert_close(op(img, *func_params), op_module(img)) + + def test_module_sigma_tensor(self, device, dtype): + func_params = [(3, 3), torch.tensor([1.5, 1.5]).view(1, -1)] + params = [(3, 3), torch.tensor((1.5, 1.5))] + op = kornia.filters.gaussian_blur2d + op_module = RandomGaussianBlur(*params) + + img = torch.ones(1, 3, 5, 5, device=device, dtype=dtype) + self.assert_close(op(img, *func_params), op_module(img)) + + @pytest.mark.slow + def test_dynamo(self, device, dtype, torch_optimizer): + kernel_size = (3, 3) + sigma = (1.5, 2.1) + img = torch.rand(1, 3, 5, 5, device=device, dtype=dtype) + aug = RandomGaussianBlur(kernel_size, sigma, "replicate") + aug = aug.compile(fullgraph=True) + actual = aug(img) + assert actual.shape == img.shape + + +class TestRandomInvert(BaseTester): + def test_smoke(self, device, dtype): + img = torch.ones(1, 3, 4, 5, device=device, dtype=dtype) + self.assert_close(RandomInvert(p=1.0)(img), torch.zeros_like(img)) + + +class TestRandomChannelShuffle(BaseTester): + def test_smoke(self, device, dtype): + torch.manual_seed(0) + img = torch.arange(1 * 3 * 2 * 2, device=device, dtype=dtype).view(1, 3, 2, 2) + + out_expected = torch.tensor( + [ + [ + [[8.0, 9.0], [10.0, 11.0]], + [[0.0, 1.0], [2.0, 3.0]], + [[4.0, 5.0], [6.0, 7.0]], + ] + ], + device=device, + dtype=dtype, + ) + + aug = RandomChannelShuffle(p=1.0) + out = aug(img) + self.assert_close(out, out_expected) + + def test_same_on_batch(self, device, dtype): + input_tensor = torch.rand(1, 3, 5, 5, device=device, dtype=dtype).repeat(2, 1, 1, 1) + transform = RandomChannelShuffle(p=1.0, same_on_batch=True) + output_tensor = transform(input_tensor) + self.assert_close(output_tensor[0], output_tensor[1]) + + +class TestRandomClahe(BaseTester): + def test_smoke(self, device, dtype): + img = torch.arange(36, device=device, dtype=dtype).reshape(2, 2, 3, 3) / 36 + expected = torch.tensor(22.4588, device=device, dtype=dtype) + self.assert_close(RandomClahe(p=1.0, grid_size=(2, 2))(img).sum(), expected) + + @pytest.mark.parametrize("batch_shape", [(1, 3, 5, 7), (3, 1, 5, 7)]) + def test_cardinality(self, batch_shape, device, dtype): + input_data = (torch.arange(105).reshape(*batch_shape) / 105).to(device=device, dtype=dtype) + output_data = RandomClahe(p=1.0, grid_size=(2, 2))(input_data) + assert output_data.shape == batch_shape + + +class TestRandomGaussianNoise(BaseTester): + def test_smoke(self, device, dtype): + torch.manual_seed(0) + img = torch.rand(1, 1, 2, 2, device=device, dtype=dtype) + aug = RandomGaussianNoise(p=1.0) + assert img.shape == aug(img).shape + + def test_same_on_batch(self, device, dtype): + input_tensor = torch.rand(1, 1, 5, 5, device=device, dtype=dtype).repeat(2, 1, 1, 1) + transform = RandomGaussianNoise(p=1.0, same_on_batch=True) + output_tensor = transform(input_tensor) + self.assert_close(output_tensor[0], output_tensor[1]) + assert not torch.allclose(input_tensor, output_tensor) + + +class TestRandomSaltAndPepperNoise(BaseTester): + @pytest.mark.parametrize( + "expected", + [ + torch.tensor( + [ + [ + [ + [1.0000, 1.0000, 0.5000, 0.5000, 0.5000], + [0.5000, 0.5000, 0.5000, 0.5000, 0.5000], + [0.0000, 0.0000, 0.5000, 0.5000, 0.5000], + [0.5000, 1.0000, 0.5000, 0.5000, 0.5000], + [0.5000, 0.5000, 0.5000, 0.5000, 0.5000], + ] + ] + ] + ) + ], + ) + def test_smoke(self, expected, device, dtype): + torch.manual_seed(0) + input_tensor = torch.ones(1, 1, 5, 5, device=device, dtype=dtype) * 0.5 + expected = expected.to(device, dtype=dtype) + aug = RandomSaltAndPepperNoise(amount=0.2, salt_vs_pepper=0.5, p=1.0) + res = aug(input_tensor) + assert input_tensor.shape == res.shape + self.assert_close(res, expected) + + def test_exception(self, device, dtype): + with pytest.raises(ValueError, match="salt_vs_pepper must be a tuple or a float"): + RandomSaltAndPepperNoise(salt_vs_pepper=[0.4, 0.6]) + with pytest.raises( + ValueError, + match=r"The length of salt_vs_pepper must be greater than 0\s+and less than or equal to 2, " + r"and it should be a tuple\.", + ): + RandomSaltAndPepperNoise(salt_vs_pepper=(0.1, 0.2, 0.3)) + from kornia.core.exceptions import BaseError + + with pytest.raises( + BaseError, + match=r"Salt_vs_pepper values must be between 0 and 1\.\s+" + r"Recommended value 0\.5\.", + ): + RandomSaltAndPepperNoise(salt_vs_pepper=(0.4, 3)) + with pytest.raises(ValueError, match="amount must be a tuple or a float"): + RandomSaltAndPepperNoise(amount=[0.01, 0.06]) + with pytest.raises( + ValueError, + match=r"The length of amount must be greater than 0\s+and less than or equal to 2, " + r"and it should be a tuple\.", + ): + RandomSaltAndPepperNoise(amount=()) + with pytest.raises( + BaseError, + match=r"amount of noise values must be between 0 and 1\.\s+" + r"Recommended values less than 0\.2\.", + ): + RandomSaltAndPepperNoise(amount=(0.05, 3)) + + with pytest.raises(ValueError, match="amount must be a tuple or a float"): + RandomSaltAndPepperNoise(amount=[0.01, 0.06]) + + with pytest.raises(ValueError, match="amount must be a tuple or a float"): + RandomSaltAndPepperNoise(amount=[0.01, 0.06]) + + @pytest.mark.parametrize("batch_shape", [1, 3, 3, 5]) + @pytest.mark.parametrize("channel_shape", [1, 1, 3, 3]) + def test_cardinality(self, batch_shape, channel_shape, device, dtype): + input_tensor = torch.ones(batch_shape, channel_shape, 16, 16, device=device, dtype=dtype) * 0.5 + transform = RandomSaltAndPepperNoise(p=1.0) + output_tensor = transform(input_tensor) + assert input_tensor.shape[0] == output_tensor.shape[0] + assert input_tensor.shape[1] == output_tensor.shape[1] + + def test_same_on_batch(self, device, dtype): + input_tensor = torch.ones(2, 1, 5, 5, device=device, dtype=dtype) * 0.5 + transform = RandomSaltAndPepperNoise(p=1.0, same_on_batch=True) + output_tensor = transform(input_tensor) + self.assert_close(output_tensor[0], output_tensor[1]) + + +class TestRandomGaussianIllumination(BaseTester): + def _get_expected(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.726599991321564, 1.000000000000000, 0.726599991321564], + [0.662100017070770, 0.912100017070770, 0.662100017070770], + [0.500000000000000, 0.691100001335144, 0.500000000000000], + ], + [ + [0.726599991321564, 1.000000000000000, 0.726599991321564], + [0.662100017070770, 0.912100017070770, 0.662100017070770], + [0.500000000000000, 0.691100001335144, 0.500000000000000], + ], + [ + [0.726599991321564, 1.000000000000000, 0.726599991321564], + [0.662100017070770, 0.912100017070770, 0.662100017070770], + [0.500000000000000, 0.691100001335144, 0.500000000000000], + ], + ] + ], + device=device, + dtype=dtype, + ) + + def test_smoke(self, device, dtype): + torch.manual_seed(1) + input_tensor = torch.ones(1, 3, 3, 3, device=device, dtype=dtype) * 0.5 + expected = self._get_expected(device=device, dtype=dtype) + aug = RandomGaussianIllumination(gain=0.5, p=1.0) + res = aug(input_tensor) + assert input_tensor.shape == res.shape + self.assert_close(res, expected, rtol=1e-4, atol=1e-4) + + def test_exception(self, device, dtype): + with pytest.raises(ValueError, match="sign must be a tuple or a float"): + RandomGaussianIllumination(sign=3) + + with pytest.raises(ValueError, match="center must be a tuple or a float"): + RandomGaussianIllumination(center=3) + + with pytest.raises(ValueError, match="sigma must be a tuple or a float"): + RandomGaussianIllumination(sigma=[0.01, 0.06]) + + with pytest.raises(ValueError, match="gain must be a tuple or a float"): + RandomGaussianIllumination(gain=[0.01, 0.06]) + + with pytest.raises( + Exception, + match=r"gain values must be between 0 and 1. Recommended values less than 0.2.", + ): + RandomGaussianIllumination(gain=(0.01, 2)) + + with pytest.raises(Exception, match=r"sigma of gaussian value must be between 0 and 1."): + RandomGaussianIllumination(sigma=(0.01, 2)) + + with pytest.raises(Exception, match=r"center of gaussian value must be between 0 and 1."): + RandomGaussianIllumination(center=(0.01, 2)) + + with pytest.raises(Exception, match=r"sign of gaussian value must be between -1 and 1."): + RandomGaussianIllumination(sign=(0.01, 2)) + + @pytest.mark.parametrize("channel_shape, batch_shape", [(1, 1), (3, 2), (5, 3)]) + def test_cardinality(self, batch_shape, channel_shape, device, dtype): + input_tensor = torch.ones(batch_shape, channel_shape, 16, 16, device=device, dtype=dtype) * 0.5 + transform = RandomGaussianIllumination(p=1.0) + output_tensor = transform(input_tensor) + assert input_tensor.shape[0] == output_tensor.shape[0] + assert input_tensor.shape[1] == output_tensor.shape[1] + + def test_same_on_batch(self, device, dtype): + input_tensor = torch.ones(2, 1, 5, 5, device=device, dtype=dtype) * 0.5 + transform = RandomGaussianIllumination(p=1.0, same_on_batch=True) + output_tensor = transform(input_tensor) + self.assert_close(output_tensor[0], output_tensor[1]) + + @pytest.mark.slow + def test_dynamo(self, device, dtype, torch_optimizer): + input_tensor = torch.ones(1, 3, 3, 3, device=device, dtype=dtype) * 0.5 + aug = RandomGaussianIllumination(gain=0.5, p=1.0) + aug = aug.compile(fullgraph=True) + actual = aug(input_tensor) + assert actual.shape == input_tensor.shape + + +class TestRandomLinearIllumination(BaseTester): + def _get_expected(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.2500000000, 0.2500000000, 0.2500000000], + [0.3750000000, 0.3750000000, 0.3750000000], + [0.5000000000, 0.5000000000, 0.5000000000], + ], + [ + [0.2500000000, 0.2500000000, 0.2500000000], + [0.3750000000, 0.3750000000, 0.3750000000], + [0.5000000000, 0.5000000000, 0.5000000000], + ], + [ + [0.2500000000, 0.2500000000, 0.2500000000], + [0.3750000000, 0.3750000000, 0.3750000000], + [0.5000000000, 0.5000000000, 0.5000000000], + ], + ] + ], + device=device, + dtype=dtype, + ) + + def test_smoke(self, device, dtype): + torch.manual_seed(1) + input_tensor = torch.ones(1, 3, 3, 3, device=device, dtype=dtype) * 0.5 + expected = self._get_expected(device=device, dtype=dtype) + aug = RandomLinearIllumination(gain=0.25, p=1.0) + res = aug(input_tensor) + assert input_tensor.shape == res.shape + self.assert_close(res, expected, rtol=1e-4, atol=1e-4) + + def test_exception(self, device, dtype): + with pytest.raises(ValueError, match="sign must be a tuple or a float"): + RandomLinearIllumination(sign=3) + + with pytest.raises(ValueError, match="gain must be a tuple or a float"): + RandomLinearIllumination(gain=[0.01, 0.06]) + + with pytest.raises( + Exception, + match=r"gain values must be between 0 and 1. Recommended values less than 0.2.", + ): + RandomLinearIllumination(gain=(0.01, 2)) + + with pytest.raises(Exception, match=r"sign of linear value must be between -1 and 1."): + RandomLinearIllumination(sign=(0.01, 2)) + + @pytest.mark.parametrize("channel_shape, batch_shape", [(1, 1), (3, 2), (5, 3)]) + def test_cardinality(self, batch_shape, channel_shape, device, dtype): + input_tensor = torch.ones(batch_shape, channel_shape, 16, 16, device=device, dtype=dtype) * 0.5 + transform = RandomGaussianIllumination(p=1.0) + output_tensor = transform(input_tensor) + assert input_tensor.shape[0] == output_tensor.shape[0] + assert input_tensor.shape[1] == output_tensor.shape[1] + + def test_same_on_batch(self, device, dtype): + input_tensor = torch.ones(2, 1, 5, 5, device=device, dtype=dtype) * 0.5 + transform = RandomGaussianIllumination(p=1.0, same_on_batch=True) + output_tensor = transform(input_tensor) + self.assert_close(output_tensor[0], output_tensor[1]) + + +class TestRandomLinearCornerIllumination(BaseTester): + def _get_expected(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.3750000596, 0.4375000298, 0.5000000000], + [0.3125000894, 0.3750000596, 0.4375000298], + [0.2500001192, 0.3125000894, 0.3750000596], + ], + [ + [0.3750000596, 0.4375000298, 0.5000000000], + [0.3125000894, 0.3750000596, 0.4375000298], + [0.2500001192, 0.3125000894, 0.3750000596], + ], + [ + [0.3750000596, 0.4375000298, 0.5000000000], + [0.3125000894, 0.3750000596, 0.4375000298], + [0.2500001192, 0.3125000894, 0.3750000596], + ], + ] + ], + device=device, + dtype=dtype, + ) + + def test_smoke(self, device, dtype): + torch.manual_seed(1) + input_tensor = torch.ones(1, 3, 3, 3, device=device, dtype=dtype) * 0.5 + expected = self._get_expected(device=device, dtype=dtype) + aug = RandomLinearCornerIllumination(gain=0.25, p=1.0) + res = aug(input_tensor) + assert input_tensor.shape == res.shape + self.assert_close(res, expected, rtol=1e-4, atol=1e-4) + + def test_exception(self, device, dtype): + with pytest.raises(ValueError, match="sign must be a tuple or a float"): + RandomLinearCornerIllumination(sign=3) + + with pytest.raises(ValueError, match="gain must be a tuple or a float"): + RandomLinearCornerIllumination(gain=[0.01, 0.06]) + + with pytest.raises( + Exception, + match=r"gain values must be between 0 and 1. Recommended values less than 0.2.", + ): + RandomLinearCornerIllumination(gain=(0.01, 2)) + + with pytest.raises(Exception, match=r"sign of linear value must be between -1 and 1."): + RandomLinearCornerIllumination(sign=(0.01, 2)) + + @pytest.mark.parametrize("channel_shape, batch_shape", [(1, 1), (3, 2), (5, 3)]) + def test_cardinality(self, batch_shape, channel_shape, device, dtype): + input_tensor = torch.ones(batch_shape, channel_shape, 16, 16, device=device, dtype=dtype) * 0.5 + transform = RandomLinearCornerIllumination(p=1.0) + output_tensor = transform(input_tensor) + assert input_tensor.shape[0] == output_tensor.shape[0] + assert input_tensor.shape[1] == output_tensor.shape[1] + + def test_same_on_batch(self, device, dtype): + input_tensor = torch.ones(2, 1, 5, 5, device=device, dtype=dtype) * 0.5 + transform = RandomLinearCornerIllumination(p=1.0, same_on_batch=True) + output_tensor = transform(input_tensor) + self.assert_close(output_tensor[0], output_tensor[1]) + + +class TestRandomChannelDropout(BaseTester): + def _get_expected(self, device, dtype): + return torch.tensor( + [ + [ + [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]], + ] + ], + device=device, + dtype=dtype, + ) + + def test_smoke(self, device, dtype): + torch.manual_seed(1) + input_tensor = torch.ones(1, 3, 3, 3, device=device, dtype=dtype) + expected = self._get_expected(device=device, dtype=dtype) + aug = RandomChannelDropout(p=1.0) + res = aug(input_tensor) + assert input_tensor.shape == res.shape + self.assert_close(res, expected, rtol=1e-4, atol=1e-4) + + def test_exception(self, device, dtype): + from kornia.core.exceptions import BaseError, TypeCheckError + + num_drop_channels = 2.0 + with pytest.raises(TypeCheckError, match=f"`num_drop_channels` must be an int. Got: {type(num_drop_channels)}"): + RandomChannelDropout(num_drop_channels=num_drop_channels, fill_value=0.0) + + num_drop_channels = 0 + with pytest.raises( + BaseError, + match=f"Invalid value in `num_drop_channels`. Must be an int greater than 1. Got: {num_drop_channels}", + ): + RandomChannelDropout(num_drop_channels=num_drop_channels, fill_value=0.0) + + num_drop_channels = 5 + input_tensor = torch.ones(1, 3, 3, 3, device=device, dtype=dtype) + with pytest.raises( + Exception, + match=r"Invalid value in `num_drop_channels`. Cannot be greater than the number of channels of `input`.", + ): + RandomChannelDropout(num_drop_channels=num_drop_channels, p=1.0)(input_tensor) + + fill_value = 2.0 + with pytest.raises( + BaseError, match=f"Invalid value in `fill_value`. Must be a float between 0 and 1. Got: {fill_value}" + ): + RandomChannelDropout(fill_value=fill_value) + + fill_value = 1 + with pytest.raises(TypeCheckError, match=f"`fill_value` must be a float. Got: {type(fill_value)}"): + RandomChannelDropout(fill_value=fill_value) + + @pytest.mark.parametrize("channel_shape, batch_shape", [(3, 1), (1, 1), (5, 5)]) + def test_cardinality(self, batch_shape, channel_shape, device, dtype): + input_tensor = torch.ones(batch_shape, channel_shape, 5, 5, device=device, dtype=dtype) + transform = RandomChannelDropout(p=1.0) + output_tensor = transform(input_tensor) + assert input_tensor.shape[0] == output_tensor.shape[0] + assert input_tensor.shape[1] == output_tensor.shape[1] + + def test_same_on_batch(self, device, dtype): + input_tensor = torch.ones(2, 3, 5, 5, device=device, dtype=dtype) + transform = RandomChannelDropout(p=1.0, same_on_batch=True) + output_tensor = transform(input_tensor) + self.assert_close(output_tensor[0], output_tensor[1]) + + +class TestNormalize(BaseTester): + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self, device, dtype): + f = Normalize(mean=torch.tensor([1.0]), std=torch.tensor([1.0])) + repr = "Normalize(mean=torch.tensor([1.]), std=torch.tensor([1.]), p=1., p_batch=1.0, same_on_batch=False)" + assert str(f) == repr + + @pytest.mark.parametrize( + "mean, std", + [ + ((1.0, 1.0, 1.0), (0.5, 0.5, 0.5)), + (1.0, 0.5), + (torch.tensor([1.0]), torch.tensor([0.5])), + ], + ) + def test_random_normalize_different_parameter_types(self, mean, std): + f = Normalize(mean=mean, std=std, p=1) + data = torch.ones(2, 3, 256, 313) + if isinstance(mean, float): + expected = (data - torch.as_tensor(mean)) / torch.as_tensor(std) + else: + expected = (data - torch.as_tensor(mean[0])) / torch.as_tensor(std[0]) + self.assert_close(f(data), expected) + + @staticmethod + @pytest.mark.parametrize( + "mean, std", + [((1.0, 1.0, 1.0, 1.0), (0.5, 0.5, 0.5, 0.5)), ((1.0, 1.0), (0.5, 0.5))], + ) + def test_random_normalize_invalid_parameter_shape(mean, std): + f = Normalize(mean=mean, std=std, p=1.0) + inputs = torch.arange(0.0, 16.0, step=1).reshape(1, 4, 4).unsqueeze(0) + with pytest.raises(ValueError): + f(inputs) + + def test_random_normalize(self, device, dtype): + f = Normalize(mean=torch.tensor([1.0]), std=torch.tensor([0.5]), p=1.0) + f1 = Normalize(mean=torch.tensor([1.0]), std=torch.tensor([0.5]), p=0.0) + + inputs = torch.arange(0.0, 16.0, step=1, device=device, dtype=dtype).reshape(1, 4, 4).unsqueeze(0) + + expected = (inputs - 1) * 2 + + identity = kornia.core.ops.eye_like(3, expected) + + self.assert_close(f(inputs), expected) + self.assert_close(f.transform_matrix, identity) + self.assert_close(f1(inputs), inputs) + self.assert_close(f1.transform_matrix, identity) + + def test_batch_random_normalize(self, device, dtype): + f = Normalize(mean=torch.tensor([1.0]), std=torch.tensor([0.5]), p=1.0) + f1 = Normalize(mean=torch.tensor([1.0]), std=torch.tensor([0.5]), p=0.0) + + inputs = torch.arange(0.0, 16.0 * 2, step=1, device=device, dtype=dtype).reshape(2, 1, 4, 4) + + expected = (inputs - 1) * 2 + + identity = kornia.core.ops.eye_like(3, expected) + + self.assert_close(f(inputs), expected) + self.assert_close(f.transform_matrix, identity) + self.assert_close(f1(inputs), inputs) + self.assert_close(f1.transform_matrix, identity) + + @pytest.mark.slow + def test_gradcheck(self, device): + torch.manual_seed(0) # for random reproductibility + + input = torch.rand((3, 3, 3), device=device, dtype=torch.float64) # 3 x 3 x 3 + self.gradcheck( + Normalize(mean=torch.tensor([1.0]), std=torch.tensor([1.0]), p=1.0), + (input,), + ) + + +class TestDenormalize(BaseTester): + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self, device, dtype): + f = Denormalize(mean=torch.tensor([1.0]), std=torch.tensor([1.0])) + repr = "Denormalize(mean=torch.tensor([1.]), std=torch.tensor([1.]), p=1., p_batch=1.0, same_on_batch=False)" + assert str(f) == repr + + def test_random_denormalize(self, device, dtype): + f = Denormalize(mean=torch.tensor([1.0]), std=torch.tensor([0.5]), p=1.0) + f1 = Denormalize(mean=torch.tensor([1.0]), std=torch.tensor([0.5]), p=0.0) + + inputs = torch.arange(0.0, 16.0, step=1, device=device, dtype=dtype).reshape(1, 4, 4).unsqueeze(0) + + expected = inputs / 2 + 1 + + identity = kornia.core.ops.eye_like(3, expected) + + self.assert_close(f(inputs), expected) + self.assert_close(f.transform_matrix, identity) + self.assert_close(f1(inputs), inputs) + self.assert_close(f1.transform_matrix, identity) + + def test_batch_random_denormalize(self, device, dtype): + f = Denormalize(mean=torch.tensor([1.0]), std=torch.tensor([0.5]), p=1.0) + f1 = Denormalize(mean=torch.tensor([1.0]), std=torch.tensor([0.5]), p=0.0) + + inputs = torch.arange(0.0, 16.0 * 2, step=1, device=device, dtype=dtype).reshape(2, 1, 4, 4) + + expected = inputs / 2 + 1 + + identity = kornia.core.ops.eye_like(3, expected) + + self.assert_close(f(inputs), expected) + self.assert_close(f.transform_matrix, identity) + self.assert_close(f1(inputs), inputs) + self.assert_close(f1.transform_matrix, identity) + + @pytest.mark.slow + def test_gradcheck(self, device): + torch.manual_seed(0) # for random reproductibility + + input = torch.rand((3, 3, 3), device=device, dtype=torch.float64) # 3 x 3 x 3 + self.gradcheck( + Denormalize(mean=torch.tensor([1.0]), std=torch.tensor([1.0]), p=1.0), + (input,), + ) + + +class TestRandomFisheye(BaseTester): + def test_smoke(self, device, dtype): + torch.manual_seed(0) + center_x = torch.tensor([-0.3, 0.3]) + center_y = torch.tensor([-0.3, 0.3]) + gamma = torch.tensor([-1.0, 1.0]) + img = torch.rand(1, 1, 2, 2, device=device, dtype=dtype) + aug = RandomFisheye(center_x, center_y, gamma, p=1.0) + assert img.shape == aug(img).shape + + def test_same_on_batch(self, device, dtype): + torch.manual_seed(0) + center_x = torch.tensor([-0.3, 0.3], device=device, dtype=dtype) + center_y = torch.tensor([-0.3, 0.3], device=device, dtype=dtype) + gamma = torch.tensor([-1.0, 1.0], device=device, dtype=dtype) + img = torch.rand(1, 1, 2, 2, device=device, dtype=dtype) + aug = RandomFisheye(center_x, center_y, gamma, same_on_batch=True, p=1.0) + assert img.shape == aug(img).shape + + @pytest.mark.skip(reason="RuntimeError: Jacobian mismatch for output 0 with respect to input 0") + def test_gradcheck(self, device, dtype): + img = torch.rand(1, 1, 3, 3, device=device, dtype=dtype) + center_x = torch.tensor([-0.3, 0.3], device=device, dtype=dtype) + center_y = torch.tensor([-0.3, 0.3], device=device, dtype=dtype) + gamma = torch.tensor([-1.0, 1.0], device=device, dtype=dtype) + self.gradcheck(RandomFisheye(center_x, center_y, gamma), (img,)) + + +class TestRandomElasticTransform(BaseTester): + def test_smoke(self, device, dtype): + img = torch.rand(1, 1, 2, 2, device=device, dtype=dtype) + aug = RandomElasticTransform(p=1.0) + assert img.shape == aug(img).shape + + def test_same_on_batch(self, device, dtype): + f = RandomElasticTransform(p=1.0, same_on_batch=True) + input = torch.eye(3, device=device, dtype=dtype).unsqueeze(dim=0).unsqueeze(dim=0).repeat(2, 1, 1, 1) + res = f(input) + self.assert_close(res[0], res[1]) + + def test_mask_transform(self, device, dtype): + torch.manual_seed(0) + features = torch.rand(1, 1, 4, 4, dtype=dtype, device=device) + labels = torch.ones(1, 1, 4, 4, dtype=dtype, device=device) * 10 + labels[:, :, :, :2] = 0 + labels[:, :, :2, :] = 0 + + compose = AugmentationSequential(RandomElasticTransform(alpha=(10, 10))) + + # Use an example which would produce invalid label values + torch.manual_seed(0) + labels_transformed = compose(features, labels, data_keys=["input", "input"])[1] + assert len(labels_transformed.unique()) > 2 + + # The transformed values are fine if we use mask input type + labels_transformed = compose(features, labels, data_keys=["input", "mask"])[1] + self.assert_close( + labels_transformed.unique(), + torch.tensor([0, 10], dtype=dtype, device=device), + ) + + @pytest.mark.parametrize("batch_prob", [[True, True], [False, True], [False, False]]) + @pytest.mark.skipif( + torch_version() in {"1.11.0", "1.12.1"} and sys.version_info.minor == 10, + reason="failing because no gaussian mean", + ) + def test_apply(self, batch_prob, device, dtype): + torch.manual_seed(0) + + aug_list = AugmentationSequential(RandomElasticTransform(sigma=(2, 2), alpha=(2, 2))) + features = torch.rand(2, 3, 10, 10, dtype=dtype, device=device) + labels = torch.randint(0, 10, (2, 1, 10, 10), dtype=dtype, device=device) + + # Make sure the transformation works correctly even if only applied to some images in the batch + to_apply = torch.tensor(batch_prob, device=device) + with patch.object(aug_list[0], "__batch_prob_generator__", return_value=to_apply): + features_transformed, labels_transformed = aug_list(features, labels, data_keys=["input", "mask"]) + self.assert_close(aug_list._params[0].data["batch_prob"], to_apply) + + # Images should remain unchanged if the transformation is not applied + self.assert_close(features_transformed[~to_apply], features[~to_apply]) + self.assert_close(labels_transformed[~to_apply], labels[~to_apply]) + + # At least one value in the images should change if the transformation is applied + if to_apply.any(): + assert features_transformed[to_apply].ne(features[to_apply]).any() + assert labels_transformed[to_apply].ne(labels[to_apply]).any() + + +class TestRandomBoxBlur: + def test_smoke(self, device, dtype): + img = torch.rand(1, 1, 2, 2, device=device, dtype=dtype) + aug = RandomBoxBlur(p=1.0) + assert img.shape == aug(img).shape + + +class TestPadTo(BaseTester): + def test_smoke(self, device, dtype): + img = torch.rand(1, 1, 2, 2, device=device, dtype=dtype) + aug = PadTo(size=(4, 5)) + out = aug(img) + assert out.shape == (1, 1, 4, 5) + self.assert_close(aug.inverse(out), img) + + +class TestResize: + def test_smoke(self, device, dtype): + img = torch.rand(1, 1, 4, 6, device=device, dtype=dtype) + aug = Resize(size=(4, 5)) + out = aug(img) + assert out.shape == (1, 1, 4, 5) + assert aug.inverse(out).shape == (1, 1, 4, 6) + + +class TestSmallestMaxSize: + def test_smoke(self, device, dtype): + img_A = torch.rand(1, 1, 4, 6, device=device, dtype=dtype) + img_B = torch.rand(1, 1, 9, 6, device=device, dtype=dtype) + aug = SmallestMaxSize(max_size=2) + + assert aug(img_A).shape == (1, 1, 2, 3) + assert aug(img_B).shape == (1, 1, 3, 2) + + aug = SmallestMaxSize(max_size=2) + assert aug(img_B).shape == (1, 1, 3, 2) + assert aug(img_A).shape == (1, 1, 2, 3) + + +class TestLongestMaxSize: + def test_smoke(self, device, dtype): + img_A = torch.rand(1, 1, 4, 6, device=device, dtype=dtype) + img_B = torch.rand(1, 1, 8, 6, device=device, dtype=dtype) + aug = LongestMaxSize(max_size=3) + + assert aug(img_A).shape == (1, 1, 2, 3) + assert aug(img_B).shape == (1, 1, 3, 2) + + aug = LongestMaxSize(max_size=3) + assert aug(img_B).shape == (1, 1, 3, 2) + assert aug(img_A).shape == (1, 1, 2, 3) + + +class TestRandomPosterize: + def test_smoke(self, device, dtype): + img = torch.rand(1, 1, 4, 5, device=device, dtype=dtype) + aug = RandomPosterize(bits=6, p=1.0).to(device, dtype) + out = aug(img) + assert out.shape == (1, 1, 4, 5) + + +class TestRandomPlasma: + def test_plasma_shadow(self, device, dtype): + img = torch.rand(2, 3, 4, 5, device=device, dtype=dtype) + aug = RandomPlasmaShadow(p=1.0).to(device) + out = aug(img) + assert out.shape == (2, 3, 4, 5) + + def test_plasma_brightness(self, device, dtype): + img = torch.rand(2, 3, 4, 5, device=device, dtype=dtype) + aug = RandomPlasmaBrightness(p=1.0).to(device) + out = aug(img) + assert out.shape == (2, 3, 4, 5) + + def test_plasma_contrast(self, device, dtype): + img = torch.rand(2, 3, 4, 5, device=device, dtype=dtype) + aug = RandomPlasmaContrast(p=1.0).to(device) + out = aug(img) + assert out.shape == (2, 3, 4, 5) + + +class TestPlanckianJitter(BaseTester): + def _get_expected_output_blackbody(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.7350, 1.0000, 0.1311, 0.1955], + [0.4553, 0.9391, 0.7258, 1.0000], + [0.6748, 0.9364, 0.5167, 0.5949], + [0.0330, 0.2501, 0.4353, 0.7679], + ], + [ + [0.6977, 0.8000, 0.1610, 0.2823], + [0.6816, 0.9152, 0.3971, 0.8742], + [0.4194, 0.5529, 0.9527, 0.0362], + [0.1852, 0.3734, 0.3051, 0.9320], + ], + [ + [0.0691, 0.1059, 0.0592, 0.0124], + [0.0817, 0.3650, 0.2839, 0.2914], + [0.2066, 0.0957, 0.2295, 0.0130], + [0.0545, 0.0951, 0.3202, 0.3114], + ], + ] + ], + device=device, + dtype=dtype, + ) + + def _get_expected_output_cied(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.6058, 0.9377, 0.1080, 0.1611], + [0.3752, 0.7740, 0.5982, 1.0000], + [0.5561, 0.7718, 0.4259, 0.4903], + [0.0272, 0.2062, 0.3587, 0.6329], + ], + [ + [0.6977, 0.8000, 0.1610, 0.2823], + [0.6816, 0.9152, 0.3971, 0.8742], + [0.4194, 0.5529, 0.9527, 0.0362], + [0.1852, 0.3734, 0.3051, 0.9320], + ], + [ + [0.1149, 0.1762, 0.0984, 0.0207], + [0.1359, 0.6072, 0.4722, 0.4848], + [0.3437, 0.1592, 0.3818, 0.0217], + [0.0906, 0.1582, 0.5326, 0.5180], + ], + ] + ], + device=device, + dtype=dtype, + ) + + def _get_expected_output_batch(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.7350, 1.0000, 0.1311, 0.1955], + [0.4553, 0.9391, 0.7258, 1.0000], + [0.6748, 0.9364, 0.5167, 0.5949], + [0.0330, 0.2501, 0.4353, 0.7679], + ], + [ + [0.6977, 0.8000, 0.1610, 0.2823], + [0.6816, 0.9152, 0.3971, 0.8742], + [0.4194, 0.5529, 0.9527, 0.0362], + [0.1852, 0.3734, 0.3051, 0.9320], + ], + [ + [0.0691, 0.1059, 0.0592, 0.0124], + [0.0817, 0.3650, 0.2839, 0.2914], + [0.2066, 0.0957, 0.2295, 0.0130], + [0.0545, 0.0951, 0.3202, 0.3114], + ], + ], + [ + [ + [0.4963, 0.7682, 0.0885, 0.1320], + [0.3074, 0.6341, 0.4901, 0.8964], + [0.4556, 0.6323, 0.3489, 0.4017], + [0.0223, 0.1689, 0.2939, 0.5185], + ], + [ + [0.6977, 0.8000, 0.1610, 0.2823], + [0.6816, 0.9152, 0.3971, 0.8742], + [0.4194, 0.5529, 0.9527, 0.0362], + [0.1852, 0.3734, 0.3051, 0.9320], + ], + [ + [0.1759, 0.2698, 0.1507, 0.0317], + [0.2081, 0.9298, 0.7231, 0.7423], + [0.5263, 0.2437, 0.5846, 0.0332], + [0.1387, 0.2422, 0.8155, 0.7932], + ], + ], + ], + device=device, + dtype=dtype, + ) + + def _get_expected_output_same_on_batch(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.3736, 0.5783, 0.0666, 0.0994], + [0.2314, 0.4774, 0.3690, 0.6749], + [0.3430, 0.4760, 0.2627, 0.3024], + [0.0168, 0.1272, 0.2213, 0.3904], + ], + [ + [0.6977, 0.8000, 0.1610, 0.2823], + [0.6816, 0.9152, 0.3971, 0.8742], + [0.4194, 0.5529, 0.9527, 0.0362], + [0.1852, 0.3734, 0.3051, 0.9320], + ], + [ + [0.2621, 0.4020, 0.2245, 0.0472], + [0.3101, 1.0000, 1.0000, 1.0000], + [0.7842, 0.3631, 0.8711, 0.0495], + [0.2067, 0.3609, 1.0000, 1.0000], + ], + ], + [ + [ + [0.3736, 0.5783, 0.0666, 0.0994], + [0.2314, 0.4774, 0.3690, 0.6749], + [0.3430, 0.4760, 0.2627, 0.3024], + [0.0168, 0.1272, 0.2213, 0.3904], + ], + [ + [0.6977, 0.8000, 0.1610, 0.2823], + [0.6816, 0.9152, 0.3971, 0.8742], + [0.4194, 0.5529, 0.9527, 0.0362], + [0.1852, 0.3734, 0.3051, 0.9320], + ], + [ + [0.2621, 0.4020, 0.2245, 0.0472], + [0.3101, 1.0000, 1.0000, 1.0000], + [0.7842, 0.3631, 0.8711, 0.0495], + [0.2067, 0.3609, 1.0000, 1.0000], + ], + ], + ], + device=device, + dtype=dtype, + ) + + def _get_input(self, device, dtype): + return torch.tensor( + [ + [ + [ + [0.4963, 0.7682, 0.0885, 0.1320], + [0.3074, 0.6341, 0.4901, 0.8964], + [0.4556, 0.6323, 0.3489, 0.4017], + [0.0223, 0.1689, 0.2939, 0.5185], + ], + [ + [0.6977, 0.8000, 0.1610, 0.2823], + [0.6816, 0.9152, 0.3971, 0.8742], + [0.4194, 0.5529, 0.9527, 0.0362], + [0.1852, 0.3734, 0.3051, 0.9320], + ], + [ + [0.1759, 0.2698, 0.1507, 0.0317], + [0.2081, 0.9298, 0.7231, 0.7423], + [0.5263, 0.2437, 0.5846, 0.0332], + [0.1387, 0.2422, 0.8155, 0.7932], + ], + ] + ], + device=device, + dtype=dtype, + ) + + def test_planckian_jitter_blackbody(self, device, dtype): + torch.manual_seed(0) + f = RandomPlanckianJitter(select_from=1) + input = self._get_input(device, dtype) + expected = self._get_expected_output_blackbody(device, dtype) + self.assert_close(f(input), expected, low_tolerance=True) + + def test_planckian_jitter_cied(self, device, dtype): + torch.manual_seed(0) + f = RandomPlanckianJitter(mode="CIED", select_from=1) + input = self._get_input(device, dtype) + expected = self._get_expected_output_cied(device, dtype) + self.assert_close(f(input), expected, low_tolerance=True) + + def test_planckian_jitter_batch(self, device, dtype): + torch.manual_seed(0) + input = self._get_input(device, dtype).repeat(2, 1, 1, 1) + + select_from = [1, 2, 24] + f = RandomPlanckianJitter(select_from=select_from) + expected = self._get_expected_output_batch(device, dtype) + self.assert_close(f(input), expected, low_tolerance=True) + + def test_planckian_jitter_same_on_batch(self, device, dtype): + torch.manual_seed(0) + input = self._get_input(device, dtype).repeat(2, 1, 1, 1) + + select_from = [1, 2, 24, 3, 4, 5] + f = RandomPlanckianJitter(select_from=select_from, same_on_batch=True, p=1.0) + expected = self._get_expected_output_same_on_batch(device, dtype) + self.assert_close(f(input), expected, low_tolerance=True) + + +class TestRandomRGBShift(BaseTester): + def test_smoke(self, device, dtype): + img = torch.rand(2, 3, 4, 5, device=device, dtype=dtype) + aug = RandomRGBShift(p=1.0).to(device) + out = aug(img) + assert out.shape == (2, 3, 4, 5) + + def test_random_rgb_shift(self, device, dtype): + if device.type != "cpu": + pytest.skip("Random parameters are device-dependent; expected values were computed on CPU") + torch.manual_seed(0) + input = torch.tensor( + [ + [[[0.2, 0.0]], [[0.3, 0.5]], [[0.4, 0.7]]], + [[[0.2, 0.7]], [[0.0, 0.8]], [[0.2, 0.3]]], + ], + device=device, + dtype=dtype, + ) + + f = RandomRGBShift(p=1.0).to(device) + expected = torch.tensor( + [ + [[[0.19626, 0.00000]], [[0.00000, 0.08848]], [[0.20742, 0.50742]]], + [[[0.46822, 0.96822]], [[0.00000, 0.43203]], [[0.33408, 0.43408]]], + ], + device=device, + dtype=dtype, + ) + self.assert_close(f(input), expected, rtol=1e-4, atol=1e-4) + + def test_random_rgb_shift_same_batch(self, device, dtype): + if device.type != "cpu": + pytest.skip("Random parameters are device-dependent; expected values were computed on CPU") + torch.manual_seed(0) + input = torch.tensor( + [ + [[[0.2, 0.0]], [[0.3, 0.5]], [[0.4, 0.7]]], + [[[0.2, 0.7]], [[0.0, 0.8]], [[0.2, 0.3]]], + ], + device=device, + dtype=dtype, + ) + + f = RandomRGBShift(p=1.0, same_on_batch=True).to(device) + expected = torch.tensor( + [ + [[[0.19626, 0.00000]], [[0.56822, 0.76822]], [[0.00000, 0.28848]]], + [[[0.19626, 0.69626]], [[0.26822, 1.00000]], [[0.00000, 0.00000]]], + ], + device=device, + dtype=dtype, + ) + self.assert_close(f(input), expected, rtol=1e-4, atol=1e-4) + + +class TestRandomTranslate(BaseTester): + torch.manual_seed(0) # for random reproductibility + + def test_smoke_no_transform(self, device): + x_data = torch.rand(1, 2, 8, 9).to(device) + aug = kornia.augmentation.RandomTranslate((0.5, 0.5)) + out = aug(x_data) + assert out.shape == x_data.shape + assert aug.inverse(out).shape == x_data.shape + assert aug.inverse(out, aug._params).shape == x_data.shape + + @pytest.mark.slow + def test_gradcheck(self, device): + input = torch.rand(1, 2, 5, 7, device=device) + self.gradcheck(kornia.augmentation.RandomTranslate((0.5, 0.5), p=1.0), (input,)) + + +class TestRandomAutoContrast(BaseTester): + torch.manual_seed(0) # for random reproductibility + + def test_smoke_no_transform(self, device): + x_data = torch.rand(1, 2, 8, 9).to(device) + aug = kornia.augmentation.RandomAutoContrast() + out = aug(x_data) + assert out.shape == x_data.shape + + @pytest.mark.slow + def test_gradcheck(self, device): + input = torch.rand(1, 2, 5, device=device, dtype=torch.float64) + # TODO: turned off with p=0 + self.gradcheck(kornia.augmentation.RandomAutoContrast(p=1.0), (input,)) + + +class TestRandomSnow(BaseTester): + torch.manual_seed(0) # for random reproductibility + + def _get_exception_test_data(self, device, dtype): + err_msg_sw_coef = "Snow coefficient values must be between 0 and 1." + err_msg_brght_coef = "Brightness values must be greater than 1." + err_msg_wrong_ch = "Number of color channels should be 3." + err_msg_wrong_sh = "Input size must have a shape of either (H, W), (C, H, W) or (*, C, H, W)." + + return [ + ( + err_msg_sw_coef, + (-0.3, 0.6), + (1.2, 3.4), + torch.rand(1, 3, 3, device=device, dtype=dtype), + ), + ( + err_msg_sw_coef, + (0.3, -0.9), + (1.3, 2.5), + torch.rand(1, 3, 4, device=device, dtype=dtype), + ), + ( + err_msg_sw_coef, + (-0.6, -0.9), + (1.1, 3.1), + torch.rand(2, 3, 4, device=device, dtype=dtype), + ), + ( + err_msg_brght_coef, + (0.1, 0.7), + (0.3, 1.5), + torch.rand(1, 3, 5, device=device, dtype=dtype), + ), + ( + err_msg_brght_coef, + (0.4, 0.6), + (1.3, 0.8), + torch.rand(1, 3, 6, device=device, dtype=dtype), + ), + ( + err_msg_brght_coef, + (0.3, 0.9), + (0.5, 0.7), + torch.rand(1, 3, 7, device=device, dtype=dtype), + ), + ( + err_msg_wrong_ch, + (0.2, 0.8), + (1.6, 3.7), + torch.rand(1, 4, 7, device=device, dtype=dtype), + ), + ( + err_msg_wrong_sh, + (0.1, 0.5), + (1.1, 2.5), + torch.rand(1, 2, 3, 4, 5, device=device, dtype=dtype), + ), + ] + + @pytest.mark.parametrize("batch_shape", [(1, 3, 4, 7), (1, 3, 6, 9)]) + def test_cardinality(self, batch_shape, device, dtype): + input_data = torch.rand(batch_shape, device=device, dtype=dtype) + aug = RandomSnow(p=1.0, snow_coefficient=(0.0, 0.5), brightness=(1.0, 3.0)) + output_data = aug(input_data) + assert output_data.shape == batch_shape + + def test_smoke(self, device, dtype): + input_data = torch.rand(1, 3, 8, 9, device=device, dtype=dtype) + aug = RandomSnow(p=1.0, snow_coefficient=(0.0, 0.5), brightness=(1.0, 2.0)) + output_data = aug(input_data) + assert output_data.shape == input_data.shape + + @pytest.mark.slow + def test_gradcheck(self, device): + input_data = torch.rand(1, 3, 6, 8, device=device, dtype=torch.float64) + self.gradcheck(RandomSnow(p=1.0), (input_data,)) + + def test_exception(self, device, dtype): + exception_test_data = self._get_exception_test_data(device, dtype) + for err_msg, snow_coef, brght_coef, input_data in exception_test_data: + with pytest.raises(Exception) as errinfo: + aug = RandomSnow(p=1.0, snow_coefficient=snow_coef, brightness=brght_coef) + aug(input_data) + + assert err_msg in str(errinfo) + + +class TestRandomMedianBlur(BaseTester): + def test_smoke(self, device, dtype): + image = torch.rand(1, 1, 2, 2, device=device, dtype=dtype) + aug = RandomMedianBlur(p=0.8) + assert image.shape == aug(image).shape + + def test_feature_median_blur(self, device, dtype): + torch.manual_seed(0) + + img = torch.ones(1, 1, 4, 4, device=device, dtype=dtype) + out = RandomMedianBlur((3, 3), p=0.5)(img) + + expected = torch.tensor( + [ + [ + [ + [0.0, 1.0, 1.0, 0.0], + [1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0], + [0.0, 1.0, 1.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + self.assert_close(out, expected) + + +class TestRandomRain(BaseTester): + torch.manual_seed(0) # for random reproductibility + + def _get_exception_test_data(self, device, dtype): + err_msg_height_bigger = "Height of drop should be greater than zero and less than image height." + err_msg_width_bigger = "Width of drop should be less than image width" + err_msg_wrong_ch = "Number of color channels should be 1 or 3." + + return [ + ( + err_msg_height_bigger, + (-2, 0), + (2, 3), + torch.rand(1, 5, 5, device=device, dtype=dtype), + ), + ( + err_msg_height_bigger, + (6, 6), + (2, 3), + torch.rand(1, 5, 5, device=device, dtype=dtype), + ), + ( + err_msg_width_bigger, + (2, 2), + (6, 6), + torch.rand(1, 5, 5, device=device, dtype=dtype), + ), + ( + err_msg_wrong_ch, + (1, 2), + (1, 2), + torch.rand(2, 4, 5, device=device, dtype=dtype), + ), + ] + + @pytest.mark.parametrize("batch_shape", [(1, 3, 5, 7), (1, 3, 6, 9), (5, 7)]) + @pytest.mark.parametrize("keepdim", [True, False]) + def test_cardinality(self, batch_shape, keepdim, device, dtype): + input_data = torch.rand(batch_shape, device=device, dtype=dtype) + output_data = RandomRain( + p=1.0, + drop_height=(3, 4), + drop_width=(2, 3), + number_of_drops=(1, 3), + keepdim=keepdim, + )(input_data) + if keepdim: + assert output_data.shape == batch_shape + else: + assert (*(1,) * (4 - len(batch_shape)), *batch_shape) == output_data.shape + + def test_smoke(self, device, dtype): + input_data = torch.rand(1, 3, 8, 9, device=device, dtype=dtype) + aug = RandomRain(p=1.0, drop_height=(2, 3), drop_width=(2, 3), number_of_drops=(1, 3)) + output_data = aug(input_data) + assert output_data.shape == input_data.shape + input_data = torch.rand(1, 3, 8, 9, device=device, dtype=dtype) + aug = RandomRain(p=1.0, drop_height=(2, 3), drop_width=(-3, -2), number_of_drops=(1, 3)) + output_data = aug(input_data) + assert output_data.shape == input_data.shape + + def test_exception(self, device, dtype): + exception_test_data = self._get_exception_test_data(device, dtype) + for err_msg, drop_height, drop_width, input_data in exception_test_data: + with pytest.raises(Exception) as errinfo: + aug = RandomRain(p=1.0, drop_height=drop_height, drop_width=drop_width) + aug(input_data) + + assert err_msg in str(errinfo) + + def test_zero_probability(self, device): + input_data = torch.rand(10, 3, 8, 8, device=device) + aug = RandomRain(p=0.0, drop_height=(2, 3), drop_width=(2, 3), number_of_drops=(1, 3)) + aug(input_data) + + +class TestMultiprocessing: + torch.manual_seed(0) # for random reproductibility + + @pytest.mark.slow + @pytest.mark.parametrize("context", ["spawn", "forkserver", "fork"] if os.name != "nt" else ["spawn"]) + def test_spawn_multiprocessing_context(self, context: str): + dataset = DummyMPDataset(context=context) + dataloader = torch.utils.data.DataLoader( + dataset, + batch_size=4, + num_workers=4, + pin_memory=True, + persistent_workers=True, + multiprocessing_context=context, + ) + + for _ in dataloader: + pass + + torch.cuda.empty_cache() + + +@pytest.mark.slow +class TestRandomJPEG(BaseTester): + torch.manual_seed(0) # for random reproductibility + + def test_smoke(self): + images = torch.rand(4, 3, 16, 16) + aug = RandomJPEG(jpeg_quality=(1.0, 100.0), p=1.0) + images_aug = aug(images) + assert images_aug.shape == images.shape + + def test_same_on_batch(self, device, dtype): + images = torch.rand(1, 3, 16, 16).repeat(2, 1, 1, 1) + aug = RandomJPEG(jpeg_quality=(1.0, 100.0), same_on_batch=True) + images_aug = aug(images) + self.assert_close(images_aug[0], images_aug[1]) + + def test_single_jpeg_quality(self, device, dtype): + images = torch.rand(4, 3, 16, 16) + aug = RandomJPEG(jpeg_quality=19.04, p=1.0) + images_aug = aug(images) + assert images_aug.shape == images.shape + + def test_single_image(self, device, dtype): + images = torch.rand(3, 16, 16) + aug = RandomJPEG(jpeg_quality=19.04, p=1.0, keepdim=True) + images_aug = aug(images) + assert images_aug.shape == images.shape + + @pytest.mark.slow + def test_gradcheck(self, device): + B, H, W = 1, 16, 16 + img = torch.zeros(B, 3, H, W, device=device, dtype=torch.float) + img[..., 0, 4:-4, 4:-4] = 1.0 + img[..., 1, 4:-4, 4:-4] = 0.5 + img[..., 2, 4:-4, 4:-4] = 0.5 + img.requires_grad = True + aug = RandomJPEG(jpeg_quality=(10.0, 10.0)) + img_jpeg = aug(img) + (img_jpeg - torch.zeros_like(img_jpeg)).abs().sum().backward() + # Numbers generated based on reference implementation + img_jpeg_mean_grad_ref = torch.tensor([0.1919], device=device) + # We use a slightly higher tolerance since our implementation varies from the reference implementation + self.assert_close(img.grad.mean().view(-1), img_jpeg_mean_grad_ref, rtol=0.01, atol=0.01) + + +@pytest.mark.slow +@pytest.mark.skipif( + torch_version_le(2, 0, 1), + reason="Test requires distributed tensor support introduced in PyTorch > 2.0.1 for transformers clip model.", +) +class TestRandomDissolving(BaseTester): + torch.manual_seed(0) # for random reproductibility + + def test_batch_proc(self, device, dtype): + images = torch.rand(4, 3, 16, 16) + aug = RandomDissolving(p=1.0, version="1.5", cache_dir="weights/") + images_aug = aug(images) + assert images_aug.shape == images.shape + + def test_single_proc(self, device, dtype): + images = torch.rand(3, 16, 16) + aug = RandomDissolving(p=1.0, keepdim=True, version="1.5", cache_dir="weights/") + images_aug = aug(images) + assert images_aug.shape == images.shape + + +class TestRandomThinPlateSpline(CommonTests): + possible_params: Dict["str", Tuple] = {} + _augmentation_cls = RandomThinPlateSpline + _default_param_set: Dict["str", Any] = {} + + @pytest.fixture(params=[_default_param_set], scope="class") + def param_set(self, request): + return request.param + + # Disable unsupported base tests + def test_smoke(self, param_set): + pytest.skip("RandomThinPlateSpline does not implement compute_transformation") + + def test_module(self): + pytest.skip("RandomThinPlateSpline does not expose transform_matrix") + + def test_random_p_1(self): + pytest.skip("RandomThinPlateSpline does not expose transform_matrix") + + def test_inverse_coordinate_check(self): + pytest.skip("RandomThinPlateSpline does not expose transform_matrix") + + def test_exception(self): + pytest.skip("RandomThinPlateSpline does not expose transform_matrix") + + def test_batch(self): + pytest.skip("RandomThinPlateSpline does not expose transform_matrix") + + def test_same_on_batch_true(self): + torch.manual_seed(0) + x = torch.randn(4, 3, 64, 64, device=self.device, dtype=self.dtype) + aug = self._augmentation_cls(p=1.0, same_on_batch=True) + _ = aug(x) + params = aug._params + assert (params["src"][0] == params["src"][1]).all() + for j in range(1, 4): + torch.testing.assert_close(params["dst"][0], params["dst"][j]) + + def test_same_on_batch_false(self): + torch.manual_seed(0) + x = torch.randn(4, 3, 64, 64, device=self.device, dtype=self.dtype) + aug = self._augmentation_cls(p=1.0, same_on_batch=False) + _ = aug(x) + params = aug._params + diffs = [(params["dst"][0] - params["dst"][j]).abs().sum().item() for j in range(1, 4)] + + assert any(d > 0 for d in diffs) + + @pytest.mark.slow + def _test_gradcheck_implementation(self, params): + # RandomThinPlateSpline generates fresh random control points on every forward call, + # which makes the standard gradcheck non-deterministic (numerical Jacobian sees a + # different warp each perturbation step). Fix the params from a single forward pass + # so that gradcheck only tests gradient flow through the deterministic warp path. + # fork_rng saves/restores CPU RNG state so this seed doesn't affect other tests. + # (generate_parameters uses rsample on CPU then .to(device), so CPU RNG governs both + # input_tensor and the TPS control-point sampling.) + aug = self._create_augmentation_from_params(**params, p=1.0) + with torch.random.fork_rng(): + torch.manual_seed(0) + input_tensor = torch.rand((1, 3, 5, 5), device=self.device, dtype=self.dtype) + _ = aug(input_tensor) + fixed_params = aug._params + + def forward_with_fixed_params(x: torch.Tensor) -> torch.Tensor: + return aug(x, params=fixed_params) + + self.gradcheck(forward_with_fixed_params, (input_tensor,)) diff --git a/tests/augmentation/test_augmentation_mix.py b/tests/augmentation/test_augmentation_mix.py new file mode 100644 index 0000000..ec7c0e2 --- /dev/null +++ b/tests/augmentation/test_augmentation_mix.py @@ -0,0 +1,651 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import copy + +import pytest +import torch + +from kornia.augmentation import ( + AugmentationSequential, + RandomCutMixV2, + RandomJigsaw, + RandomMixUpV2, + RandomMosaic, + RandomTransplantation, + RandomTransplantation3D, +) + +from testing.base import BaseTester + + +class TestRandomMixUpV2(BaseTester): + def test_smoke(self, device, dtype): + f = RandomMixUpV2() + repr = "RandomMixUpV2(lambda_val=None, p=1.0, p_batch=1.0, same_on_batch=False)" + assert str(f) == repr, str(f) + + def test_random_mixup_p1(self, device, dtype): + torch.manual_seed(0) + f = RandomMixUpV2(p=1.0, data_keys=["input", "class"]) + + input = torch.stack( + [torch.ones(1, 3, 4, device=device, dtype=dtype), torch.zeros(1, 3, 4, device=device, dtype=dtype)] + ) + label = torch.tensor([1, 0], device=device, dtype=dtype) + lam = torch.tensor([0.1320, 0.3074], device=device, dtype=dtype) + + expected = torch.stack( + [ + torch.ones(1, 3, 4, device=device, dtype=dtype) * (1 - lam[0]), + torch.ones(1, 3, 4, device=device, dtype=dtype) * lam[1], + ] + ) + + out_image, out_label = f(input, label) + + self.assert_close(out_image, expected, rtol=1e-4, atol=1e-4) + self.assert_close(out_label[:, 0], label) + self.assert_close(out_label[:, 1], torch.tensor([0, 1], device=device, dtype=dtype)) + self.assert_close(out_label[:, 2], lam, rtol=1e-4, atol=1e-4) + + def test_random_mixup_p0(self, device, dtype): + torch.manual_seed(0) + f = RandomMixUpV2(p=0.0, data_keys=["input", "class"]) + + input = torch.stack( + [torch.ones(1, 3, 4, device=device, dtype=dtype), torch.zeros(1, 3, 4, device=device, dtype=dtype)] + ) + label = torch.tensor([1, 0], device=device) + lam = torch.tensor([0.0, 0.0], device=device, dtype=dtype) + + expected = input.clone() + + out_image, out_label = f(input, label) + + self.assert_close(out_image, expected, rtol=1e-4, atol=1e-4) + self.assert_close(out_label[:, 2], lam, rtol=1e-4, atol=1e-4) + + def test_random_mixup_lam0(self, device, dtype): + torch.manual_seed(0) + f = RandomMixUpV2(lambda_val=(0.0, 0.0), p=1.0, data_keys=["input", "class"]) + + input = torch.stack( + [torch.ones(1, 3, 4, device=device, dtype=dtype), torch.zeros(1, 3, 4, device=device, dtype=dtype)] + ) + label = torch.tensor([1, 0], device=device, dtype=dtype) + lam = torch.tensor([0.0, 0.0], device=device, dtype=dtype) + + expected = input.clone() + + out_image, out_label = f(input, label) + + self.assert_close(out_image, expected, rtol=1e-4, atol=1e-4) + self.assert_close(out_label[:, 0], label) + self.assert_close(out_label[:, 1], torch.tensor([0, 1], device=device, dtype=dtype)) + self.assert_close(out_label[:, 2], lam, rtol=1e-4, atol=1e-4) + + def test_random_mixup_same_on_batch(self, device, dtype): + torch.manual_seed(0) + f = RandomMixUpV2(same_on_batch=True, p=1.0, data_keys=["input", "class"]) + + input = torch.stack( + [torch.ones(1, 3, 4, device=device, dtype=dtype), torch.zeros(1, 3, 4, device=device, dtype=dtype)] + ) + label = torch.tensor([1, 0], device=device, dtype=dtype) + lam = torch.tensor([0.0885, 0.0885], device=device, dtype=dtype) + + expected = torch.stack( + [ + torch.ones(1, 3, 4, device=device, dtype=dtype) * (1 - lam[0]), + torch.ones(1, 3, 4, device=device, dtype=dtype) * lam[1], + ] + ) + + out_image, out_label = f(input, label) + self.assert_close(out_image, expected, rtol=1e-4, atol=1e-4) + self.assert_close(out_label[:, 0], label) + self.assert_close(out_label[:, 1], torch.tensor([0, 1], device=device, dtype=dtype)) + self.assert_close(out_label[:, 2], lam, rtol=1e-4, atol=1e-4) + + +class TestRandomCutMixV2(BaseTester): + def test_smoke(self): + f = RandomCutMixV2(data_keys=["input", "class"], use_correct_lambda=True) + expected_repr = "RandomCutMixV2(cut_size=None, beta=None, num_mix=1, p=1.0, p_batch=1.0, same_on_batch=False)" + assert str(f) == expected_repr + + def test_random_mixup_p1(self, device, dtype): + torch.manual_seed(76) + f = RandomCutMixV2(p=1.0, data_keys=["input", "class"], use_correct_lambda=True) + + input = torch.stack( + [torch.ones(1, 3, 4, device=device, dtype=dtype), torch.zeros(1, 3, 4, device=device, dtype=dtype)] + ) + label = torch.tensor([1, 0], device=device, dtype=dtype) + + expected = torch.tensor( + [ + [[[0.0, 0.0, 0.0, 1.0], [0.0, 0.0, 0.0, 1.0], [1.0, 1.0, 1.0, 1.0]]], + [[[1.0, 1.0, 1.0, 0.0], [1.0, 1.0, 1.0, 0.0], [0.0, 0.0, 0.0, 0.0]]], + ], + device=device, + dtype=dtype, + ) + + out_image, out_label = f(input, label) + + self.assert_close(out_image, expected, rtol=1e-4, atol=1e-4) + self.assert_close(out_label[0, :, 0], label) + self.assert_close(out_label[0, :, 1], torch.tensor([0, 1], device=device, dtype=dtype)) + self.assert_close(out_label[0, :, 2], torch.tensor([0.5, 0.5], device=device, dtype=dtype)) + + def test_random_mixup_p0(self, device, dtype): + torch.manual_seed(76) + f = RandomCutMixV2(p=0.0, data_keys=["input", "class"], use_correct_lambda=True) + + input = torch.stack( + [torch.ones(1, 3, 4, device=device, dtype=dtype), torch.zeros(1, 3, 4, device=device, dtype=dtype)] + ) + label = torch.tensor([1, 0], device=device) + + expected = input.clone() + exp_label = torch.tensor([[[1, 1, 0], [0, 0, 0]]], device=device, dtype=dtype) + + out_image, out_label = f(input, label) + + self.assert_close(out_image, expected, rtol=1e-4, atol=1e-4) + self.assert_close(out_label, exp_label) + + def test_random_mixup_beta0(self, device, dtype): + torch.manual_seed(76) + # beta 0 => resample 0.5 area + # beta cannot be 0 after torch 1.8.0 + f = RandomCutMixV2(beta=1e-7, p=1.0, data_keys=["input", "class"], use_correct_lambda=True) + + input = torch.stack( + [torch.ones(1, 3, 4, device=device, dtype=dtype), torch.zeros(1, 3, 4, device=device, dtype=dtype)] + ) + label = torch.tensor([1, 0], device=device, dtype=dtype) + + expected = torch.tensor( + [ + [[[0.0, 0.0, 1.0, 1.0], [0.0, 0.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]], + [[[1.0, 1.0, 0.0, 0.0], [1.0, 1.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]]], + ], + device=device, + dtype=dtype, + ) + + out_image, out_label = f(input, label) + + self.assert_close(out_image, expected, rtol=1e-4, atol=1e-4) + self.assert_close(out_label[0, :, 0], label) + self.assert_close(out_label[0, :, 1], torch.tensor([0, 1], device=device, dtype=dtype)) + # cut area = 4 / 12, but with use_correct_lambda=True the lambda calculation is different + self.assert_close( + out_label[0, :, 2], + torch.tensor([0.66667, 0.66667], device=device, dtype=dtype), + rtol=1e-4, + atol=1e-4, + ) + + def test_random_mixup_num2(self, device, dtype): + torch.manual_seed(76) + f = RandomCutMixV2(num_mix=5, p=1.0, data_keys=["input", "class"], use_correct_lambda=True) + + input = torch.stack( + [torch.ones(1, 3, 4, device=device, dtype=dtype), torch.zeros(1, 3, 4, device=device, dtype=dtype)] + ) + label = torch.tensor([1, 0], device=device, dtype=dtype) + + expected = torch.tensor( + [ + [[[0.0, 0.0, 0.0, 1.0], [0.0, 0.0, 0.0, 1.0], [1.0, 1.0, 1.0, 1.0]]], + [[[1.0, 1.0, 0.0, 0.0], [1.0, 1.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]]], + ], + device=device, + dtype=dtype, + ) + + out_image, out_label = f(input, label) + + self.assert_close(out_image, expected, rtol=1e-4, atol=1e-4) + self.assert_close(out_label[:, :, 0], label.view(1, -1).expand(5, 2)) + self.assert_close( + out_label[:, :, 1], torch.tensor([[1, 0], [1, 0], [1, 0], [1, 0], [0, 1]], device=device, dtype=dtype) + ) + # Updated expected values for use_correct_lambda=True + self.assert_close( + out_label[:, :, 2], + torch.tensor( + [[0.9167, 0.6667], [1.0, 0.8333], [0.5, 0.9167], [0.9167, 1.0], [0.5, 0.6667]], + device=device, + dtype=dtype, + ), + rtol=1e-4, + atol=1e-4, + ) + + def test_random_mixup_same_on_batch(self, device, dtype): + torch.manual_seed(42) + f = RandomCutMixV2(same_on_batch=True, p=1.0, data_keys=["input", "class"], use_correct_lambda=True) + + input = torch.stack( + [torch.ones(1, 3, 4, device=device, dtype=dtype), torch.zeros(1, 3, 4, device=device, dtype=dtype)] + ) + label = torch.tensor([1, 0], device=device, dtype=dtype) + + expected = torch.tensor( + [ + [[[0.0, 0.0, 0.0, 1.0], [0.0, 0.0, 0.0, 1.0], [1.0, 1.0, 1.0, 1.0]]], + [[[1.0, 1.0, 1.0, 0.0], [1.0, 1.0, 1.0, 0.0], [0.0, 0.0, 0.0, 0.0]]], + ], + device=device, + dtype=dtype, + ) + + out_image, out_label = f(input, label) + + self.assert_close(out_image, expected, rtol=1e-4, atol=1e-4) + self.assert_close(out_label[0, :, 0], label) + self.assert_close(out_label[0, :, 1], torch.tensor([0, 1], device=device, dtype=dtype)) + self.assert_close( + out_label[0, :, 2], torch.tensor([0.5000, 0.5000], device=device, dtype=dtype), rtol=1e-4, atol=1e-4 + ) + + +class TestRandomMosaic(BaseTester): + def test_smoke(self): + f = RandomMosaic(data_keys=["input", "class"]) + repr = ( + "RandomMosaic(output_size=None, mosaic_grid=(2, 2), start_ratio_range=(0.3, 0.7), p=0.7," + " p_batch=1.0, same_on_batch=False, mosaic_grid=(2, 2), output_size=None, min_bbox_size=0.0," + " padding_mode=constant, resample=bilinear, align_corners=True, cropping_mode=slice)" + ) + assert str(f) == repr + + def test_numerical(self, device, dtype): + torch.manual_seed(76) + f = RandomMosaic(p=1.0, data_keys=["input", "bbox_xyxy"]) + + input = torch.stack( + [torch.ones(1, 8, 8, device=device, dtype=dtype), torch.zeros(1, 8, 8, device=device, dtype=dtype)] + ) + boxes = torch.tensor([[[4, 5, 6, 7], [1, 2, 3, 4]], [[2, 2, 6, 6], [0, 0, 0, 0]]], device=device, dtype=dtype) + + out_image, out_box = f(input, boxes) + + expected = torch.tensor( + [ + [ + [ + [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0], + ] + ], + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ] + ], + ], + device=device, + dtype=dtype, + ) + + expected_box = torch.tensor( + [ + [ + [0.7074, 0.7099, 2.7074, 2.7099], + [0.0000, 0.0000, 1.0000, 1.0000], + [0.0000, 5.7099, 2.7074, 8.0000], + [0.0000, 2.7099, 1.0000, 4.7099], + [7.0000, 0.7099, 8.0000, 2.7099], + [5.7074, 0.0000, 7.7074, 1.0000], + [7.0000, 7.0000, 8.0000, 8.0000], + [5.7074, 5.7099, 7.7074, 7.7099], + ], + [ + [0.0000, 0.0000, 1.0000, 2.8313], + [0.0000, 0.0000, 1.0000, 1.0000], + [0.0000, 7.0000, 1.0000, 8.0000], + [0.0000, 6.8313, 1.0000, 8.0000], + [4.5036, 0.0000, 8.0000, 2.8313], + [1.5036, 0.0000, 3.5036, 1.0000], + [4.5036, 6.8313, 8.0000, 8.0000], + [1.5036, 3.8313, 3.5036, 5.8313], + ], + ], + device=device, + dtype=dtype, + ) + + self.assert_close(out_image, expected, rtol=1e-4, atol=1e-4) + self.assert_close(out_box, expected_box, rtol=1e-4, atol=1e-4) + + @pytest.mark.parametrize("p", [0.0, 0.5, 1.0]) + def test_p(self, p, device, dtype): + torch.manual_seed(76) + f = RandomMosaic(output_size=(300, 300), p=p, data_keys=["input", "bbox_xyxy"]) + + input = torch.randn((2, 3, 224, 224), device=device, dtype=dtype) + boxes = torch.tensor( + [ + # image 1 + [[70.0, 5, 150, 100], [60, 180, 175, 220]], # head # feet + # image 2 + [[75, 30, 175, 140], [0, 0, 0, 0]], # head # placeholder + ], + device=device, + dtype=dtype, + ) + + f(input, boxes) + + +class TestRandomJigsaw(BaseTester): + def test_smoke(self, device, dtype): + f = RandomJigsaw(data_keys=["input"]) + repr = "RandomJigsaw(grid=(4, 4), p=0.5, p_batch=1.0, same_on_batch=False, grid=(4, 4))" + assert str(f) == repr + + # Test square and non-square images. + f = RandomJigsaw(grid=(2, 2), p=1.0, data_keys=["input"]) + input = torch.arange(64, device=device, dtype=dtype).reshape(2, 1, 4, 8) + f(input) + input = torch.arange(64, device=device, dtype=dtype).reshape(2, 1, 8, 4) + f(input) + input = torch.arange(32, device=device, dtype=dtype).reshape(2, 1, 4, 4) + f(input) + + def test_numerical(self, device, dtype): + torch.manual_seed(22) + f = RandomJigsaw(grid=(2, 2), p=1.0, data_keys=["input"]) + + input = torch.arange(32, device=device, dtype=dtype).reshape(2, 1, 4, 4) + + out_image = f(input) + + expected = torch.tensor( + [ + [[[2.0, 3.0, 0.0, 1.0], [6.0, 7.0, 4.0, 5.0], [8.0, 9.0, 10.0, 11.0], [12.0, 13.0, 14.0, 15.0]]], + [ + [ + [16.0, 17.0, 18.0, 19.0], + [20.0, 21.0, 22.0, 23.0], + [24.0, 25.0, 26.0, 27.0], + [28.0, 29.0, 30.0, 31.0], + ] + ], + ], + device=device, + dtype=dtype, + ) + + self.assert_close(out_image, expected, rtol=1e-4, atol=1e-4) + + @pytest.mark.parametrize("p", [0.0, 0.5, 1.0]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_p(self, p, same_on_batch, device, dtype): + torch.manual_seed(76) + f = RandomJigsaw(p=p, data_keys=["input"], same_on_batch=same_on_batch) + + input = torch.randn((12, 3, 256, 256), device=device, dtype=dtype) + + f(input) + + +class TestRandomTransplantation(BaseTester): + def test_smoke(self, device, dtype): + torch.manual_seed(22) + + mask = torch.zeros(2, 3, 3, device=device, dtype=dtype) + mask[0, 0:2, 0:2] = 1 + mask[1, 1:2, 1:2] = 2 + image = mask.clone().unsqueeze(dim=1) + + f = RandomTransplantation(p=1, excluded_labels=[0]) + image_out, mask_out = f(image, mask) + + mask_out_expected = torch.tensor( + [[[1, 1, 0], [1, 2, 0], [0, 0, 0]], [[1, 1, 0], [1, 1, 0], [0, 0, 0]]], device=device, dtype=dtype + ) + + self.assert_close(mask_out, mask_out_expected) + self.assert_close(image_out, mask_out_expected.unsqueeze(dim=1)) + + def test_mask_only(self, device, dtype): + torch.manual_seed(22) + + mask = torch.zeros(2, 3, 3, device=device, dtype=dtype) + mask[0, 0:2, 0:2] = 1 + mask[1, 1:2, 1:2] = 2 + + f = RandomTransplantation(p=1, excluded_labels=[0], data_keys=["mask"]) + mask_out = f(mask) + + mask_out_expected = torch.tensor( + [[[1, 1, 0], [1, 2, 0], [0, 0, 0]], [[1, 1, 0], [1, 1, 0], [0, 0, 0]]], device=device, dtype=dtype + ) + + self.assert_close(mask_out, mask_out_expected) + + @pytest.mark.parametrize("n_spatial", [2, 3, 4]) + def test_module(self, n_spatial, device, dtype): + torch.manual_seed(22) + + spatial_dimensions = [10] * n_spatial + image = torch.rand(4, 3, *spatial_dimensions, device=device, dtype=dtype) + mask = torch.zeros(4, *spatial_dimensions, device=device, dtype=dtype) + mask_additional = torch.randint(0, 2, (4, *spatial_dimensions), device=device, dtype=dtype) + + selection = torch.zeros(*spatial_dimensions, device=device, dtype=torch.bool) + selection[[slice(0, 5)] * n_spatial] = True + assert selection.sum() == 5**n_spatial + + # Transplant rectangle from the (i - 1)-th to the i-th image + for i in range(4): + mask[i, selection] = i + 1 + + image_copy = image.clone() + mask_copy = mask.clone() + mask_additional_copy = mask_additional.clone() + + f = RandomTransplantation(p=1, excluded_labels=[0]) + image_out, mask_out, mask_additional_out = f(image, mask, mask_additional, data_keys=["input", "mask", "mask"]) + + self.assert_close(image, image_copy) + self.assert_close(mask, mask_copy) + self.assert_close(mask_additional, mask_additional_copy) + + for i in range(4): + selection_moved = mask_out[i, selection] + selection_unchanged = mask_out[i, ~selection] + self.assert_close(selection_moved, torch.full_like(selection_moved, (i - 1) % 4 + 1)) + self.assert_close(selection_unchanged, torch.full_like(selection_unchanged, 0)) + self.assert_close(image_out[i, :, selection], image[(i - 1) % 4, :, selection]) + self.assert_close(image_out[i, :, ~selection], image[i, :, ~selection]) + self.assert_close(mask_additional_out[i, selection], mask_additional[(i - 1) % 4, selection]) + self.assert_close(mask_additional_out[i, ~selection], mask_additional[i, ~selection]) + + def test_apply_none(self, device, dtype): + torch.manual_seed(22) + image = torch.rand(4, 3, 10, 10, device=device, dtype=dtype) + mask = torch.randint(0, 2, (4, 10, 10), device=device, dtype=dtype) + + f = RandomTransplantation(p=0) + image_out, mask_out = f(image, mask) + + assert torch.all(f._params["batch_prob"] == 0) + assert len(f._params["selected_labels"]) == 0 + + self.assert_close(image_out, image) + self.assert_close(mask_out, mask) + + @pytest.mark.parametrize("wrapper", [AugmentationSequential, lambda x: x]) + def test_repeating(self, wrapper, device, dtype): + torch.manual_seed(22) + image = torch.rand(4, 3, 10, 10, device=device, dtype=dtype) + mask = torch.randint(0, 2, (4, 10, 10), device=device, dtype=dtype) + + f = wrapper(RandomTransplantation(p=0.5)) + image_out, mask_out = f(image, mask, data_keys=["input", "mask"]) + image_out_same, mask_out_same = f(image, mask, params=f._params, data_keys=["input", "mask"]) + image_out_different, mask_out_different = f(image, mask, data_keys=["input", "mask"]) + + self.assert_close(image_out, image_out_same) + self.assert_close(mask_out, mask_out_same) + with pytest.raises(AssertionError): + self.assert_close(image_out, image_out_different) + with pytest.raises(AssertionError): + self.assert_close(mask_out, mask_out_different) + + @pytest.mark.parametrize("wrapper", [AugmentationSequential, lambda x: x]) + def test_data_keys(self, wrapper, device, dtype): + torch.manual_seed(22) + image = torch.rand(4, 3, 10, 10, device=device, dtype=dtype) + mask = torch.randint(0, 2, (4, 10, 10), device=device, dtype=dtype) + + f = wrapper(RandomTransplantation(p=1)) + torch.manual_seed(22) + image_out, mask_out = f(image, mask, data_keys=["input", "mask"]) + torch.manual_seed(22) + mask_out2, image_out2 = f(mask, image, data_keys=["mask", "input"]) + + self.assert_close(image_out, image_out2) + self.assert_close(mask_out, mask_out2) + + @pytest.mark.parametrize("wrapper", [AugmentationSequential]) + def test_dict_input(self, wrapper, device, dtype): + torch.manual_seed(22) + image = torch.rand(4, 3, 10, 10, device=device, dtype=dtype) + mask = torch.randint(0, 2, (4, 10, 10), device=device, dtype=dtype) + + f = wrapper(RandomTransplantation(p=1), data_keys=None) + torch.manual_seed(22) + dict_input = {"image": image, "mask": mask} + aug_dict_output = f(dict_input) + torch.manual_seed(22) + dict_input2 = {"mask": mask, "image": image} + aug_dict_output2 = f(dict_input2) + + image_out = aug_dict_output["image"] + mask_out = aug_dict_output["mask"] + image_out2 = aug_dict_output2["image"] + mask_out2 = aug_dict_output2["mask"] + + self.assert_close(image_out, image_out2) + self.assert_close(mask_out, mask_out2) + + @pytest.mark.parametrize("n_spatial", [2, 3]) + def test_sequential(self, n_spatial, device, dtype): + torch.manual_seed(22) + spatial_dimensions = [10] * n_spatial + image = torch.rand(4, 3, *spatial_dimensions, device=device, dtype=dtype) + mask = torch.randint(0, 2, (4, *spatial_dimensions), device=device, dtype=dtype) + + if n_spatial == 2: + f = RandomTransplantation(p=1) + elif n_spatial == 3: + f = RandomTransplantation3D(p=1) + else: + raise ValueError("n_spatial must be 2 or 3 since AugmentationSequential only supports 2D and 3D input") + + torch.manual_seed(22) + image_out, mask_out = f(image, mask) + + torch.manual_seed(22) + image_out2, mask_out2 = AugmentationSequential(f)(image, mask, data_keys=["image", "mask"]) + + self.assert_close(image_out, image_out2) + self.assert_close(mask_out, mask_out2) + + @pytest.mark.parametrize( + "input_shape_image, input_shape_mask, target_shape_image", + [ + [(1, 2, 3, 4), (1, 3, 4), (1, 2, 3, 4)], # (B, C, H, W) + [(1, 2, 5, 3, 4), (1, 5, 3, 4), (1, 2, 5, 3, 4)], # (B, C, D, H, W) + [(1, 1, 1, 1), (1, 1, 1), (1, 1, 1, 1)], # (B, C, H, W) + ], + ) + def test_cardinality(self, input_shape_image, input_shape_mask, target_shape_image, device, dtype): + torch.manual_seed(22) + image = torch.rand(input_shape_image, device=device, dtype=dtype) + mask = torch.randint(0, 2, input_shape_mask, device=device, dtype=dtype) + + f = RandomTransplantation(p=1) + image_out, mask_out = f(image, mask) + + assert image_out.shape == target_shape_image + assert mask_out.shape == torch.Size([s for i, s in enumerate(target_shape_image) if i != 1]) + + def test_gradcheck(self, device): + torch.manual_seed(22) + image = torch.rand(1, 3, 2, 2, device=device, dtype=torch.float64) + mask = torch.randint(0, 2, (1, 2, 2), device=device, dtype=torch.float64) + + self.gradcheck(RandomTransplantation(p=1.0), (image, mask)) + + def test_exception(self, device, dtype): + if device.type == "mps": + pytest.skip("MPS does not support float64") + torch.manual_seed(22) + image = torch.rand(1, 3, 2, 2, device=device, dtype=torch.float64) + mask = torch.randint(0, 2, (1, 2, 2), device=device, dtype=torch.float64) + f = RandomTransplantation(p=1.0) + f(image, mask) + params = f._params + + with pytest.raises(Exception, match="excluded_labels must be a 1-dimensional"): + RandomTransplantation(p=1.0, excluded_labels=torch.tensor([[0, 1]], device=device, dtype=dtype)) + + with pytest.raises(Exception, match=r"Length of keys.*does not match number of inputs"): + f = RandomTransplantation(p=1.0) + f(image, mask, data_keys=["input", "mask", "mask"]) + + with pytest.raises(Exception, match=r"selected_labels must be a 1-dimensional torch\.tensor"): + params_copy = copy.deepcopy(params) + params_copy["selected_labels"] = torch.tensor([[0, 1]], device=device, dtype=dtype) + del params_copy["selection"] + f(image, mask, params=params_copy) + + with pytest.raises(Exception, match="There cannot be more selected labels"): + params_copy = copy.deepcopy(params) + params_copy["selected_labels"] = torch.tensor([0, 1], device=device, dtype=dtype) + del params_copy["selection"] + f(image, mask, params=params_copy) + + with pytest.raises(Exception, match="Every image input must have one additional dimension"): + f(image.unsqueeze(dim=-1), mask) + + with pytest.raises(Exception, match="The dimensions of the input image and segmentation mask must match"): + image = torch.rand(1, 3, 2, 5, device=device, dtype=torch.float64) + f(image, mask) diff --git a/tests/augmentation/test_auto_operation.py b/tests/augmentation/test_auto_operation.py new file mode 100644 index 0000000..22f68c1 --- /dev/null +++ b/tests/augmentation/test_auto_operation.py @@ -0,0 +1,144 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import inspect +from typing import List + +import pytest +import torch + +from kornia.augmentation.auto.autoaugment import AutoAugment +from kornia.augmentation.auto.operations import OperationBase, ops +from kornia.augmentation.auto.rand_augment.rand_augment import RandAugment +from kornia.augmentation.auto.rand_augment.rand_augment import default_policy as randaug_config +from kornia.augmentation.auto.trivial_augment import TrivialAugment +from kornia.augmentation.container import AugmentationSequential +from kornia.geometry.bbox import bbox_to_mask + +from testing.augmentation.utils import reproducibility_test +from testing.base import BaseTester + + +def _find_all_ops() -> List[OperationBase]: + _ops = [op for _, op in inspect.getmembers(ops, inspect.isclass)] + return [op() for op in _ops if issubclass(op, OperationBase) and op != OperationBase] + + +def _test_sequential(augment_method, device, dtype): + inp = torch.rand(1, 3, 1000, 500, device=device, dtype=dtype) + bbox = torch.tensor([[[355, 10], [660, 10], [660, 250], [355, 250]]], device=device, dtype=dtype) + keypoints = torch.tensor([[[465, 115], [545, 116]]], device=device, dtype=dtype) + mask = bbox_to_mask( + torch.tensor([[[155, 0], [900, 0], [900, 400], [155, 400]]], device=device, dtype=dtype), 1000, 500 + )[:, None] + aug = AugmentationSequential(augment_method, data_keys=["input", "mask", "bbox", "keypoints"]) + out = aug(inp, mask, bbox, keypoints) + assert out[0].shape == inp.shape + assert out[1].shape == mask.shape + assert out[2].shape == bbox.shape + assert out[3].shape == keypoints.shape + assert set(out[1].unique().tolist()).issubset(set(mask.unique().tolist())) + + out_inv = aug.inverse(*out) + assert out_inv[0].shape == inp.shape + assert out_inv[1].shape == mask.shape + assert out_inv[2].shape == bbox.shape + assert out_inv[3].shape == keypoints.shape + assert set(out_inv[1].unique().tolist()).issubset(set(mask.unique().tolist())) + + reproducibility_test((inp, mask, bbox, keypoints), aug) + + +class TestAutoAugment(BaseTester): + @pytest.mark.parametrize("policy", ["imagenet", "cifar10", "svhn", [[("shear_x", 0.9, 4), ("invert", 0.2, None)]]]) + def test_smoke(self, policy, device, dtype): + aug = AutoAugment(policy) + in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True) + aug(in_tensor) + aug.is_intensity_only() + + def test_transform_mat(self, device, dtype): + aug = AutoAugment([[("shear_x", 0.9, 4), ("invert", 0.2, None)]], transformation_matrix_mode="silence") + in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True) + aug(in_tensor) + trans = aug.get_transformation_matrix(in_tensor, params=aug._params) + self.assert_close(trans, aug.transform_matrix) + + def test_reproduce(self, device, dtype): + aug = AutoAugment() + in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True) + out_tensor = aug(in_tensor) + out_tensor_2 = aug(in_tensor, params=aug._params) + self.assert_close(out_tensor, out_tensor_2) + + def test_sequential(augment_method, device, dtype): + _test_sequential(AutoAugment(), device=device, dtype=dtype) + + +class TestRandAugment(BaseTester): + @pytest.mark.parametrize("policy", [None, [[("translate_y", -0.5, 0.5)]]]) + def test_smoke(self, policy): + if policy is None: + n = len(randaug_config) + else: + n = 1 + aug = RandAugment(n=n, m=15, policy=policy) + in_tensor = torch.rand(10, 3, 50, 50, requires_grad=True) + aug(in_tensor) + + def test_transform_mat(self, device, dtype): + aug = RandAugment(n=3, m=15) + in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True) + aug(in_tensor) + trans = aug.get_transformation_matrix(in_tensor, params=aug._params) + self.assert_close(trans, aug.transform_matrix) + + def test_reproduce(self, device, dtype): + aug = RandAugment(n=3, m=15) + in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True) + out_tensor = aug(in_tensor) + out_tensor_2 = aug(in_tensor, params=aug._params) + self.assert_close(out_tensor, out_tensor_2) + + def test_sequential(augment_method, device, dtype): + _test_sequential(RandAugment(n=3, m=15), device=device, dtype=dtype) + + +class TestTrivialAugment(BaseTester): + @pytest.mark.parametrize("policy", [None, [[("translate_y", -0.5, 0.5)]]]) + def test_smoke(self, policy): + aug = TrivialAugment(policy=policy) + in_tensor = torch.rand(10, 3, 50, 50, requires_grad=True) + aug(in_tensor) + + def test_transform_mat(self, device, dtype): + aug = TrivialAugment() + in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True) + aug(in_tensor) + aug(in_tensor) + trans = aug.get_transformation_matrix(in_tensor, params=aug._params) + self.assert_close(trans, aug.transform_matrix) + + def test_reproduce(self, device, dtype): + aug = TrivialAugment() + in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True) + out_tensor = aug(in_tensor) + out_tensor_2 = aug(in_tensor, params=aug._params) + self.assert_close(out_tensor, out_tensor_2) + + def test_sequential(augment_method, device, dtype): + _test_sequential(TrivialAugment(), device=device, dtype=dtype) diff --git a/tests/augmentation/test_backward.py b/tests/augmentation/test_backward.py new file mode 100644 index 0000000..3ff9db6 --- /dev/null +++ b/tests/augmentation/test_backward.py @@ -0,0 +1,513 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +from torch import nn + +from kornia.augmentation import ( + ColorJiggle, + ColorJitter, + RandomAffine, + RandomErasing, + RandomMotionBlur, + RandomPerspective, + RandomResizedCrop, + RandomRotation, + RandomSharpness, +) + + +@pytest.mark.skip(reason="To be deprecated") +class TestColorJiggleBackward: + @pytest.mark.parametrize("brightness", [0.8, torch.tensor(0.8), torch.tensor([0.8, 1.2])]) + @pytest.mark.parametrize("contrast", [0.8, torch.tensor(0.8), torch.tensor([0.8, 1.2])]) + @pytest.mark.parametrize("saturation", [0.8, torch.tensor(0.8), torch.tensor([0.8, 1.2])]) + @pytest.mark.parametrize("hue", [0.1, torch.tensor(0.1), torch.tensor([-0.1, 0.1])]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_param(self, brightness, contrast, saturation, hue, same_on_batch, device, dtype): + _brightness = ( + brightness + if isinstance(brightness, (int, float)) + else nn.Parameter(brightness.clone().to(device=device, dtype=dtype)) + ) + _contrast = ( + contrast + if isinstance(contrast, (int, float)) + else nn.Parameter(contrast.clone().to(device=device, dtype=dtype)) + ) + _saturation = ( + saturation + if isinstance(saturation, (int, float)) + else nn.Parameter(saturation.clone().to(device=device, dtype=dtype)) + ) + _hue = hue if isinstance(hue, (int, float)) else nn.Parameter(hue.clone().to(device=device, dtype=dtype)) + + torch.manual_seed(0) + input = torch.randint(255, (2, 3, 10, 10), device=device, dtype=dtype) / 255.0 + aug = ColorJiggle(_brightness, _contrast, _saturation, _hue, same_on_batch) + + output = aug(input) + + if len(list(aug.parameters())) != 0: + mse = nn.MSELoss() + opt = torch.optim.SGD(aug.parameters(), lr=10) + loss = mse(output, torch.ones_like(output) * 2) + loss.backward() + opt.step() + + if not isinstance(brightness, (int, float)): + assert isinstance(aug.brightness, torch.Tensor) + # Assert if param not updated + assert (brightness.to(device=device, dtype=dtype) - aug.brightness.data).sum() != 0 + if not isinstance(contrast, (int, float)): + assert isinstance(aug.contrast, torch.Tensor) + # Assert if param not updated + assert (contrast.to(device=device, dtype=dtype) - aug.contrast.data).sum() != 0 + if not isinstance(saturation, (int, float)): + assert isinstance(aug.saturation, torch.Tensor) + # Assert if param not updated + assert (saturation.to(device=device, dtype=dtype) - aug.saturation.data).sum() != 0 + if not isinstance(hue, (int, float)): + assert isinstance(aug.hue, torch.Tensor) + # Assert if param not updated + assert (hue.to(device=device, dtype=dtype) - aug.hue.data).sum() != 0 + + +@pytest.mark.skip(reason="To be deprecated") +class TestColorJitterBackward: + @pytest.mark.parametrize("brightness", [0.8, torch.tensor(0.8), torch.tensor([0.8, 1.2])]) + @pytest.mark.parametrize("contrast", [0.8, torch.tensor(0.8), torch.tensor([0.8, 1.2])]) + @pytest.mark.parametrize("saturation", [0.8, torch.tensor(0.8), torch.tensor([0.8, 1.2])]) + @pytest.mark.parametrize("hue", [0.1, torch.tensor(0.1), torch.tensor([-0.1, 0.1])]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_param(self, brightness, contrast, saturation, hue, same_on_batch, device, dtype): + _brightness = ( + brightness + if isinstance(brightness, (int, float)) + else nn.Parameter(brightness.clone().to(device=device, dtype=dtype)) + ) + _contrast = ( + contrast + if isinstance(contrast, (int, float)) + else nn.Parameter(contrast.clone().to(device=device, dtype=dtype)) + ) + _saturation = ( + saturation + if isinstance(saturation, (int, float)) + else nn.Parameter(saturation.clone().to(device=device, dtype=dtype)) + ) + _hue = hue if isinstance(hue, (int, float)) else nn.Parameter(hue.clone().to(device=device, dtype=dtype)) + + torch.manual_seed(0) + input = torch.randint(255, (2, 3, 10, 10), device=device, dtype=dtype) / 255.0 + aug = ColorJitter(_brightness, _contrast, _saturation, _hue, same_on_batch) + + output = aug(input) + + if len(list(aug.parameters())) != 0: + mse = nn.MSELoss() + opt = torch.optim.SGD(aug.parameters(), lr=10) + loss = mse(output, torch.ones_like(output) * 2) + loss.backward() + opt.step() + + if not isinstance(brightness, (int, float)): + assert isinstance(aug.brightness, torch.Tensor) + # Assert if param not updated + assert (brightness.to(device=device, dtype=dtype) - aug.brightness.data).sum() != 0 + if not isinstance(contrast, (int, float)): + assert isinstance(aug.contrast, torch.Tensor) + # Assert if param not updated + assert (contrast.to(device=device, dtype=dtype) - aug.contrast.data).sum() != 0 + if not isinstance(saturation, (int, float)): + assert isinstance(aug.saturation, torch.Tensor) + # Assert if param not updated + assert (saturation.to(device=device, dtype=dtype) - aug.saturation.data).sum() != 0 + if not isinstance(hue, (int, float)): + assert isinstance(aug.hue, torch.Tensor) + # Assert if param not updated + assert (hue.to(device=device, dtype=dtype) - aug.hue.data).sum() != 0 + + +@pytest.mark.skip(reason="To be deprecated") +class TestRandomAffineBackward: + @pytest.mark.parametrize("degrees", [10, [10.0, 20.0], (10, 20), torch.tensor(10.0), torch.tensor([10, 20])]) + @pytest.mark.parametrize("translate", [[0.1, 0.2], torch.tensor([0.1, 0.2])]) + @pytest.mark.parametrize( + "scale", [[0.1, 0.2], [0.1, 0.2, 0.3, 0.4], torch.tensor([0.1, 0.2]), torch.tensor([0.1, 0.2, 0.3, 0.4])] + ) + @pytest.mark.parametrize( + "shear", [[10.0, 20.0], [10.0, 20.0, 30.0, 40.0], torch.tensor([10, 20]), torch.tensor([10, 20, 30, 40])] + ) + @pytest.mark.parametrize("resample", ["bilinear"]) # TODO: Ignore nearest for now. + @pytest.mark.parametrize("align_corners", [True, False]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_param(self, degrees, translate, scale, shear, resample, align_corners, same_on_batch, device, dtype): + _degrees = ( + degrees + if isinstance(degrees, (int, float, list, tuple)) + else nn.Parameter(degrees.clone().to(device=device, dtype=dtype)) + ) + _translate = ( + translate + if isinstance(translate, (int, float, list, tuple)) + else nn.Parameter(translate.clone().to(device=device, dtype=dtype)) + ) + _scale = ( + scale + if isinstance(scale, (int, float, list, tuple)) + else nn.Parameter(scale.clone().to(device=device, dtype=dtype)) + ) + _shear = ( + shear + if isinstance(shear, (int, float, list, tuple)) + else nn.Parameter(shear.clone().to(device=device, dtype=dtype)) + ) + + torch.manual_seed(0) + input = torch.randint(255, (2, 3, 10, 10), device=device, dtype=dtype) / 255.0 + aug = RandomAffine( + _degrees, + _translate, + _scale, + _shear, + resample, + align_corners=align_corners, + same_on_batch=same_on_batch, + p=1.0, + ) + + output = aug(input) + + if len(list(aug.parameters())) != 0: + mse = nn.MSELoss() + opt = torch.optim.SGD(aug.parameters(), lr=10) + loss = mse(output, torch.ones_like(output) * 2) + try: + loss.backward() + except Exception as e: + raise AssertionError(list(aug.named_parameters())) from e + opt.step() + + if not isinstance(degrees, (int, float, list, tuple)): + assert isinstance(aug._param_generator.degrees, torch.Tensor) + # Assert if param not updated + if resample == "nearest" and aug._param_generator.degrees.is_cuda: + # grid_sample in nearest mode and cuda device returns nan than 0 + pass + elif resample == "nearest" or torch.all(aug._param_generator.degrees._grad == 0.0): + # grid_sample will return grad = 0 for resample nearest + # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 + assert (degrees.to(device=device, dtype=dtype) - aug._param_generator.degrees.data).sum() == 0 + else: + assert (degrees.to(device=device, dtype=dtype) - aug._param_generator.degrees.data).sum() != 0 + if not isinstance(translate, (int, float, list, tuple)): + assert isinstance(aug._param_generator.translate, torch.Tensor) + # Assert if param not updated + if resample == "nearest" and aug._param_generator.translate.is_cuda: + # grid_sample in nearest mode and cuda device returns nan than 0 + pass + elif resample == "nearest" or torch.all(aug._param_generator.translate._grad == 0.0): + # grid_sample will return grad = 0 for resample nearest + # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 + assert (translate.to(device=device, dtype=dtype) - aug._param_generator.translate.data).sum() == 0 + else: + assert (translate.to(device=device, dtype=dtype) - aug._param_generator.translate.data).sum() != 0 + if not isinstance(scale, (int, float, list, tuple)): + assert isinstance(aug._param_generator.scale, torch.Tensor) + # Assert if param not updated + if resample == "nearest" and aug._param_generator.scale.is_cuda: + # grid_sample in nearest mode and cuda device returns nan than 0 + pass + elif resample == "nearest" or torch.all(aug._param_generator.scale._grad == 0.0): + # grid_sample will return grad = 0 for resample nearest + # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 + assert (scale.to(device=device, dtype=dtype) - aug._param_generator.scale.data).sum() == 0 + else: + assert (scale.to(device=device, dtype=dtype) - aug._param_generator.scale.data).sum() != 0 + if not isinstance(shear, (int, float, list, tuple)): + assert isinstance(aug._param_generator.shear, torch.Tensor) + # Assert if param not updated + if resample == "nearest" and aug._param_generator.shear.is_cuda: + # grid_sample in nearest mode and cuda device returns nan than 0 + pass + elif resample == "nearest" or torch.all(aug._param_generator.shear._grad == 0.0): + # grid_sample will return grad = 0 for resample nearest + # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 + assert (shear.to(device=device, dtype=dtype) - aug._param_generator.shear.data).sum() == 0 + else: + assert (shear.to(device=device, dtype=dtype) - aug._param_generator.shear.data).sum() != 0 + + +@pytest.mark.skip(reason="To be deprecated") +class TestRandomRotationBackward: + @pytest.mark.parametrize("degrees", [10, [10.0, 20.0], (10, 20), torch.tensor(10.0), torch.tensor([10, 20])]) + @pytest.mark.parametrize("resample", ["bilinear"]) # TODO: Ignore nearest for now. + @pytest.mark.parametrize("align_corners", [True, False]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_param(self, degrees, resample, align_corners, same_on_batch, device, dtype): + _degrees = ( + degrees + if isinstance(degrees, (int, float, list, tuple)) + else nn.Parameter(degrees.clone().to(device=device, dtype=dtype)) + ) + + torch.manual_seed(0) + input = torch.randint(255, (2, 3, 10, 10), device=device, dtype=dtype) / 255.0 + aug = RandomRotation(_degrees, resample, align_corners=align_corners, same_on_batch=same_on_batch) + + output = aug(input) + + if len(list(aug.parameters())) != 0: + mse = nn.MSELoss() + opt = torch.optim.SGD(aug.parameters(), lr=10) + loss = mse(output, torch.ones_like(output) * 2) + loss.backward() + opt.step() + + if not isinstance(degrees, (int, float, list, tuple)): + assert isinstance(aug._param_generator.degrees, torch.Tensor) + # Assert if param not updated + if resample == "nearest" and aug._param_generator.degrees.is_cuda: + # grid_sample in nearest mode and cuda device returns nan than 0 + pass + elif resample == "nearest" or torch.all(aug._param_generator.degrees._grad == 0.0): + # grid_sample will return grad = 0 for resample nearest + # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 + assert (degrees.to(device=device, dtype=dtype) - aug._param_generator.degrees.data).sum() == 0 + else: + assert (degrees.to(device=device, dtype=dtype) - aug._param_generator.degrees.data).sum() != 0 + + +@pytest.mark.skip(reason="To be deprecated") +class TestRandomPerspectiveBackward: + @pytest.mark.parametrize("distortion_scale", [0.5, torch.tensor(0.5)]) + @pytest.mark.parametrize("resample", ["bilinear"]) # TODO: Ignore nearest for now. + @pytest.mark.parametrize("align_corners", [True, False]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_param(self, distortion_scale, resample, align_corners, same_on_batch, device, dtype): + _distortion_scale = ( + distortion_scale + if isinstance(distortion_scale, (float, int)) + else nn.Parameter(distortion_scale.clone().to(device=device, dtype=dtype)) + ) + + torch.manual_seed(0) + input = torch.randint(255, (2, 3, 10, 10), device=device, dtype=dtype) / 255.0 + aug = RandomPerspective( + _distortion_scale, resample=resample, same_on_batch=same_on_batch, align_corners=align_corners, p=1.0 + ) + + output = aug(input) + + if len(list(aug.parameters())) != 0: + mse = nn.MSELoss() + opt = torch.optim.SGD(aug.parameters(), lr=0.1) + loss = mse(output, torch.ones_like(output) * 2) + loss.backward() + opt.step() + + if not isinstance(distortion_scale, (float, int)): + assert isinstance(aug._param_generator.distortion_scale, torch.Tensor) + # Assert if param not updated + if resample == "nearest" and aug._param_generator.distortion_scale.is_cuda: + # grid_sample in nearest mode and cuda device returns nan than 0 + pass + elif resample == "nearest" or torch.all(aug._param_generator.distortion_scale._grad == 0.0): + # grid_sample will return grad = 0 for resample nearest + # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 + assert ( + distortion_scale.to(device=device, dtype=dtype) - aug._param_generator.distortion_scale.data + ).sum() == 0 + else: + assert ( + distortion_scale.to(device=device, dtype=dtype) - aug._param_generator.distortion_scale.data + ).sum() != 0 + + +@pytest.mark.skip(reason="To be deprecated") +class TestRandomMotionBlurBackward: + @pytest.mark.parametrize("angle", [20.0, torch.tensor([-20.0, 20.0])]) + @pytest.mark.parametrize("direction", [[-0.5, 0.5], torch.tensor([-0.5, 0.5])]) + @pytest.mark.parametrize("border_type", ["constant", "reflect", "replicate", "circular"]) + @pytest.mark.parametrize("resample", ["bilinear"]) # TODO: Ignore nearest for now. + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_param(self, angle, direction, border_type, resample, same_on_batch, device, dtype): + _angle = ( + angle + if isinstance(angle, (float, int, list, tuple)) + else nn.Parameter(angle.clone().to(device=device, dtype=dtype)) + ) + _direction = ( + direction + if isinstance(direction, (list, tuple)) + else nn.Parameter(direction.clone().to(device=device, dtype=dtype)) + ) + + torch.manual_seed(0) + input = torch.randint(255, (2, 3, 10, 10), device=device, dtype=dtype) / 255.0 + aug = RandomMotionBlur((3, 3), _angle, _direction, border_type, resample, same_on_batch, p=1.0) + + output = aug(input) + + if len(list(aug.parameters())) != 0: + mse = nn.MSELoss() + opt = torch.optim.SGD(aug.parameters(), lr=0.1) + loss = mse(output, torch.ones_like(output) * 2) + loss.backward() + opt.step() + + if not isinstance(angle, (float, int, list, tuple)): + assert isinstance(aug._param_generator.angle, torch.Tensor) + if resample == "nearest" and aug._param_generator.angle.is_cuda: + # grid_sample in nearest mode and cuda device returns nan than 0 + pass + elif resample == "nearest" or torch.all(aug._param_generator.angle._grad == 0.0): + # grid_sample will return grad = 0 for resample nearest + # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 + assert (angle.to(device=device, dtype=dtype) - aug._param_generator.angle.data).sum() == 0 + else: + # Assert if param not updated + assert (angle.to(device=device, dtype=dtype) - aug._param_generator.angle.data).sum() != 0 + if not isinstance(direction, (list, tuple)): + assert isinstance(aug._param_generator.direction, torch.Tensor) + if torch.all(aug._param_generator.direction._grad == 0.0): + # grid_sample will return grad = 0 for resample nearest + # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 + assert (direction.to(device=device, dtype=dtype) - aug._param_generator.direction.data).sum() == 0 + else: + # Assert if param not updated + assert (direction.to(device=device, dtype=dtype) - aug._param_generator.direction.data).sum() != 0 + + +@pytest.mark.skip(reason="To be deprecated") +class TestRandomSharpnessBackward: + @pytest.mark.parametrize("sharpness", [0.5, [0, 0.5], torch.tensor([0, 0.5])]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_param(self, sharpness, same_on_batch, device, dtype): + _sharpness = ( + sharpness + if isinstance(sharpness, (float, int, list, tuple)) + else nn.Parameter(sharpness.clone().to(device=device, dtype=dtype)) + ) + + torch.manual_seed(0) + input = torch.randint(255, (2, 3, 10, 10), device=device, dtype=dtype) / 255.0 + aug = RandomSharpness(_sharpness, same_on_batch=same_on_batch) + + output = aug(input) + + if len(list(aug.parameters())) != 0: + mse = nn.MSELoss() + opt = torch.optim.SGD(aug.parameters(), lr=0.1) + loss = mse(output, torch.ones_like(output) * 2) + loss.backward() + opt.step() + + if not isinstance(sharpness, (float, int, list, tuple)): + assert isinstance(aug._param_generator.sharpness, torch.Tensor) + # Assert if param not updated + assert (sharpness.to(device=device, dtype=dtype) - aug._param_generator.sharpness.data).sum() != 0 + + +@pytest.mark.skip(reason="To be deprecated") +class TestRandomResizedCropBackward: + @pytest.mark.skip("Param gen is probably breaking grads.") + @pytest.mark.parametrize("scale", [[0.08, 1.0], torch.tensor([0.08, 1.0])]) + @pytest.mark.parametrize("ratio", [[3.0 / 4.0, 4.0 / 3.0], torch.tensor([3.0 / 4.0, 4.0 / 3.0])]) + @pytest.mark.parametrize("resample", ["bilinear"]) # TODO: Ignore nearest for now. + @pytest.mark.parametrize("align_corners", [True, False]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_param(self, scale, ratio, resample, align_corners, same_on_batch, device, dtype): + _scale = ( + scale if isinstance(scale, (list, tuple)) else nn.Parameter(scale.clone().to(device=device, dtype=dtype)) + ) + _ratio = ( + ratio if isinstance(ratio, (list, tuple)) else nn.Parameter(ratio.clone().to(device=device, dtype=dtype)) + ) + + torch.manual_seed(0) + input = torch.randint(255, (2, 3, 10, 10), device=device, dtype=dtype) / 255.0 + aug = RandomResizedCrop( + (8, 8), _scale, _ratio, resample=resample, same_on_batch=same_on_batch, align_corners=align_corners + ) + + output = aug(input) + + if len(list(aug.parameters())) != 0: + mse = nn.MSELoss() + opt = torch.optim.SGD(aug.parameters(), lr=0.1) + loss = mse(output, torch.ones_like(output) * 2) + loss.backward() + opt.step() + + if not isinstance(scale, (list, tuple)): + assert isinstance(aug.scale, torch.Tensor) + # Assert if param not updated + assert (scale.to(device=device, dtype=dtype) - aug.scale.data).sum() != 0 + if not isinstance(ratio, (list, tuple)): + assert isinstance(aug.ratio, torch.Tensor) + # Assert if param not updated + assert (ratio.to(device=device, dtype=dtype) - aug.ratio.data).sum() != 0 + + +@pytest.mark.skip(reason="To be deprecated") +class TestRandomErasingBackward: + @pytest.mark.skip("Need differentiable indexing.") + @pytest.mark.parametrize("scale", [[0.02, 0.33], torch.tensor([0.02, 0.33])]) + @pytest.mark.parametrize("ratio", [[0.3, 3.3], torch.tensor([0.3, 3.3])]) + @pytest.mark.parametrize("value", [0.0]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_param(self, scale, ratio, value, same_on_batch, device, dtype): + _scale = ( + scale if isinstance(scale, (list, tuple)) else nn.Parameter(scale.clone().to(device=device, dtype=dtype)) + ) + _ratio = ( + ratio if isinstance(ratio, (list, tuple)) else nn.Parameter(ratio.clone().to(device=device, dtype=dtype)) + ) + + torch.manual_seed(0) + input = torch.randint(255, (2, 3, 10, 10), device=device, dtype=dtype) / 255.0 + aug = RandomErasing(_scale, _ratio, value, same_on_batch) + output = aug(input) + + if len(list(aug.parameters())) != 0: + mse = nn.MSELoss() + opt = torch.optim.SGD(aug.parameters(), lr=0.1) + loss = mse(output, torch.ones_like(output) * 2) + loss.backward() + opt.step() + + if not isinstance(scale, (list, tuple)): + assert isinstance(aug.scale, torch.Tensor) + if torch.all(aug.scale._grad == 0.0): + # grid_sample will return grad = 0 for resample nearest + # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 + assert (scale.to(device=device, dtype=dtype) - aug.scale.data).sum() == 0 + else: + # Assert if param not updated + assert (scale.to(device=device, dtype=dtype) - aug.scale.data).sum() != 0 + if not isinstance(ratio, (list, tuple)): + assert isinstance(aug.ratio, torch.Tensor) + if torch.all(aug.ratio._grad == 0.0): + # grid_sample will return grad = 0 for resample nearest + # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 + assert (ratio.to(device=device, dtype=dtype) - aug.ratio.data).sum() == 0 + else: + # Assert if param not updated + assert (ratio.to(device=device, dtype=dtype) - aug.ratio.data).sum() != 0 diff --git a/tests/augmentation/test_backward_3d.py b/tests/augmentation/test_backward_3d.py new file mode 100644 index 0000000..7489fa4 --- /dev/null +++ b/tests/augmentation/test_backward_3d.py @@ -0,0 +1,308 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +from torch import nn + +from kornia.augmentation import RandomAffine3D, RandomMotionBlur3D, RandomPerspective3D, RandomRotation3D + + +@pytest.mark.skip(reason="To be deprecated") +class TestRandomAffine3DBackward: + @pytest.mark.parametrize( + "degrees", + [ + 10, + [10.0, 20.0], + [10.0, 20.0, 30.0], + [(10, 20), (10, 20), (10, 20)], + torch.tensor(10.0), + torch.tensor([10.0, 20.0]), + torch.tensor([10, 20, 30]), + torch.tensor([(10, 20), (10, 20), (10, 20)]), + ], + ) + @pytest.mark.parametrize("translate", [[0.1, 0.2, 0.3], torch.tensor([0.1, 0.2, 0.3])]) + @pytest.mark.parametrize( + "scale", + [ + [0.1, 0.2], + [(0.1, 0.2), (0.1, 0.2), (0.1, 0.2)], + torch.tensor([0.1, 0.2]), + torch.tensor([(0.1, 0.2), (0.1, 0.2), (0.1, 0.2)]), + ], + ) + @pytest.mark.parametrize( + "shear", + [ + 10.0, + [10.0, 20.0], + [10.0, 20.0, 30.0, 40.0, 50.0, 60.0], + [(-10.0, 10.0), (-10.0, 10.0), (-10.0, 10.0), (-10.0, 10.0), (-10.0, 10.0), (-10.0, 10.0)], + torch.tensor(10), + torch.tensor([10, 20]), + torch.tensor([10.0, 20.0, 30.0, 40.0, 50.0, 60.0]), + torch.tensor([(-10.0, 10.0), (-10.0, 10.0), (-10.0, 10.0), (-10.0, 10.0), (-10.0, 10.0), (-10.0, 10.0)]), + ], + ) + @pytest.mark.parametrize("resample", ["bilinear"]) # TODO: Ignore nearest for now. + @pytest.mark.parametrize("align_corners", [True, False]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_param(self, degrees, translate, scale, shear, resample, align_corners, same_on_batch, device, dtype): + _degrees = ( + degrees + if isinstance(degrees, (int, float, list, tuple)) + else nn.Parameter(degrees.clone().to(device=device, dtype=dtype)) + ) + _translate = ( + translate + if isinstance(translate, (int, float, list, tuple)) + else nn.Parameter(translate.clone().to(device=device, dtype=dtype)) + ) + _scale = ( + scale + if isinstance(scale, (int, float, list, tuple)) + else nn.Parameter(scale.clone().to(device=device, dtype=dtype)) + ) + _shear = ( + shear + if isinstance(shear, (int, float, list, tuple)) + else nn.Parameter(shear.clone().to(device=device, dtype=dtype)) + ) + + torch.manual_seed(0) + input = torch.randint(255, (2, 3, 10, 10, 10), device=device, dtype=dtype) / 255.0 + aug = RandomAffine3D( + _degrees, + _translate, + _scale, + _shear, + resample, + align_corners=align_corners, + same_on_batch=same_on_batch, + p=1.0, + ) + + output = aug(input) + + if len(list(aug.parameters())) != 0: + mse = nn.MSELoss() + opt = torch.optim.SGD(aug.parameters(), lr=10) + loss = mse(output, torch.ones_like(output) * 2) # to ensure that a big loss value could be obtained + loss.backward() + opt.step() + + if not isinstance(degrees, (int, float, list, tuple)): + assert isinstance(aug.degrees, torch.Tensor) + # Assert if param not updated + if resample == "nearest" and aug.degrees.is_cuda: + # grid_sample in nearest mode and cuda device returns nan than 0 + pass + elif resample == "nearest" or torch.all(aug.degrees._grad == 0.0): + # grid_sample will return grad = 0 for resample nearest + # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 + assert (degrees.to(device=device, dtype=dtype) - aug.degrees.data).sum() == 0 + else: + assert (degrees.to(device=device, dtype=dtype) - aug.degrees.data).sum() != 0 + if not isinstance(translate, (int, float, list, tuple)): + assert isinstance(aug.translate, torch.Tensor) + # Assert if param not updated + if resample == "nearest" and aug.translate.is_cuda: + # grid_sample in nearest mode and cuda device returns nan than 0 + pass + elif resample == "nearest" or torch.all(aug.translate._grad == 0.0): + # grid_sample will return grad = 0 for resample nearest + # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 + assert (translate.to(device=device, dtype=dtype) - aug.translate.data).sum() == 0 + else: + assert (translate.to(device=device, dtype=dtype) - aug.translate.data).sum() != 0 + if not isinstance(scale, (int, float, list, tuple)): + assert isinstance(aug.scale, torch.Tensor) + # Assert if param not updated + if resample == "nearest" and aug.scale.is_cuda: + # grid_sample in nearest mode and cuda device returns nan than 0 + pass + elif resample == "nearest" or torch.all(aug.scale._grad == 0.0): + # grid_sample will return grad = 0 for resample nearest + # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 + assert (scale.to(device=device, dtype=dtype) - aug.scale.data).sum() == 0 + else: + assert (scale.to(device=device, dtype=dtype) - aug.scale.data).sum() != 0 + if not isinstance(shear, (int, float, list, tuple)): + assert isinstance(aug.shears, torch.Tensor) + # Assert if param not updated + if resample == "nearest" and aug.shears.is_cuda: + # grid_sample in nearest mode and cuda device returns nan than 0 + pass + elif resample == "nearest" or torch.all(aug.shears._grad == 0.0): + # grid_sample will return grad = 0 for resample nearest + # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 + assert (shear.to(device=device, dtype=dtype) - aug.shears.data).sum() == 0 + else: + assert (shear.to(device=device, dtype=dtype) - aug.shears.data).sum() != 0 + + +class TestRandomRotation3DBackward: + @pytest.mark.parametrize( + "degrees", + [ + 10, + [10.0, 20.0], + [10.0, 20.0, 30.0], + [(10, 20), (10, 20), (10, 20)], + torch.tensor(10.0), + torch.tensor([10.0, 20.0]), + torch.tensor([10, 20, 30]), + torch.tensor([(10, 20), (10, 20), (10, 20)]), + ], + ) + @pytest.mark.parametrize("resample", ["bilinear"]) # TODO: Ignore nearest for now. + @pytest.mark.parametrize("align_corners", [True, False]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_param(self, degrees, resample, align_corners, same_on_batch, device, dtype): + _degrees = ( + degrees + if isinstance(degrees, (int, float, list, tuple)) + else nn.Parameter(degrees.clone().to(device=device, dtype=dtype)) + ) + + torch.manual_seed(0) + input = torch.randint(255, (2, 3, 10, 10, 10), device=device, dtype=dtype) / 255.0 + aug = RandomRotation3D(_degrees, resample, align_corners=align_corners, same_on_batch=same_on_batch, p=1.0) + + output = aug(input) + + if len(list(aug.parameters())) != 0: + mse = nn.MSELoss() + opt = torch.optim.SGD(aug.parameters(), lr=10) + loss = mse(output, torch.ones_like(output) * 2) # to ensure that a big loss value could be obtained + loss.backward() + opt.step() + + if not isinstance(degrees, (int, float, list, tuple)): + assert isinstance(aug._param_generator.degrees, torch.Tensor) + # Assert if param not updated + if resample == "nearest" and aug._param_generator.degrees.is_cuda: + # grid_sample in nearest mode and cuda device returns nan than 0 + pass + elif resample == "nearest" or torch.all(aug._param_generator.degrees._grad == 0.0): + # grid_sample will return grad = 0 for resample nearest + # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 + assert (degrees.to(device=device, dtype=dtype) - aug._param_generator.degrees.data).sum() == 0 + else: + assert (degrees.to(device=device, dtype=dtype) - aug._param_generator.degrees.data).sum() != 0 + + +class TestRandomPerspective3DBackward: + @pytest.mark.parametrize("distortion_scale", [0.5, torch.tensor(0.5)]) + @pytest.mark.parametrize("resample", ["bilinear"]) # TODO: Ignore nearest for now. + @pytest.mark.parametrize("align_corners", [True, False]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_param(self, distortion_scale, resample, align_corners, same_on_batch, device, dtype): + _distortion_scale = ( + distortion_scale + if isinstance(distortion_scale, (float, int)) + else nn.Parameter(distortion_scale.clone().to(device=device, dtype=dtype)) + ) + + torch.manual_seed(0) + input = torch.randint(255, (2, 3, 10, 10, 10), device=device, dtype=dtype) / 255.0 + aug = RandomPerspective3D( + _distortion_scale, resample=resample, same_on_batch=same_on_batch, align_corners=align_corners, p=1.0 + ) + + output = aug(input) + + if len(list(aug.parameters())) != 0: + mse = nn.MSELoss() + opt = torch.optim.SGD(aug.parameters(), lr=10) + loss = mse(output, torch.ones_like(output) * 2) # to ensure that a big loss value could be obtained + loss.backward() + opt.step() + + if not isinstance(distortion_scale, (float, int)): + assert isinstance(aug._param_generator.distortion_scale, torch.Tensor) + # Assert if param not updated + if resample == "nearest" and aug._param_generator.distortion_scale.is_cuda: + # grid_sample in nearest mode and cuda device returns nan than 0 + pass + elif resample == "nearest" or torch.all(aug._param_generator.distortion_scale._grad == 0.0): + # grid_sample will return grad = 0 for resample nearest + # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 + assert ( + distortion_scale.to(device=device, dtype=dtype) - aug._param_generator.distortion_scale.data + ).sum() == 0 + else: + assert ( + distortion_scale.to(device=device, dtype=dtype) - aug._param_generator.distortion_scale.data + ).sum() != 0 + + +class TestRandomMotionBlur3DBackward: + @pytest.mark.parametrize("angle", [20.0, torch.tensor(20.0), torch.tensor([20.0])]) + @pytest.mark.parametrize("direction", [[-0.5, 0.5], torch.tensor([-0.5, 0.5])]) + # 'reflect' is not implemented by torch. + @pytest.mark.parametrize("border_type", ["constant", "replicate", "circular"]) + @pytest.mark.parametrize("resample", ["bilinear"]) # TODO: Ignore nearest for now. + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_param(self, angle, direction, border_type, resample, same_on_batch, device, dtype): + _angle = ( + angle + if isinstance(angle, (float, int, list, tuple)) + else nn.Parameter(angle.clone().to(device=device, dtype=dtype)) + ) + _direction = ( + direction + if isinstance(direction, (list, tuple)) + else nn.Parameter(direction.clone().to(device=device, dtype=dtype)) + ) + + torch.manual_seed(0) + input = torch.randint(255, (2, 3, 10, 10, 10), device=device, dtype=dtype) / 255.0 + aug = RandomMotionBlur3D((3, 3), _angle, _direction, border_type, resample, same_on_batch, p=1.0) + + output = aug(input) + + if len(list(aug.parameters())) != 0: + mse = nn.MSELoss() + opt = torch.optim.SGD(aug.parameters(), lr=10) + loss = mse(output, torch.ones_like(output) * 2) # to ensure that a big loss value could be obtained + loss.backward() + opt.step() + + if not isinstance(angle, (float, int, list, tuple)): + assert isinstance(aug._param_generator.angle, torch.Tensor) + if resample == "nearest" and aug._param_generator.angle.is_cuda: + # grid_sample in nearest mode and cuda device returns nan than 0 + pass + elif resample == "nearest" or torch.all(aug._param_generator.angle._grad == 0.0): + # grid_sample will return grad = 0 for resample nearest + # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 + assert (angle.to(device=device, dtype=dtype) - aug._param_generator.angle.data).sum() == 0 + else: + # Assert if param not updated + assert (angle.to(device=device, dtype=dtype) - aug._param_generator.angle.data).sum() != 0 + if not isinstance(direction, (list, tuple)): + assert isinstance(aug._param_generator.direction, torch.Tensor) + if torch.all(aug._param_generator.direction._grad == 0.0): + # grid_sample will return grad = 0 for resample nearest + # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 + assert (direction.to(device=device, dtype=dtype) - aug._param_generator.direction.data).sum() == 0 + else: + # Assert if param not updated + assert (direction.to(device=device, dtype=dtype) - aug._param_generator.direction.data).sum() != 0 diff --git a/tests/augmentation/test_base.py b/tests/augmentation/test_base.py new file mode 100644 index 0000000..ac8f067 --- /dev/null +++ b/tests/augmentation/test_base.py @@ -0,0 +1,260 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from unittest.mock import patch + +import pytest +import torch + +from kornia.augmentation._2d.base import AugmentationBase2D +from kornia.augmentation._2d.geometric.affine import RandomAffine +from kornia.augmentation._2d.intensity.gaussian_blur import RandomGaussianBlur +from kornia.augmentation._3d.geometric.affine import RandomAffine3D +from kornia.augmentation._3d.intensity.motion_blur import RandomMotionBlur3D +from kornia.augmentation.base import _BasicAugmentationBase + +from testing.base import BaseTester + + +class TestBasicAugmentationBase(BaseTester): + def test_smoke(self): + base = _BasicAugmentationBase(p=0.5, p_batch=1.0, same_on_batch=True) + __repr__ = "_BasicAugmentationBase(p=0.5, p_batch=1.0, same_on_batch=True)" + assert str(base) == __repr__ + + def test_infer_input(self, device, dtype): + input = torch.rand((2, 3, 4, 5), device=device, dtype=dtype) + augmentation = _BasicAugmentationBase(p=1.0, p_batch=1) + with patch.object(augmentation, "transform_tensor", autospec=True) as transform_tensor: + transform_tensor.side_effect = lambda x: x.unsqueeze(dim=2) + output = augmentation.transform_tensor(input) + assert output.shape == torch.Size([2, 3, 1, 4, 5]) + self.assert_close(input, output[:, :, 0, :, :]) + + @pytest.mark.parametrize( + "p,p_batch,same_on_batch,num,seed", + [ + (1.0, 1.0, False, 12, 1), + (1.0, 0.0, False, 0, 1), + (0.0, 1.0, False, 0, 1), + (0.0, 0.0, False, 0, 1), + (0.5, 0.1, False, 7, 3), + (0.5, 0.1, True, 12, 3), + (0.3, 1.0, False, 2, 1), + (0.3, 1.0, True, 0, 1), + ], + ) + def test_forward_params(self, p, p_batch, same_on_batch, num, seed, device, dtype): + input_shape = (12,) + torch.manual_seed(seed) + augmentation = _BasicAugmentationBase(p, p_batch, same_on_batch) + with patch.object(augmentation, "generate_parameters", autospec=True) as generate_parameters: + generate_parameters.side_effect = lambda shape: { + "degrees": torch.arange(0, shape[0], device=device, dtype=dtype) + } + output = augmentation.forward_parameters(input_shape) + assert "batch_prob" in output + # generate_parameters is now called with the full batch shape (ONNX-friendly contract). + assert len(output["degrees"]) == input_shape[0] + assert output["batch_prob"].sum().item() == num + + @pytest.mark.parametrize("keepdim", (True, False)) + def test_forward(self, device, dtype, keepdim): + torch.manual_seed(42) + input = torch.rand((12, 3, 4, 5), device=device, dtype=dtype) + expected_output = input[..., :2, :2] if keepdim else input.unsqueeze(dim=0)[..., :2, :2] + augmentation = _BasicAugmentationBase(p=0.3, p_batch=1.0, keepdim=keepdim) + with ( + patch.object(augmentation, "apply_transform", autospec=True) as apply_transform, + patch.object(augmentation, "generate_parameters", autospec=True) as generate_parameters, + patch.object(augmentation, "transform_tensor", autospec=True) as transform_tensor, + patch.object(augmentation, "transform_output_tensor", autospec=True) as transform_output_tensor, + ): + generate_parameters.side_effect = lambda shape: { + "degrees": torch.arange(0, shape[0], device=device, dtype=dtype) + } + transform_tensor.side_effect = lambda x: x.unsqueeze(dim=0) + transform_output_tensor.side_effect = lambda x, y: x.squeeze() + apply_transform.side_effect = lambda input, params, flags: input[..., :2, :2] + # check_batching.side_effect = lambda input: None + output = augmentation(input) + assert output.shape == expected_output.shape + self.assert_close(output, expected_output) + + @pytest.mark.parametrize("p", [0.0, 1.0]) + def test_deterministic_p_skips_bernoulli(self, p): + """When p is 0 or 1 the outcome is deterministic — no Bernoulli sampler should be created.""" + base = _BasicAugmentationBase(p=p, p_batch=0.5) + assert not isinstance(getattr(base, "_p_gen", None), torch.distributions.Bernoulli) + + @pytest.mark.parametrize("p_batch", [0.0, 1.0]) + def test_deterministic_p_batch_skips_bernoulli(self, p_batch): + """When p_batch is 0 or 1 the outcome is deterministic — no Bernoulli sampler should be created.""" + base = _BasicAugmentationBase(p=0.5, p_batch=p_batch) + assert not isinstance(getattr(base, "_p_batch_gen", None), torch.distributions.Bernoulli) + + +class TestAugmentationBase2D(BaseTester): + def test_forward(self, device, dtype): + torch.manual_seed(42) + input = torch.rand((2, 3, 4, 5), device=device, dtype=dtype) + # input_transform = torch.rand((2, 3, 3), device=device, dtype=dtype) + expected_output = torch.rand((2, 3, 4, 5), device=device, dtype=dtype) + augmentation = AugmentationBase2D(p=1.0) + + with ( + patch.object(augmentation, "apply_transform", autospec=True) as apply_transform, + patch.object(augmentation, "generate_parameters", autospec=True) as generate_parameters, + ): + # Calling the augmentation with a single tensor shall return the expected tensor using the generated params. + params = {"params": {}, "flags": {"foo": 0}} + generate_parameters.return_value = params + apply_transform.return_value = expected_output + output = augmentation(input) + # RuntimeError: Boolean value of Tensor with more than one value is ambiguous + # Not an easy fix, happens on verifying torch.tensor([True, True]) + # _params = {'batch_prob': torch.tensor([True, True]), 'params': {}, 'flags': {'foo': 0}} + # apply_transform.assert_called_once_with(input, _params) + # Identity check relaxed to value equality: the where-blend always materialises + # a fresh tensor, so output is never the same object as apply_transform's return. + assert torch.equal(output, expected_output) + + # Calling the augmentation with a tensor and set return_transform shall + # return the expected tensor and transformation. + output = augmentation(input) + # Identity check relaxed to value equality: the where-blend always materialises + # a fresh tensor, so output is never the same object as apply_transform's return. + assert torch.equal(output, expected_output) + + # Calling the augmentation with a tensor and params shall return the expected tensor using the given params. + params = {"params": {}, "flags": {"bar": 1}} + apply_transform.reset_mock() + generate_parameters.return_value = None + output = augmentation(input, params=params) + # RuntimeError: Boolean value of Tensor with more than one value is ambiguous + # Not an easy fix, happens on verifying torch.tensor([True, True]) + # _params = {'batch_prob': torch.tensor([True, True]), 'params': {}, 'flags': {'foo': 0}} + # apply_transform.assert_called_once_with(input, _params) + # Identity check relaxed to value equality: the where-blend always materialises + # a fresh tensor, so output is never the same object as apply_transform's return. + assert torch.equal(output, expected_output) + + # Calling the augmentation with a tensor,a transformation and set + # return_transform shall return the expected tensor and the proper + # transformation matrix. + # expected_final_transformation = expected_transform @ input_transform + # output = augmentation((input, input_transform)) + # assert output is expected_output + + def test_gradcheck(self, device): + torch.manual_seed(42) + + input = torch.rand((1, 1, 3, 3), device=device, dtype=torch.float64) + output = torch.rand((1, 1, 3, 3), device=device, dtype=torch.float64) + input_transform = torch.rand((1, 3, 3), device=device, dtype=torch.float64) + + input_param = {"batch_prob": torch.tensor([True]), "x": input_transform, "y": {}} + + augmentation = AugmentationBase2D(p=1.0) + + with patch.object(augmentation, "apply_transform", autospec=True) as apply_transform: + apply_transform.return_value = output + self.gradcheck(augmentation, ((input, input_param))) + + +class TestGeometricAugmentationBase2D: + @pytest.mark.parametrize("batch_prob", [[True, True], [False, True], [False, False]]) + def test_autocast(self, batch_prob, device, dtype): + if not hasattr(torch, "autocast"): + pytest.skip("PyTorch version without autocast support") + + # Uses some subclass of `GeometricAugmentationBase2D` which perform some op which can mismatch the dtype + # Will cover AugmentationBase2D and RigidAffineAugmentationBase2D too + aug = RandomAffine(0.5, (0.1, 0.5), (0.5, 1.5), 1.2, p=1.0) + x = torch.rand(len(batch_prob), 5, 10, 7, dtype=dtype, device=device) + + to_apply = torch.tensor(batch_prob, device=device) + with patch.object(aug, "__batch_prob_generator__", return_value=to_apply): + params = aug.forward_parameters(x.shape) + + with torch.autocast(device.type): + res = aug(x, params) + + assert res.dtype == dtype, "The output dtype should match the input dtype" + + +class TestIntensityAugmentationBase2D: + @pytest.mark.parametrize("batch_prob", [[True, True], [False, True], [False, False]]) + def test_autocast(self, batch_prob, device, dtype): + if not hasattr(torch, "autocast"): + pytest.skip("PyTorch version without autocast support") + + # Uses some subclass of `IntensityAugmentationBase2D` which perform some op which can mismatch the dtype + # Will cover AugmentationBase2D and RigidAffineAugmentationBase2D too + aug = RandomGaussianBlur((3, 3), (0.1, 3), p=1) + x = torch.rand(len(batch_prob), 5, 10, 7, dtype=dtype, device=device) + + to_apply = torch.tensor(batch_prob, device=device) + with patch.object(aug, "__batch_prob_generator__", return_value=to_apply): + params = aug.forward_parameters(x.shape) + + with torch.autocast(device.type): + res = aug(x, params) + + assert res.dtype == dtype, "The output dtype should match the input dtype" + + +class TestIntensityAugmentationBase3D: + @pytest.mark.parametrize("batch_prob", [[True, True], [False, True], [False, False]]) + def test_autocast(self, batch_prob, device, dtype): + if not hasattr(torch, "autocast"): + pytest.skip("PyTorch version without autocast support") + + # Uses some subclass of `IntensityAugmentationBase3D` which perform some op which can mismatch the dtype + # Will cover RigidAffineAugmentationBase3D and AugmentationBase3D too + aug = RandomMotionBlur3D(3, 35.0, 0.5, p=1) + x = torch.rand(len(batch_prob), 1, 3, 10, 7, dtype=dtype, device=device) + + to_apply = torch.tensor(batch_prob, device=device) + with patch.object(aug, "__batch_prob_generator__", return_value=to_apply): + params = aug.forward_parameters(x.shape) + + with torch.autocast(device.type): + res = aug(x, params) + + assert res.dtype == dtype, "The output dtype should match the input dtype" + + +class TestGeometricAugmentationBase3D: + @pytest.mark.parametrize("batch_prob", [[True, True], [False, True], [False, False]]) + def test_autocast(self, batch_prob, device, dtype): + if not hasattr(torch, "autocast"): + pytest.skip("PyTorch version without autocast support") + + # Uses some subclass of `GeometricAugmentationBase3D` which perform some op which can mismatch the dtype + # Will cover RigidAffineAugmentationBase3D and AugmentationBase3D too + aug = RandomAffine3D((15.0, 20.0, 20.0), p=1) + x = torch.rand(len(batch_prob), 1, 3, 10, 7, dtype=dtype, device=device) + + to_apply = torch.tensor(batch_prob, device=device) + with patch.object(aug, "__batch_prob_generator__", return_value=to_apply): + params = aug.forward_parameters(x.shape) + + with torch.autocast(device.type): + res = aug(x, params) + + assert res.dtype == dtype, "The output dtype should match the input dtype" diff --git a/tests/augmentation/test_dist_mapper.py b/tests/augmentation/test_dist_mapper.py new file mode 100644 index 0000000..340539f --- /dev/null +++ b/tests/augmentation/test_dist_mapper.py @@ -0,0 +1,33 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch +from torch import nn +from torch.distributions import Normal + +from kornia.augmentation.random_generator import DistributionWithMapper + +from testing.base import assert_close + + +class TestDistMapper: + def test_mapper(self): + _ = torch.manual_seed(0) + dist = DistributionWithMapper(Normal(0.0, 1.0), map_fn=nn.Sigmoid()) + out = dist.rsample((8,)) + exp = torch.tensor([0.8236, 0.4272, 0.1017, 0.6384, 0.2527, 0.1980, 0.5995, 0.6980]) + assert_close(out, exp, rtol=1e-4, atol=1e-4) diff --git a/tests/augmentation/test_motionblur.py b/tests/augmentation/test_motionblur.py new file mode 100644 index 0000000..494ad01 --- /dev/null +++ b/tests/augmentation/test_motionblur.py @@ -0,0 +1,160 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.augmentation import RandomMotionBlur, RandomMotionBlur3D +from kornia.filters import motion_blur, motion_blur3d + +from testing.base import BaseTester + + +class TestRandomMotionBlur(BaseTester): + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a torch.Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self, device): + f = RandomMotionBlur(kernel_size=(3, 5), angle=(10, 30), direction=0.5) + repr = ( + "RandomMotionBlur(kernel_size=(3, 5), angle=tensor([10., 30.]), direction=tensor([-0.5000, 0.5000]), " + "border_type='constant', p=0.5, p_batch=1.0, same_on_batch=False, return_transform=None)" + ) + assert str(f) == repr + + @pytest.mark.parametrize("kernel_size", [(3, 5), (7, 21)]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + @pytest.mark.parametrize("p", [0.0, 1.0]) + def test_random_motion_blur(self, kernel_size, same_on_batch, p, device, dtype): + f = RandomMotionBlur(kernel_size=kernel_size, angle=(10, 30), direction=0.5, same_on_batch=same_on_batch, p=p) + torch.manual_seed(0) + batch_size = 2 + input = torch.randn(1, 3, 5, 6, device=device, dtype=dtype).repeat(batch_size, 1, 1, 1) + + output = f(input) + + if same_on_batch: + self.assert_close(output[0], output[1], rtol=1e-4, atol=1e-4) + elif p == 0: + self.assert_close(output, input, rtol=1e-4, atol=1e-4) + else: + assert not torch.allclose(output[0], output[1], rtol=1e-4, atol=1e-4) + + assert output.shape == torch.Size([batch_size, 3, 5, 6]) + + @pytest.mark.parametrize("input_shape", [(1, 1, 5, 5), (2, 1, 5, 5)]) + def test_against_functional(self, input_shape): + input = torch.randn(*input_shape) + + f = RandomMotionBlur(kernel_size=(3, 5), angle=(10, 30), direction=0.5, p=1.0) + output = f(input) + + expected = motion_blur( + input, + f._params["ksize_factor"].unique().item(), + f._params["angle_factor"], + f._params["direction_factor"], + f.flags["border_type"].name.lower(), + ) + + self.assert_close(output, expected, rtol=1e-4, atol=1e-4) + + @pytest.mark.slow + def test_gradcheck(self, device): + torch.manual_seed(0) # for random reproductibility + inp = torch.rand((1, 3, 11, 7), device=device, dtype=torch.float64) + # TODO: Gradcheck for param random gen failed. Suspect get_motion_kernel2d issue. + params = { + "batch_prob": torch.tensor([True]), + "ksize_factor": torch.tensor([31]), + "angle_factor": torch.tensor([30.0]), + "direction_factor": torch.tensor([-0.5]), + "border_type": torch.tensor([0]), + "idx": torch.tensor([0]), + } + self.gradcheck( + RandomMotionBlur(kernel_size=3, angle=(10, 30), direction=(-0.5, 0.5), p=1.0), + (inp, params), + fast_mode=False, + ) + + +class TestRandomMotionBlur3D(BaseTester): + # TODO: improve and implement more meaningful smoke tests e.g check for a consistent + # return values such a torch.Tensor variable. + @pytest.mark.xfail(reason="might fail under windows OS due to printing preicision.") + def test_smoke(self, device, dtype): + f = RandomMotionBlur3D(kernel_size=(3, 5), angle=(10, 30), direction=0.5) + repr = ( + "RandomMotionBlur3D(kernel_size=(3, 5), angle=tensor([[10., 30.]," + "\n [10., 30.],\n [10., 30.]]), direction=tensor([-0.5000, 0.5000]), " + "border_type='constant', p=0.5, p_batch=1.0, same_on_batch=False, return_transform=None)" + ) + assert str(f) == repr + + @pytest.mark.parametrize("same_on_batch", [True, False]) + @pytest.mark.parametrize("p", [0.0, 1.0]) + def test_random_motion_blur(self, same_on_batch, p, device, dtype): + f = RandomMotionBlur3D(kernel_size=(3, 5), angle=(10, 30), direction=0.5, same_on_batch=same_on_batch, p=p) + batch_size = 2 + input = torch.randn(1, 3, 5, 6, 7, device=device, dtype=dtype).repeat(batch_size, 1, 1, 1, 1) + + output = f(input) + + if same_on_batch: + self.assert_close(output[0], output[1], rtol=1e-4, atol=1e-4) + elif p == 0: + self.assert_close(output, input, rtol=1e-4, atol=1e-4) + else: + assert not torch.allclose(output[0], output[1], rtol=1e-4, atol=1e-4) + + assert output.shape == torch.Size([batch_size, 3, 5, 6, 7]) + + @pytest.mark.parametrize("input_shape", [(1, 1, 5, 6, 7), (2, 1, 5, 6, 7)]) + def test_against_functional(self, input_shape): + input = torch.randn(*input_shape) + + f = RandomMotionBlur3D(kernel_size=(3, 5), angle=(10, 30), direction=0.5, p=1.0) + output = f(input) + + expected = motion_blur3d( + input, + f._params["ksize_factor"].unique().item(), + f._params["angle_factor"], + f._params["direction_factor"], + f.flags["border_type"].name.lower(), + ) + + self.assert_close(output, expected, rtol=1e-4, atol=1e-4) + + @pytest.mark.slow + def test_gradcheck(self, device): + torch.manual_seed(0) # for random reproductibility + inp = torch.rand((1, 3, 6, 7), device=device, dtype=torch.float64) + params = { + "batch_prob": torch.tensor([True]), + "ksize_factor": torch.tensor([31]), + "angle_factor": torch.tensor([[30.0, 30.0, 30.0]]), + "direction_factor": torch.tensor([-0.5]), + "border_type": torch.tensor([0]), + "idx": torch.tensor([0]), + } + self.gradcheck( + RandomMotionBlur3D(kernel_size=3, angle=(10, 30), direction=(-0.5, 0.5), p=1.0), + (inp, params), + fast_mode=False, + ) diff --git a/tests/augmentation/test_onnx_export.py b/tests/augmentation/test_onnx_export.py new file mode 100644 index 0000000..a44abe6 --- /dev/null +++ b/tests/augmentation/test_onnx_export.py @@ -0,0 +1,320 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""ONNX export regression tests for kornia.augmentation modules. + +These tests pin the set of augmentations that successfully export through +``torch.onnx.export`` (legacy TorchScript-tracer path) at opset 20, so future +changes to the augmentation base classes do not silently regress export +support. + +Opset 20 was chosen as the floor because: +- ``aten::affine_grid_generator`` (used by every geometric augmentation that + goes through ``warp_affine``) only lowers from opset 20 upward. +- ``aten::grid_sampler`` is supported from opset 16, well below 20. +- Earlier opsets work for non-geometric augmentations, but standardising on a + single opset keeps the test matrix small. + +What this file does NOT test: +- Numerical equivalence between the exported ONNX graph and the eager forward. + That requires onnxruntime, which is not a hard dependency of kornia. Add a + separate test file (e.g. ``test_onnx_export_numerical.py``) gated on the + optional ``onnxruntime`` import when that work lands. +- The ``dynamo=True`` export path. That is tracked separately because it has + different op coverage (notably, ``aten::linalg_solve`` lowers cleanly under + dynamo but not under the legacy tracer). +- Augmentations that currently cannot export under the legacy tracer due to + missing ONNX op support, see ``XFAIL_OPS`` below. + +Adding an augmentation to ``EXPORTABLE_OPS`` is the way to advertise that it +cleanly exports today; adding to ``XFAIL_OPS`` marks a known gap with the +underlying op so we don't lose the signal that it's still pending. +""" + +from __future__ import annotations + +import io +import warnings +from typing import Any, Callable, Tuple + +import pytest +import torch + +import kornia.augmentation as K + + +def _hflip() -> torch.nn.Module: + return K.RandomHorizontalFlip(p=1.0) + + +def _vflip() -> torch.nn.Module: + return K.RandomVerticalFlip(p=1.0) + + +def _color_jiggle() -> torch.nn.Module: + return K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0) + + +def _brightness() -> torch.nn.Module: + return K.RandomBrightness(brightness=(0.5, 1.5), p=1.0) + + +def _grayscale() -> torch.nn.Module: + return K.RandomGrayscale(p=1.0) + + +def _invert() -> torch.nn.Module: + return K.RandomInvert(p=1.0) + + +def _gaussian_blur() -> torch.nn.Module: + return K.RandomGaussianBlur((3, 3), (0.1, 2.0), p=1.0) + + +def _erasing() -> torch.nn.Module: + return K.RandomErasing(p=1.0) + + +def _posterize() -> torch.nn.Module: + return K.RandomPosterize(3, p=1.0) + + +def _solarize() -> torch.nn.Module: + return K.RandomSolarize(0.1, p=1.0) + + +def _normalize() -> torch.nn.Module: + return K.Normalize(mean=torch.zeros(3), std=torch.ones(3)) + + +def _affine() -> torch.nn.Module: + return K.RandomAffine(degrees=10.0, p=1.0) + + +def _rotation() -> torch.nn.Module: + return K.RandomRotation(degrees=10.0, p=1.0) + + +def _perspective() -> torch.nn.Module: + return K.RandomPerspective(p=1.0) + + +def _augmentation_sequential() -> torch.nn.Module: + """A torchgeo-typical pipeline: per-element random flips followed by normalize.""" + return K.AugmentationSequential( + K.RandomHorizontalFlip(p=0.5), + K.RandomVerticalFlip(p=0.5), + K.Normalize(mean=torch.zeros(3), std=torch.ones(3)), + data_keys=["input"], + ) + + +# Augmentations that export today through the legacy TorchScript tracer at opset 20. +EXPORTABLE_OPS: list[Tuple[str, Callable[[], torch.nn.Module]]] = [ + ("RandomHorizontalFlip", _hflip), + ("RandomVerticalFlip", _vflip), + ("ColorJiggle", _color_jiggle), + ("RandomBrightness", _brightness), + ("RandomGrayscale", _grayscale), + ("RandomInvert", _invert), + ("RandomGaussianBlur", _gaussian_blur), + ("RandomErasing", _erasing), + ("RandomPosterize", _posterize), + ("RandomSolarize", _solarize), + ("Normalize", _normalize), + # Geometric augmentations enabled by: + # - bumping opset to 20 (for ``affine_grid_generator``) + # - the closed-form 3x3 inverse in ``kornia.core.utils._inverse_3x3_closed_form`` + # that replaces ``aten::linalg_inv`` during ONNX export. + ("RandomAffine", _affine), + ("RandomRotation", _rotation), + # ``RandomPerspective`` exports via the closed-form Heckbert decomposition in + # ``kornia.geometry.transform.imgwarp._get_perspective_transform_closed_form``, + # which replaces the 8x8 ``torch.linalg.solve`` with two unit-square-to-quad + # constructions and one closed-form 3x3 inverse. + ("RandomPerspective", _perspective), + # Container: pinned because torchgeo and similar callers wrap their pipelines + # in ``AugmentationSequential``. The container's ``forward`` strips trailing + # ``None`` positional args injected by the legacy ONNX tracer. + ("AugmentationSequential", _augmentation_sequential), +] + +# Currently no augmentations are blocked at export time. Augmentations that export +# but produce numerically different results from eager are tracked separately in +# ``ONNX_NUMERICAL_KNOWN_DRIFT`` below. +XFAIL_OPS: list[Tuple[str, Callable[[], torch.nn.Module], str]] = [] + + +def _try_export(module: torch.nn.Module, x: torch.Tensor) -> int: + """Run ``torch.onnx.export`` against an in-memory buffer and return the byte count.""" + module.eval() + buf = io.BytesIO() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + torch.onnx.export( + module, + (x,), + buf, + opset_version=20, + input_names=["input"], + output_names=["output"], + dynamo=False, + ) + return len(buf.getvalue()) + + +@pytest.mark.parametrize("name,factory", EXPORTABLE_OPS, ids=[n for n, _ in EXPORTABLE_OPS]) +def test_onnx_export_exportable(name: str, factory: Callable[[], torch.nn.Module]) -> None: + """Augmentation exports cleanly via ``torch.onnx.export`` (legacy tracer, opset 20).""" + torch.manual_seed(0) + x = torch.randn(2, 3, 32, 32) + size = _try_export(factory(), x) + assert size > 0, f"{name}: ONNX graph was empty" + + +@pytest.mark.parametrize("name,factory,reason", XFAIL_OPS, ids=[n for n, _, _ in XFAIL_OPS]) +def test_onnx_export_known_blocked(name: str, factory: Callable[[], torch.nn.Module], reason: str) -> None: + """Augmentation cannot export today; pinned so we notice if it starts working.""" + torch.manual_seed(0) + x = torch.randn(2, 3, 32, 32) + with pytest.raises(Exception): + _try_export(factory(), x) + + +# ---------------------------------------------------------------------------- +# Numerical-equivalence checks (require onnxruntime). +# +# Each entry pins a concrete deterministic configuration of the augmentation is +# fixed angles, fixed brightness, etc. So comparing eager and ONNX is a +# clean apples-to-apples test of "did the export faithfully capture the +# computation". Augmentations with random parameter ranges are deliberately +# excluded from the numerical check because ``torch.onnx.export`` consumes RNG +# during tracing in ways that don't line up with a separate eager call, even +# under the same ``torch.manual_seed``, that mismatch is RNG bookkeeping and +# would mask real bugs. The export-success tests above cover the random case. +# ---------------------------------------------------------------------------- + +ort = pytest.importorskip("onnxruntime", reason="onnxruntime is required for numerical-equivalence checks") +np = pytest.importorskip("numpy") + + +# Deterministic factories for numerical tests. Tuple ranges of the form ``(v, v)`` +# evaluate to a single fixed value under uniform sampling. Geometric augmentations +# additionally rely on the in-place-mutation fix in +# ``kornia.geometry.conversions.normal_transform_pixel`` and the ``deg2rad`` +# inlining in the affine/shear ``compute_transformation`` paths. +def _hflip_det() -> torch.nn.Module: + return K.RandomHorizontalFlip(p=1.0) + + +def _vflip_det() -> torch.nn.Module: + return K.RandomVerticalFlip(p=1.0) + + +def _grayscale_det() -> torch.nn.Module: + return K.RandomGrayscale(p=1.0) + + +def _invert_det() -> torch.nn.Module: + return K.RandomInvert(p=1.0) + + +def _normalize_det() -> torch.nn.Module: + return K.Normalize(mean=torch.zeros(3), std=torch.ones(3)) + + +def _brightness_det() -> torch.nn.Module: + return K.RandomBrightness(brightness=(1.2, 1.2), p=1.0) + + +def _color_jiggle_det() -> torch.nn.Module: + return K.ColorJiggle(brightness=(1.2, 1.2), p=1.0) + + +def _gaussian_blur_det() -> torch.nn.Module: + return K.RandomGaussianBlur((3, 3), (1.5, 1.5), p=1.0) + + +def _solarize_det() -> torch.nn.Module: + return K.RandomSolarize(thresholds=(0.5, 0.5), additions=(0.0, 0.0), p=1.0) + + +def _rotation_det() -> torch.nn.Module: + return K.RandomRotation(degrees=(15.0, 15.0), p=1.0) + + +def _affine_det() -> torch.nn.Module: + return K.RandomAffine(degrees=(15.0, 15.0), translate=(0.0, 0.0), scale=(1.0, 1.0), shear=(0.0, 0.0), p=1.0) + + +def _affine_with_shear_det() -> torch.nn.Module: + return K.RandomAffine(degrees=(0.0, 0.0), shear=(5.0, 5.0), p=1.0) + + +# Augmentations whose exported graph produces numerically equivalent results to +# eager when configured with deterministic parameters. ``max diff < 1e-3`` +# threshold is conservative, most are float32 precision (< 1e-5). +ONNX_NUMERICAL_EQUIVALENT: list[Tuple[str, Callable[[], torch.nn.Module]]] = [ + # Parameter-free / no random sampling + ("RandomHorizontalFlip", _hflip_det), + ("RandomVerticalFlip", _vflip_det), + ("RandomGrayscale", _grayscale_det), + ("RandomInvert", _invert_det), + ("Normalize", _normalize_det), + # Random params pinned to deterministic ranges + ("RandomBrightness", _brightness_det), + ("ColorJiggle", _color_jiggle_det), + ("RandomGaussianBlur", _gaussian_blur_det), + ("RandomSolarize", _solarize_det), + # Geometric augmentations, exercise the warp_affine / grid_sample path. + # Previously diverged due to in-place mutation in normal_transform_pixel + # losing the scale factors under tracing. + ("RandomRotation", _rotation_det), + ("RandomAffine", _affine_det), + ("RandomAffine_with_shear", _affine_with_shear_det), +] + + +def _run_onnx(module: torch.nn.Module, x: torch.Tensor) -> Any: + module.eval() + buf = io.BytesIO() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + torch.onnx.export( + module, + (x,), + buf, + opset_version=20, + dynamo=False, + input_names=["input"], + output_names=["output"], + ) + sess = ort.InferenceSession(buf.getvalue()) + return sess.run(["output"], {"input": x.numpy()})[0] + + +@pytest.mark.parametrize("name,factory", ONNX_NUMERICAL_EQUIVALENT, ids=[n for n, _ in ONNX_NUMERICAL_EQUIVALENT]) +def test_onnx_export_numerically_matches_eager(name: str, factory: Callable[[], torch.nn.Module]) -> None: + """Exported graph produces the same numbers as eager for the deterministic configuration.""" + torch.manual_seed(0) + x = torch.randn(2, 3, 32, 32) + aug = factory() + aug.eval() + eager = aug(x).numpy() + onnx_out = _run_onnx(aug, x) + assert eager.shape == onnx_out.shape, f"{name}: shape mismatch {eager.shape} vs {onnx_out.shape}" + max_diff = float(np.abs(eager - onnx_out).max()) + assert max_diff < 1e-3, f"{name}: max abs diff {max_diff:.4f} exceeds 1e-3" diff --git a/tests/augmentation/test_param_validation.py b/tests/augmentation/test_param_validation.py new file mode 100644 index 0000000..0d42839 --- /dev/null +++ b/tests/augmentation/test_param_validation.py @@ -0,0 +1,166 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import cast + +import pytest +import torch + +from kornia.augmentation.utils.param_validation import ( + _common_param_check, + _range_bound, + _tuple_range_reader, +) + + +class TestParamValidation: + @pytest.mark.parametrize( + "batch_size, same_on_batch", + [ + (1, True), + (0, False), + (1, None), + ], + ) + def test_common_param_check_valid(self, batch_size, same_on_batch): + """Valid combinations of batch_size and same_on_batch should not raise.""" + _common_param_check(batch_size=batch_size, same_on_batch=same_on_batch) + + @pytest.mark.parametrize("batch_size", [-1]) + def test_common_param_check_invalid_batch_size(self, batch_size): + """Negative batch_size should raise an assertion error.""" + with pytest.raises(AssertionError): + _common_param_check(batch_size=batch_size) + + @pytest.mark.parametrize("same_on_batch", [cast(bool, "invalid")]) + def test_common_param_check_invalid_same_on_batch(self, same_on_batch): + """ + Invalid runtime values for same_on_batch should raise. + + typing.cast is used to inject an invalid value at runtime + without breaking static type checking of the test itself. + """ + with pytest.raises(AssertionError): + _common_param_check(batch_size=1, same_on_batch=same_on_batch) + + @pytest.mark.parametrize( + "input_param, target_size, expected", + [ + (10.0, 2, torch.tensor([[-10.0, 10.0], [-10.0, 10.0]], dtype=torch.float32)), + ((5.0, 10.0), 2, torch.tensor([[5.0, 10.0], [5.0, 10.0]], dtype=torch.float32)), + (torch.tensor([5.0, 10.0]), 2, torch.tensor([[5.0, 10.0], [5.0, 10.0]], dtype=torch.float32)), + ([5.0, 10.0], 2, torch.tensor([[5.0, 10.0], [5.0, 10.0]], dtype=torch.float32)), + (torch.tensor([1.0, 2.0]), 2, torch.tensor([[1.0, 2.0], [1.0, 2.0]], dtype=torch.float32)), + ([(5.0, 10.0), (3.0, 8.0)], 2, torch.tensor([[5.0, 10.0], [3.0, 8.0]], dtype=torch.float32)), + (10.0, 1, torch.tensor([[-10.0, 10.0]], dtype=torch.float32)), + ( + torch.tensor([[5.0, 10.0], [3.0, 8.0]]), + 2, + torch.tensor([[5.0, 10.0], [3.0, 8.0]], dtype=torch.float32), + ), + ], + ids=[ + "float-symmetric-2", + "tuple-2", + "tensor-1d", + "list", + "tensor-1d-alt", + "list-of-tuples", + "float-symmetric-1", + "tensor-2d", + ], + ) + @pytest.mark.parametrize( + "device", + [ + torch.device("cpu"), + pytest.param( + torch.device("cuda"), + marks=pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available"), + ), + ], + ) + def test_tuple_range_reader_valid(self, input_param, target_size, expected, device): + """Supported input formats should expand correctly across devices.""" + res = _tuple_range_reader(input_param, target_size, device=device) + assert res.shape == (target_size, 2) + torch.testing.assert_close(res, expected.to(device)) + + @pytest.mark.parametrize( + "args, kwargs, expected_exception, match_msg", + [ + ((-10, 2), {}, ValueError, None), + (("invalid", 2), {}, TypeError, None), + ((torch.rand(2, 3), 2), {}, ValueError, "Degrees must be a"), + (([1, 2, 3], 2), {}, TypeError, "If not pass a torch.tensor"), + ((["a", 1.0], 2), {}, TypeError, "If not pass a torch.tensor"), + ], + ) + def test_tuple_range_reader_errors(self, args, kwargs, expected_exception, match_msg): + """Invalid inputs should raise the appropriate exception.""" + if match_msg is None: + with pytest.raises(expected_exception): + _tuple_range_reader(*args, **kwargs) + else: + with pytest.raises(expected_exception, match=match_msg): + _tuple_range_reader(*args, **kwargs) + + @pytest.mark.parametrize( + "factor, center, bounds, check, expected_exception, match_msg", + [ + (-1.0, 0, (-10, 10), "singular", ValueError, None), + (10.0, 0, None, "singular", ValueError, "`center` and `bounds` cannot be None"), + ((-10, 10), 0, (-5, 5), "singular", ValueError, "param out of bounds"), + ((10, 5), 0, None, "joint", ValueError, "should be smaller than"), + ("invalid", 0, (-10, 10), "singular", TypeError, None), + ((-10.0, 10.0), 0, (-5, 5), "singular", ValueError, "param out of bounds"), + ], + ) + def test_range_bound_errors(self, factor, center, bounds, check, expected_exception, match_msg): + """Invalid parameter combinations should raise.""" + if match_msg is None: + with pytest.raises(expected_exception): + _range_bound(factor, "param", center=center, bounds=bounds, check=check) + else: + with pytest.raises(expected_exception, match=match_msg): + _range_bound(factor, "param", center=center, bounds=bounds, check=check) + + @pytest.mark.parametrize( + "factor, center, bounds, check, expected", + [ + (10.0, 0, (-10, 10), "singular", torch.tensor([-10.0, 10.0], dtype=torch.float32)), + (10.0, 0, (-5, 5), "singular", torch.tensor([-5.0, 5.0], dtype=torch.float32)), + (0.2, 1.0, (0, 2), "singular", torch.tensor([0.8, 1.2], dtype=torch.float32)), + ((5.0, 10.0), 0, None, "singular", torch.tensor([5.0, 10.0], dtype=torch.float32)), + ([-5.0, 5.0], 0, (-10, 10), "singular", torch.tensor([-5.0, 5.0], dtype=torch.float32)), + (torch.tensor([5.0, 10.0]), 0, None, "singular", torch.tensor([5.0, 10.0], dtype=torch.float32)), + ((10.0, 5.0), 0, None, "singular", torch.tensor([10.0, 5.0], dtype=torch.float32)), + ], + ids=[ + "float-clamp-full", + "float-clamp-partial", + "float-center-offset", + "tuple-input", + "list-input", + "tensor-input", + "singular-min-gt-max", + ], + ) + def test_range_bound_valid(self, factor, center, bounds, check, expected): + """Valid inputs should produce the expected bounded range.""" + res = _range_bound(factor, "param", center=center, bounds=bounds, check=check) + torch.testing.assert_close(res, expected) diff --git a/tests/augmentation/test_perspective_rand.py b/tests/augmentation/test_perspective_rand.py new file mode 100644 index 0000000..78e17e5 --- /dev/null +++ b/tests/augmentation/test_perspective_rand.py @@ -0,0 +1,227 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestRandomPerspective(BaseTester): + torch.manual_seed(0) # for random reproductibility + + def test_smoke_no_transform_float(self, device): + x_data = torch.rand(1, 2, 8, 9).to(device) + + aug = kornia.augmentation.RandomPerspective(0.5, p=0.5) + + out_perspective = aug(x_data) + + assert out_perspective.shape == x_data.shape + assert aug.inverse(out_perspective).shape == x_data.shape + + def test_smoke_no_transform(self, device, dtype): + x_data = torch.rand(1, 2, 8, 9, dtype=dtype).to(device) + + aug = kornia.augmentation.RandomPerspective(torch.tensor(0.5, device=device, dtype=dtype), p=0.5) + + out_perspective = aug(x_data) + + assert out_perspective.shape == x_data.shape + assert aug.inverse(out_perspective).shape == x_data.shape + + def test_smoke_no_transform_batch(self, device, dtype): + x_data = torch.rand(2, 2, 8, 9, dtype=dtype).to(device) + + aug = kornia.augmentation.RandomPerspective(torch.tensor(0.5, device=device, dtype=dtype), p=0.5) + + out_perspective = aug(x_data) + + assert out_perspective.shape == x_data.shape + assert aug.inverse(out_perspective).shape == x_data.shape + + def test_smoke_transform(self, device, dtype): + x_data = torch.rand(1, 2, 4, 5, dtype=dtype).to(device) + + aug = kornia.augmentation.RandomPerspective(torch.tensor(0.5, device=device, dtype=dtype), p=0.5) + + out_perspective = aug(x_data) + + assert out_perspective.shape == x_data.shape + assert aug.transform_matrix.shape == torch.Size([1, 3, 3]) + assert aug.inverse(out_perspective).shape == x_data.shape + + def test_smoke_transform_sampling_method(self, device, dtype): + x_data = torch.rand(1, 2, 4, 5, dtype=dtype).to(device) + + aug = kornia.augmentation.RandomPerspective( + torch.tensor(0.5, device=device, dtype=dtype), p=0.5, sampling_method="area_preserving" + ) + + out_perspective = aug(x_data) + + assert out_perspective.shape == x_data.shape + assert aug.transform_matrix.shape == torch.Size([1, 3, 3]) + assert aug.inverse(out_perspective).shape == x_data.shape + + def test_no_transform_module(self, device, dtype): + x_data = torch.rand(1, 2, 8, 9, dtype=dtype).to(device) + aug = kornia.augmentation.RandomPerspective(torch.tensor(0.5, device=device, dtype=dtype)) + out_perspective = aug(x_data) + assert out_perspective.shape == x_data.shape + assert aug.inverse(out_perspective).shape == x_data.shape + + def test_transform_module_should_return_identity(self, device, dtype): + torch.manual_seed(0) + x_data = torch.rand(1, 2, 4, 5, dtype=dtype).to(device) + + aug = kornia.augmentation.RandomPerspective(torch.tensor(0.5, device=device, dtype=dtype), p=0.0) + + out_perspective = aug(x_data) + assert out_perspective.shape == x_data.shape + assert aug.transform_matrix.shape == (1, 3, 3) + self.assert_close(out_perspective, x_data) + self.assert_close(aug.transform_matrix, torch.eye(3, device=device, dtype=dtype)[None]) + assert aug.inverse(out_perspective).shape == x_data.shape + + def test_transform_module_should_return_expected_transform(self, device, dtype): + torch.manual_seed(0) + x_data = torch.rand(1, 2, 4, 5).to(device).type(dtype) + + expected_output = torch.tensor( + [ + [ + [ + [0.0000, 0.0000, 0.0000, 0.0197, 0.0429], + [0.0000, 0.5632, 0.5322, 0.3677, 0.1430], + [0.0000, 0.3083, 0.4032, 0.1761, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + ], + [ + [0.0000, 0.0000, 0.0000, 0.1189, 0.0586], + [0.0000, 0.7087, 0.5420, 0.3995, 0.0863], + [0.0000, 0.2695, 0.5981, 0.5888, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + ], + ] + ], + device=device, + dtype=x_data.dtype, + ) + + expected_transform = torch.tensor( + [[[1.0523, 0.3493, 0.3046], [-0.1066, 1.0426, 0.5846], [0.0351, 0.1213, 1.0000]]], + device=device, + dtype=x_data.dtype, + ) + + aug = kornia.augmentation.RandomPerspective( + torch.tensor(0.5, device=device, dtype=dtype), p=0.99999999 + ) # step one the random state + + out_perspective = aug(x_data) + + assert out_perspective.shape == x_data.shape + assert aug.transform_matrix.shape == (1, 3, 3) + self.assert_close(out_perspective, expected_output, atol=1e-4, rtol=1e-4) + self.assert_close(aug.transform_matrix, expected_transform, atol=1e-4, rtol=1e-4) + assert aug.inverse(out_perspective).shape == x_data.shape + + def test_gradcheck(self, device, dtype): + input = torch.rand(1, 2, 5, 7, dtype=torch.float64, device=device) + # TODO: turned off with p=0 + self.gradcheck( + kornia.augmentation.RandomPerspective(torch.tensor(0.5, device=device, dtype=dtype), p=0.0), + (input,), + ) + + +class TestRandomAffine(BaseTester): + torch.manual_seed(0) # for random reproductibility + + def test_smoke_no_transform(self, device): + x_data = torch.rand(1, 2, 8, 9).to(device) + aug = kornia.augmentation.RandomAffine(0.0) + out = aug(x_data) + assert out.shape == x_data.shape + assert aug.inverse(out).shape == x_data.shape + assert aug.inverse(out, aug._params).shape == x_data.shape + + def test_smoke_no_transform_batch(self, device): + x_data = torch.rand(2, 2, 8, 9).to(device) + aug = kornia.augmentation.RandomAffine(0.0) + out = aug(x_data) + assert out.shape == x_data.shape + # assert False, (aug.transform_matrix.shape, out.shape, aug._params) + assert aug.inverse(out).shape == x_data.shape + assert aug.inverse(out, aug._params).shape == x_data.shape + + @pytest.mark.parametrize("degrees", [45.0, (-45.0, 45.0), torch.tensor([45.0, 45.0])]) + @pytest.mark.parametrize("translate", [(0.1, 0.1), torch.tensor([0.1, 0.1])]) + @pytest.mark.parametrize( + "scale", [(0.8, 1.2), (0.8, 1.2, 0.9, 1.1), torch.tensor([0.8, 1.2]), torch.tensor([0.8, 1.2, 0.7, 1.3])] + ) + @pytest.mark.parametrize( + "shear", + [ + 5.0, + (-5.0, 5.0), + (-5.0, 5.0, -3.0, 3.0), + torch.tensor(5.0), + torch.tensor([-5.0, 5.0]), + torch.tensor([-5.0, 5.0, -3.0, 3.0]), + ], + ) + def test_batch_multi_params(self, degrees, translate, scale, shear, device, dtype): + x_data = torch.rand(2, 2, 8, 9).to(device) + aug = kornia.augmentation.RandomAffine(degrees=degrees, translate=translate, scale=scale, shear=shear) + out = aug(x_data) + assert out.shape == x_data.shape + assert aug.inverse(out).shape == x_data.shape + + def test_smoke_transform(self, device): + x_data = torch.rand(1, 2, 4, 5).to(device) + aug = kornia.augmentation.RandomAffine(0.0) + out = aug(x_data) + + assert out.shape == x_data.shape + assert aug.transform_matrix.shape == torch.Size([1, 3, 3]) + assert aug.inverse(out).shape == x_data.shape + + def test_gradcheck(self, device): + input = torch.rand(1, 2, 5, 7, device=device, dtype=torch.float64) + # TODO: turned off with p=0 + self.gradcheck(kornia.augmentation.RandomAffine(10, p=0.0), (input,)) + + +class TestRandomShear(BaseTester): + torch.manual_seed(0) # for random reproductibility + + def test_smoke_no_transform(self, device): + x_data = torch.rand(1, 2, 8, 9).to(device) + aug = kornia.augmentation.RandomShear((10.0, 10.0)) + out = aug(x_data) + assert out.shape == x_data.shape + assert aug.inverse(out).shape == x_data.shape + assert aug.inverse(out, aug._params).shape == x_data.shape + + def test_gradcheck(self, device): + input = torch.rand(1, 2, 5, 7, device=device, dtype=torch.float64) + # TODO: turned off with p=0 + self.gradcheck(kornia.augmentation.RandomShear((10.0, 10.0), p=1.0), (input,)) diff --git a/tests/augmentation/test_presets.py b/tests/augmentation/test_presets.py new file mode 100644 index 0000000..26f5694 --- /dev/null +++ b/tests/augmentation/test_presets.py @@ -0,0 +1,106 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import pytest +import torch + +from kornia.augmentation.presets.ada import AdaptiveDiscriminatorAugmentation + +from testing.base import BaseTester + + +class PresetTests(BaseTester): + pass + + +@pytest.mark.usefixtures("device", "dtype") +class TestAdaptiveDiscriminatorAugmentation(PresetTests): + def test_initial_hyper_params(self): + ada_preset = AdaptiveDiscriminatorAugmentation() + assert ada_preset.adjustment_speed > 0 + assert 0 <= ada_preset.target_real_acc <= 1 + assert 0 <= ada_preset.ema_lambda <= 1 + assert ada_preset.update_every >= 1 + assert 0 <= ada_preset.max_p <= 1 + assert 0 <= ada_preset.p <= ada_preset.max_p # initial p + assert ada_preset._num_calls == 0 + self.assert_close(ada_preset.real_acc_ema, 0.5) + + transforms = list(ada_preset.children()) + expected_transforms = [ + "RandomHorizontalFlip", + "RandomRotation90", + "RandomErasing", + "RandomAffine", + "ColorJitter", + "RandomGaussianNoise", + ] + + assert len(transforms) == len(expected_transforms) + for t, et in zip(transforms, expected_transforms): + assert et == str(t.__class__.__name__) + + def test_transforms_behaviour(self, device, dtype): + ada_preset = AdaptiveDiscriminatorAugmentation().to(device) + inputs = torch.randn(2, 3, 32, 32).to(device) + outputs = ada_preset(inputs) + assert outputs.dtype == inputs.dtype + assert outputs.shape == inputs.shape + + ada_preset.p = 0 + ada_outputs = ada_preset(inputs) + self.assert_close(inputs, ada_outputs) + + def test_adaptive_probability(self, device, dtype): + inputs = torch.randn(2, 3, 32, 32) + n_runs = 3 + initial_p = 0.5 + update_every = 3 + ada = AdaptiveDiscriminatorAugmentation( + initial_p=initial_p, + adjustment_speed=0.01, + max_p=0.8, + update_every=update_every, + target_real_acc=0.9, + ema_lambda=0, + ) + + # p increasing, without reaching max_p + for i in range(ada.update_every * n_runs): + self.assert_close(ada.p, initial_p + (i // update_every) * ada.adjustment_speed) + ada(inputs, real_acc=ada.target_real_acc + 0.1) + self.assert_close(ada.p, initial_p + n_runs * ada.adjustment_speed) + + # decreasing without reaching 0 + initial_p = ada.p + for i in range(ada.update_every * n_runs): + self.assert_close(ada.p, initial_p - (i // update_every) * ada.adjustment_speed) + ada(inputs, real_acc=ada.target_real_acc - 0.1) + self.assert_close(ada.p, initial_p - n_runs * ada.adjustment_speed) + + # p clamped at 0 + ada.p = ada.adjustment_speed / 2 + for _ in range(ada.update_every): + ada(inputs, real_acc=ada.target_real_acc - 0.1) + self.assert_close(ada.p, 0) + + # p clamped at max_p + ada.p = ada.max_p - ada.adjustment_speed / 2 + for _ in range(ada.update_every): + ada(inputs, real_acc=ada.target_real_acc + 0.1) + self.assert_close(ada.p, ada.max_p) diff --git a/tests/augmentation/test_random_generator.py b/tests/augmentation/test_random_generator.py new file mode 100644 index 0000000..3f77b77 --- /dev/null +++ b/tests/augmentation/test_random_generator.py @@ -0,0 +1,1556 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +from torch import Tensor + +from kornia.augmentation.random_generator import ( + AffineGenerator, + ColorJiggleGenerator, + ColorJitterGenerator, + CropGenerator, + CutmixGenerator, + MixupGenerator, + MotionBlurGenerator, + PerspectiveGenerator, + PlainUniformGenerator, + PosterizeGenerator, + ProbabilityGenerator, + RectangleEraseGenerator, + ResizedCropGenerator, + center_crop_generator, +) +from kornia.core._compat import torch_version_ge + +from testing.base import assert_close + + +class RandomGeneratorBaseTests: + def test_valid_param_combinations(self, device, dtype): + raise NotImplementedError + + def test_invalid_param_combinations(self, device, dtype): + raise NotImplementedError + + def test_random_gen(self, device, dtype): + raise NotImplementedError + + def test_same_on_batch(self, device, dtype): + raise NotImplementedError + + +class TestRandomProbGen(RandomGeneratorBaseTests): + @pytest.mark.parametrize("p", [0.0, 0.5, 1.0]) + @pytest.mark.parametrize("batch_size", [0, 1, 8]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_valid_param_combinations(self, p, batch_size, same_on_batch): + ProbabilityGenerator(p)(torch.Size([batch_size]), same_on_batch) + + @pytest.mark.parametrize( + "p", + [ + # Should be failed if p > 1. or p < 0. + (-1.0), + (2.0), + ], + ) + def test_invalid_param_combinations(self, p): + with pytest.raises(Exception): + ProbabilityGenerator(p)(torch.Size([8])) + + @pytest.mark.parametrize( + "p,expected", + [ + (0.0, [False] * 8), + (0.5, [False, False, True, False, True, False, True, False]), + (1.0, [True] * 8), + ], + ) + def test_random_gen(self, p, expected, device, dtype): + torch.manual_seed(42) + batch_size = 8 + res = ProbabilityGenerator(p)(torch.Size([batch_size])) + assert (res["probs"] == torch.tensor(expected)).long().sum() == batch_size + + @pytest.mark.parametrize("seed,expected", [(42, [False] * 8), (0, [True] * 8)]) + def test_same_on_batch(self, seed, expected, device, dtype): + torch.manual_seed(seed) + batch_size = 8 + res = ProbabilityGenerator(0.5)(torch.Size([batch_size]), True) + assert (res["probs"] == torch.tensor(expected)).long().sum() == batch_size + + +class TestColorJiggleGen(RandomGeneratorBaseTests): + @pytest.mark.parametrize("brightness", [None, torch.tensor([0.8, 1.2])]) + @pytest.mark.parametrize("contrast", [None, torch.tensor([0.8, 1.2])]) + @pytest.mark.parametrize("saturation", [None, torch.tensor([0.8, 1.2])]) + @pytest.mark.parametrize("hue", [None, torch.tensor([-0.1, 0.1])]) + @pytest.mark.parametrize("batch_size", [0, 1, 8]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_valid_param_combinations( + self, + brightness, + contrast, + saturation, + hue, + batch_size, + same_on_batch, + device, + dtype, + ): + ColorJiggleGenerator( + torch.as_tensor( + brightness if brightness is not None else torch.tensor([0.0, 0.0]), + device=device, + dtype=dtype, + ), + torch.as_tensor( + contrast if contrast is not None else torch.tensor([0.0, 0.0]), + device=device, + dtype=dtype, + ), + torch.as_tensor( + saturation if saturation is not None else torch.tensor([0.0, 0.0]), + device=device, + dtype=dtype, + ), + torch.as_tensor( + hue if hue is not None else torch.tensor([0.0, 0.0]), + device=device, + dtype=dtype, + ), + )(torch.Size([batch_size]), same_on_batch) + + @pytest.mark.parametrize( + "brightness,contrast,saturation,hue", + [ + # Should be failed if value out of bounds or tensor.shape != [1, 2] + (torch.tensor([-1.0, 2.0]), None, None, None), + (torch.tensor([0.0, 3.0]), None, None, None), + (torch.tensor([0.0]), None, None, None), + (torch.tensor([0.0, 1.0, 2.0]), None, None, None), + (None, torch.tensor([-1.0, 2.0]), None, None), + (None, torch.tensor([0.0]), None, None), + (None, torch.tensor([0.0, 1.0, 2.0]), None, None), + (None, None, torch.tensor([-1.0, 2.0]), None), + (None, None, torch.tensor([0.0]), None), + (None, None, torch.tensor([0.0, 1.0, 2.0]), None), + (None, None, None, torch.tensor([-1.0, 0.0])), + (None, None, None, torch.tensor([0, 1.0])), + (None, None, None, torch.tensor([0.0])), + (None, None, None, torch.tensor([0.0, 1.0, 2.0])), + ], + ) + def test_invalid_param_combinations(self, brightness, contrast, saturation, hue, device, dtype): + with pytest.raises(Exception): + ColorJiggleGenerator( + torch.as_tensor( + brightness if brightness is not None else torch.tensor([0.0, 0.0]), + device=device, + dtype=dtype, + ), + torch.as_tensor( + contrast if contrast is not None else torch.tensor([0.0, 0.0]), + device=device, + dtype=dtype, + ), + torch.as_tensor( + saturation if saturation is not None else torch.tensor([0.0, 0.0]), + device=device, + dtype=dtype, + ), + torch.as_tensor( + hue if hue is not None else torch.tensor([0.0, 0.0]), + device=device, + dtype=dtype, + ), + )(torch.Size([8])) + + def test_random_gen(self, device, dtype): + torch.manual_seed(42) + batch_size = 8 + jitter_params = ColorJiggleGenerator( + brightness=torch.tensor([0.8, 1.2], device=device, dtype=dtype), + contrast=torch.tensor([0.7, 1.3], device=device, dtype=dtype), + saturation=torch.tensor([0.6, 1.4], device=device, dtype=dtype), + hue=torch.tensor([-0.1, 0.1], device=device, dtype=dtype), + )(torch.Size([batch_size])) + + expected_jitter_params = { + "brightness_factor": torch.tensor( + [1.1529, 1.1660, 0.9531, 1.1837, 0.9562, 1.0404, 0.9026, 1.1175], + device=device, + dtype=dtype, + ), + "contrast_factor": torch.tensor( + [1.2645, 0.7799, 1.2608, 1.0561, 1.2216, 1.0406, 1.1447, 0.9576], + device=device, + dtype=dtype, + ), + "hue_factor": torch.tensor( + [0.0771, 0.0148, -0.0467, 0.0255, -0.0461, -0.0117, -0.0406, 0.0663], + device=device, + dtype=dtype, + ), + "saturation_factor": torch.tensor( + [0.6843, 0.8156, 0.8871, 0.7595, 1.0378, 0.6049, 1.3612, 0.6602], + device=device, + dtype=dtype, + ), + "order": torch.tensor([3, 2, 0, 1], device=device, dtype=torch.long), + } + + assert set(jitter_params.keys()) == { + "brightness_factor", + "contrast_factor", + "hue_factor", + "saturation_factor", + "order", + }, ( + "Redundant keys found apart from \ + 'brightness_factor', 'contrast_factor', 'hue_factor', 'saturation_factor', 'order'" + ) + + assert_close( + jitter_params["brightness_factor"], + expected_jitter_params["brightness_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["contrast_factor"], + expected_jitter_params["contrast_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["hue_factor"], + expected_jitter_params["hue_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["saturation_factor"], + expected_jitter_params["saturation_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["order"], + expected_jitter_params["order"], + rtol=1e-4, + atol=1e-4, + ) + + def test_random_gen_accumulative_additive_additive(self, device, dtype): + torch.manual_seed(42) + batch_size = 8 + jitter_params = ColorJiggleGenerator( + brightness=torch.tensor([0.8, 1.2], device=device, dtype=dtype), + contrast=torch.tensor([0.7, 1.3], device=device, dtype=dtype), + saturation=torch.tensor([0.6, 1.4], device=device, dtype=dtype), + hue=torch.tensor([-0.1, 0.1], device=device, dtype=dtype), + )(torch.Size([batch_size])) + + expected_jitter_params = { + "brightness_factor": torch.tensor( + [1.1529, 1.1660, 0.9531, 1.1837, 0.9562, 1.0404, 0.9026, 1.1175], + device=device, + dtype=dtype, + ), + "contrast_factor": torch.tensor( + [1.2645, 0.7799, 1.2608, 1.0561, 1.2216, 1.0406, 1.1447, 0.9576], + device=device, + dtype=dtype, + ), + "hue_factor": torch.tensor( + [0.0771, 0.0148, -0.0467, 0.0255, -0.0461, -0.0117, -0.0406, 0.0663], + device=device, + dtype=dtype, + ), + "saturation_factor": torch.tensor( + [0.6843, 0.8156, 0.8871, 0.7595, 1.0378, 0.6049, 1.3612, 0.6602], + device=device, + dtype=dtype, + ), + "order": torch.tensor([3, 2, 0, 1], device=device, dtype=torch.long), + } + + assert set(jitter_params.keys()) == { + "brightness_factor", + "contrast_factor", + "hue_factor", + "saturation_factor", + "order", + }, ( + "Redundant keys found apart from \ + 'brightness_factor', 'contrast_factor', 'hue_factor', 'saturation_factor', 'order'" + ) + + assert_close( + jitter_params["brightness_factor"], + expected_jitter_params["brightness_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["contrast_factor"], + expected_jitter_params["contrast_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["hue_factor"], + expected_jitter_params["hue_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["saturation_factor"], + expected_jitter_params["saturation_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["order"], + expected_jitter_params["order"], + rtol=1e-4, + atol=1e-4, + ) + + def test_same_on_batch(self, device, dtype): + torch.manual_seed(42) + batch_size = 8 + jitter_params = ColorJiggleGenerator( + brightness=torch.tensor([0.8, 1.2], device=device, dtype=dtype), + contrast=torch.tensor([0.7, 1.3], device=device, dtype=dtype), + saturation=torch.tensor([0.6, 1.4], device=device, dtype=dtype), + hue=torch.tensor([-0.1, 0.1], device=device, dtype=dtype), + )(torch.Size([batch_size]), same_on_batch=True) + + expected_res = { + "brightness_factor": torch.tensor([1.1529] * batch_size, device=device, dtype=dtype), + "contrast_factor": torch.tensor([1.2490] * batch_size, device=device, dtype=dtype), + "hue_factor": torch.tensor([-0.0234] * batch_size, device=device, dtype=dtype), + "saturation_factor": torch.tensor([1.3674] * batch_size, device=device, dtype=dtype), + "order": torch.tensor([2, 3, 0, 1], device=device, dtype=torch.long), + } + + assert_close( + jitter_params["brightness_factor"], + expected_res["brightness_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["contrast_factor"], + expected_res["contrast_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["hue_factor"], + expected_res["hue_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["saturation_factor"], + expected_res["saturation_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["order"], + expected_res["order"], + rtol=1e-4, + atol=1e-4, + ) + + +class TestColorJitterGen(RandomGeneratorBaseTests): + @pytest.mark.parametrize("brightness", [None, torch.tensor([0.8, 1.2])]) + @pytest.mark.parametrize("contrast", [None, torch.tensor([0.8, 1.2])]) + @pytest.mark.parametrize("saturation", [None, torch.tensor([0.8, 1.2])]) + @pytest.mark.parametrize("hue", [None, torch.tensor([-0.1, 0.1])]) + @pytest.mark.parametrize("batch_size", [0, 1, 8]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_valid_param_combinations( + self, + brightness, + contrast, + saturation, + hue, + batch_size, + same_on_batch, + device, + dtype, + ): + ColorJitterGenerator( + torch.as_tensor( + brightness if brightness is not None else torch.tensor([0.0, 0.0]), + device=device, + dtype=dtype, + ), + torch.as_tensor( + contrast if contrast is not None else torch.tensor([0.0, 0.0]), + device=device, + dtype=dtype, + ), + torch.as_tensor( + saturation if saturation is not None else torch.tensor([0.0, 0.0]), + device=device, + dtype=dtype, + ), + torch.as_tensor( + hue if hue is not None else torch.tensor([0.0, 0.0]), + device=device, + dtype=dtype, + ), + )(torch.Size([batch_size]), same_on_batch) + + @pytest.mark.parametrize( + "brightness,contrast,saturation,hue", + [ + # Should be failed if value out of bounds or tensor.shape != [1, 2] + (torch.tensor([-1.0, 2.0]), None, None, None), + (torch.tensor([0.0]), None, None, None), + (torch.tensor([0.0, 1.0, 2.0]), None, None, None), + (None, torch.tensor([-1.0, 2.0]), None, None), + (None, torch.tensor([0.0]), None, None), + (None, torch.tensor([0.0, 1.0, 2.0]), None, None), + (None, None, torch.tensor([-1.0, 2.0]), None), + (None, None, torch.tensor([0.0]), None), + (None, None, torch.tensor([0.0, 1.0, 2.0]), None), + (None, None, None, torch.tensor([-1.0, 0.0])), + (None, None, None, torch.tensor([0, 1.0])), + (None, None, None, torch.tensor([0.0])), + (None, None, None, torch.tensor([0.0, 1.0, 2.0])), + ], + ) + def test_invalid_param_combinations(self, brightness, contrast, saturation, hue, device, dtype): + with pytest.raises(Exception): + ColorJitterGenerator( + torch.as_tensor( + brightness if brightness is not None else torch.tensor([0.0, 0.0]), + device=device, + dtype=dtype, + ), + torch.as_tensor( + contrast if contrast is not None else torch.tensor([0.0, 0.0]), + device=device, + dtype=dtype, + ), + torch.as_tensor( + saturation if saturation is not None else torch.tensor([0.0, 0.0]), + device=device, + dtype=dtype, + ), + torch.as_tensor( + hue if hue is not None else torch.tensor([0.0, 0.0]), + device=device, + dtype=dtype, + ), + )(torch.Size([8])) + + def test_random_gen(self, device, dtype): + # TODO(jian): crashes with pytorch 1.10, cuda and fp64 + if (torch_version_ge(1, 10) and "cuda" in str(device)) or dtype == torch.float64: + pytest.skip("AssertionError: cannot reproduce the same result") + torch.manual_seed(42) + batch_size = 8 + gen = ColorJitterGenerator( + brightness=torch.tensor([0.8, 1.2], device=device, dtype=dtype), + contrast=torch.tensor([0.7, 1.3], device=device, dtype=dtype), + saturation=torch.tensor([0.6, 1.4], device=device, dtype=dtype), + hue=torch.tensor([-0.1, 0.1], device=device, dtype=dtype), + ).to(device, dtype) + jitter_params = gen(torch.Size([batch_size])) + + expected_jitter_params = { + "brightness_factor": torch.tensor( + [1.1529, 1.1660, 0.9531, 1.1837, 0.9562, 1.0404, 0.9026, 1.1175], + device=device, + dtype=dtype, + ), + "contrast_factor": torch.tensor( + [1.2645, 0.7799, 1.2608, 1.0561, 1.2216, 1.0406, 1.1447, 0.9576], + device=device, + dtype=dtype, + ), + "hue_factor": torch.tensor( + [0.0771, 0.0148, -0.0467, 0.0255, -0.0461, -0.0117, -0.0406, 0.0663], + device=device, + dtype=dtype, + ), + "saturation_factor": torch.tensor( + [0.6843, 0.8156, 0.8871, 0.7595, 1.0378, 0.6049, 1.3612, 0.6602], + device=device, + dtype=dtype, + ), + "order": torch.tensor([3, 2, 0, 1], device=device, dtype=torch.long), + } + + assert set(jitter_params.keys()) == { + "brightness_factor", + "contrast_factor", + "hue_factor", + "saturation_factor", + "order", + }, ( + "Redundant keys found apart from \ + 'brightness_factor', 'contrast_factor', 'hue_factor', 'saturation_factor', 'order'" + ) + + assert_close( + jitter_params["brightness_factor"], + expected_jitter_params["brightness_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["contrast_factor"], + expected_jitter_params["contrast_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["hue_factor"], + expected_jitter_params["hue_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["saturation_factor"], + expected_jitter_params["saturation_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["order"], + expected_jitter_params["order"], + rtol=1e-4, + atol=1e-4, + ) + + def test_same_on_batch(self, device, dtype): + if "cuda" in str(device) or dtype == torch.float64: + pytest.skip("AssertionError: cannot reproduce the same result") + torch.manual_seed(42) + batch_size = 8 + gen = ColorJitterGenerator( + brightness=torch.tensor([0.8, 1.2], device=device, dtype=dtype), + contrast=torch.tensor([0.7, 1.3], device=device, dtype=dtype), + saturation=torch.tensor([0.6, 1.4], device=device, dtype=dtype), + hue=torch.tensor([-0.1, 0.1], device=device, dtype=dtype), + ).to(device, dtype) + jitter_params = gen(torch.Size([batch_size]), same_on_batch=True) + + expected_res = { + "brightness_factor": torch.tensor([1.1529] * batch_size, device=device, dtype=dtype), + "contrast_factor": torch.tensor([1.2490] * batch_size, device=device, dtype=dtype), + "hue_factor": torch.tensor([-0.0234] * batch_size, device=device, dtype=dtype), + "saturation_factor": torch.tensor([1.3674] * batch_size, device=device, dtype=dtype), + "order": torch.tensor([2, 3, 0, 1], device=device, dtype=torch.long), + } + + assert_close( + jitter_params["brightness_factor"], + expected_res["brightness_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["contrast_factor"], + expected_res["contrast_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["hue_factor"], + expected_res["hue_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["saturation_factor"], + expected_res["saturation_factor"], + rtol=1e-4, + atol=1e-4, + ) + assert_close( + jitter_params["order"], + expected_res["order"], + rtol=1e-4, + atol=1e-4, + ) + + +class TestRandomPerspectiveGen(RandomGeneratorBaseTests): + @pytest.mark.parametrize("height,width", [(200, 200)]) + @pytest.mark.parametrize("distortion_scale", [torch.tensor(0.0), torch.tensor(0.5), torch.tensor(1.0)]) + @pytest.mark.parametrize("batch_size", [0, 1, 8]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_valid_param_combinations(self, height, width, distortion_scale, batch_size, same_on_batch, device, dtype): + PerspectiveGenerator(distortion_scale.to(device=device, dtype=dtype))( + torch.Size([batch_size, 1, height, width]), same_on_batch + ) + + @pytest.mark.parametrize( + "height,width,distortion_scale", + [ + # Should be failed if distortion_scale > 1. or distortion_scale < 0. + (-100, 100, torch.tensor(0.5)), + (100, -100, torch.tensor(0.5)), + (100, 100, torch.tensor(-0.5)), + (100, 100, torch.tensor(1.5)), + (100, 100, torch.tensor([0.0, 0.5])), + ], + ) + def test_invalid_param_combinations(self, height, width, distortion_scale, device, dtype): + with pytest.raises(Exception): + PerspectiveGenerator(distortion_scale.to(device=device, dtype=dtype))(torch.Size([8, 1, height, width])) + + def test_random_gen(self, device, dtype): + torch.manual_seed(42) + batch_size = 2 + res = PerspectiveGenerator(torch.tensor(0.5, device=device, dtype=dtype))(torch.Size([batch_size, 1, 200, 200])) + + expected = { + "start_points": torch.tensor( + [ + [[0.0, 0.0], [199.0, 0.0], [199.0, 199.0], [0.0, 199.0]], + [[0.0, 0.0], [199.0, 0.0], [199.0, 199.0], [0.0, 199.0]], + ], + device=device, + dtype=dtype, + ), + "end_points": torch.tensor( + [ + [ + [44.1135, 45.7502], + [179.8568, 47.9653], + [179.4776, 168.9552], + [12.8286, 159.3179], + ], + [ + [47.0386, 6.6593], + [152.2701, 29.6790], + [155.5298, 170.6142], + [37.0547, 177.5298], + ], + ], + device=device, + dtype=dtype, + ), + } + assert res.keys() == expected.keys() + assert_close(res["start_points"], expected["start_points"]) + assert_close(res["end_points"], expected["end_points"]) + + def test_same_on_batch(self, device, dtype): + torch.manual_seed(42) + batch_size = 2 + res = PerspectiveGenerator(torch.tensor(0.5, device=device, dtype=dtype))( + torch.Size([batch_size, 1, 200, 200]), same_on_batch=True + ) + expected = { + "start_points": torch.tensor( + [[[0.0, 0.0], [199.0, 0.0], [199.0, 199.0], [0.0, 199.0]]], + device=device, + dtype=dtype, + ).repeat(2, 1, 1), + "end_points": torch.tensor( + [ + [ + [44.1135, 45.7502], + [179.8568, 47.9653], + [179.4776, 168.9552], + [12.8286, 159.3179], + ] + ], + device=device, + dtype=dtype, + ).repeat(2, 1, 1), + } + assert res.keys() == expected.keys() + assert_close(res["start_points"], expected["start_points"]) + assert_close(res["end_points"], expected["end_points"]) + + def test_sampling_method(self, device, dtype): + torch.manual_seed(42) + batch_size = 2 + res = PerspectiveGenerator( + torch.tensor(0.5, device=device, dtype=dtype), + sampling_method="area_preserving", + )(torch.Size([batch_size, 1, 200, 200])) + + expected = { + "start_points": torch.tensor( + [ + [[0.0, 0.0], [199.0, 0.0], [199.0, 199.0], [0.0, 199.0]], + [[0.0, 0.0], [199.0, 0.0], [199.0, 199.0], [0.0, 199.0]], + ], + device=device, + dtype=dtype, + ), + "end_points": torch.tensor( + [ + [ + [38.2269, 41.5004], + [187.2864, 45.9306], + [188.0448, 209.0895], + [-24.3428, 228.3641], + ], + [ + [44.0771, -36.6814], + [242.4598, 9.3580], + [235.9404, 205.7715], + [24.1094, 191.9404], + ], + ], + device=device, + dtype=dtype, + ), + } + assert res.keys() == expected.keys() + assert_close(res["start_points"], expected["start_points"]) + assert_close(res["end_points"], expected["end_points"]) + + def test_not_implemented_sampling_method(self, device, dtype): + batch_size = 2 + with pytest.raises(NotImplementedError): + PerspectiveGenerator( + torch.tensor(0.5, device=device, dtype=dtype), + sampling_method="non_existing_method", + )(torch.Size([batch_size, 1, 200, 200])) + + +class TestRandomAffineGen(RandomGeneratorBaseTests): + @pytest.mark.parametrize("batch_size", [0, 1, 4]) + @pytest.mark.parametrize("height", [200]) + @pytest.mark.parametrize("width", [300]) + @pytest.mark.parametrize("degrees", [torch.tensor([0, 30])]) + @pytest.mark.parametrize("translate", [None, torch.tensor([0.1, 0.1])]) + @pytest.mark.parametrize("scale", [None, torch.tensor([0.7, 1.2])]) + @pytest.mark.parametrize("shear", [None, torch.tensor([[0, 20], [0, 20]])]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_valid_param_combinations( + self, + batch_size, + height, + width, + degrees, + translate, + scale, + shear, + same_on_batch, + device, + dtype, + ): + AffineGenerator( + degrees=degrees.to(device=device, dtype=dtype), + translate=(translate.to(device=device, dtype=dtype) if translate is not None else None), + scale=scale.to(device=device, dtype=dtype) if scale is not None else None, + shear=shear.to(device=device, dtype=dtype) if shear is not None else None, + )(torch.Size([batch_size, 1, height, width]), same_on_batch) + + @pytest.mark.parametrize( + "height,width,degrees,translate,scale,shear", + [ + (-100, 100, torch.tensor([10, 20]), None, None, None), + (100, -100, torch.tensor([10, 20]), None, None, None), + (100, 100, 0.5, None, None, None), + (100, 100, torch.tensor([10, 20, 30]), None, None, None), + (100, 100, torch.tensor([10, 20]), torch.tensor([0.1]), None, None), + (10, 10, torch.tensor([1, 2]), torch.tensor([0.1, 0.2, 0.3]), None, None), + (100, 100, torch.tensor([10, 20]), None, torch.tensor([1]), None), + (100, 100, torch.tensor([10, 20]), None, torch.tensor([1, 2, 3]), None), + (100, 100, torch.tensor([10, 20]), None, None, torch.tensor([1])), + (10, 10, torch.tensor([1, 2]), None, None, torch.tensor([1, 2, 3])), + (10, 10, torch.tensor([1, 2]), None, None, torch.tensor([1, 2, 3, 4, 5])), + ], + ) + def test_invalid_param_combinations(self, height, width, degrees, translate, scale, shear, device, dtype): + with pytest.raises(Exception): + AffineGenerator( + degrees=degrees.to(device=device, dtype=dtype), + translate=(translate.to(device=device, dtype=dtype) if translate is not None else None), + scale=(scale.to(device=device, dtype=dtype) if scale is not None else None), + shear=(shear.to(device=device, dtype=dtype) if shear is not None else None), + )(torch.Size([8, 1, height, width])) + + def test_random_gen(self, device, dtype): + torch.manual_seed(42) + degrees = torch.tensor([10, 20], device=device, dtype=dtype) + translate = torch.tensor([0.1, 0.1], device=device, dtype=dtype) + scale = torch.tensor([0.7, 1.2], device=device, dtype=dtype) + shear = torch.tensor([[10, 20], [10, 20]], device=device, dtype=dtype) + res = AffineGenerator( + degrees=degrees.to(device=device, dtype=dtype), + translate=(translate.to(device=device, dtype=dtype) if translate is not None else None), + scale=scale.to(device=device, dtype=dtype) if scale is not None else None, + shear=shear.to(device=device, dtype=dtype) if shear is not None else None, + )(torch.Size([2, 1, 200, 200])) + expected = { + "translations": torch.tensor([[-4.3821, -9.7371], [4.0358, 11.7457]], device=device, dtype=dtype), + "center": torch.tensor([[99.5000, 99.5000], [99.5000, 99.5000]], device=device, dtype=dtype), + "scale": torch.tensor([[0.8914, 0.8914], [1.1797, 1.1797]], device=device, dtype=dtype), + "angle": torch.tensor([18.8227, 19.1500], device=device, dtype=dtype), + "shear_x": torch.tensor([19.4077, 11.3319], device=device, dtype=dtype), + "shear_y": torch.tensor([19.3460, 15.9358], device=device, dtype=dtype), + } + assert res.keys() == expected.keys() + assert_close(res["translations"], expected["translations"], rtol=1e-4, atol=1e-4) + assert_close(res["center"], expected["center"], rtol=1e-4, atol=1e-4) + assert_close(res["scale"], expected["scale"], rtol=1e-4, atol=1e-4) + assert_close(res["angle"], expected["angle"], rtol=1e-4, atol=1e-4) + assert_close(res["shear_x"], expected["shear_x"], rtol=1e-4, atol=1e-4) + assert_close(res["shear_y"], expected["shear_y"], rtol=1e-4, atol=1e-4) + + def test_same_on_batch(self, device, dtype): + torch.manual_seed(42) + degrees = torch.tensor([10, 20], device=device, dtype=dtype) + translate = torch.tensor([0.1, 0.1], device=device, dtype=dtype) + scale = torch.tensor([0.7, 1.2], device=device, dtype=dtype) + shear = torch.tensor([[10, 20], [10, 20]], device=device, dtype=dtype) + res = AffineGenerator( + degrees=degrees.to(device=device, dtype=dtype), + translate=(translate.to(device=device, dtype=dtype) if translate is not None else None), + scale=scale.to(device=device, dtype=dtype) if scale is not None else None, + shear=shear.to(device=device, dtype=dtype) if shear is not None else None, + )(torch.Size([2, 1, 200, 200]), True) + expected = { + "translations": torch.tensor([[-4.6854, 18.3722], [-4.6854, 18.3722]], device=device, dtype=dtype), + "center": torch.tensor([[99.5000, 99.5000], [99.5000, 99.5000]], device=device, dtype=dtype), + "scale": torch.tensor([[1.1575, 1.1575], [1.1575, 1.1575]], device=device, dtype=dtype), + "angle": torch.tensor([18.8227, 18.8227], device=device, dtype=dtype), + "shear_x": torch.tensor([13.9045, 13.9045], device=device, dtype=dtype), + "shear_y": torch.tensor([16.0090, 16.0090], device=device, dtype=dtype), + } + assert res.keys() == expected.keys() + assert_close(res["translations"], expected["translations"], rtol=1e-4, atol=1e-4) + assert_close(res["center"], expected["center"], rtol=1e-4, atol=1e-4) + assert_close(res["scale"], expected["scale"], rtol=1e-4, atol=1e-4) + assert_close(res["angle"], expected["angle"], rtol=1e-4, atol=1e-4) + assert_close(res["shear_x"], expected["shear_x"], rtol=1e-4, atol=1e-4) + assert_close(res["shear_y"], expected["shear_y"], rtol=1e-4, atol=1e-4) + + +class TestRandomCropGen(RandomGeneratorBaseTests): + @pytest.mark.parametrize("batch_size", [0, 2]) + @pytest.mark.parametrize("input_size", [(200, 200)]) + @pytest.mark.parametrize("size", [(100, 100), torch.tensor([50, 50])]) + @pytest.mark.parametrize("resize_to", [None, (100, 100)]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_valid_param_combinations(self, batch_size, input_size, size, resize_to, same_on_batch, device, dtype): + if isinstance(size, Tensor): + size = size.repeat(batch_size, 1).to(device=device, dtype=dtype) + CropGenerator(size, resize_to)(torch.Size([batch_size, 1, *input_size]), same_on_batch) + + @pytest.mark.parametrize( + "input_size,size,resize_to", + [ + ((-300, 300), (200, 200), (100, 100)), + ((200, 200), torch.tensor([50, 50]), (100, 100)), + ], + ) + def test_invalid_param_combinations(self, input_size, size, resize_to, device, dtype): + batch_size = 2 + with pytest.raises(Exception): + CropGenerator( + (size.to(device=device, dtype=dtype) if isinstance(size, Tensor) else size), + resize_to, + )(torch.Size([batch_size, 1, *input_size])) + + def test_random_gen(self, device, dtype): + torch.manual_seed(42) + res = CropGenerator(torch.tensor([[50, 60], [70, 80]], device=device, dtype=dtype), (200, 200))( + torch.Size([2, 1, 100, 100]) + ) + expected = { + "src": torch.tensor( + [ + [[36, 19], [95, 19], [95, 68], [36, 68]], + [[19, 29], [98, 29], [98, 98], [19, 98]], + ], + device=device, + dtype=dtype, + ), + "dst": torch.tensor( + [ + [[0, 0], [199, 0], [199, 199], [0, 199]], + [[0, 0], [199, 0], [199, 199], [0, 199]], + ], + device=device, + dtype=dtype, + ), + "input_size": torch.tensor([[100, 100], [100, 100]], device=device, dtype=torch.long), + "output_size": torch.tensor([[200, 200], [200, 200]], device=device, dtype=torch.long), + } + assert res.keys() == expected.keys() + assert_close(res["src"], expected["src"]) + assert_close(res["dst"], expected["dst"]) + + def test_same_on_batch(self, device, dtype): + torch.manual_seed(42) + res = CropGenerator(torch.tensor([[50, 60], [70, 80]], device=device, dtype=dtype), (200, 200))( + torch.Size([2, 1, 100, 100]), True + ) + expected = { + "src": torch.tensor( + [ + [[36, 46], [95, 46], [95, 95], [36, 95]], + [[36, 46], [115, 46], [115, 115], [36, 115]], + ], + device=device, + dtype=dtype, + ), + "dst": torch.tensor( + [ + [[0, 0], [199, 0], [199, 199], [0, 199]], + [[0, 0], [199, 0], [199, 199], [0, 199]], + ], + device=device, + dtype=dtype, + ), + "input_size": torch.tensor([[100, 100], [100, 100]], device=device, dtype=torch.long), + "output_size": torch.tensor([[200, 200], [200, 200]], device=device, dtype=torch.long), + } + assert res.keys() == expected.keys() + assert_close(res["src"], expected["src"]) + assert_close(res["dst"], expected["dst"]) + + +class TestRandomCropSizeGen(RandomGeneratorBaseTests): + @pytest.mark.parametrize("batch_size", [0, 1, 8]) + @pytest.mark.parametrize("size", [(200, 200)]) + @pytest.mark.parametrize("scale", [torch.tensor([0.7, 1.3])]) + @pytest.mark.parametrize("ratio", [torch.tensor([0.9, 1.1])]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_valid_param_combinations(self, batch_size, size, scale, ratio, same_on_batch, device, dtype): + ResizedCropGenerator( + size, + torch.as_tensor(scale, device=device, dtype=dtype), + torch.as_tensor(ratio, device=device, dtype=dtype), + )(torch.Size([batch_size, 1, 300, 300]), same_on_batch) + + @pytest.mark.parametrize( + "size,scale,ratio", + [ + ((100), torch.tensor([0.7, 1.3]), torch.tensor([0.9, 1.1])), + ((100, 100, 100), torch.tensor([0.7, 1.3]), torch.tensor([0.9, 1.1])), + ((100, 100), torch.tensor([0.7]), torch.tensor([0.9, 1.1])), + ((100, 100), torch.tensor([0.7, 1.3, 1.5]), torch.tensor([0.9, 1.1])), + ((100, 100), torch.tensor([0.7, 1.3]), torch.tensor([0.9])), + ((100, 100), torch.tensor([0.7, 1.3]), torch.tensor([0.9, 1.1, 1.3])), + ], + ) + def test_invalid_param_combinations(self, size, scale, ratio, device, dtype): + batch_size = 2 + with pytest.raises(Exception): + ResizedCropGenerator( + size, + torch.as_tensor(scale, device=device, dtype=dtype), + torch.as_tensor(ratio, device=device, dtype=dtype), + )(torch.Size([batch_size, 1, 300, 300])) + + def test_random_gen(self, device, dtype): + torch.manual_seed(42) + res = ResizedCropGenerator( + (100, 100), + torch.tensor([0.7, 1.3], device=device, dtype=dtype), + torch.tensor([0.9, 1.1], device=device, dtype=dtype), + )(torch.Size([2, 1, 300, 300])) + expected = { + "src": torch.tensor( + [ + [[11.0, 3.0], [293.0, 3.0], [293.0, 298.0], [11.0, 298.0]], + [[8.0, 22.0], [286.0, 22.0], [286.0, 298.0], [8.0, 298.0]], + ], + device=device, + dtype=dtype, + ), + "dst": torch.tensor( + [ + [[0.0, 0.0], [99.0, 0.0], [99.0, 99.0], [0.0, 99.0]], + [[0.0, 0.0], [99.0, 0.0], [99.0, 99.0], [0.0, 99.0]], + ], + device=device, + dtype=dtype, + ), + "input_size": torch.tensor([[300, 300], [300, 300]], device=device, dtype=torch.int64), + "output_size": torch.tensor([[100, 100], [100, 100]], device=device, dtype=torch.long), + } + assert res.keys() == expected.keys() + assert_close(res["src"], expected["src"]) + assert_close(res["dst"], expected["dst"]) + assert_close(res["input_size"], expected["input_size"]) + + def test_same_on_batch(self, device, dtype): + torch.manual_seed(42) + res = ResizedCropGenerator( + (100, 100), + torch.tensor([0.7, 1.3], device=device, dtype=dtype), + torch.tensor([0.9, 1.1], device=device, dtype=dtype), + )(torch.Size([2, 1, 300, 300]), same_on_batch=True) + expected = { + "src": torch.tensor( + [ + [[0.0, 9.0], [298.0, 9.0], [298.0, 287.0], [0.0, 287.0]], + [[0.0, 9.0], [298.0, 9.0], [298.0, 287.0], [0.0, 287.0]], + ], + device=device, + dtype=dtype, + ), + "dst": torch.tensor( + [ + [[0.0, 0.0], [99.0, 0.0], [99.0, 99.0], [0.0, 99.0]], + [[0.0, 0.0], [99.0, 0.0], [99.0, 99.0], [0.0, 99.0]], + ], + device=device, + dtype=dtype, + ), + "input_size": torch.tensor([[300, 300], [300, 300]], device=device, dtype=torch.int64), + "output_size": torch.tensor([[100, 100], [100, 100]], device=device, dtype=torch.long), + } + assert res.keys() == expected.keys() + assert_close(res["src"], expected["src"]) + assert_close(res["dst"], expected["dst"]) + assert_close(res["input_size"], expected["input_size"]) + + +class TestRandomRectangleGen(RandomGeneratorBaseTests): + @pytest.mark.parametrize("batch_size", [0, 1, 8]) + @pytest.mark.parametrize("height", [200]) + @pytest.mark.parametrize("width", [300]) + @pytest.mark.parametrize("scale", [torch.tensor([0.7, 1.1])]) + @pytest.mark.parametrize("ratio", [torch.tensor([0.7, 1.1])]) + @pytest.mark.parametrize("value", [0]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_valid_param_combinations( + self, + batch_size, + height, + width, + scale, + ratio, + value, + same_on_batch, + device, + dtype, + ): + RectangleEraseGenerator( + scale=scale.to(device=device, dtype=dtype), + ratio=ratio.to(device=device, dtype=dtype), + value=value, + )(torch.Size([batch_size, 1, height, width]), same_on_batch) + + @pytest.mark.parametrize( + "height,width,scale,ratio,value", + [ + (-100, 100, torch.tensor([0.7, 1.3]), torch.tensor([0.7, 1.3]), 0), + (100, -100, torch.tensor([0.7, 1.3]), torch.tensor([0.7, 1.3]), 0), + (100, -100, torch.tensor([0.7]), torch.tensor([0.7, 1.3]), 0), + (100, 100, torch.tensor([0.7, 1.3, 1.5]), torch.tensor([0.7, 1.3]), 0), + (100, 100, torch.tensor([0.7, 1.3]), torch.tensor([0.7]), 0), + (100, 100, torch.tensor([0.7, 1.3]), torch.tensor([0.7, 1.3, 1.5]), 0), + (100, 100, torch.tensor([0.7, 1.3]), torch.tensor([0.7, 1.3]), -1), + (100, 100, torch.tensor([0.7, 1.3]), torch.tensor([0.7, 1.3]), 2), + ( + 100, + 100, + torch.tensor([0.5, 0.7]), + torch.tensor([0.7, 0.9]), + torch.tensor(0.5), + ), + ], + ) + def test_invalid_param_combinations(self, height, width, scale, ratio, value, device, dtype): + batch_size = 8 + with pytest.raises(Exception): + RectangleEraseGenerator( + scale=scale.to(device=device, dtype=dtype), + ratio=ratio.to(device=device, dtype=dtype), + value=value, + )(torch.Size([batch_size, 1, height, width])) + + def test_random_gen(self, device, dtype): + torch.manual_seed(42) + width, height = 100, 150 + scale = torch.tensor([0.7, 1.3], device=device, dtype=dtype) + ratio = torch.tensor([0.7, 1.3], device=device, dtype=dtype) + value = 0.5 + res = RectangleEraseGenerator( + scale=scale.to(device=device, dtype=dtype), + ratio=ratio.to(device=device, dtype=dtype), + value=value, + )(torch.Size([2, 1, height, width])) + expected = { + "widths": torch.tensor([100, 100], device=device, dtype=dtype), + "heights": torch.tensor([0, 0], device=device, dtype=dtype), + "xs": torch.tensor([0, 0], device=device, dtype=dtype), + "ys": torch.tensor([6, 8], device=device, dtype=dtype), + "values": torch.tensor([0.5000, 0.5000], device=device, dtype=dtype), + } + assert res.keys() == expected.keys() + assert_close(res["widths"], expected["widths"]) + assert_close(res["widths"], expected["widths"]) + assert_close(res["xs"], expected["xs"]) + assert_close(res["ys"], expected["ys"]) + assert_close(res["values"], expected["values"]) + + def test_same_on_batch(self, device, dtype): + torch.manual_seed(42) + width, height = 100, 150 + scale = torch.tensor([0.7, 1.3], device=device, dtype=dtype) + ratio = torch.tensor([0.7, 1.3], device=device, dtype=dtype) + value = 0.5 + res = RectangleEraseGenerator( + scale=scale.to(device=device, dtype=dtype), + ratio=ratio.to(device=device, dtype=dtype), + value=value, + )(torch.Size([2, 1, height, width]), True) + expected = { + "widths": torch.tensor([100, 100], device=device, dtype=dtype), + "heights": torch.tensor([0, 0], device=device, dtype=dtype), + "xs": torch.tensor([0, 0], device=device, dtype=dtype), + "ys": torch.tensor([10, 10], device=device, dtype=dtype), + "values": torch.tensor([0.5000, 0.5000], device=device, dtype=dtype), + } + assert res.keys() == expected.keys() + assert_close(res["widths"], expected["widths"]) + assert_close(res["widths"], expected["widths"]) + assert_close(res["xs"], expected["xs"]) + assert_close(res["ys"], expected["ys"]) + assert_close(res["values"], expected["values"]) + + +class TestCenterCropGen(RandomGeneratorBaseTests): + @pytest.mark.parametrize("batch_size", [0, 2]) + @pytest.mark.parametrize("height", [200]) + @pytest.mark.parametrize("width", [200]) + @pytest.mark.parametrize("size", [(100, 100)]) + def test_valid_param_combinations(self, batch_size, height, width, size, device, dtype): + center_crop_generator(batch_size=batch_size, height=height, width=width, size=size) + + @pytest.mark.parametrize( + "height,width,size", + [ + (200, -200, (100, 100)), + (-200, 200, (100, 100)), + (100, 100, (120, 120)), + (150, 100, (120, 120)), + (100, 150, (120, 120)), + ], + ) + def test_invalid_param_combinations(self, height, width, size, device, dtype): + batch_size = 2 + with pytest.raises(Exception): + center_crop_generator(batch_size=batch_size, height=height, width=width, size=size) + + def test_random_gen(self, device, dtype): + torch.manual_seed(42) + res = center_crop_generator(batch_size=2, height=200, width=200, size=(120, 150)) + expected = { + "src": torch.tensor( + [ + [[25, 40], [174, 40], [174, 159], [25, 159]], + [[25, 40], [174, 40], [174, 159], [25, 159]], + ], + device=device, + dtype=torch.long, + ), + "dst": torch.tensor( + [ + [[0, 0], [149, 0], [149, 119], [0, 119]], + [[0, 0], [149, 0], [149, 119], [0, 119]], + ], + device=device, + dtype=torch.long, + ), + "input_size": torch.tensor([[200, 200], [200, 200]], device=device, dtype=torch.long), + "output_size": torch.tensor([[120, 150], [120, 150]], device=device, dtype=torch.long), + } + assert res.keys() == expected.keys() + assert_close(res["src"].to(device=device), expected["src"]) + assert_close(res["dst"].to(device=device), expected["dst"]) + + def test_same_on_batch(self, device, dtype): + pass + + +class TestRandomMotionBlur(RandomGeneratorBaseTests): + @pytest.mark.parametrize("batch_size", [0, 1, 8]) + @pytest.mark.parametrize("kernel_size", [3, (3, 5)]) + @pytest.mark.parametrize("angle", [torch.tensor([10, 30])]) + @pytest.mark.parametrize("direction", [torch.tensor([-1, -1]), torch.tensor([1, 1])]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_valid_param_combinations(self, batch_size, kernel_size, angle, direction, same_on_batch, device, dtype): + MotionBlurGenerator( + kernel_size=kernel_size, + angle=angle.to(device=device, dtype=dtype), + direction=direction.to(device=device, dtype=dtype), + )(torch.Size([batch_size]), same_on_batch) + + @pytest.mark.parametrize( + "kernel_size,angle,direction", + [ + (4, torch.tensor([30, 100]), torch.tensor([-1, 1])), + (1, torch.tensor([30, 100]), torch.tensor([-1, 1])), + ((1, 2, 3), torch.tensor([30, 100]), torch.tensor([-1, 1])), + (3, torch.tensor([30, 100]), torch.tensor([-2, 1])), + (3, torch.tensor([30, 100]), torch.tensor([-1, 2])), + ], + ) + def test_invalid_param_combinations(self, kernel_size, angle, direction, device, dtype): + with pytest.raises(Exception): + MotionBlurGenerator( + kernel_size=kernel_size, + angle=angle.to(device=device, dtype=dtype), + direction=direction.to(device=device, dtype=dtype), + )(torch.Size([8])) + + def test_random_gen(self, device, dtype): + torch.manual_seed(42) + angle = torch.tensor([30, 90]) + direction = torch.tensor([-1, 1]) + res = MotionBlurGenerator( + kernel_size=3, + angle=angle.to(device=device, dtype=dtype), + direction=direction.to(device=device, dtype=dtype), + )(torch.Size([2])) + expected = { + "ksize_factor": torch.tensor([3, 3], device=device, dtype=torch.int32), + "angle_factor": torch.tensor([82.9362, 84.9002], device=device, dtype=dtype), + "direction_factor": torch.tensor([-0.2343, 0.9186], device=device, dtype=dtype), + } + assert res.keys() == expected.keys() + assert_close(res["ksize_factor"], expected["ksize_factor"], rtol=1e-4, atol=1e-4) + assert_close(res["angle_factor"], expected["angle_factor"], rtol=1e-4, atol=1e-4) + assert_close(res["direction_factor"], expected["direction_factor"], rtol=1e-4, atol=1e-4) + + def test_same_on_batch(self, device, dtype): + torch.manual_seed(42) + angle = torch.tensor([30, 90]) + direction = torch.tensor([-1, 1]) + res = MotionBlurGenerator( + kernel_size=3, + angle=angle.to(device=device, dtype=dtype), + direction=direction.to(device=device, dtype=dtype), + )(torch.Size([2]), True) + expected = { + "ksize_factor": torch.tensor([3, 3], device=device, dtype=torch.int32), + "angle_factor": torch.tensor([82.9362, 82.9362], device=device, dtype=dtype), + "direction_factor": torch.tensor([0.8300, 0.8300], device=device, dtype=dtype), + } + assert res.keys() == expected.keys() + assert_close(res["ksize_factor"], expected["ksize_factor"], rtol=1e-4, atol=1e-4) + assert_close(res["angle_factor"], expected["angle_factor"], rtol=1e-4, atol=1e-4) + assert_close(res["direction_factor"], expected["direction_factor"], rtol=1e-4, atol=1e-4) + + +class TestRandomPosterizeGen(RandomGeneratorBaseTests): + @pytest.mark.parametrize("batch_size", [0, 1, 8]) + @pytest.mark.parametrize("bits", [torch.tensor([0, 8])]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_valid_param_combinations(self, batch_size, bits, same_on_batch, device, dtype): + PosterizeGenerator(bits)(torch.Size([batch_size]), same_on_batch) + + @pytest.mark.parametrize( + "bits", + [ + (torch.tensor([-1, 1])), + (torch.tensor([0, 9])), + (torch.tensor([3])), + ([0, 8],), + ], + ) + def test_invalid_param_combinations(self, bits, device, dtype): + with pytest.raises(Exception): + if isinstance(bits, Tensor): + PosterizeGenerator(bits.to(device=device, dtype=dtype))(torch.Size([3])) + else: + PosterizeGenerator(bits)(torch.Size([3])) + + def test_random_gen(self, device, dtype): + torch.manual_seed(9) + batch_size = 8 + res = PosterizeGenerator(bits=torch.tensor([0, 8], device=device, dtype=dtype))(torch.Size([batch_size])) + expected = {"bits_factor": torch.tensor([5, 2, 4, 6, 7, 7, 2, 8], device=device, dtype=torch.int32)} + assert res.keys() == expected.keys() + assert_close(res["bits_factor"], expected["bits_factor"], rtol=1e-4, atol=1e-4) + + def test_same_on_batch(self, device, dtype): + torch.manual_seed(9) + batch_size = 8 + res = PosterizeGenerator(bits=torch.tensor([0, 8], device=device, dtype=dtype))( + torch.Size([batch_size]), same_on_batch=True + ) + expected = {"bits_factor": torch.tensor([5, 5, 5, 5, 5, 5, 5, 5], device=device, dtype=torch.int32)} + assert res.keys() == expected.keys() + assert_close(res["bits_factor"], expected["bits_factor"], rtol=1e-4, atol=1e-4) + + +class TestPlainUniformGenerator(RandomGeneratorBaseTests): + @pytest.mark.parametrize("batch_size", [0, 1, 8]) + @pytest.mark.parametrize("sharpness", [torch.tensor([0.0, 1.0])]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_valid_param_combinations(self, batch_size, sharpness, same_on_batch, device, dtype): + PlainUniformGenerator( + (sharpness.to(device=device, dtype=dtype), "sharpness", None, None), + (sharpness.to(device=device, dtype=dtype), "sharpness_xx", 0.0, (0.0, 1.0)), + )(torch.Size([batch_size, 1]), same_on_batch) + + @pytest.mark.parametrize("sharpness", [(torch.tensor([-1, 5])), (torch.tensor([3])), ([0, 1.0])]) + def test_invalid_param_combinations(self, sharpness, device, dtype): + with pytest.raises(Exception): + PlainUniformGenerator( + (sharpness.to(device=device, dtype=dtype), "sharpness", None, None), + ( + sharpness.to(device=device, dtype=dtype), + "sharpness", + 0.0, + (0.0, 1.0), + ), + )(torch.Size([8, 1])) + + def test_random_gen(self, device, dtype): + torch.manual_seed(42) + batch_size = 8 + res = PlainUniformGenerator( + ( + torch.tensor([0.0, 1.0], device=device, dtype=dtype), + "sharpness_factor", + None, + None, + ) + )(torch.Size([batch_size, 1])) + expected = { + "sharpness_factor": torch.tensor( + [0.8823, 0.9150, 0.3829, 0.9593, 0.3904, 0.6009, 0.2566, 0.7936], + device=device, + dtype=dtype, + ) + } + assert res.keys() == expected.keys() + assert_close(res["sharpness_factor"], expected["sharpness_factor"], rtol=1e-4, atol=1e-4) + + def test_same_on_batch(self, device, dtype): + torch.manual_seed(42) + batch_size = 8 + res = PlainUniformGenerator( + ( + torch.tensor([0.0, 1.0], device=device, dtype=dtype), + "sharpness_factor", + None, + None, + ) + )(torch.Size([batch_size, 1]), True) + expected = { + "sharpness_factor": torch.tensor( + [0.8823, 0.8823, 0.8823, 0.8823, 0.8823, 0.8823, 0.8823, 0.8823], + device=device, + dtype=dtype, + ) + } + assert res.keys() == expected.keys() + assert_close(res["sharpness_factor"], expected["sharpness_factor"], rtol=1e-4, atol=1e-4) + + +class TestRandomMixUpGen(RandomGeneratorBaseTests): + @pytest.mark.parametrize("batch_size", [0, 1, 8]) + @pytest.mark.parametrize("p", [0.0, 0.5, 1.0]) + @pytest.mark.parametrize("lambda_val", [None, torch.tensor([0.0, 1.0])]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_valid_param_combinations(self, batch_size, p, lambda_val, same_on_batch, device, dtype): + MixupGenerator( + p=p, + lambda_val=(lambda_val.to(device=device, dtype=dtype) if isinstance(lambda_val, (Tensor)) else lambda_val), + )(torch.Size([batch_size, 3, 200, 200]), same_on_batch=same_on_batch) + + @pytest.mark.parametrize( + "lambda_val", + [ + (torch.tensor([-1, 1])), + (torch.tensor([0, 2])), + (torch.tensor([0, 0.5, 1])), + ([0.0, 1.0]), + ], + ) + def test_invalid_param_combinations(self, lambda_val, device, dtype): + with pytest.raises(Exception): + MixupGenerator(p=1.0, lambda_val=lambda_val.to(device=device, dtype=dtype))( + torch.Size([8, 3, 200, 200]), same_on_batch=False + ) + + def test_random_gen(self, device, dtype): + if device.type != "cpu": + pytest.skip( + "Random number sequences differ between CPU and non-CPU devices; expected values computed on CPU" + ) + torch.manual_seed(42) + batch_size = 8 + res = MixupGenerator(p=0.5, lambda_val=torch.tensor([0.0, 1.0], device=device, dtype=dtype))( + torch.Size([batch_size, 3, 200, 200]), same_on_batch=False + ) + expected = { + "mixup_pairs": torch.tensor([6, 1, 0, 7, 2, 5, 3, 4], device=device, dtype=torch.long), + "mixup_lambdas": torch.tensor( + [0.0000, 0.0000, 0.5739, 0.0000, 0.6274, 0.0000, 0.4414, 0.0000], + device=device, + dtype=dtype, + ), + } + assert res.keys() == expected.keys() + assert_close(res["mixup_pairs"], expected["mixup_pairs"], rtol=1e-4, atol=1e-4) + assert_close(res["mixup_lambdas"], expected["mixup_lambdas"], rtol=1e-4, atol=1e-4) + + def test_same_on_batch(self, device, dtype): + if device.type != "cpu": + pytest.skip( + "Random number sequences differ between CPU and non-CPU devices; expected values computed on CPU" + ) + torch.manual_seed(9) + batch_size = 8 + res = MixupGenerator(p=0.999999, lambda_val=torch.tensor([0.0, 1.0], device=device, dtype=dtype))( + torch.Size([batch_size, 3, 200, 200]), same_on_batch=True + ) + expected = { + "mixup_pairs": torch.tensor([4, 6, 7, 5, 0, 1, 3, 2], device=device, dtype=torch.long), + "mixup_lambdas": torch.tensor( + [0.3804, 0.3804, 0.3804, 0.3804, 0.3804, 0.3804, 0.3804, 0.3804], + device=device, + dtype=dtype, + ), + } + assert res.keys() == expected.keys() + assert_close(res["mixup_pairs"], expected["mixup_pairs"], rtol=1e-4, atol=1e-4) + assert_close(res["mixup_lambdas"], expected["mixup_lambdas"], rtol=1e-4, atol=1e-4) + + +class TestRandomCutMixGen(RandomGeneratorBaseTests): + @pytest.mark.parametrize("batch_size", [0, 1, 8]) + @pytest.mark.parametrize("p", [0, 0.5, 1.0]) + @pytest.mark.parametrize("width,height", [(200, 200)]) + @pytest.mark.parametrize("num_mix", [1, 3]) + @pytest.mark.parametrize("beta", [None, torch.tensor(1e-15), torch.tensor(1.0)]) + @pytest.mark.parametrize("cut_size", [None, torch.tensor([0.0, 1.0]), torch.tensor([0.3, 0.6])]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_valid_param_combinations( + self, + batch_size, + p, + width, + height, + num_mix, + beta, + cut_size, + same_on_batch, + device, + dtype, + ): + CutmixGenerator( + p=p, + num_mix=num_mix, + beta=(beta.to(device=device, dtype=dtype) if isinstance(beta, (Tensor)) else beta), + cut_size=(cut_size.to(device=device, dtype=dtype) if isinstance(cut_size, (Tensor)) else cut_size), + )(torch.Size([batch_size, 3, height, width]), same_on_batch=same_on_batch) + + @pytest.mark.parametrize( + "width,height,num_mix,beta,cut_size", + [ + (200, -200, 1, None, None), + (-200, 200, 1, None, None), + (200, 200, 0, None, None), + (200, 200, 1.5, None, None), + (200, 200, 1, torch.tensor([0.0, 1.0]), None), + (200, 200, 1, None, torch.tensor([-1.0, 1.0])), + (200, 200, 1, None, torch.tensor([0.0, 2.0])), + ], + ) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_invalid_param_combinations(self, width, height, num_mix, beta, cut_size, same_on_batch, device, dtype): + with pytest.raises(Exception): + CutmixGenerator( + p=0.5, + num_mix=num_mix, + beta=(beta.to(device=device, dtype=dtype) if isinstance(beta, (Tensor)) else beta), + cut_size=(cut_size.to(device=device, dtype=dtype) if isinstance(cut_size, (Tensor)) else cut_size), + )(torch.Size([8, 3, height, width]), same_on_batch=same_on_batch) + + def test_random_gen(self, device, dtype): + torch.manual_seed(42) + image_shape = torch.Size([8, 3, 200, 200]) + res = CutmixGenerator( + p=0.5, + num_mix=1, + beta=torch.tensor(1.0, device=device, dtype=dtype), + cut_size=torch.tensor([0.0, 1.0], device=device, dtype=dtype), + )(image_shape, same_on_batch=False) + + expected = { + "mix_pairs": torch.tensor([[1, 7, 5, 3, 6, 4, 2, 0]], device=device, dtype=torch.long), + "crop_src": torch.tensor( + [ + [ + [[48.0, 31.0], [47.0, 31.0], [47.0, 30.0], [48.0, 30.0]], + [[48.0, 31.0], [47.0, 31.0], [47.0, 30.0], [48.0, 30.0]], + [[17.0, 11.0], [141.0, 11.0], [141.0, 135.0], [17.0, 135.0]], + [[48.0, 31.0], [47.0, 31.0], [47.0, 30.0], [48.0, 30.0]], + [[16.0, 10.0], [147.0, 10.0], [147.0, 141.0], [16.0, 141.0]], + [[48.0, 31.0], [47.0, 31.0], [47.0, 30.0], [48.0, 30.0]], + [[8.0, 5.0], [171.0, 5.0], [171.0, 168.0], [8.0, 168.0]], + [[48.0, 31.0], [47.0, 31.0], [47.0, 30.0], [48.0, 30.0]], + ] + ], + device=device, + dtype=dtype, + ), + "image_shape": image_shape, + } + assert res.keys() == expected.keys(), res.keys() + assert_close(res["mix_pairs"], expected["mix_pairs"], rtol=1e-4, atol=1e-4) + assert_close(res["crop_src"], expected["crop_src"], rtol=1e-4, atol=1e-4) + + def test_same_on_batch(self, device, dtype): + torch.manual_seed(42) + image_shape = torch.Size([2, 3, 200, 200]) + res = CutmixGenerator( + p=0.5, + num_mix=1, + beta=torch.tensor(1.0, device=device, dtype=dtype), + cut_size=torch.tensor([0.0, 1.0], device=device, dtype=dtype), + )(image_shape, same_on_batch=True) + expected = { + "mix_pairs": torch.tensor([[1, 0]], device=device, dtype=torch.long), + "crop_src": torch.tensor( + [ + [ + [[114, 53], [113, 53], [113, 52], [114, 52]], + [[114, 53], [113, 53], [113, 52], [114, 52]], + ] + ], + device=device, + dtype=dtype, + ), + "image_shape": image_shape, + } + assert res.keys() == expected.keys(), res.keys() + assert_close(res["mix_pairs"], expected["mix_pairs"], rtol=1e-4, atol=1e-4) + assert_close(res["crop_src"], expected["crop_src"], rtol=1e-4, atol=1e-4) diff --git a/tests/augmentation/test_random_generator_3d.py b/tests/augmentation/test_random_generator_3d.py new file mode 100644 index 0000000..589b0f5 --- /dev/null +++ b/tests/augmentation/test_random_generator_3d.py @@ -0,0 +1,693 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.augmentation.random_generator import ( + AffineGenerator3D, + CropGenerator3D, + MotionBlurGenerator3D, + PerspectiveGenerator3D, + RotationGenerator3D, + center_crop_generator3d, +) + +from testing.base import assert_close + + +class RandomGeneratorBaseTests: + def test_valid_param_combinations(self, device, dtype): + raise NotImplementedError + + def test_invalid_param_combinations(self, device, dtype): + raise NotImplementedError + + def test_random_gen(self, device, dtype): + raise NotImplementedError + + def test_same_on_batch(self, device, dtype): + raise NotImplementedError + + +class TestRandomPerspectiveGen3D(RandomGeneratorBaseTests): + @pytest.mark.parametrize("batch_size", [0, 1, 8]) + @pytest.mark.parametrize("depth,height,width", [(200, 200, 200)]) + @pytest.mark.parametrize("distortion_scale", [torch.tensor(0.0), torch.tensor(0.5), torch.tensor(1.0)]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_valid_param_combinations( + self, depth, height, width, distortion_scale, batch_size, same_on_batch, device, dtype + ): + param_gen = PerspectiveGenerator3D(distortion_scale=distortion_scale.to(device=device, dtype=dtype)) + param_gen(batch_shape=torch.Size((batch_size, depth, height, width)), same_on_batch=same_on_batch) + + @pytest.mark.parametrize( + "depth,height,width,distortion_scale", + [ + # Should be failed if distortion_scale > 1. or distortion_scale < 0. + (100, 100, -100, torch.tensor(-0.5)), + (100, 100, 100, torch.tensor(1.5)), + (100, 100, 100, torch.tensor([0.0, 0.5])), + ], + ) + def test_invalid_param_combinations(self, depth, height, width, distortion_scale, device, dtype): + with pytest.raises(Exception): + param_gen = PerspectiveGenerator3D(distortion_scale=distortion_scale.to(device=device, dtype=dtype)) + param_gen(batch_shape=torch.Size((2, depth, height, width))) + + def test_random_gen(self, device, dtype): + torch.manual_seed(42) + batch_size = 2 + + param_gen = PerspectiveGenerator3D(distortion_scale=torch.tensor(0.5, device=device, dtype=dtype)) + res = param_gen(batch_shape=torch.Size((batch_size, 200, 200, 200))) + + expected = { + "start_points": torch.tensor( + [ + [ + [0.0, 0.0, 0.0], + [199.0, 0.0, 0.0], + [199.0, 199.0, 0.0], + [0.0, 199.0, 0.0], + [0.0, 0.0, 199.0], + [199.0, 0.0, 199.0], + [199.0, 199.0, 199.0], + [0.0, 199.0, 199.0], + ], + [ + [0.0, 0.0, 0.0], + [199.0, 0.0, 0.0], + [199.0, 199.0, 0.0], + [0.0, 199.0, 0.0], + [0.0, 0.0, 199.0], + [199.0, 0.0, 199.0], + [199.0, 199.0, 199.0], + [0.0, 199.0, 199.0], + ], + ], + device=device, + dtype=dtype, + ), + "end_points": torch.tensor( + [ + [ + [44.1135, 45.7502, 19.1432], + [151.0347, 19.5224, 30.0448], + [186.1714, 159.3179, 47.0386], + [6.6593, 152.2701, 29.6790], + [43.4702, 28.3858, 161.9453], + [177.5298, 44.2721, 170.3048], + [185.6710, 167.6275, 185.5184], + [22.0682, 184.1540, 157.4157], + ], + [ + [5.2657, 13.4747, 17.9406], + [189.0318, 27.3596, 0.3080], + [151.4223, 195.2367, 44.3007], + [29.1605, 182.1176, 40.4487], + [28.8963, 45.1991, 171.2670], + [181.8843, 31.7171, 180.7795], + [163.4786, 151.6794, 159.5485], + [14.0707, 159.5684, 169.5268], + ], + ], + device=device, + dtype=dtype, + ), + } + assert res.keys() == expected.keys() + assert_close(res["start_points"], expected["start_points"], atol=1e-4, rtol=1e-4) + assert_close(res["end_points"], expected["end_points"], atol=1e-4, rtol=1e-4) + + def test_same_on_batch(self, device, dtype): + torch.manual_seed(42) + batch_size = 2 + + param_gen = PerspectiveGenerator3D(distortion_scale=torch.tensor(0.5, device=device, dtype=dtype)) + res = param_gen(batch_shape=torch.Size((batch_size, 200, 200, 200)), same_on_batch=True) + + expected = { + "start_points": torch.tensor( + [ + [ + [0.0, 0.0, 0.0], + [199.0, 0.0, 0.0], + [199.0, 199.0, 0.0], + [0.0, 199.0, 0.0], + [0.0, 0.0, 199.0], + [199.0, 0.0, 199.0], + [199.0, 199.0, 199.0], + [0.0, 199.0, 199.0], + ], + [ + [0.0, 0.0, 0.0], + [199.0, 0.0, 0.0], + [199.0, 199.0, 0.0], + [0.0, 199.0, 0.0], + [0.0, 0.0, 199.0], + [199.0, 0.0, 199.0], + [199.0, 199.0, 199.0], + [0.0, 199.0, 199.0], + ], + ], + device=device, + dtype=dtype, + ), + "end_points": torch.tensor( + [ + [ + [44.1135, 45.7502, 19.1432], + [151.0347, 19.5224, 30.0448], + [186.1714, 159.3179, 47.0386], + [6.6593, 152.2701, 29.6790], + [43.4702, 28.3858, 161.9453], + [177.5298, 44.2721, 170.3048], + [185.6710, 167.6275, 185.5184], + [22.0682, 184.1540, 157.4157], + ], + [ + [44.1135, 45.7502, 19.1432], + [151.0347, 19.5224, 30.0448], + [186.1714, 159.3179, 47.0386], + [6.6593, 152.2701, 29.6790], + [43.4702, 28.3858, 161.9453], + [177.5298, 44.2721, 170.3048], + [185.6710, 167.6275, 185.5184], + [22.0682, 184.1540, 157.4157], + ], + ], + device=device, + dtype=dtype, + ), + } + assert res.keys() == expected.keys() + assert_close(res["start_points"], expected["start_points"], atol=1e-4, rtol=1e-4) + assert_close(res["end_points"], expected["end_points"], atol=1e-4, rtol=1e-4) + + +class TestRandomAffineGen3D(RandomGeneratorBaseTests): + @pytest.mark.parametrize("batch_shape", [(0, 200, 300, 400), (1, 200, 300, 400), (8, 200, 300, 400)]) + @pytest.mark.parametrize("degrees", [torch.tensor([(0.0, 30.0), (0.0, 30.0), (0.0, 30.0)])]) + @pytest.mark.parametrize("translate", [None, torch.tensor([0.1, 0.1, 0.1])]) + @pytest.mark.parametrize( + "scale", [None, torch.tensor([[0.7, 1.2], [0.7, 1.2], [0.7, 1.2]]), torch.tensor([0.7, 1.2])] + ) + @pytest.mark.parametrize( + "shear", [None, torch.tensor([[0.0, 20.0], [0.0, 20.0], [0.0, 20.0], [0.0, 20.0], [0.0, 20.0], [0.0, 20.0]])] + ) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_valid_param_combinations( + self, batch_shape, degrees, translate, scale, shear, same_on_batch, device, dtype + ): + if isinstance(degrees, torch.Tensor): + degrees.to(dtype=dtype, device=device) + if isinstance(translate, torch.Tensor): + translate.to(dtype=dtype, device=device) + if isinstance(scale, torch.Tensor): + scale.to(dtype=dtype, device=device) + if isinstance(shear, torch.Tensor): + shear.to(dtype=dtype, device=device) + + param_gen = AffineGenerator3D(degrees=degrees, translate=translate, scale=scale, shears=shear) + param_gen(batch_shape=torch.Size(batch_shape), same_on_batch=same_on_batch) + + @pytest.mark.parametrize( + "depth,height,width,degrees,translate,scale,shear", + [ + (-100, 100, 100, torch.tensor([[0, 9], [0, 9], [0, 9]]), None, None, None), + (100, -100, 100, torch.tensor([[0, 9], [0, 9], [0, 9]]), None, None, None), + (100, 100, -100, torch.tensor([[0, 9], [0, 9], [0, 9]]), None, None, None), + # (100, 100, 100, torch.tensor([0, 9]), None, None, None), + (100, 100, 100, torch.tensor([[0, 9], [0, 9], [0, 9]]), torch.tensor([0.1, 0.2]), None, None), + (100, 100, 100, torch.tensor([[0, 9], [0, 9], [0, 9]]), torch.tensor([0.1, 0.2]), None, None), + (100, 100, 100, torch.tensor([[0, 9], [0, 9], [0, 9]]), torch.tensor([0.1]), None, None), + (100, 100, 100, torch.tensor([[0, 9], [0, 9], [0, 9]]), None, torch.tensor([[0.2, 0.2, 0.2]]), None), + (100, 100, 100, torch.tensor([[0, 9], [0, 9], [0, 9]]), None, torch.tensor([0.2]), None), + (100, 100, 100, torch.tensor([[0, 9], [0, 9], [0, 9]]), None, None, torch.tensor([[20, 20, 30]])), + # (100, 100, 100, torch.tensor([[0, 9], [0, 9], [0, 9]]), None, None, torch.tensor([20])), + ], + ) + def test_invalid_param_combinations(self, depth, height, width, degrees, translate, scale, shear, device, dtype): + if isinstance(degrees, torch.Tensor): + degrees.to(dtype=dtype, device=device) + if isinstance(translate, torch.Tensor): + translate.to(dtype=dtype, device=device) + if isinstance(scale, torch.Tensor): + scale.to(dtype=dtype, device=device) + if isinstance(shear, torch.Tensor): + shear.to(dtype=dtype, device=device) + + with pytest.raises(Exception): + param_gen = AffineGenerator3D(degrees=degrees, translate=translate, scale=scale, shears=shear) + param_gen(batch_shape=torch.Size((2, depth, height, width))) + + def test_random_gen(self, device, dtype): + torch.manual_seed(42) + degrees = torch.tensor([[10, 20], [10, 20], [10, 20]], device=device, dtype=dtype) + translate = torch.tensor([0.1, 0.1, 0.1], device=device, dtype=dtype) + scale = torch.tensor([[0.7, 1.2], [0.7, 1.2], [0.7, 1.2]], device=device, dtype=dtype) + shear = torch.tensor([[0, 20], [0, 20], [0, 20], [0, 20], [0, 20], [0, 20]], device=device, dtype=dtype) + + param_gen = AffineGenerator3D(degrees=degrees, translate=translate, scale=scale, shears=shear) + res = param_gen(batch_shape=torch.Size((2, 200, 200, 200))) + + expected = { + "translations": torch.tensor( + [[14.7762, 9.6438, 15.4177], [2.7086, -2.8238, 2.9562]], device=device, dtype=dtype + ), + "center": torch.tensor( + [[99.5000, 99.5000, 99.5000], [99.5000, 99.5000, 99.5000]], device=device, dtype=dtype + ), + "scale": torch.tensor([[0.8283, 1.1704, 1.1673], [1.0968, 0.7666, 0.9968]], device=device, dtype=dtype), + "angles": torch.tensor( + [[18.8227, 13.8286, 13.9045], [19.1500, 19.5931, 16.0090]], device=device, dtype=dtype + ), + "sxy": torch.tensor([5.3316, 12.5490], device=device, dtype=dtype), + "sxz": torch.tensor([5.3926, 8.8273], device=device, dtype=dtype), + "syx": torch.tensor([5.9384, 16.6337], device=device, dtype=dtype), + "syz": torch.tensor([2.1063, 5.3899], device=device, dtype=dtype), + "szx": torch.tensor([7.1763, 3.9873], device=device, dtype=dtype), + "szy": torch.tensor([10.9438, 0.1232], device=device, dtype=dtype), + } + assert res.keys() == expected.keys() + assert_close(res["translations"], expected["translations"], rtol=1e-4, atol=1e-4) + assert_close(res["center"], expected["center"], rtol=1e-4, atol=1e-4) + assert_close(res["scale"], expected["scale"], rtol=1e-4, atol=1e-4) + assert_close(res["angles"], expected["angles"], rtol=1e-4, atol=1e-4) + assert_close(res["sxy"], expected["sxy"], rtol=1e-4, atol=1e-4) + assert_close(res["sxz"], expected["sxz"], rtol=1e-4, atol=1e-4) + assert_close(res["syx"], expected["syx"], rtol=1e-4, atol=1e-4) + assert_close(res["syz"], expected["syz"], rtol=1e-4, atol=1e-4) + assert_close(res["szx"], expected["szx"], rtol=1e-4, atol=1e-4) + assert_close(res["szy"], expected["szy"], rtol=1e-4, atol=1e-4) + + def test_same_on_batch(self, device, dtype): + torch.manual_seed(42) + degrees = torch.tensor([[10, 20], [10, 20], [10, 20]], device=device, dtype=dtype) + translate = torch.tensor([0.1, 0.1, 0.1], device=device, dtype=dtype) + scale = torch.tensor([[0.7, 1.2], [0.7, 1.2], [0.7, 1.2]], device=device, dtype=dtype) + shear = torch.tensor([[0, 20], [0, 20], [0, 20], [0, 20], [0, 20], [0, 20]], device=device, dtype=dtype) + + param_gen = AffineGenerator3D(degrees=degrees, translate=translate, scale=scale, shears=shear) + res = param_gen(batch_shape=torch.Size((2, 200, 200, 200)), same_on_batch=True) + + expected = { + "translations": torch.tensor( + [[-9.7371, 11.7457, 17.6309], [-9.7371, 11.7457, 17.6309]], device=device, dtype=dtype + ), + "center": torch.tensor( + [[99.5000, 99.5000, 99.5000], [99.5000, 99.5000, 99.5000]], device=device, dtype=dtype + ), + "scale": torch.tensor([[1.1797, 0.8952, 1.0004], [1.1797, 0.8952, 1.0004]], device=device, dtype=dtype), + "angles": torch.tensor( + [[18.8227, 19.1500, 13.8286], [18.8227, 19.1500, 13.8286]], device=device, dtype=dtype + ), + "sxy": torch.tensor([2.6637, 2.6637], device=device, dtype=dtype), + "sxz": torch.tensor([18.6920, 18.6920], device=device, dtype=dtype), + "syx": torch.tensor([11.8716, 11.8716], device=device, dtype=dtype), + "syz": torch.tensor([17.3881, 17.3881], device=device, dtype=dtype), + "szx": torch.tensor([11.3543, 11.3543], device=device, dtype=dtype), + "szy": torch.tensor([14.8219, 14.8219], device=device, dtype=dtype), + } + assert res.keys() == expected.keys() + assert_close(res["translations"], expected["translations"], rtol=1e-4, atol=1e-4) + assert_close(res["center"], expected["center"], rtol=1e-4, atol=1e-4) + assert_close(res["scale"], expected["scale"], rtol=1e-4, atol=1e-4) + assert_close(res["angles"], expected["angles"], rtol=1e-4, atol=1e-4) + assert_close(res["sxy"], expected["sxy"], rtol=1e-4, atol=1e-4) + assert_close(res["sxz"], expected["sxz"], rtol=1e-4, atol=1e-4) + assert_close(res["syx"], expected["syx"], rtol=1e-4, atol=1e-4) + assert_close(res["syz"], expected["syz"], rtol=1e-4, atol=1e-4) + assert_close(res["szx"], expected["szx"], rtol=1e-4, atol=1e-4) + assert_close(res["szy"], expected["szy"], rtol=1e-4, atol=1e-4) + + +class TestRandomRotationGen3D(RandomGeneratorBaseTests): + @pytest.mark.parametrize("batch_size", [0, 1, 8]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_valid_param_combinations(self, batch_size, same_on_batch, device, dtype): + degrees = torch.tensor([[0.0, 30.0], [0.0, 30.0], [0.0, 30.0]], device=device, dtype=dtype) + param_gen = RotationGenerator3D(degrees=degrees.to(device=device, dtype=dtype)) + param_gen(torch.Size((batch_size,)), same_on_batch=same_on_batch) + + @pytest.mark.parametrize( + "degrees", + [(torch.tensor(-10)), (torch.tensor([-10])), (torch.tensor([[0, 30]])), (torch.tensor([[0, 30], [0, 30]]))], + ) + def test_invalid_param_combinations(self, degrees, device, dtype): + with pytest.raises(Exception): + param_gen = RotationGenerator3D(degrees=degrees.to(device=device, dtype=dtype)) + param_gen(torch.Size((2,))) + + def test_random_gen(self, device, dtype): + torch.manual_seed(42) + degrees = torch.tensor([[0.0, 30.0], [0.0, 30.0], [0.0, 30.0]], device=device, dtype=dtype) + param_gen = RotationGenerator3D(degrees=degrees) + res = param_gen(torch.Size((2,)), same_on_batch=False) + + expected = { + "yaw": torch.tensor([26.4681, 27.4501], device=device, dtype=dtype), + "pitch": torch.tensor([11.4859, 28.7792], device=device, dtype=dtype), + "roll": torch.tensor([11.7134, 18.0269], device=device, dtype=dtype), + } + assert res.keys() == expected.keys() + assert_close(res["yaw"], expected["yaw"], atol=1e-4, rtol=1e-4) + assert_close(res["pitch"], expected["pitch"], atol=1e-4, rtol=1e-4) + assert_close(res["roll"], expected["roll"], atol=1e-4, rtol=1e-4) + + def test_same_on_batch(self, device, dtype): + torch.manual_seed(42) + degrees = torch.tensor([[0.0, 30.0], [0.0, 30.0], [0.0, 30.0]], device=device, dtype=dtype) + param_gen = RotationGenerator3D(degrees=degrees) + res = param_gen(torch.Size((2,)), same_on_batch=True) + + expected = { + "yaw": torch.tensor([26.4681, 26.4681], device=device, dtype=dtype), + "pitch": torch.tensor([27.4501, 27.4501], device=device, dtype=dtype), + "roll": torch.tensor([11.4859, 11.4859], device=device, dtype=dtype), + } + assert res.keys() == expected.keys() + assert_close(res["yaw"], expected["yaw"], atol=1e-4, rtol=1e-4) + assert_close(res["pitch"], expected["pitch"], atol=1e-4, rtol=1e-4) + assert_close(res["roll"], expected["roll"], atol=1e-4, rtol=1e-4) + + +class TestRandomCropGen3D(RandomGeneratorBaseTests): + @pytest.mark.parametrize("batch_size", [0, 2]) + @pytest.mark.parametrize("input_size", [(200, 200, 200)]) + @pytest.mark.parametrize("size", [(100, 100, 100), torch.tensor([50, 60, 70])]) + @pytest.mark.parametrize("resize_to", [None, (100, 100, 100)]) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_valid_param_combinations(self, batch_size, input_size, size, resize_to, same_on_batch, device, dtype): + if isinstance(size, torch.Tensor): + size = size.repeat(batch_size, 1).to(device=device, dtype=dtype) + + param_gen = CropGenerator3D(size=size, resize_to=resize_to) + param_gen(batch_shape=torch.Size((batch_size, 1, *input_size)), same_on_batch=same_on_batch) + + @pytest.mark.parametrize( + "input_size,size,resize_to", + [ + ((-300, 300, 300), (200, 200, 200), (100, 100, 100)), + ((100, 100, 100), (200, 200, 200), (100, 100, 100)), + ((200, 200, 200), torch.tensor([50, 50, 50]), (100, 100, 100)), + ((100, 100, 100), torch.tensor([[50, 60, 70], [50, 60, 70]]), (100, 100)), + ], + ) + def test_invalid_param_combinations(self, input_size, size, resize_to, device, dtype): + with pytest.raises(Exception): + param_gen = CropGenerator3D( + size=size.to(device=device, dtype=dtype) if isinstance(size, torch.Tensor) else size, + resize_to=resize_to, + ) + param_gen(batch_shape=torch.Size((2, 1, *input_size))) + + def test_random_gen(self, device, dtype): + torch.manual_seed(42) + param_gen = CropGenerator3D( + size=torch.tensor([[50, 60, 70], [50, 60, 70]], device=device, dtype=dtype), resize_to=(100, 100, 100) + ) + res = param_gen(batch_shape=torch.Size((2, 1, 200, 200, 200))) + + expected = { + "src": torch.tensor( + [ + [ + [115, 53, 58], + [184, 53, 58], + [184, 112, 58], + [115, 112, 58], + [115, 53, 107], + [184, 53, 107], + [184, 112, 107], + [115, 112, 107], + ], + [ + [119, 135, 90], + [188, 135, 90], + [188, 194, 90], + [119, 194, 90], + [119, 135, 139], + [188, 135, 139], + [188, 194, 139], + [119, 194, 139], + ], + ], + device=device, + dtype=dtype, + ), + "dst": torch.tensor( + [ + [ + [0, 0, 0], + [99, 0, 0], + [99, 99, 0], + [0, 99, 0], + [0, 0, 99], + [99, 0, 99], + [99, 99, 99], + [0, 99, 99], + ], + [ + [0, 0, 0], + [99, 0, 0], + [99, 99, 0], + [0, 99, 0], + [0, 0, 99], + [99, 0, 99], + [99, 99, 99], + [0, 99, 99], + ], + ], + device=device, + dtype=dtype, + ), + } + assert res.keys() == expected.keys() + assert_close(res["src"], expected["src"], atol=1e-4, rtol=1e-4) + assert_close(res["dst"], expected["dst"], atol=1e-4, rtol=1e-4) + + def test_same_on_batch(self, device, dtype): + torch.manual_seed(42) + + param_gen = CropGenerator3D( + size=torch.tensor([[50, 60, 70], [50, 60, 70]], device=device, dtype=dtype), resize_to=(100, 100, 100) + ) + res = param_gen(batch_shape=torch.Size((2, 1, 200, 200, 200)), same_on_batch=True) + + expected = { + "src": torch.tensor( + [ + [ + [115, 129, 57], + [184, 129, 57], + [184, 188, 57], + [115, 188, 57], + [115, 129, 106], + [184, 129, 106], + [184, 188, 106], + [115, 188, 106], + ], + [ + [115, 129, 57], + [184, 129, 57], + [184, 188, 57], + [115, 188, 57], + [115, 129, 106], + [184, 129, 106], + [184, 188, 106], + [115, 188, 106], + ], + ], + device=device, + dtype=dtype, + ), + "dst": torch.tensor( + [ + [ + [0, 0, 0], + [99, 0, 0], + [99, 99, 0], + [0, 99, 0], + [0, 0, 99], + [99, 0, 99], + [99, 99, 99], + [0, 99, 99], + ], + [ + [0, 0, 0], + [99, 0, 0], + [99, 99, 0], + [0, 99, 0], + [0, 0, 99], + [99, 0, 99], + [99, 99, 99], + [0, 99, 99], + ], + ], + device=device, + dtype=dtype, + ), + } + assert res.keys() == expected.keys() + assert_close(res["src"], expected["src"], atol=1e-4, rtol=1e-4) + assert_close(res["dst"], expected["dst"], atol=1e-4, rtol=1e-4) + + +class TestCenterCropGen3D(RandomGeneratorBaseTests): + @pytest.mark.parametrize("batch_size", [0, 2]) + @pytest.mark.parametrize("depth,height,width", [(200, 200, 200)]) + @pytest.mark.parametrize("size", [(100, 100, 100)]) + def test_valid_param_combinations(self, batch_size, depth, height, width, size, device, dtype): + center_crop_generator3d(batch_size=batch_size, depth=depth, height=height, width=width, size=size) + + @pytest.mark.parametrize( + "depth,height,width,size", + [ + (200, 200, -200, (100, 100, 100)), + (200, -200, 200, (100, 100)), + (200, 100, 100, (300, 120, 100)), + (200, 150, 100, (120, 180, 100)), + (200, 100, 150, (120, 80, 200)), + ], + ) + def test_invalid_param_combinations(self, depth, height, width, size, device, dtype): + with pytest.raises(Exception): + center_crop_generator3d(batch_size=2, depth=depth, height=height, width=width, size=size) + + def test_random_gen(self, device, dtype): + torch.manual_seed(42) + res = center_crop_generator3d(batch_size=2, depth=200, height=200, width=200, size=(120, 150, 100)) + expected = { + "src": torch.tensor( + [ + [ + [50, 25, 40], + [149, 25, 40], + [149, 174, 40], + [50, 174, 40], + [50, 25, 159], + [149, 25, 159], + [149, 174, 159], + [50, 174, 159], + ] + ], + device=device, + dtype=torch.long, + ).repeat(2, 1, 1), + "dst": torch.tensor( + [ + [ + [0, 0, 0], + [99, 0, 0], + [99, 149, 0], + [0, 149, 0], + [0, 0, 119], + [99, 0, 119], + [99, 149, 119], + [0, 149, 119], + ] + ], + device=device, + dtype=torch.long, + ).repeat(2, 1, 1), + } + assert res.keys() == expected.keys() + assert_close(res["src"].to(device=device), expected["src"], atol=1e-4, rtol=1e-4) + assert_close(res["dst"].to(device=device), expected["dst"], atol=1e-4, rtol=1e-4) + + def test_same_on_batch(self, device, dtype): + pass + + +class TestRandomMotionBlur3D(RandomGeneratorBaseTests): + @pytest.mark.parametrize("batch_size", [0, 1, 8]) + @pytest.mark.parametrize("kernel_size", [3, (3, 5)]) + @pytest.mark.parametrize("angle", [torch.tensor([(10.0, 30.0), (30.0, 60.0), (60.0, 90.0)])]) + @pytest.mark.parametrize( + "direction", [torch.tensor([-1.0, -1.0]), torch.tensor([-1.0, 1.0]), torch.tensor([1.0, 1.0])] + ) + @pytest.mark.parametrize("same_on_batch", [True, False]) + def test_valid_param_combinations(self, batch_size, kernel_size, angle, direction, same_on_batch, device, dtype): + param_gen = MotionBlurGenerator3D( + kernel_size=kernel_size, + angle=angle.to(device=device, dtype=dtype), + direction=direction.to(device=device, dtype=dtype), + ) + + param_gen(batch_shape=torch.Size((batch_size,)), same_on_batch=same_on_batch) + + @pytest.mark.parametrize( + "kernel_size,angle,direction", + [ + (4, torch.tensor([(10, 30), (30, 60), (60, 90)]), torch.tensor([-1, 1])), + (1, torch.tensor([(10, 30), (30, 60), (60, 90)]), torch.tensor([-1, 1])), + ((3, 4, 5), torch.tensor([(10, 30), (30, 60), (60, 90)]), torch.tensor([-1, 1])), + (3, torch.tensor([(10, 30), (30, 60), (60, 90)]), torch.tensor([-2, 1])), + (3, torch.tensor([(10, 30), (30, 60), (60, 90)]), torch.tensor([-1, 2])), + ], + ) + def test_invalid_param_combinations(self, kernel_size, angle, direction, device, dtype): + with pytest.raises(Exception): + param_gen = MotionBlurGenerator3D( + kernel_size=kernel_size, + angle=angle.to(device=device, dtype=dtype), + direction=direction.to(device=device, dtype=dtype), + ) + + param_gen(batch_shape=torch.Size((2,))) + + def test_random_gen(self, device, dtype): + torch.manual_seed(42) + angle = torch.tensor([(10, 30), (30, 60), (60, 90)], device=device, dtype=dtype) + direction = torch.tensor([-1, 1], device=device, dtype=dtype) + param_gen = MotionBlurGenerator3D(kernel_size=3, angle=angle, direction=direction) + + res = param_gen(batch_shape=torch.Size((2,)), same_on_batch=False) + + expected = { + "ksize_factor": torch.tensor([3, 3], device=device, dtype=torch.int32), + "angle_factor": torch.tensor( + [[27.6454, 41.4859, 71.7134], [28.3001, 58.7792, 78.0269]], device=device, dtype=dtype + ), + "direction_factor": torch.tensor([-0.4869, 0.5873], device=device, dtype=dtype), + } + assert res.keys() == expected.keys() + assert_close(res["ksize_factor"], expected["ksize_factor"], rtol=1e-4, atol=1e-4) + assert_close(res["angle_factor"], expected["angle_factor"], rtol=1e-4, atol=1e-4) + assert_close(res["direction_factor"], expected["direction_factor"], rtol=1e-4, atol=1e-4) + + def test_same_on_batch(self, device, dtype): + torch.manual_seed(42) + angle = torch.tensor([(10, 30), (30, 60), (60, 90)], device=device, dtype=dtype) + direction = torch.tensor([-1, 1], device=device, dtype=dtype) + param_gen = MotionBlurGenerator3D(kernel_size=3, angle=angle, direction=direction) + + res = param_gen(batch_shape=torch.Size((2,)), same_on_batch=True) + + expected = { + "ksize_factor": torch.tensor([3, 3], device=device, dtype=torch.int32), + "angle_factor": torch.tensor( + [[27.6454, 57.4501, 71.4859], [27.6454, 57.4501, 71.4859]], device=device, dtype=dtype + ), + "direction_factor": torch.tensor([0.9186, 0.9186], device=device, dtype=dtype), + } + assert res.keys() == expected.keys() + assert_close(res["ksize_factor"], expected["ksize_factor"], rtol=1e-4, atol=1e-4) + assert_close(res["angle_factor"], expected["angle_factor"], rtol=1e-4, atol=1e-4) + assert_close(res["direction_factor"], expected["direction_factor"], rtol=1e-4, atol=1e-4) diff --git a/tests/benchmark.py b/tests/benchmark.py new file mode 100644 index 0000000..8349344 --- /dev/null +++ b/tests/benchmark.py @@ -0,0 +1,47 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from itertools import product + +import torch +from torch.utils import benchmark + +# Compare takes a list of measurements which we'll save in results. +results = [] + +sizes = [1, 64, 1024] +for b, _ in product(sizes, sizes): + # label and sub_label are the rows + # description is the column + label = "get_perspective_transform" + sub_label = f"[{b}, {4}, {2}]" + x = torch.rand((b, 4, 2)) + for num_threads in [1, 4, 16, 32]: + results.append( + benchmark.Timer( + stmt="get_perspective_transform(x, x)", + setup="from kornia.geometry import get_perspective_transform", + globals={"x": x}, + num_threads=num_threads, + label=label, + sub_label=sub_label, + description="get_perspective_transform", + ).blocked_autorange(min_run_time=1) + ) + +compare = benchmark.Compare(results) +compare.print() diff --git a/tests/color/test_colormap.py b/tests/color/test_colormap.py new file mode 100644 index 0000000..a4a01a2 --- /dev/null +++ b/tests/color/test_colormap.py @@ -0,0 +1,123 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.color import ApplyColorMap, ColorMap, ColorMapType, apply_colormap + +from testing.base import BaseTester, assert_close + + +def test_autumn(device, dtype): + cm = ColorMap(base="autumn", num_colors=64, device=device, dtype=dtype) + colors = cm.colors + + actual = colors[..., 0] + expected = torch.tensor([1, 0, 0], device=device, dtype=dtype) + assert_close(actual, expected) + + actual = colors[..., 32] + expected = torch.tensor([1.0, 0.5079365079365079, 0.0], device=device, dtype=dtype) + assert_close(actual, expected) + + actual = colors[..., -1] + expected = torch.tensor([1, 1, 0], device=device, dtype=dtype) + assert_close(actual, expected) + + +class TestApplyColorMap(BaseTester): + def test_smoke(self, device, dtype): + input_tensor = torch.tensor([[[0, 1, 2], [15, 25, 33], [128, 158, 188]]], device=device, dtype=dtype) + expected_tensor = torch.tensor( + [ + [ + [ + [1.0000000000, 1.0000000000, 1.0000000000], + [1.0000000000, 1.0000000000, 1.0000000000], + [1.0000000000, 1.0000000000, 1.0000000000], + ], + [ + [0.0000000000, 0.0158730168, 0.0158730168], + [0.0634920672, 0.1111111119, 0.1428571492], + [0.5079365373, 0.6190476418, 0.7301587462], + ], + [ + [0.0000000000, 0.0000000000, 0.0000000000], + [0.0000000000, 0.0000000000, 0.0000000000], + [0.0000000000, 0.0000000000, 0.0000000000], + ], + ] + ], + device=device, + dtype=dtype, + ) + cm = ColorMap(base="autumn", device=device, dtype=dtype) + actual = apply_colormap(input_tensor, cm) + + self.assert_close(actual, expected_tensor) + + def test_exception(self, device, dtype): + cm = ColorMap(base="autumn", device=device, dtype=dtype) + with pytest.raises(Exception): + apply_colormap(torch.rand(size=(3, 3), dtype=dtype, device=device), cm) + + with pytest.raises(Exception): + apply_colormap(torch.rand(size=(3), dtype=dtype, device=device), cm) + + with pytest.raises(Exception): + apply_colormap(torch.rand(size=(3), dtype=dtype, device=device).item(), cm) + + @pytest.mark.parametrize("shape", [(2, 1, 3, 3), (1, 3, 3, 3), (1, 3, 3)]) + @pytest.mark.parametrize("cmap_base", ColorMapType) + def test_cardinality(self, shape, device, dtype, cmap_base): + cm = ColorMap(base=cmap_base, num_colors=256, device=device, dtype=dtype) + input_tensor = torch.randint(0, 256, shape, device=device, dtype=dtype) + actual = apply_colormap(input_tensor, cm) + + if len(shape) == 4: + expected_shape = (shape[-4], shape[-3] * 3, shape[-2], shape[-1]) + else: + expected_shape = (1, shape[-3] * 3, shape[-2], shape[-1]) + + assert actual.shape == expected_shape + + @pytest.mark.skip(reason="jacobian mismatch") + def test_gradcheck(self, device): + # TODO: implement differentiability + cm = ColorMap(base="autumn", device=device, dtype=torch.float64) + input_tensor = torch.randint(0, 63, (1, 2, 1), device=device, dtype=torch.float64) + + self.gradcheck(apply_colormap, (input_tensor, cm)) + + def test_dynamo(self, device, dtype, torch_optimizer): + op = apply_colormap + op_script = torch_optimizer(op) + + cm = ColorMap(base="autumn", device=device, dtype=dtype) + img = torch.ones(1, 3, 3, device=device, dtype=dtype) + + self.assert_close(op(img, cm), op_script(img, cm)) + + def test_module(self, device, dtype): + op = apply_colormap + cm = ColorMap(base="autumn", device=device, dtype=dtype) + op_module = ApplyColorMap(colormap=cm) + + img = torch.ones(1, 3, 3, device=device, dtype=dtype) + + self.assert_close(op(img, colormap=cm), op_module(img)) diff --git a/tests/color/test_gray.py b/tests/color/test_gray.py new file mode 100644 index 0000000..c72c8d7 --- /dev/null +++ b/tests/color/test_gray.py @@ -0,0 +1,343 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +from torch.autograd import gradcheck + +import kornia +from kornia.core._compat import torch_version + +from testing.base import BaseTester, assert_close + + +class TestGrayscaleToRgb(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 1, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.grayscale_to_rgb(img), torch.Tensor) + + @pytest.mark.parametrize("batch_size, height, width", [(1, 3, 4), (2, 2, 4), (3, 4, 1)]) + def test_cardinality(self, device, dtype, batch_size, height, width): + img = torch.ones(batch_size, 1, height, width, device=device, dtype=dtype) + assert kornia.color.grayscale_to_rgb(img).shape == (batch_size, 3, height, width) + + def test_exception(self, device, dtype): + from kornia.core.exceptions import TypeCheckError + + with pytest.raises(TypeCheckError): + assert kornia.color.grayscale_to_rgb([0.0]) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.grayscale_to_rgb(img) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.grayscale_to_rgb(img) + + with pytest.raises(ValueError): + img = torch.ones(1, 3, 1, 1, device=device, dtype=dtype) + assert kornia.color.grayscale_to_rgb(img) + + def test_opencv(self, device, dtype): + data = torch.tensor( + [ + [ + [0.4684734, 0.8954562, 0.6064363, 0.5236061, 0.6106016], + [0.1709944, 0.5133104, 0.7915002, 0.5745703, 0.1680204], + [0.5279005, 0.6092287, 0.3034387, 0.5333768, 0.6064113], + [0.3503858, 0.5720159, 0.7052018, 0.4558409, 0.3261529], + [0.6988886, 0.5897652, 0.6532392, 0.7234108, 0.7218805], + ] + ], + device=device, + dtype=dtype, + ) + + # Output data generated with OpenCV 4.5.2: cv2.cvtColor(img_np, cv2.COLOR_GRAY2RGB) + expected = torch.tensor( + [ + [ + [0.4684734, 0.8954562, 0.6064363, 0.5236061, 0.6106016], + [0.1709944, 0.5133104, 0.7915002, 0.5745703, 0.1680204], + [0.5279005, 0.6092287, 0.3034387, 0.5333768, 0.6064113], + [0.3503858, 0.5720159, 0.7052018, 0.4558409, 0.3261529], + [0.6988886, 0.5897652, 0.6532392, 0.7234108, 0.7218805], + ], + [ + [0.4684734, 0.8954562, 0.6064363, 0.5236061, 0.6106016], + [0.1709944, 0.5133104, 0.7915002, 0.5745703, 0.1680204], + [0.5279005, 0.6092287, 0.3034387, 0.5333768, 0.6064113], + [0.3503858, 0.5720159, 0.7052018, 0.4558409, 0.3261529], + [0.6988886, 0.5897652, 0.6532392, 0.7234108, 0.7218805], + ], + [ + [0.4684734, 0.8954562, 0.6064363, 0.5236061, 0.6106016], + [0.1709944, 0.5133104, 0.7915002, 0.5745703, 0.1680204], + [0.5279005, 0.6092287, 0.3034387, 0.5333768, 0.6064113], + [0.3503858, 0.5720159, 0.7052018, 0.4558409, 0.3261529], + [0.6988886, 0.5897652, 0.6532392, 0.7234108, 0.7218805], + ], + ], + device=device, + dtype=dtype, + ) + + img_rgb = kornia.color.grayscale_to_rgb(data) + assert_close(img_rgb, expected) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 1, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.grayscale_to_rgb, (img,), raise_exception=True, fast_mode=True) + + def test_dynamo(self, device, dtype, torch_optimizer): + B, C, H, W = 2, 1, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.grayscale_to_rgb + op_optimized = torch_optimizer(op) + assert_close(op(img), op_optimized(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 1, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + gray_ops = kornia.color.GrayscaleToRgb().to(device, dtype) + gray_fcn = kornia.color.grayscale_to_rgb + assert_close(gray_ops(img), gray_fcn(img)) + + +class TestRgbToGrayscale(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + out = kornia.color.rgb_to_grayscale(img) + assert out.device == img.device + assert out.dtype == img.dtype + + def test_smoke_byte(self, device): + C, H, W = 3, 4, 5 + img = torch.randint(0, 255, (C, H, W), device=device, dtype=torch.uint8) + out = kornia.color.rgb_to_grayscale(img) + assert out.device == img.device + assert out.dtype == img.dtype + + @pytest.mark.parametrize("batch_size, height, width", [(1, 3, 4), (2, 2, 4), (3, 4, 1)]) + def test_cardinality(self, device, dtype, batch_size, height, width): + img = torch.ones(batch_size, 3, height, width, device=device, dtype=dtype) + assert kornia.color.rgb_to_grayscale(img).shape == (batch_size, 1, height, width) + + def test_exception(self, device, dtype): + from kornia.core.exceptions import TypeCheckError + + with pytest.raises(TypeCheckError): + assert kornia.color.rgb_to_grayscale([0.0]) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_grayscale(img) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_grayscale(img) + + with pytest.raises(ValueError): + img = torch.ones(3, 1, 1, device=device, dtype=dtype) + rgb_weights = torch.tensor([0.2, 0.8]) + assert kornia.color.rgb_to_grayscale(img, rgb_weights=rgb_weights) + + def test_unsupported_dtype_raises(self, device): + # int32 is not uint8 and not a float dtype -> TypeError + img = torch.ones(3, 4, 4, device=device, dtype=torch.int32) + with pytest.raises(TypeError, match="Unknown data type"): + kornia.color.rgb_to_grayscale(img) + + def test_opencv(self, device, dtype): + data = torch.tensor( + [ + [ + [0.3944633, 0.8597369, 0.1670904, 0.2825457, 0.0953912], + [0.1251704, 0.8020709, 0.8933256, 0.9170977, 0.1497008], + [0.2711633, 0.1111478, 0.0783281, 0.2771807, 0.5487481], + [0.0086008, 0.8288748, 0.9647092, 0.8922020, 0.7614344], + [0.2898048, 0.1282895, 0.7621747, 0.5657831, 0.9918593], + ], + [ + [0.5414237, 0.9962701, 0.8947155, 0.5900949, 0.9483274], + [0.0468036, 0.3933847, 0.8046577, 0.3640994, 0.0632100], + [0.6171775, 0.8624780, 0.4126036, 0.7600935, 0.7279997], + [0.4237089, 0.5365476, 0.5591233, 0.1523191, 0.1382165], + [0.8932794, 0.8517839, 0.7152701, 0.8983801, 0.5905426], + ], + [ + [0.2869580, 0.4700376, 0.2743714, 0.8135023, 0.2229074], + [0.9306560, 0.3734594, 0.4566821, 0.7599275, 0.7557513], + [0.7415742, 0.6115875, 0.3317572, 0.0379378, 0.1315770], + [0.8692724, 0.0809556, 0.7767404, 0.8742208, 0.1522012], + [0.7708948, 0.4509611, 0.0481175, 0.2358997, 0.6900532], + ], + ], + device=device, + dtype=dtype, + ) + + # Output data generated with OpenCV 4.1.1: cv2.cvtColor(img_np, cv2.COLOR_RGB2GRAY) + expected = torch.tensor( + [ + [ + [0.4684734, 0.8954562, 0.6064363, 0.5236061, 0.6106016], + [0.1709944, 0.5133104, 0.7915002, 0.5745703, 0.1680204], + [0.5279005, 0.6092287, 0.3034387, 0.5333768, 0.6064113], + [0.3503858, 0.5720159, 0.7052018, 0.4558409, 0.3261529], + [0.6988886, 0.5897652, 0.6532392, 0.7234108, 0.7218805], + ] + ], + device=device, + dtype=dtype, + ) + + img_gray = kornia.color.rgb_to_grayscale(data) + assert_close(img_gray, expected) + + def test_custom_rgb_weights(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + + rgb_weights = torch.tensor([0.5, 0.25, 0.25]) + img_gray = kornia.color.rgb_to_grayscale(img, rgb_weights=rgb_weights) + assert img_gray.device == device + assert img_gray.dtype == dtype + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.rgb_to_grayscale, (img,), raise_exception=True, fast_mode=True) + + @pytest.mark.slow + @pytest.mark.skipif(torch_version() in {"2.0.0", "2.0.1"}, reason="Not working on 2.0") + def test_dynamo(self, device, dtype, torch_optimizer): + # TODO: investigate the problem with dynamo + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.rgb_to_grayscale + op_optimized = torch_optimizer(op) + assert_close(op(img), op_optimized(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + gray_ops = kornia.color.RgbToGrayscale().to(device, dtype) + gray_fcn = kornia.color.rgb_to_grayscale + assert_close(gray_ops(img), gray_fcn(img)) + + +class TestBgrToGrayscale(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert kornia.color.bgr_to_grayscale(img) is not None + + @pytest.mark.parametrize("batch_size, height, width", [(1, 3, 4), (2, 2, 4), (3, 4, 1)]) + def test_cardinality(self, device, dtype, batch_size, height, width): + img = torch.ones(batch_size, 3, height, width, device=device, dtype=dtype) + assert kornia.color.bgr_to_grayscale(img).shape == (batch_size, 1, height, width) + + def test_exception(self, device, dtype): + from kornia.core.exceptions import TypeCheckError + + with pytest.raises(TypeCheckError): + assert kornia.color.bgr_to_grayscale([0.0]) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.bgr_to_grayscale(img) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.bgr_to_grayscale(img) + + def test_opencv(self, device, dtype): + data = torch.tensor( + [ + [ + [0.3944633, 0.8597369, 0.1670904, 0.2825457, 0.0953912], + [0.1251704, 0.8020709, 0.8933256, 0.9170977, 0.1497008], + [0.2711633, 0.1111478, 0.0783281, 0.2771807, 0.5487481], + [0.0086008, 0.8288748, 0.9647092, 0.8922020, 0.7614344], + [0.2898048, 0.1282895, 0.7621747, 0.5657831, 0.9918593], + ], + [ + [0.5414237, 0.9962701, 0.8947155, 0.5900949, 0.9483274], + [0.0468036, 0.3933847, 0.8046577, 0.3640994, 0.0632100], + [0.6171775, 0.8624780, 0.4126036, 0.7600935, 0.7279997], + [0.4237089, 0.5365476, 0.5591233, 0.1523191, 0.1382165], + [0.8932794, 0.8517839, 0.7152701, 0.8983801, 0.5905426], + ], + [ + [0.2869580, 0.4700376, 0.2743714, 0.8135023, 0.2229074], + [0.9306560, 0.3734594, 0.4566821, 0.7599275, 0.7557513], + [0.7415742, 0.6115875, 0.3317572, 0.0379378, 0.1315770], + [0.8692724, 0.0809556, 0.7767404, 0.8742208, 0.1522012], + [0.7708948, 0.4509611, 0.0481175, 0.2358997, 0.6900532], + ], + ], + device=device, + dtype=dtype, + ) + + # Output data generated with OpenCV 4.1.1: cv2.cvtColor(img_np, cv2.COLOR_BGR2GRAY) + expected = torch.tensor( + [ + [ + [0.4485849, 0.8233618, 0.6262833, 0.6218331, 0.6341921], + [0.3200093, 0.4340172, 0.7107211, 0.5454938, 0.2801398], + [0.6149265, 0.7018101, 0.3503231, 0.4891168, 0.5292346], + [0.5096100, 0.4336508, 0.6704276, 0.4525143, 0.2134447], + [0.7878902, 0.6494595, 0.5211386, 0.6623823, 0.6660464], + ] + ], + device=device, + dtype=dtype, + ) + + img_gray = kornia.color.bgr_to_grayscale(data) + assert_close(img_gray, expected) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.bgr_to_grayscale, (img,), raise_exception=True, fast_mode=True) + + @pytest.mark.slow + @pytest.mark.skipif(torch_version() in {"2.0.0", "2.0.1"}, reason="Not working on 2.0") + def test_dynamo(self, device, dtype, torch_optimizer): + # TODO: investigate the problem with dynamo + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.rgb_to_grayscale + op_optimized = torch_optimizer(op) + assert_close(op(img), op_optimized(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + gray_ops = kornia.color.BgrToGrayscale().to(device, dtype) + gray_fcn = kornia.color.bgr_to_grayscale + assert_close(gray_ops(img), gray_fcn(img)) diff --git a/tests/color/test_hls.py b/tests/color/test_hls.py new file mode 100644 index 0000000..25f6c1b --- /dev/null +++ b/tests/color/test_hls.py @@ -0,0 +1,270 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math + +import pytest +import torch +from torch.autograd import gradcheck + +import kornia + +from testing.base import BaseTester + + +class TestRgbToHls(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.rgb_to_hls(img), torch.Tensor) + + @pytest.mark.parametrize("shape", [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1), (3, 2, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.ones(shape, device=device, dtype=dtype) + assert kornia.color.rgb_to_hls(img).shape == shape + + def test_exception(self, device, dtype): + with pytest.raises(TypeError): + assert kornia.color.rgb_to_hls([0.0]) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_hls(img) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_hls(img) + + def test_unit(self, device, dtype): + data = torch.tensor( + [ + [ + [0.4237059, 0.1935902, 0.8585021, 0.3790484, 0.1389151], + [0.5933651, 0.0474544, 0.2801555, 0.1691061, 0.9221829], + [0.2351739, 0.5852075, 0.5789326, 0.8411915, 0.5960411], + [0.0290176, 0.6459382, 0.8581501, 0.4755400, 0.7735767], + [0.9497226, 0.0919441, 0.5462211, 0.7836787, 0.6403612], + ], + [ + [0.2280025, 0.1352853, 0.7999730, 0.6658246, 0.4910861], + [0.3499791, 0.1250734, 0.6315800, 0.4785843, 0.8477826], + [0.3646359, 0.2415122, 0.5301932, 0.0782518, 0.8710389], + [0.6957581, 0.6162295, 0.6259052, 0.1753750, 0.6737530], + [0.7678874, 0.9825978, 0.0234877, 0.2485284, 0.8159551], + ], + [ + [0.7330830, 0.9015747, 0.0229067, 0.4280063, 0.5400181], + [0.0037299, 0.3259412, 0.3467951, 0.9575506, 0.1525899], + [0.9660432, 0.5287710, 0.6654660, 0.3797526, 0.4981400], + [0.7422802, 0.9926301, 0.5334370, 0.7852844, 0.4397180], + [0.2281681, 0.2560037, 0.5134379, 0.5800887, 0.8685090], + ], + ], + device=device, + dtype=dtype, + ) + + # OpenCV + expected = torch.tensor( + [ + [ + [4.59454770, 4.26846900, 0.97384680, 2.27317070, 3.26934400], + [0.61494170, 3.89691880, 2.29297200, 3.77774720, 0.94595980], + [4.00329600, 5.40794320, 4.56610100, 5.86935100, 1.81946310], + [3.20989560, 4.27144400, 0.29820946, 4.70416550, 0.73408560], + [0.78329855, 2.28729030, 5.30166340, 5.63437900, 3.38281500], + ], + [ + [0.48054275, 0.51843000, 0.44070444, 0.52243650, 0.33946657], + [0.29854750, 0.18669781, 0.45586777, 0.56332830, 0.53738640], + [0.60060860, 0.41335985, 0.59782960, 0.45972168, 0.68458940], + [0.38564888, 0.80442977, 0.69579350, 0.48032972, 0.60664740], + [0.58894540, 0.53727096, 0.28485440, 0.51610350, 0.75443510], + ], + [ + [0.52553130, 0.79561585, 0.94802250, 0.30024928, 0.59078425], + [0.98750657, 0.74582230, 0.38544560, 0.90278864, 0.83178820], + [0.91497860, 0.41573380, 0.16817844, 0.82978433, 0.59113250], + [0.92475650, 0.96231550, 0.53370523, 0.63488615, 0.42437580], + [0.87768690, 0.96239233, 0.91754496, 0.55295944, 0.46453667], + ], + ], + device=device, + dtype=dtype, + ) + + self.assert_close(kornia.color.rgb_to_hls(data), expected, low_tolerance=True) + + def test_nan_rgb_to_hls(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + data = torch.ones(2, 3, 5, 5, device=device, dtype=dtype) + + # OpenCV + expected = torch.cat( + [ + torch.zeros(2, 1, 5, 5, device=device, dtype=dtype), + torch.ones(2, 1, 5, 5, device=device, dtype=dtype), + torch.zeros(2, 1, 5, 5, device=device, dtype=dtype), + ], + dim=1, + ) + self.assert_close(kornia.color.rgb_to_hls(data), expected) + + def test_nan_random_extreme_values(self, device, dtype): + # generate extreme colors randomly + ext_rand_slice = (torch.rand((1, 3, 32, 32), dtype=dtype, device=device) >= 0.5).float() + assert not kornia.color.rgb_to_hls(ext_rand_slice).isnan().any() + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.rand(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.rgb_to_hls, (img,), raise_exception=True, fast_mode=True) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.rgb_to_hls + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.RgbToHls().to(device, dtype) + fcn = kornia.color.rgb_to_hls + self.assert_close(ops(img), fcn(img)) + + +class TestHlsToRgb(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.hls_to_rgb(img), torch.Tensor) + + @pytest.mark.parametrize("shape", [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1), (3, 2, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.ones(shape, device=device, dtype=dtype) + assert kornia.color.hls_to_rgb(img).shape == shape + + def test_exception(self, device, dtype): + with pytest.raises(TypeError): + assert kornia.color.hls_to_rgb([0.0]) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.hls_to_rgb(img) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.hls_to_rgb(img) + + def test_unit(self, device, dtype): + data = torch.tensor( + [ + [ + [ + [0.5513626, 0.8487718, 0.1822479, 0.2851745, 0.2669488], + [0.7596772, 0.4565057, 0.6181599, 0.3852497, 0.7746902], + [0.5742747, 0.1957062, 0.7530835, 0.2104362, 0.9449323], + [0.9918052, 0.2437515, 0.4718738, 0.8502576, 0.1675640], + [0.9210159, 0.0538564, 0.5801026, 0.6110542, 0.3768399], + ], + [ + [0.4111853, 0.0183454, 0.7832276, 0.2975794, 0.1139528], + [0.6207729, 0.1073406, 0.8335325, 0.5700451, 0.2594557], + [0.7520493, 0.5097187, 0.4719872, 0.9477938, 0.1640292], + [0.8973427, 0.6455371, 0.7567374, 0.3159562, 0.8135307], + [0.0855004, 0.6645504, 0.9923756, 0.6209313, 0.2356791], + ], + [ + [0.4734681, 0.0422099, 0.7405791, 0.9671807, 0.1793800], + [0.8221875, 0.7219887, 0.3627397, 0.4403201, 0.0024084], + [0.0803350, 0.9432759, 0.0241543, 0.8292291, 0.7745832], + [0.3707901, 0.0851424, 0.5805428, 0.1098685, 0.4238486], + [0.1058410, 0.0816052, 0.5792874, 0.9578886, 0.6281684], + ], + ] + ], + device=device, + dtype=dtype, + ) + + data[:, 0] = 2 * math.pi * data[:, 0] + + # OpenCV + expected = torch.tensor( + [ + [ + [ + [0.21650219, 0.01911971, 0.91374826, 0.17609520, 0.10979544], + [0.65698080, 0.02984191, 0.77314806, 0.38072730, 0.25964087], + [0.73213010, 0.81102980, 0.47240910, 0.96834683, 0.29108350], + [0.93540700, 0.64780010, 0.61551300, 0.35066980, 0.89171433], + [0.09454980, 0.69192480, 0.98795897, 0.25782573, 0.08763295], + ], + [ + [0.48587522, 0.01757100, 0.94376480, 0.58539250, 0.13439366], + [0.30897713, 0.18483935, 0.80829670, 0.75936294, 0.25883088], + [0.75421450, 0.97218925, 0.46058673, 0.99108470, 0.03697497], + [0.85927840, 0.67571700, 0.89796180, 0.28124255, 0.89256540], + [0.07645091, 0.65486740, 0.99254686, 0.50014400, 0.38372523], + ], + [ + [0.60586834, 0.01897625, 0.62269044, 0.00976634, 0.09351197], + [0.93256867, 0.14439031, 0.89391685, 0.49867177, 0.26008060], + [0.77196840, 0.04724807, 0.48338777, 0.90450300, 0.12093388], + [0.86302150, 0.61535730, 0.85029656, 0.34361976, 0.73449594], + [0.08502806, 0.63717590, 0.99679226, 0.98403690, 0.16492467], + ], + ] + ], + device=device, + dtype=dtype, + ) + + f = kornia.color.hls_to_rgb + self.assert_close(f(data), expected, low_tolerance=True) + + data[:, 0] += 2 * math.pi + self.assert_close(f(data), expected, low_tolerance=True) + + data[:, 0] -= 4 * math.pi + self.assert_close(f(data), expected, low_tolerance=True) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.rand(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.hls_to_rgb, (img,), raise_exception=True, fast_mode=True) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.hls_to_rgb + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.HlsToRgb().to(device, dtype) + fcn = kornia.color.hls_to_rgb + self.assert_close(ops(img), fcn(img)) diff --git a/tests/color/test_hsv.py b/tests/color/test_hsv.py new file mode 100644 index 0000000..0444c4d --- /dev/null +++ b/tests/color/test_hsv.py @@ -0,0 +1,253 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math + +import pytest +import torch +from torch.autograd import gradcheck + +import kornia + +from testing.base import BaseTester + + +class TestRgbToHsv(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.rgb_to_hsv(img), torch.Tensor) + + @pytest.mark.parametrize("shape", [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1), (3, 2, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.ones(shape, device=device, dtype=dtype) + assert kornia.color.rgb_to_hsv(img).shape == shape + + def test_exception(self, device, dtype): + with pytest.raises(TypeError): + assert kornia.color.rgb_to_hsv([0.0]) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_hsv(img) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_hsv(img) + + def test_unit(self, device, dtype): + data = torch.tensor( + [ + [ + [0.3944633, 0.8597369, 0.1670904, 0.2825457, 0.0953912], + [0.1251704, 0.8020709, 0.8933256, 0.9170977, 0.1497008], + [0.2711633, 0.1111478, 0.0783281, 0.2771807, 0.5487481], + [0.0086008, 0.8288748, 0.9647092, 0.8922020, 0.7614344], + [0.2898048, 0.1282895, 0.7621747, 0.5657831, 0.9918593], + ], + [ + [0.5414237, 0.9962701, 0.8947155, 0.5900949, 0.9483274], + [0.0468036, 0.3933847, 0.8046577, 0.3640994, 0.0632100], + [0.6171775, 0.8624780, 0.4126036, 0.7600935, 0.7279997], + [0.4237089, 0.5365476, 0.5591233, 0.1523191, 0.1382165], + [0.8932794, 0.8517839, 0.7152701, 0.8983801, 0.5905426], + ], + [ + [0.2869580, 0.4700376, 0.2743714, 0.8135023, 0.2229074], + [0.9306560, 0.3734594, 0.4566821, 0.7599275, 0.7557513], + [0.7415742, 0.6115875, 0.3317572, 0.0379378, 0.1315770], + [0.8692724, 0.0809556, 0.7767404, 0.8742208, 0.1522012], + [0.7708948, 0.4509611, 0.0481175, 0.2358997, 0.6900532], + ], + ], + device=device, + dtype=dtype, + ) + + # OpenCV + expected = torch.tensor( + [ + [ + [1.6519808, 1.31889750, 2.24879380, 3.58221600, 2.25095400], + [4.2816400, 0.04868213, 0.83454597, 5.53361700, 4.31957400], + [3.4185164, 2.79190370, 2.88832240, 1.74746920, 1.36192720], + [3.6837196, 0.63789610, 5.72131160, 5.26143740, 6.25968700], + [2.9292210, 2.56143520, 0.97840965, 1.57294110, 6.02352240], + ], + [ + [0.4699935, 0.52820253, 0.81324730, 0.65267974, 0.89941100], + [0.9497089, 0.53438100, 0.48878422, 0.60298723, 0.91636120], + [0.6343409, 0.87112963, 0.81016120, 0.95008780, 0.81926220], + [0.9901055, 0.90233060, 0.42042294, 0.82927720, 0.81847864], + [0.6755719, 0.84938710, 0.93686795, 0.73741645, 0.40461043], + ], + [ + [0.5414237, 0.99627006, 0.89471555, 0.81350225, 0.94832740], + [0.9306560, 0.80207086, 0.89332560, 0.91709770, 0.75575125], + [0.7415741, 0.86247796, 0.41260356, 0.76009345, 0.72799970], + [0.8692723, 0.82887480, 0.96470920, 0.89220200, 0.76143440], + [0.8932794, 0.85178390, 0.76217470, 0.89838010, 0.99185926], + ], + ], + device=device, + dtype=dtype, + ) + + self.assert_close(kornia.color.rgb_to_hsv(data), expected) + + def test_nan_rgb_to_hsv(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + data = torch.zeros(3, 5, 5, device=device, dtype=dtype) # 3x5x5 + expected = torch.zeros_like(data) # 3x5x5 + self.assert_close(kornia.color.rgb_to_hsv(data), expected) + + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.rand(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.rgb_to_hsv, (img,), raise_exception=True, fast_mode=True) + + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.rgb_to_hsv + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.RgbToHsv().to(device, dtype) + fcn = kornia.color.rgb_to_hsv + self.assert_close(ops(img), fcn(img)) + + +class TestHsvToRgb(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.hsv_to_rgb(img), torch.Tensor) + + @pytest.mark.parametrize("shape", [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1), (3, 2, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.ones(shape, device=device, dtype=dtype) + assert kornia.color.hsv_to_rgb(img).shape == shape + + def test_exception(self, device, dtype): + with pytest.raises(TypeError): + assert kornia.color.hsv_to_rgb([0.0]) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.hsv_to_rgb(img) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.hsv_to_rgb(img) + + def test_unit(self, device, dtype): + data = torch.tensor( + [ + [ + [ + [3.5433271, 5.6390061, 1.3766849, 2.5384088, 4.6848912], + [5.7209363, 5.3262630, 6.2059994, 4.1164689, 2.3872600], + [0.6370091, 3.6186798, 5.9170871, 2.8275447, 5.4289737], + [0.2751994, 1.6632686, 1.0049511, 0.7046204, 1.3791083], + [0.7863123, 4.4852505, 4.3064494, 2.5573561, 5.9083076], + ], + [ + [0.5026655, 0.9453601, 0.5929778, 0.2632897, 0.4590443], + [0.6201433, 0.5610679, 0.9653260, 0.0830478, 0.5000827], + [0.6067343, 0.6422323, 0.6777940, 0.7705711, 0.6050767], + [0.5495264, 0.5573426, 0.4683768, 0.2268902, 0.2116482], + [0.6525245, 0.0022379, 0.4909980, 0.1682271, 0.6327152], + ], + [ + [0.8471680, 0.9302199, 0.3265766, 0.7944570, 0.7038843], + [0.4833369, 0.2088473, 0.1169234, 0.4966302, 0.6448684], + [0.2713015, 0.5893380, 0.6015301, 0.6801558, 0.2322258], + [0.5704236, 0.6797268, 0.4755683, 0.4811209, 0.5317836], + [0.3236262, 0.0999796, 0.3614958, 0.5117705, 0.8194097], + ], + ] + ], + device=device, + dtype=dtype, + ) + + # OpenCV + expected = torch.tensor( + [ + [ + [ + [0.42132590, 0.93021995, 0.26564622, 0.58528465, 0.53384290], + [0.48333693, 0.20884734, 0.11692339, 0.45538613, 0.32238087], + [0.27130150, 0.21084610, 0.60153013, 0.15604737, 0.23222584], + [0.57042360, 0.45685310, 0.47556830, 0.48112088, 0.49611038], + [0.32362622, 0.09981924, 0.20394461, 0.42567685, 0.81940967], + ], + [ + [0.68380290, 0.05082710, 0.32657660, 0.79445700, 0.38077020], + [0.18359877, 0.09166980, 0.00405421, 0.45823452, 0.64486840], + [0.20682439, 0.41690278, 0.19381660, 0.68015575, 0.09171140], + [0.33933756, 0.67972680, 0.46658220, 0.44541004, 0.53178360], + [0.27101707, 0.09975589, 0.18400209, 0.51177055, 0.30095676], + ], + [ + [0.84716797, 0.59178180, 0.13292392, 0.67397410, 0.70388430], + [0.34453064, 0.19874583, 0.01237347, 0.49663020, 0.41256943], + [0.10669357, 0.58933800, 0.33635240, 0.52297890, 0.20633064], + [0.25696078, 0.30088606, 0.25282317, 0.37195927, 0.41923255], + [0.11245217, 0.09997964, 0.36149580, 0.46373847, 0.48655340], + ], + ] + ], + device=device, + dtype=dtype, + ) + + f = kornia.color.hsv_to_rgb + self.assert_close(f(data), expected) + + data[:, 0] += 2 * math.pi + self.assert_close(f(data), expected, low_tolerance=True) + + data[:, 0] -= 4 * math.pi + self.assert_close(f(data), expected, low_tolerance=True) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.rand(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.hsv_to_rgb, (img,), raise_exception=True, fast_mode=True) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.hsv_to_rgb + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.HsvToRgb().to(device, dtype) + fcn = kornia.color.hsv_to_rgb + self.assert_close(ops(img), fcn(img)) diff --git a/tests/color/test_lab.py b/tests/color/test_lab.py new file mode 100644 index 0000000..318beb8 --- /dev/null +++ b/tests/color/test_lab.py @@ -0,0 +1,283 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +from torch.autograd import gradcheck + +import kornia + +from testing.base import BaseTester + + +class TestRgbToLab(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.rgb_to_lab(img), torch.Tensor) + + @pytest.mark.parametrize("shape", [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1), (3, 2, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.ones(shape, device=device, dtype=dtype) + assert kornia.color.rgb_to_lab(img).shape == shape + + def test_exception(self, device, dtype): + with pytest.raises(TypeError): + assert kornia.color.rgb_to_lab([0.0]) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_lab(img) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_lab(img) + + def test_unit(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + data = torch.tensor( + [ + [ + [0.0, 1.0, 0.69396782, 0.63581685, 0.09902618], + [0.59459005, 0.74215373, 0.89662376, 0.25920381, 0.89937686], + [0.29857584, 0.28139791, 0.16441015, 0.55507519, 0.06124221], + [0.40908658, 0.10261389, 0.01691456, 0.76006799, 0.32971736], + ], + [ + [0.0, 1.0, 0.79009938, 0.91742945, 0.60044175], + [0.42812678, 0.18552390, 0.04186043, 0.38030245, 0.15420346], + [0.13552373, 0.53955473, 0.79102736, 0.49050815, 0.75271446], + [0.39861023, 0.80680277, 0.82823833, 0.54438462, 0.22063386], + ], + [ + [0.0, 1.0, 0.84317145, 0.59529881, 0.15297393], + [0.59235313, 0.36617295, 0.34600773, 0.40304737, 0.61720451], + [0.46040250, 0.42006640, 0.54765106, 0.48982632, 0.13914755], + [0.58402964, 0.89597990, 0.98276161, 0.25019163, 0.69285921], + ], + ], + device=device, + dtype=dtype, + ) + + # Reference output generated using skimage: rgb2lab(data) + expected = torch.tensor( + [ + [ + [0.0, 100.0, 79.75208576, 86.38913217, 55.25164186], + [51.66668553, 43.81214392, 48.93865503, 39.03804484, 52.55152607], + [23.7114063, 52.38661792, 72.54607218, 53.89587489, 67.94892652], + [45.02897165, 75.98315061, 78.257619, 61.85069778, 33.77972627], + ], + [ + [0.0, -0.002454937, -5.40909568, -37.74958445, -55.02172792], + [24.16049084, 58.53088654, 75.33566652, -9.65827726, 76.94753157], + [36.53113547, -28.57665427, -54.16269089, 6.2586262, -67.69290198], + [12.32708756, -33.04781428, -29.29282657, 13.46090338, 42.98737069], + ], + [ + [0.0, 0.00465342, -9.49591204, 32.9931831, 47.80929165], + [-16.11189945, 7.72083678, 19.17820444, -6.90801653, -17.46468994], + [-39.99097133, 9.92432127, 19.90687976, 2.40429413, 61.24066709], + [-25.45166461, -22.94347485, -31.32259433, 47.2621717, -60.05694598], + ], + ], + device=device, + dtype=dtype, + ) + + self.assert_close(kornia.color.rgb_to_lab(data), expected, low_tolerance=True) + + def test_forth_and_back(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + data = torch.rand(3, 4, 5, device=device, dtype=dtype) + lab = kornia.color.rgb_to_lab + rgb = kornia.color.lab_to_rgb + + data_out = lab(rgb(data, clip=False)) + self.assert_close(data_out, data) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.rand(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.rgb_to_lab, (img,), raise_exception=True, fast_mode=True) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.rgb_to_lab + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.RgbToLab().to(device, dtype) + fcn = kornia.color.rgb_to_lab + self.assert_close(ops(img), fcn(img)) + + +class TestLabToRgb(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.lab_to_rgb(img), torch.Tensor) + + @pytest.mark.parametrize("shape", [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1), (3, 2, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.ones(shape, device=device, dtype=dtype) + assert kornia.color.lab_to_rgb(img).shape == shape + + def test_exception(self, device, dtype): + with pytest.raises(TypeError): + assert kornia.color.lab_to_rgb([0.0]) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.lab_to_rgb(img) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.lab_to_rgb(img) + + def test_unit(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + data = torch.tensor( + [ + [ + [ + [50.21928787, 23.29810143, 14.98279190, 62.50927353, 72.78904724], + [70.86846924, 68.75330353, 52.81696701, 76.17090607, 88.63134003], + [46.87160873, 72.38699341, 37.71450806, 82.57386780, 74.79967499], + [77.33016968, 47.39180374, 61.76217651, 90.83254242, 86.96239471], + ], + [ + [65.81327057, -3.69859719, 0.16971001, 14.86583614, -65.54960632], + [-41.03258133, -19.52661896, 64.16155243, -58.53935242, -71.78411102], + [112.05227661, -60.13330460, 43.07910538, -51.01456833, -58.25787354], + [-62.37575531, 50.88882065, -39.27450943, 17.00958824, -24.93779755], + ], + [ + [-69.53346252, -73.34986877, -11.47461891, 66.73863220, 70.43983459], + [51.92737579, 58.77009583, 45.97863388, 24.44452858, 98.81991577], + [-7.60597992, 78.97976685, -69.31867218, 67.33953857, 14.28889370], + [92.31149292, -85.91405487, -32.83668518, -23.45091820, 69.99038696], + ], + ] + ], + device=device, + dtype=dtype, + ) + + # Reference output generated using skimage: lab2rgb(data) + expected = torch.tensor( + [ + [ + [ + [0.63513142, 0.0, 0.10660624, 0.79048697, 0.26823414], + [0.48903025, 0.64529494, 0.91140099, 0.15877841, 0.45987959], + [1.0, 0.36069696, 0.29236125, 0.55744393, 0.0], + [0.41710863, 0.3198324, 0.0, 0.94256868, 0.82748892], + ], + [ + [0.28210726, 0.26080003, 0.15027717, 0.54540429, 0.80323837], + [0.748392, 0.68774842, 0.24204415, 0.83695682, 0.9902132], + [0.0, 0.79101603, 0.26633725, 0.89223337, 0.82301254], + [0.84857086, 0.34455393, 0.66555314, 0.86168397, 0.8948667], + ], + [ + [0.94172458, 0.66390044, 0.21043296, 0.02453515, 0.04169043], + [0.28233233, 0.20235374, 0.19803933, 0.55069441, 0.0], + [0.50205101, 0.0, 0.79745394, 0.25376936, 0.6114783], + [0.0, 1.0, 0.80867314, 1.0, 0.28778443], + ], + ] + ], + device=device, + dtype=dtype, + ) + + expected_unclipped = torch.tensor( + [ + [ + [ + [0.63513142, -1.78708635, 0.10660624, 0.79048697, 0.26823414], + [0.48903025, 0.64529494, 0.91140099, 0.15877841, 0.45987959], + [1.01488435, 0.36069696, 0.29236125, 0.55744393, -0.28090181], + [0.41710863, 0.3198324, -1.81087917, 0.94256868, 0.82748892], + ], + [ + [0.28210726, 0.26080003, 0.15027717, 0.54540429, 0.80323837], + [0.748392, 0.68774842, 0.24204415, 0.83695682, 0.9902132], + [-1.37862046, 0.79101603, 0.26633725, 0.89223337, 0.82301254], + [0.84857086, 0.34455393, 0.66555314, 0.86168397, 0.8948667], + ], + [ + [0.94172458, 0.66390044, 0.21043296, 0.02453515, 0.04169043], + [0.28233233, 0.20235374, 0.19803933, 0.55069441, -0.62707704], + [0.50205101, -0.25005965, 0.79745394, 0.25376936, 0.6114783], + [-0.55802926, 1.0223477, 0.80867314, 1.07334156, 0.28778443], + ], + ] + ], + device=device, + dtype=dtype, + ) + + self.assert_close(kornia.color.lab_to_rgb(data), expected) + self.assert_close(kornia.color.lab_to_rgb(data, clip=False), expected_unclipped) + + def test_forth_and_back(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + data = torch.rand(3, 4, 5, device=device, dtype=dtype) + lab = kornia.color.rgb_to_lab + rgb = kornia.color.lab_to_rgb + + unclipped_data_out = rgb(lab(data), clip=False) + self.assert_close(unclipped_data_out, data) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.rand(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + img = kornia.color.rgb_to_lab(img) + assert gradcheck(kornia.color.lab_to_rgb, (img,), raise_exception=True, fast_mode=True) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.lab_to_rgb + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.LabToRgb().to(device, dtype) + fcn = kornia.color.lab_to_rgb + self.assert_close(ops(img), fcn(img)) diff --git a/tests/color/test_luv.py b/tests/color/test_luv.py new file mode 100644 index 0000000..3d5088e --- /dev/null +++ b/tests/color/test_luv.py @@ -0,0 +1,259 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +from torch.autograd import gradcheck + +import kornia + +from testing.base import BaseTester + + +class TestRgbToLuv(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.rgb_to_luv(img), torch.Tensor) + + @pytest.mark.parametrize("shape", [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1), (3, 2, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.ones(shape, device=device, dtype=dtype) + assert kornia.color.rgb_to_luv(img).shape == shape + + def test_exception(self, device, dtype): + with pytest.raises(TypeError): + assert kornia.color.rgb_to_luv([0.0]) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_luv(img) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_luv(img) + + def test_unit(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + data = torch.tensor( + [ + [ + [0.0, 1.0, 0.69396782, 0.63581685, 0.09902618], + [0.59459005, 0.74215373, 0.89662376, 0.25920381, 0.89937686], + [0.29857584, 0.28139791, 0.16441015, 0.55507519, 0.06124221], + [0.40908658, 0.10261389, 0.01691456, 0.76006799, 0.32971736], + ], + [ + [0.0, 1.0, 0.79009938, 0.91742945, 0.60044175], + [0.42812678, 0.18552390, 0.04186043, 0.38030245, 0.15420346], + [0.13552373, 0.53955473, 0.79102736, 0.49050815, 0.75271446], + [0.39861023, 0.80680277, 0.82823833, 0.54438462, 0.22063386], + ], + [ + [0.0, 1.0, 0.84317145, 0.59529881, 0.15297393], + [0.59235313, 0.36617295, 0.34600773, 0.40304737, 0.61720451], + [0.46040250, 0.42006640, 0.54765106, 0.48982632, 0.13914755], + [0.58402964, 0.89597990, 0.98276161, 0.25019163, 0.69285921], + ], + ], + device=device, + dtype=dtype, + ) + + # Reference output generated using skimage: rgb2luv(data) + expected = torch.tensor( + [ + [ + [0.0, 100.0, 79.75208282, 86.38912964, 55.25164032], + [51.66668701, 43.81214523, 48.93865585, 39.03804398, 52.55152512], + [23.71140671, 52.38661957, 72.54607391, 53.89587402, 67.94892883], + [45.02897263, 75.98315430, 78.25762177, 61.85069656, 33.77972794], + ], + [ + [-0.0, -0.00054950, -13.54032803, -35.42317200, -49.27433014], + [21.34596062, 94.13956451, 137.11340332, -14.69241238, 102.94833374], + [9.55611229, -30.01761436, -58.94236755, 9.83261871, -62.96137619], + [-1.55336237, -55.22497559, -56.21067810, 43.76751328, 1.46367633], + ], + [ + [-0.0, 0.00766720, -13.74480152, 52.17128372, 60.92724228], + [-27.01125526, -1.72837746, 6.57535267, -7.83582020, -38.45543289], + [-50.89970779, 17.65329361, 36.54148102, 2.25501800, 78.93702698], + [-38.39783859, -31.71204376, -46.63606644, 50.16629410, -84.74416351], + ], + ], + device=device, + dtype=dtype, + ) + + self.assert_close(kornia.color.rgb_to_luv(data), expected, low_tolerance=True) + + def test_forth_and_back(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + # Generate on CPU with a fixed seed so the test is order-independent. + torch.manual_seed(0) + data = torch.rand(3, 4, 5).to(device=device, dtype=dtype) + luv = kornia.color.rgb_to_luv + rgb = kornia.color.luv_to_rgb + + data_out = luv(rgb(data)) + self.assert_close(data_out, data) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.rand(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.rgb_to_luv, (img,), raise_exception=True, fast_mode=True) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.rgb_to_luv + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.RgbToLuv().to(device, dtype) + fcn = kornia.color.rgb_to_luv + self.assert_close(ops(img), fcn(img)) + + +class TestLuvToRgb(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.luv_to_rgb(img), torch.Tensor) + + @pytest.mark.parametrize("shape", [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1), (3, 2, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.ones(shape, device=device, dtype=dtype) + assert kornia.color.luv_to_rgb(img).shape == shape + + def test_exception(self, device, dtype): + with pytest.raises(TypeError): + assert kornia.color.luv_to_rgb([0.0]) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.luv_to_rgb(img) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.luv_to_rgb(img) + + def test_unit(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + data = torch.tensor( + [ + [ + [ + [50.21928787, 23.29810143, 14.98279190, 62.50927353, 72.78904724], + [70.86846924, 68.75330353, 52.81696701, 76.17090607, 88.63134003], + [46.87160873, 72.38699341, 37.71450806, 82.57386780, 74.79967499], + [77.33016968, 47.39180374, 61.76217651, 90.83254242, 86.96239471], + ], + [ + [65.81327057, -3.69859719, 0.16971001, 14.86583614, -65.54960632], + [-41.03258133, -19.52661896, 64.16155243, -58.53935242, -71.78411102], + [112.05227661, -60.13330460, 43.07910538, -51.01456833, -58.25787354], + [-62.37575531, 50.88882065, -39.27450943, 17.00958824, -24.93779755], + ], + [ + [-69.53346252, -73.34986877, -11.47461891, 66.73863220, 70.43983459], + [51.92737579, 58.77009583, 45.97863388, 24.44452858, 98.81991577], + [-7.60597992, 78.97976685, -69.31867218, 67.33953857, 14.28889370], + [92.31149292, -85.91405487, -32.83668518, -23.45091820, 69.99038696], + ], + ] + ], + device=device, + dtype=dtype, + ) + + # Reference output generated using skimage: luv2rgb(data) + + expected = torch.tensor( + [ + [ + [ + [0.78923208, 0.17048222, 0.14947766, 0.65528989, 0.07863078], + [0.41649094, 0.55222923, 0.72673196, 0.21939684, 0.34298307], + [0.82763243, 0.24021322, 0.58888060, 0.47255886, 0.16407511], + [0.30320778, 0.72233224, 0.21593384, 0.98893607, 0.71707106], + ], + [ + [0.20532851, 0.13188709, 0.13879408, 0.59964627, 0.80721593], + [0.75411713, 0.70656943, 0.41770950, 0.82750136, 0.99659365], + [0.12436169, 0.79804462, 0.10958754, 0.89803618, 0.81000644], + [0.85726571, 0.17667055, 0.63285238, 0.85567462, 0.91538441], + ], + [ + [0.73985511, 0.59308004, 0.21156698, 0.03804367, 0.32732114], + [0.42489606, 0.33011687, 0.12804756, 0.64905322, 0.25216782], + [0.41637793, 0.22158240, 0.63437861, 0.46121466, 0.68336427], + [0.06325728, 0.78878325, 0.74280596, 0.99514300, 0.47176042], + ], + ] + ], + device=device, + dtype=dtype, + ) + + self.assert_close(kornia.color.luv_to_rgb(data), expected) + + def test_forth_and_back(self, device, dtype): + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + # Generate on CPU with a fixed seed so the test is order-independent. + torch.manual_seed(0) + data = torch.rand(3, 4, 5).to(device=device, dtype=dtype) + luv = kornia.color.rgb_to_luv + rgb = kornia.color.luv_to_rgb + + data_out = rgb(luv(data)) + self.assert_close(data_out, data) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.rand(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + img = kornia.color.rgb_to_luv(img) + assert gradcheck(kornia.color.luv_to_rgb, (img,), raise_exception=True, fast_mode=True) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.luv_to_rgb + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.LuvToRgb().to(device, dtype) + fcn = kornia.color.luv_to_rgb + self.assert_close(ops(img), fcn(img)) diff --git a/tests/color/test_raw.py b/tests/color/test_raw.py new file mode 100644 index 0000000..b7172fc --- /dev/null +++ b/tests/color/test_raw.py @@ -0,0 +1,298 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import pytest +import torch +from torch.autograd import gradcheck + +import kornia + +from testing.base import ( + BaseTester, + assert_close, # test utils +) + + +class TestRawToRgb(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 1, 4, 6 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.raw_to_rgb(img, kornia.color.CFA.BG), torch.Tensor) + + @pytest.mark.parametrize("batch_size, height, width", [(1, 6, 4), (2, 2, 4), (3, 4, 2)]) + def test_cardinality(self, device, dtype, batch_size, height, width): + img = torch.ones(batch_size, 1, height, width, device=device, dtype=dtype) + assert kornia.color.raw_to_rgb(img, kornia.color.CFA.BG).shape == (batch_size, 3, height, width) + + def test_exception(self, device, dtype): + with pytest.raises(TypeError): + assert kornia.color.raw_to_rgb([0.0], kornia.color.CFA.BG) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.raw_to_rgb(img, kornia.color.CFA.GB) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.raw_to_rgb(img, kornia.color.CFA.RG) + + with pytest.raises(ValueError): + img = torch.ones(1, 3, 1, 1, device=device, dtype=dtype) + assert kornia.color.raw_to_rgb(img, kornia.color.CFA.GR) + + # dimensionality test + with pytest.raises(ValueError): + img = torch.ones(3, 2, 1, device=device, dtype=dtype) + assert kornia.color.raw_to_rgb(img, kornia.color.CFA.GR) + + # dimensionality test + with pytest.raises(ValueError): + img = torch.ones(3, 1, 2, device=device, dtype=dtype) + assert kornia.color.raw_to_rgb(img, kornia.color.CFA.GR) + + # With he current implementations we should get back an identical raw representation when doing raw -> rgb -> raw + # Note that with more advanced implementations this may not necessarily be true or desirable + def test_forth_and_back(self, device, dtype): # skipcq: PYL-R0201 + data = torch.rand(1, 80, 80, device=device, dtype=dtype) + raw = kornia.color.rgb_to_raw + rgb = kornia.color.raw_to_rgb + + for x in kornia.color.CFA: + data_out = raw(rgb(data, cfa=x), cfa=x) + assert_close(data_out, data) + + # make sure different cfas are actually different + def test_cfas_not_the_same(self, device, dtype): # skipcq: PYL-R0201 + data = torch.rand(1, 16, 16, device=device, dtype=dtype) + assert ( + torch.max( + kornia.color.raw_to_rgb(data, kornia.color.CFA.BG) - kornia.color.raw_to_rgb(data, kornia.color.CFA.RG) + ) + > 0.0 + ) + + # The outcome will be very different for different implementations + # Here we compare against a current baseline, it is safe to update this if the underlying algorithm changes + def test_functional(self, device, dtype): # skipcq: PYL-R0201 + data = torch.tensor( + [[[1, 0.5, 0.2, 0.4], [0.75, 0.25, 0.8, 0.3], [0.65, 0.15, 0.7, 0.2], [0.55, 0.5, 0.6, 0.1]]], + device=device, + dtype=dtype, + ) + # checked by hand as correct interpolation. Note the ugly replication that happens for Red on the last column + # and row. We shall accept to live with that + expected = torch.tensor( + [ + [ + [1.0000, 0.6000, 0.2000, 0.2000], + [0.8250, 0.6375, 0.4500, 0.4500], + [0.6500, 0.6750, 0.7000, 0.7000], + [0.6500, 0.6750, 0.7000, 0.7000], + ], + [ + [0.6250, 0.5000, 0.6250, 0.4000], + [0.7500, 0.5500, 0.8000, 0.5500], + [0.4000, 0.1500, 0.4375, 0.2000], + [0.5500, 0.3625, 0.6000, 0.4000], + ], + [ + [0.2500, 0.2500, 0.2750, 0.3000], + [0.2500, 0.2500, 0.2750, 0.3000], + [0.3750, 0.3750, 0.2875, 0.2000], + [0.5000, 0.5000, 0.3000, 0.1000], + ], + ], + device=device, + dtype=dtype, + ) + + img_rgb = kornia.color.raw_to_rgb(data, kornia.color.raw.CFA.BG) + assert_close(img_rgb, expected) + + # If we roll the data and the different CFAs they give the same result (expect on edges!) + def test_cfa_on_rolled(self, device, dtype): # skipcq: PYL-R0201 + data = torch.rand(1, 1, 8, 8, device=device, dtype=dtype) + bgres = kornia.color.raw_to_rgb(data, kornia.color.raw.CFA.BG) + gbres = kornia.color.raw_to_rgb(data.roll((0, 1), (-2, -1)), kornia.color.raw.CFA.GB) + grres = kornia.color.raw_to_rgb(data.roll((1, 0), (-2, -1)), kornia.color.raw.CFA.GR) + rgres = kornia.color.raw_to_rgb(data.roll((1, 1), (-2, -1)), kornia.color.raw.CFA.RG) + + assert_close(bgres[:, :, 1:5, 1:5], gbres[:, :, 1:5, 2:6]) + assert_close(bgres[:, :, 1:5, 1:5], grres[:, :, 2:6, 1:5]) + assert_close(bgres[:, :, 1:5, 1:5], rgres[:, :, 2:6, 2:6]) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 1, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.raw_to_rgb, (img, kornia.color.raw.CFA.BG), raise_exception=True, fast_mode=True) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 1, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.raw_to_rgb + op_jit = torch.jit.script(op) + assert_close(op(img, kornia.color.raw.CFA.BG), op_jit(img, kornia.color.raw.CFA.BG)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 1, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + raw_ops = kornia.color.RawToRgb(kornia.color.raw.CFA.BG).to(device, dtype) + raw_fcn = kornia.color.raw_to_rgb + assert_close(raw_ops(img), raw_fcn(img, kornia.color.raw.CFA.BG)) + + +class TestRgbToRaw(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 6 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.rgb_to_raw(img, kornia.color.raw.CFA.BG), torch.Tensor) + + @pytest.mark.parametrize("batch_size, height, width", [(1, 3, 4), (2, 2, 4), (3, 4, 1)]) + def test_cardinality(self, device, dtype, batch_size, height, width): + img = torch.ones(batch_size, 3, height, width, device=device, dtype=dtype) + assert kornia.color.rgb_to_raw(img, kornia.color.raw.CFA.GR).shape == (batch_size, 1, height, width) + + def test_exception(self, device, dtype): + with pytest.raises(TypeError): + assert kornia.color.rgb_to_raw([0.0], kornia.color.raw.CFA.RG) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_raw(img, kornia.color.raw.CFA.BG) + + # Reverse test in rawtorgb is sufficient functional test + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.rgb_to_raw, (img, kornia.color.raw.CFA.BG), raise_exception=True, fast_mode=True) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.rgb_to_raw + op_jit = torch.jit.script(op) + assert_close(op(img, kornia.color.raw.CFA.BG), op_jit(img, kornia.color.raw.CFA.BG)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + raw_ops = kornia.color.RgbToRaw(kornia.color.raw.CFA.BG).to(device, dtype) + raw_fcn = kornia.color.rgb_to_raw + assert_close(raw_ops(img), raw_fcn(img, kornia.color.raw.CFA.BG)) + + +class TestRawToRgb2x2Downscaled(BaseTester): + def test_smoke(self, device, dtype) -> None: + C, H, W = 1, 4, 6 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.raw_to_rgb_2x2_downscaled(img, kornia.color.CFA.BG), torch.Tensor) + + @pytest.mark.parametrize( + "batch_size, height, width", + [((1,), 6, 4), ((), 6, 4), ((), 0, 0), ((4, 5), 0, 0), ((2,), 2, 4), ((3,), 4, 2), ((4, 2, 1), 4, 2)], + ) + def test_cardinality(self, device, dtype, batch_size, height, width): + img = torch.ones(*batch_size, 1, height, width, device=device, dtype=dtype) + assert kornia.color.raw_to_rgb_2x2_downscaled(img, kornia.color.CFA.BG).shape == ( + *batch_size, + 3, + height // 2, + width // 2, + ) + + def test_exception(self, device, dtype): + from kornia.core.exceptions import BaseError, ShapeError + + with pytest.raises(BaseError) as errinf: + kornia.color.raw_to_rgb_2x2_downscaled([0.0], kornia.color.CFA.BG) + assert "Input type is not a torch.Tensor" in str(errinf.value) + + with pytest.raises(ShapeError) as errinf: + img = torch.ones(1, 1, device=device, dtype=dtype) + kornia.color.raw_to_rgb_2x2_downscaled(img, kornia.color.CFA.GB) + assert "Shape dimension mismatch" in str(errinf.value) or "Expected shape" in str(errinf.value) + + with pytest.raises(ShapeError) as errinf: + img = torch.ones(2, 2, 2, device=device, dtype=dtype) + kornia.color.raw_to_rgb_2x2_downscaled(img, kornia.color.CFA.RG) + assert "Shape dimension mismatch" in str(errinf.value) or "Expected shape" in str(errinf.value) + + with pytest.raises(Exception) as errinf: + img = torch.ones(1, 3, 2, device=device, dtype=dtype) + kornia.color.raw_to_rgb_2x2_downscaled(img, kornia.color.CFA.GR) + assert "Input H&W must be evenly disible by 2. Got" in str(errinf) + + with pytest.raises(Exception) as errinf: + img = torch.ones(1, 2, 3, device=device, dtype=dtype) + kornia.color.raw_to_rgb_2x2_downscaled(img, kornia.color.CFA.GR) + assert "Input H&W must be evenly disible by 2. Got" in str(errinf) + + with pytest.raises(ValueError) as errinf: + img = torch.ones(1, 4, 8, device=device, dtype=dtype) + nonexistent_cfa = 195162495283 + kornia.color.raw_to_rgb_2x2_downscaled(img, nonexistent_cfa) + assert "Unsupported CFA Got" in str(errinf) + + @pytest.mark.parametrize( + "cfa, expected_rgb", + [ + (kornia.color.raw.CFA.BG, [[[0.12, 0.46]], [[0.33, 0.505]], [[0.00, 0.89]]]), + (kornia.color.raw.CFA.GB, [[[0.43, 0.58]], [[0.06, 0.675]], [[0.23, 0.43]]]), + (kornia.color.raw.CFA.RG, [[[0.0, 0.89]], [[0.33, 0.505]], [[0.12, 0.46]]]), + (kornia.color.raw.CFA.GR, [[[0.23, 0.43]], [[0.06, 0.675]], [[0.43, 0.58]]]), + ], + ) + def test_functional(self, device, dtype, cfa, expected_rgb): + data = torch.tensor([[[0.12, 0.43, 0.46, 0.58], [0.23, 0.00, 0.43, 0.89]]], device=device, dtype=dtype) + expected_rgb = torch.tensor(expected_rgb, device=device, dtype=dtype) + + img_rgb = kornia.color.raw_to_rgb_2x2_downscaled(data, cfa) + self.assert_close(img_rgb, expected_rgb) + + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 1, 8, 8 + img = torch.ones(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + self.gradcheck( + kornia.color.raw_to_rgb_2x2_downscaled, (img, kornia.color.raw.CFA.BG), raise_exception=True, fast_mode=True + ) + + def test_jit(self, device, dtype): + B, C, H, W = 2, 1, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.raw_to_rgb_2x2_downscaled + op_jit = torch.jit.script(op) + self.assert_close(op(img, kornia.color.raw.CFA.BG), op_jit(img, kornia.color.raw.CFA.BG)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 1, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + raw_ops = kornia.color.RawToRgb2x2Downscaled(kornia.color.raw.CFA.BG).to(device, dtype) + raw_fcn = kornia.color.raw_to_rgb_2x2_downscaled + self.assert_close(raw_ops(img), raw_fcn(img, kornia.color.raw.CFA.BG)) + + def test_dynamo(self, device, dtype, torch_optimizer): + B, C, H, W = 2, 1, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.raw_to_rgb_2x2_downscaled + op_optimized = torch_optimizer(op) + self.assert_close(op(img, kornia.color.CFA.BG), op_optimized(img, kornia.color.CFA.BG)) diff --git a/tests/color/test_rgb.py b/tests/color/test_rgb.py new file mode 100644 index 0000000..93d70ad --- /dev/null +++ b/tests/color/test_rgb.py @@ -0,0 +1,499 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +from torch.autograd import gradcheck + +import kornia + +from testing.base import BaseTester + + +class TestRgbToBgr(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.rgb_to_bgr(img), torch.Tensor) + + @pytest.mark.parametrize("shape", [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1), (3, 2, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.ones(shape, device=device, dtype=dtype) + assert kornia.color.rgb_to_bgr(img).shape == shape + + def test_exception(self, device, dtype): + with pytest.raises(TypeError): + assert kornia.color.rgb_to_bgr([0.0]) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_bgr(img) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_bgr(img) + + with pytest.raises(TypeError): + assert kornia.color.bgr_to_rgb([0.0]) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.bgr_to_rgb(img) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.bgr_to_rgb(img) + + def test_back_and_forth(self, device, dtype): + data_bgr = torch.rand(1, 3, 3, 2, device=device, dtype=dtype) + data_rgb = kornia.color.bgr_to_rgb(data_bgr) + data_bgr_new = kornia.color.rgb_to_bgr(data_rgb) + self.assert_close(data_bgr, data_bgr_new) + + def test_unit(self, device, dtype): + data = torch.tensor( + [ + [[1.0, 1.0], [1.0, 1.0]], + [[2.0, 2.0], [2.0, 2.0]], + [[3.0, 3.0], [3.0, 3.0]], + ], + device=device, + dtype=dtype, + ) # 3x2x2 + + expected = torch.tensor( + [ + [[3.0, 3.0], [3.0, 3.0]], + [[2.0, 2.0], [2.0, 2.0]], + [[1.0, 1.0], [1.0, 1.0]], + ], + device=device, + dtype=dtype, + ) # 3x2x2 + + f = kornia.color.rgb_to_bgr + self.assert_close(f(data), expected) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.rgb_to_bgr, (img,), raise_exception=True, fast_mode=True) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.rgb_to_bgr + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.RgbToBgr().to(device, dtype) + fcn = kornia.color.rgb_to_bgr + self.assert_close(ops(img), fcn(img)) + + def test_module_bgr(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.BgrToRgb().to(device, dtype) + fcn = kornia.color.bgr_to_rgb + self.assert_close(ops(img), fcn(img)) + + +class TestRgbToRgba(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.rgb_to_rgba(img, 0.0), torch.Tensor) + + @pytest.mark.parametrize("shape", [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1), (3, 2, 1)]) + def test_cardinality(self, device, dtype, shape): + out_shape = list(shape) + out_shape[-3] += 1 + img = torch.ones(shape, device=device, dtype=dtype) + assert kornia.color.rgb_to_rgba(img, 0.0).shape == tuple(out_shape) + + def test_exception(self, device, dtype): + # rgb to rgba + with pytest.raises(TypeError): + assert kornia.color.rgb_to_rgba([0.0], 0.0) + + with pytest.raises(TypeError): + img = torch.ones(1, 3, 1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_rgba(img) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_rgba(img, 0.0) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_rgba(img, 0.0) + + with pytest.raises(TypeError): + img = torch.ones(3, 1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_rgba(img, "alpha_str") + + # rgba to rgb + with pytest.raises(TypeError): + assert kornia.color.rgba_to_rgb(0.0) + + with pytest.raises(ValueError): + img = torch.ones(1, 3, 1, 1, device=device, dtype=dtype) + assert kornia.color.rgba_to_rgb(img) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.rgba_to_rgb(img) + + def test_back_and_forth_rgb(self, device, dtype): + a_val: float = 1.0 + x_rgb = torch.ones(3, 4, 4, device=device, dtype=dtype) + x_rgba = kornia.color.rgb_to_rgba(x_rgb, a_val) + x_rgb_new = kornia.color.rgba_to_rgb(x_rgba) + self.assert_close(x_rgb, x_rgb_new) + + def test_back_and_forth_bgr(self, device, dtype): + a_val: float = 1.0 + x_bgr = torch.ones(3, 4, 4, device=device, dtype=dtype) + x_rgba = kornia.color.bgr_to_rgba(x_bgr, a_val) + x_bgr_new = kornia.color.rgba_to_bgr(x_rgba) + self.assert_close(x_bgr, x_bgr_new) + + @pytest.mark.parametrize("aval", [0.4, 45.0]) + def test_unit(self, device, dtype, aval): + data = torch.tensor( + [ + [ + [[1.0, 1.0], [1.0, 1.0]], + [[2.0, 2.0], [2.0, 2.0]], + [[3.0, 3.0], [3.0, 3.0]], + ] + ], + device=device, + dtype=dtype, + ) # Bx3x2x2 + + expected = torch.tensor( + [ + [ + [[1.0, 1.0], [1.0, 1.0]], + [[2.0, 2.0], [2.0, 2.0]], + [[3.0, 3.0], [3.0, 3.0]], + [[aval, aval], [aval, aval]], + ] + ], + device=device, + dtype=dtype, + ) # Bx4x2x2 + + self.assert_close(kornia.color.rgb_to_rgba(data, aval), expected) + + @pytest.mark.parametrize("aval", [0.4, 45.0]) + def test_unit_aval_th(self, device, dtype, aval): + data = torch.tensor( + [ + [ + [[1.0, 1.0], [1.0, 1.0]], + [[2.0, 2.0], [2.0, 2.0]], + [[3.0, 3.0], [3.0, 3.0]], + ] + ], + device=device, + dtype=dtype, + ) # Bx3x2x2 + + expected = torch.tensor( + [ + [ + [[1.0, 1.0], [1.0, 1.0]], + [[2.0, 2.0], [2.0, 2.0]], + [[3.0, 3.0], [3.0, 3.0]], + [[aval, aval], [aval, aval]], + ] + ], + device=device, + dtype=dtype, + ) # Bx4x2x2 + + aval = torch.full_like(data[:, :1], aval) # Bx1xHxW + self.assert_close(kornia.color.rgb_to_rgba(data, aval), expected) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.rgb_to_rgba, (img, 1.0), raise_exception=True, fast_mode=True) + + @pytest.mark.grad() + def test_gradcheck_th(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + aval = torch.ones(B, 1, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.rgb_to_rgba, (img, aval), raise_exception=True, fast_mode=True) + + @pytest.mark.skip(reason="unsupported Union type") + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.rgb_to_rgba + op_jit = torch.jit.script(op) + self.assert_close(op(img, 1.0), op_jit(img, 1.0)) + aval = torch.ones(B, 1, H, W, device=device, dtype=dtype) + self.assert_close(op(img, aval), op_jit(img, aval)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.RgbToRgba(1.0).to(device, dtype) + fcn = kornia.color.rgb_to_rgba + self.assert_close(ops(img), fcn(img, 1.0)) + + def test_module_bgr(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.BgrToRgba(1.0).to(device, dtype) + fcn = kornia.color.bgr_to_rgba + self.assert_close(ops(img), fcn(img, 1.0)) + + def test_module_bgra2rgb(self, device, dtype): + B, C, H, W = 2, 4, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.RgbaToRgb().to(device, dtype) + fcn = kornia.color.rgba_to_rgb + self.assert_close(ops(img), fcn(img)) + + def test_module_bgra2bgr(self, device, dtype): + B, C, H, W = 2, 4, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.RgbaToBgr().to(device, dtype) + fcn = kornia.color.rgba_to_bgr + self.assert_close(ops(img), fcn(img)) + + def test_numerical_bgr(self, device, dtype): + # input: B=1, G=0.5, R=0 (BGR) + data = torch.tensor([[[1.0]], [[0.5]], [[0.0]]], device=device, dtype=dtype) # 3x1x1 + # expected: R=0, G=0.5, B=1, A=1.0 (RGBA) + expected = torch.tensor([[[0.0]], [[0.5]], [[1.0]], [[1.0]]], device=device, dtype=dtype) + self.assert_close(kornia.color.bgr_to_rgba(data, 1.0), expected) + + def test_numerical_rgba_bgr(self, device, dtype): + # input: R=0.2, G=0.4, B=0.6, A=1.0 (RGBA) + data = torch.tensor([[[0.2]], [[0.4]], [[0.6]], [[1.0]]], device=device, dtype=dtype) # 4x1x1 + # expected: B=0.6, G=0.4, R=0.2 (BGR) + # Note: rgba_to_bgr uses rgba_to_rgb internally which does alpha compositing. + # With A=1.0, it should just be the RGB channels. + expected = torch.tensor([[[0.6]], [[0.4]], [[0.2]]], device=device, dtype=dtype) + self.assert_close(kornia.color.rgba_to_bgr(data), expected) + + +class TestLinearRgb(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.rgb_to_linear_rgb(img), torch.Tensor) + assert isinstance(kornia.color.linear_rgb_to_rgb(img), torch.Tensor) + + @pytest.mark.parametrize("shape", [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1), (3, 2, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.ones(shape, device=device, dtype=dtype) + assert kornia.color.rgb_to_linear_rgb(img).shape == shape + assert kornia.color.linear_rgb_to_rgb(img).shape == shape + + def test_exception(self, device, dtype): + with pytest.raises(TypeError): + assert kornia.color.rgb_to_linear_rgb([0.0]) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_linear_rgb(img) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_linear_rgb(img) + + with pytest.raises(TypeError): + assert kornia.color.linear_rgb_to_rgb([0.0]) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.linear_rgb_to_rgb(img) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.linear_rgb_to_rgb(img) + + def test_back_and_forth(self, device, dtype): + data_bgr = torch.rand(1, 3, 3, 2, device=device, dtype=dtype) + data_rgb = kornia.color.rgb_to_linear_rgb(data_bgr) + data_bgr_new = kornia.color.linear_rgb_to_rgb(data_rgb) + self.assert_close(data_bgr, data_bgr_new) + + def test_unit(self, device, dtype): + data = torch.tensor( + [ + [[1.0, 0.0], [0.5, 0.1]], + [[1.0, 0.0], [0.5, 0.2]], + [[1.0, 0.0], [0.5, 0.3]], + ], + device=device, + dtype=dtype, + ) # 3x2x2 + + expected = torch.tensor( + [ + [[1.00000000, 0.00000000], [0.21404116, 0.01002283]], + [[1.00000000, 0.00000000], [0.21404116, 0.03310477]], + [[1.00000000, 0.00000000], [0.21404116, 0.07323898]], + ], + device=device, + dtype=dtype, + ) # 3x2x2 + + f = kornia.color.rgb_to_linear_rgb + self.assert_close(f(data), expected) + + def test_unit_linear(self, device, dtype): + data = torch.tensor( + [ + [[1.00000000, 0.00000000], [0.21404116, 0.01002283]], + [[1.00000000, 0.00000000], [0.21404116, 0.03310477]], + [[1.00000000, 0.00000000], [0.21404116, 0.07323898]], + ], + device=device, + dtype=dtype, + ) # 3x2x2 + + expected = torch.tensor( + [ + [[1.0, 0.0], [0.5, 0.1]], + [[1.0, 0.0], [0.5, 0.2]], + [[1.0, 0.0], [0.5, 0.3]], + ], + device=device, + dtype=dtype, + ) # 3x2x2 + + f = kornia.color.linear_rgb_to_rgb + self.assert_close(f(data), expected) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.rgb_to_linear_rgb, (img,), raise_exception=True, fast_mode=True) + assert gradcheck(kornia.color.linear_rgb_to_rgb, (img,), raise_exception=True, fast_mode=True) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.rgb_to_linear_rgb + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + @pytest.mark.jit() + def test_jit_linear(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.linear_rgb_to_rgb + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.RgbToLinearRgb().to(device, dtype) + fcn = kornia.color.rgb_to_linear_rgb + self.assert_close(ops(img), fcn(img)) + + def test_module_linear(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.LinearRgbToRgb().to(device, dtype) + fcn = kornia.color.linear_rgb_to_rgb + self.assert_close(ops(img), fcn(img)) + + +class TestRgb255Normals(BaseTester): + # Smoke tests: check if the functions execute and return a tensor + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.normals_to_rgb255(img), torch.Tensor) + assert isinstance(kornia.color.rgb_to_rgb255(img), torch.Tensor) + assert isinstance(kornia.color.rgb255_to_rgb(img), torch.Tensor) + assert isinstance(kornia.color.rgb255_to_normals(img), torch.Tensor) + + # Cardinality tests: ensure the output shape is correct + @pytest.mark.parametrize("shape", [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.ones(shape, device=device, dtype=dtype) + assert kornia.color.normals_to_rgb255(img).shape == shape + assert kornia.color.rgb_to_rgb255(img).shape == shape + assert kornia.color.rgb255_to_rgb(img).shape == shape + assert kornia.color.rgb255_to_normals(img).shape == shape + + # Back and forth tests: ensure round-trip conversions preserve data + def test_back_and_forth(self, device, dtype): + data_rgb = torch.rand(1, 3, 3, 2, device=device, dtype=dtype) + rgb255 = kornia.color.rgb_to_rgb255(data_rgb) + rgb_restored = kornia.color.rgb255_to_rgb(rgb255) + torch.testing.assert_close(data_rgb, rgb_restored, atol=1e-4, rtol=1e-4) + + data_normals = torch.nn.functional.normalize(torch.rand(1, 3, 3, 2, device=device, dtype=dtype), p=2, dim=1) + rgb255_normals = kornia.color.normals_to_rgb255(data_normals) + normals_restored = kornia.color.rgb255_to_normals(rgb255_normals) + torch.testing.assert_close(data_normals, normals_restored, atol=1e-4, rtol=1e-4) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.full((B, C, H, W), 0.5, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.rgb_to_rgb255, (img,), raise_exception=True, fast_mode=True) + assert gradcheck(kornia.color.rgb255_to_rgb, (img,), raise_exception=True, fast_mode=True) + assert gradcheck(kornia.color.normals_to_rgb255, (img,), raise_exception=True, fast_mode=True) + assert gradcheck(kornia.color.rgb255_to_normals, (img,), raise_exception=True, fast_mode=True) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + for op in [ + kornia.color.normals_to_rgb255, + kornia.color.rgb_to_rgb255, + kornia.color.rgb255_to_rgb, + kornia.color.rgb255_to_normals, + ]: + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + + for fcn, Mod in [ + (kornia.color.normals_to_rgb255, kornia.color.NormalsToRgb255()), + (kornia.color.rgb255_to_rgb, kornia.color.Rgb255ToRgb()), + (kornia.color.rgb_to_rgb255, kornia.color.RgbToRgb255()), + (kornia.color.rgb255_to_normals, kornia.color.Rgb255ToNormals()), + ]: + self.assert_close(fcn(img), Mod(img)) diff --git a/tests/color/test_sepia.py b/tests/color/test_sepia.py new file mode 100644 index 0000000..4d64dc5 --- /dev/null +++ b/tests/color/test_sepia.py @@ -0,0 +1,98 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestSepia(BaseTester): + def test_smoke(self, device, dtype): + input_tensor = torch.tensor( + [[[0.1, 1.0], [0.2, 0.1]], [[0.1, 0.8], [0.2, 0.5]], [[0.1, 0.3], [0.2, 0.8]]], device=device, dtype=dtype + ) + + # With rescale + expected_tensor = torch.tensor( + [[[0.1269, 1.0], [0.2537, 0.5400]], [[0.1269, 1.0], [0.2537, 0.5403]], [[0.1269, 1.0], [0.2538, 0.5403]]], + device=device, + dtype=dtype, + ) + actual = kornia.color.sepia(input_tensor, rescale=True) + + assert actual.shape[:] == (3, 2, 2) + self.assert_close(actual, expected_tensor, rtol=1e-2, atol=1e-2) + + # Without rescale + expected_tensor = torch.tensor( + [ + [[0.1351, 1.0649], [0.2702, 0.5750]], + [[0.1203, 0.9482], [0.2406, 0.5123]], + [[0.0937, 0.7385], [0.1874, 0.3990]], + ], + device=device, + dtype=dtype, + ) + + actual = kornia.color.sepia(input_tensor, rescale=False) + assert actual.shape[:] == (3, 2, 2) + self.assert_close(actual, expected_tensor, rtol=1e-2, atol=1e-2) + + def test_exception(self, device, dtype): + with pytest.raises(ValueError): + kornia.color.sepia(torch.rand(size=(4, 1, 1), dtype=dtype, device=device)) + + @pytest.mark.parametrize("batch_shape", [(1, 3, 8, 15), (2, 3, 11, 7), (3, 8, 15)]) + def test_cardinality(self, batch_shape, device, dtype): + input_tensor = torch.rand(batch_shape, device=device, dtype=dtype) + actual = kornia.color.sepia(input_tensor) + assert actual.shape == batch_shape + + def test_noncontiguous(self, device, dtype): + batch_size = 3 + inp = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + + actual = kornia.color.sepia(inp) + + assert inp.is_contiguous() is False + assert actual.is_contiguous() + self.assert_close(actual, actual) + + def test_gradcheck(self, device): + # test parameters + batch_shape = (1, 3, 5, 5) + + # evaluate function gradient + input_tensor = torch.rand(batch_shape, device=device, dtype=torch.float64) + self.gradcheck(kornia.color.sepia, (input_tensor,)) + + def test_jit(self, device, dtype): + op = kornia.color.sepia + op_script = torch.jit.script(op) + + img = torch.ones(1, 3, 5, 5, device=device, dtype=dtype) + self.assert_close(op(img), op_script(img)) + + def test_module(self, device, dtype): + op = kornia.color.sepia + op_module = kornia.color.Sepia() + + img = torch.ones(1, 3, 5, 5, device=device, dtype=dtype) + self.assert_close(op(img), op_module(img)) diff --git a/tests/color/test_xyz.py b/tests/color/test_xyz.py new file mode 100644 index 0000000..ea47035 --- /dev/null +++ b/tests/color/test_xyz.py @@ -0,0 +1,250 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +from torch.autograd import gradcheck + +import kornia + +from testing.base import BaseTester + + +class TestRgbToXyz(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.rgb_to_xyz(img), torch.Tensor) + + @pytest.mark.parametrize("shape", [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1), (3, 2, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.ones(shape, device=device, dtype=dtype) + assert kornia.color.rgb_to_xyz(img).shape == shape + + def test_exception(self, device, dtype): + from kornia.core.exceptions import ShapeError, TypeCheckError + + with pytest.raises(TypeCheckError): + assert kornia.color.rgb_to_xyz([0.0]) + + with pytest.raises(ShapeError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_xyz(img) + + with pytest.raises(ShapeError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_xyz(img) + + def test_unit(self, device, dtype): + data = torch.tensor( + [ + [ + [ + [0.9637, 0.0586, 0.6470, 0.6212, 0.9622], + [0.8293, 0.4858, 0.8953, 0.2607, 0.3250], + [0.5314, 0.4189, 0.8388, 0.8065, 0.2211], + [0.9682, 0.2928, 0.4118, 0.2533, 0.0455], + ], + [ + [0.6936, 0.3457, 0.9466, 0.9937, 0.2692], + [0.7485, 0.7320, 0.8323, 0.6889, 0.4831], + [0.1865, 0.7439, 0.1366, 0.8858, 0.2077], + [0.6227, 0.6140, 0.3936, 0.5024, 0.4157], + ], + [ + [0.6477, 0.9269, 0.7531, 0.7349, 0.9485], + [0.4264, 0.8539, 0.9830, 0.2269, 0.1138], + [0.3988, 0.1605, 0.6220, 0.0546, 0.1106], + [0.2128, 0.5673, 0.0781, 0.1431, 0.3310], + ], + ] + ], + device=device, + dtype=dtype, + ) + + # Reference output generated using OpenCV: cv2.cvtColor(data, cv2.COLOR_RGB2XYZ) + expected = torch.tensor( + [ + [ + [ + [0.7623584, 0.31501925, 0.7412189, 0.7441359, 0.66425407], + [0.6866283, 0.61618143, 0.84423876, 0.39480132, 0.32732624], + [0.3578189, 0.4677382, 0.50703406, 0.6592388, 0.18541752], + [0.6603961, 0.44267434, 0.32468265, 0.30994105, 0.22713262], + ], + [ + [0.7477299, 0.32658678, 0.86891913, 0.89580274, 0.4656054], + [0.7424382, 0.6884378, 0.8565741, 0.5644922, 0.4228247], + [0.2751717, 0.63267857, 0.3209684, 0.8089483, 0.20354219], + [0.6665957, 0.5423198, 0.37470126, 0.42349333, 0.33085644], + ], + [ + [0.7167665, 0.92310345, 0.8409531, 0.82877415, 0.95198023], + [0.51042646, 0.9080406, 1.0505873, 0.30275896, 0.17200153], + [0.4114541, 0.24927814, 0.62354034, 0.17305644, 0.13412625], + [0.29514894, 0.6179093, 0.12908883, 0.20075734, 0.3649534], + ], + ] + ], + device=device, + dtype=dtype, + ) + + self.assert_close(kornia.color.rgb_to_xyz(data), expected) + + def test_forth_and_back(self, device, dtype): + data = torch.rand(3, 4, 5, device=device, dtype=dtype) + xyz = kornia.color.rgb_to_xyz + rgb = kornia.color.xyz_to_rgb + + data_out = xyz(rgb(data)) + self.assert_close(data_out, data) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.rand(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.rgb_to_xyz, (img,), raise_exception=True, fast_mode=True) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.rgb_to_xyz + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.RgbToXyz().to(device, dtype) + fcn = kornia.color.rgb_to_xyz + self.assert_close(ops(img), fcn(img)) + + +class TestXyzToRgb(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.xyz_to_rgb(img), torch.Tensor) + + @pytest.mark.parametrize("shape", [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1), (3, 2, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.ones(shape, device=device, dtype=dtype) + assert kornia.color.xyz_to_rgb(img).shape == shape + + def test_exception(self, device, dtype): + from kornia.core.check import ShapeError, TypeCheckError + + with pytest.raises(TypeCheckError): + assert kornia.color.xyz_to_rgb([0.0]) + + with pytest.raises(ShapeError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.xyz_to_rgb(img) + + with pytest.raises(ShapeError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.xyz_to_rgb(img) + + def test_unit(self, device, dtype): + data = torch.tensor( + [ + [ + [ + [0.6315228, 0.4196397, 0.33123854, 0.3277169, 0.20501322], + [0.44133404, 0.4823162, 0.528561, 0.51644784, 0.28001237], + [0.31131047, 0.8453884, 0.4486181, 0.6015828, 0.42048606], + [0.5472367, 0.48154795, 0.36668795, 0.39913517, 0.40271503], + ], + [ + [0.79137707, 0.501063, 0.3700857, 0.57410157, 0.15295872], + [0.570678, 0.76664513, 0.48567873, 0.47680324, 0.2583247], + [0.38080955, 0.9315215, 0.4404478, 0.50659215, 0.5984908], + [0.5388581, 0.76993656, 0.4027568, 0.5952581, 0.68663263], + ], + [ + [0.86013114, 0.17629854, 0.83010703, 0.27881518, 0.30543375], + [0.17009716, 0.61201245, 0.33521807, 0.15526368, 0.7401195], + [0.34011865, 0.6541383, 0.96909684, 0.43090558, 0.70467836], + [0.6738866, 0.47461915, 0.91508406, 0.44147202, 0.14099535], + ], + ] + ], + device=device, + dtype=dtype, + ) + + # Reference output generated using OpenCV: cv2.cvtColor(data, cv2.COLOR_RGB2XYZ) + expected = torch.tensor( + [ + [ + [ + [0.4011656, 0.5017336, 0.09065688, 0.04048038, 0.2769511], + [0.46811658, 0.07937729, 0.79911184, 0.8632158, 0.1413149], + [0.2538725, 0.98146427, 0.29357457, 0.95588684, 0.09129933], + [0.6090472, 0.14032376, 0.11294556, 0.15829873, 0.1792411], + ], + [ + [0.90825266, 0.54057753, 0.40771842, 0.7709542, 0.1009315], + [0.64988965, 0.99616426, 0.41274834, 0.40036052, 0.24396756], + [0.42678973, 0.95531154, 0.43172216, 0.3851813, 0.7444883], + [0.5084846, 0.99737406, 0.4381809, 0.7481805, 0.9036418], + ], + [ + [0.78309417, 0.10751611, 0.82060075, 0.19588977, 0.3031369], + [0.08796211, 0.51749885, 0.28474382, 0.09561294, 0.74540937], + [0.2992335, 0.5486014, 0.95973116, 0.38571155, 0.64634556], + [0.6330101, 0.3715171, 0.90575427, 0.36752605, 0.03138365], + ], + ] + ], + device=device, + dtype=dtype, + ) + + self.assert_close(kornia.color.xyz_to_rgb(data), expected, low_tolerance=True) + + def test_forth_and_back(self, device, dtype): + data = torch.rand(3, 4, 5, device=device, dtype=dtype) + xyz = kornia.color.rgb_to_xyz + rgb = kornia.color.xyz_to_rgb + + data_out = rgb(xyz(data)) + self.assert_close(data_out, data, low_tolerance=True) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.rand(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.xyz_to_rgb, (img,), raise_exception=True, fast_mode=True) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.xyz_to_rgb + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.XyzToRgb().to(device, dtype) + fcn = kornia.color.xyz_to_rgb + self.assert_close(ops(img), fcn(img)) diff --git a/tests/color/test_ycbcr.py b/tests/color/test_ycbcr.py new file mode 100644 index 0000000..af4bd0a --- /dev/null +++ b/tests/color/test_ycbcr.py @@ -0,0 +1,257 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +from torch.autograd import gradcheck + +import kornia + +from testing.base import BaseTester + + +class TestRgbToYcbcr(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.rgb_to_ycbcr(img), torch.Tensor) + + @pytest.mark.parametrize("shape", [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1), (3, 2, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.ones(shape, device=device, dtype=dtype) + assert kornia.color.rgb_to_ycbcr(img).shape == shape + + @pytest.mark.parametrize("shape", [(3, 4, 4), (2, 3, 4, 4)]) + def test_rgb_to_y(self, device, dtype, shape): + img = torch.rand(*shape, device=device, dtype=dtype) + output_y = kornia.color.rgb_to_y(img) + output_ycbcr = kornia.color.rgb_to_ycbcr(img) + assert torch.equal(output_y, output_ycbcr[..., 0:1, :, :]) + + def test_exception(self, device, dtype): + with pytest.raises(TypeError): + assert kornia.color.rgb_to_ycbcr([0.0]) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_ycbcr(img) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.rgb_to_ycbcr(img) + + def test_unit(self, device, dtype): + data = torch.tensor( + [ + [ + [ + [0.7925, 0.8704, 0.2771, 0.3279, 0.0193], + [0.2483, 0.6212, 0.9859, 0.5044, 0.5621], + [0.5762, 0.3959, 0.2931, 0.2669, 0.0243], + [0.6989, 0.0529, 0.8344, 0.6523, 0.8980], + [0.5181, 0.9341, 0.2172, 0.0520, 0.7266], + ], + [ + [0.8413, 0.0284, 0.3625, 0.8864, 0.5595], + [0.3791, 0.0235, 0.4251, 0.0619, 0.5270], + [0.3516, 0.8005, 0.9571, 0.4113, 0.6119], + [0.0632, 0.8836, 0.0261, 0.1550, 0.4923], + [0.2332, 0.7044, 0.9514, 0.2443, 0.2818], + ], + [ + [0.6899, 0.2063, 0.3179, 0.8989, 0.4378], + [0.0384, 0.5230, 0.6416, 0.9749, 0.7863], + [0.8577, 0.3115, 0.2375, 0.5446, 0.9837], + [0.3213, 0.6618, 0.5977, 0.3999, 0.4962], + [0.1385, 0.5831, 0.9756, 0.8714, 0.8017], + ], + ] + ], + device=device, + dtype=dtype, + ) + + # Reference output generated using OpenCV: cv2.cvtColor(data, cv2.COLOR_RGB2XYZ) + expected = torch.tensor( + [ + [ + [ + [0.8094629, 0.30041942, 0.33188093, 0.7208117, 0.38409105], + [0.30114722, 0.2591345, 0.6174494, 0.29830942, 0.5670594], + [0.47643244, 0.623786, 0.6765313, 0.38332558, 0.47857922], + [0.2826805, 0.60994685, 0.33298355, 0.3316514, 0.6140681], + [0.30756146, 0.75926465, 0.7346588, 0.25828302, 0.4740529], + ], + [ + [0.43258172, 0.4469004, 0.492101, 0.60042214, 0.53028214], + [0.35180065, 0.64882994, 0.51362985, 0.8815981, 0.62366176], + [0.71505237, 0.32387614, 0.2523859, 0.5909421, 0.78490996], + [0.52178127, 0.529264, 0.64931786, 0.53848785, 0.43353173], + [0.4046721, 0.40065458, 0.6358658, 0.8458253, 0.6847739], + ], + [ + [0.48791525, 0.9063825, 0.46094114, 0.21983841, 0.23989305], + [0.46231723, 0.7581379, 0.7626976, 0.6469568, 0.496467], + [0.5711212, 0.33752257, 0.22661471, 0.4169921, 0.17608929], + [0.7967522, 0.10283372, 0.85753804, 0.72865105, 0.70245713], + [0.65009415, 0.6246666, 0.13107029, 0.35291404, 0.6800583], + ], + ] + ], + device=device, + dtype=dtype, + ) + + self.assert_close(kornia.color.rgb_to_ycbcr(data), expected, low_tolerance=True) + + # TODO: investigate and implement me + # def test_forth_and_back(self, device, dtype): + # pass + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.rand(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.rgb_to_ycbcr, (img,), raise_exception=True, fast_mode=True) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.rgb_to_ycbcr + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.RgbToYcbcr().to(device, dtype) + fcn = kornia.color.rgb_to_ycbcr + self.assert_close(ops(img), fcn(img)) + + +class TestYcbcrToRgb(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 3, 4, 5 + img = torch.rand(C, H, W, device=device, dtype=dtype) + assert isinstance(kornia.color.ycbcr_to_rgb(img), torch.Tensor) + + @pytest.mark.parametrize("shape", [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1), (3, 2, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.ones(shape, device=device, dtype=dtype) + assert kornia.color.ycbcr_to_rgb(img).shape == shape + + def test_exception(self, device, dtype): + with pytest.raises(TypeError): + assert kornia.color.ycbcr_to_rgb([0.0]) + + with pytest.raises(ValueError): + img = torch.ones(1, 1, device=device, dtype=dtype) + assert kornia.color.ycbcr_to_rgb(img) + + with pytest.raises(ValueError): + img = torch.ones(2, 1, 1, device=device, dtype=dtype) + assert kornia.color.ycbcr_to_rgb(img) + + def test_unit(self, device, dtype): + data = torch.tensor( + [ + [ + [ + [0.9892, 0.2620, 0.4184, 0.5286, 0.2793], + [0.0722, 0.8828, 0.8714, 0.0657, 0.7798], + [0.8118, 0.7522, 0.0260, 0.8811, 0.5226], + [0.0644, 0.3648, 0.4448, 0.4202, 0.7316], + [0.9138, 0.1956, 0.4257, 0.6381, 0.1353], + ], + [ + [0.7408, 0.8529, 0.5119, 0.0220, 0.0226], + [0.8963, 0.5652, 0.9568, 0.6977, 0.8221], + [0.4645, 0.0478, 0.4952, 0.5492, 0.4861], + [0.9980, 0.9978, 0.0281, 0.5283, 0.8146], + [0.7789, 0.2663, 0.6437, 0.6926, 0.5627], + ], + [ + [0.7377, 0.7152, 0.3080, 0.8515, 0.4841], + [0.7192, 0.3297, 0.7337, 0.0230, 0.2464], + [0.6399, 0.8998, 0.3838, 0.3043, 0.3774], + [0.1281, 0.6731, 0.4218, 0.3963, 0.8541], + [0.2245, 0.2413, 0.2351, 0.9522, 0.8158], + ], + ] + ], + device=device, + dtype=dtype, + ) + + # Reference output generated using OpenCV: cv2.cvtColor(data, cv2.COLOR_RGB2XYZ) + expected = torch.tensor( + [ + [ + [ + [1.0000, 0.5639256, 0.14902398, 1.0000, 0.2569923], + [0.37973762, 0.64386904, 1.0000, 0.0000, 0.4239992], + [1.0000, 1.0000, 0.0000, 0.60653293, 0.3505922], + [0.0000, 0.6076593, 0.33508536, 0.27470887, 1.0000], + [0.52727354, 0.0000, 0.05404532, 1.0000, 0.5783674], + ], + [ + [0.736647, 0.0000, 0.55139434, 0.44206098, 0.4548782], + [0.0000, 0.98196536, 0.54739904, 0.33826917, 0.850068], + [0.72412336, 0.6222996, 0.110618, 1.0000, 0.614918], + [0.15862459, 0.0699634, 0.66296846, 0.4845066, 0.3705502], + [1.0000, 0.46070462, 0.5654058, 0.24897486, 0.0000], + ], + [ + [1.0000, 0.88769174, 0.4394987, 0.0000, 0.0000], + [0.77483994, 0.99839956, 1.0000, 0.41622213, 1.0000], + [0.7488585, 0.0000, 0.01748962, 0.9683316, 0.49795526], + [0.9473541, 1.0000, 0.0000, 0.47037587, 1.0000], + [1.0000, 0.0000, 0.6804801, 0.9795798, 0.24646705], + ], + ] + ], + device=device, + dtype=dtype, + ) + + self.assert_close(kornia.color.ycbcr_to_rgb(data), expected) + + # TODO: investigate and implement me + # def test_forth_and_back(self, device, dtype): + # pass + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.rand(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck(kornia.color.ycbcr_to_rgb, (img,), raise_exception=True, fast_mode=True) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.color.ycbcr_to_rgb + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + ops = kornia.color.YcbcrToRgb().to(device, dtype) + fcn = kornia.color.ycbcr_to_rgb + self.assert_close(ops(img), fcn(img)) diff --git a/tests/color/test_yuv.py b/tests/color/test_yuv.py new file mode 100644 index 0000000..ad1e42b --- /dev/null +++ b/tests/color/test_yuv.py @@ -0,0 +1,111 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +from torch.autograd import gradcheck + +import kornia +from kornia.core.exceptions import ShapeError + +from testing.base import BaseTester + + +class TestRgbToYuv(BaseTester): + def test_smoke(self, device, dtype): + img = torch.rand(3, 4, 5, device=device, dtype=dtype) + out = kornia.color.rgb_to_yuv(img) + assert isinstance(out, torch.Tensor) + + @pytest.mark.parametrize( + "shape", + [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1)], + ) + def test_cardinality(self, device, dtype, shape): + img = torch.ones(shape, device=device, dtype=dtype) + out = kornia.color.rgb_to_yuv(img) + assert out.shape == shape + + def test_exception(self, device, dtype): + with pytest.raises((TypeError, AttributeError)): + kornia.color.rgb_to_yuv([0.0]) + + with pytest.raises(ShapeError): + kornia.color.rgb_to_yuv(torch.ones(1, 1, device=device, dtype=dtype)) + + with pytest.raises(ShapeError): + kornia.color.rgb_to_yuv(torch.ones(2, 1, 1, device=device, dtype=dtype)) + + def test_unit_invariants(self, device, dtype): + rgb = torch.tensor( + [ + [1.0, 0.0, 0.0], # red + [0.0, 1.0, 0.0], # green + [0.0, 0.0, 1.0], # blue + [1.0, 1.0, 1.0], # white + [0.0, 0.0, 0.0], # black + ], + device=device, + dtype=dtype, + ).view(5, 3, 1, 1) + + yuv = kornia.color.rgb_to_yuv(rgb) + + # shape preserved + assert yuv.shape == rgb.shape + + Y = yuv[:, 0, 0, 0] + + # basic luminance ordering invariants + assert Y[3] > Y[4] # white > black + assert Y[1] > Y[2] # green generally brighter than blue + + # neutral colors have near-zero chroma + self.assert_close(yuv[3, 1:], torch.zeros_like(yuv[3, 1:]), atol=1e-4, rtol=1e-4) + self.assert_close(yuv[4], torch.zeros_like(yuv[4]), atol=1e-4, rtol=1e-4) + + def test_round_trip_rgb_yuv_rgb(self, device, dtype): + rgb = torch.rand(3, 4, 5, device=device, dtype=dtype) + + yuv = kornia.color.rgb_to_yuv(rgb) + rgb_back = kornia.color.yuv_to_rgb(yuv) + + atol = 1e-3 if dtype in (torch.float32, torch.float64) else 1e-2 + + self.assert_close(rgb_back, rgb, atol=atol, rtol=1e-3) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + img = torch.rand(2, 3, 4, 4, device=device, dtype=torch.float64, requires_grad=True) + assert gradcheck( + kornia.color.rgb_to_yuv, + (img,), + raise_exception=True, + fast_mode=True, + ) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + img = torch.ones(2, 3, 4, 4, device=device, dtype=dtype) + op = kornia.color.rgb_to_yuv + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + def test_module(self, device, dtype): + img = torch.ones(2, 3, 4, 4, device=device, dtype=dtype) + module = kornia.color.RgbToYuv().to(device, dtype) + self.assert_close(module(img), kornia.color.rgb_to_yuv(img)) diff --git a/tests/contrib/__init__.py b/tests/contrib/__init__.py new file mode 100644 index 0000000..4d273d5 --- /dev/null +++ b/tests/contrib/__init__.py @@ -0,0 +1,16 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/contrib/test_combine_tensor_patch.py b/tests/contrib/test_combine_tensor_patch.py new file mode 100644 index 0000000..5b3abd6 --- /dev/null +++ b/tests/contrib/test_combine_tensor_patch.py @@ -0,0 +1,133 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestCombineTensorPatches(BaseTester): + def test_smoke(self, device, dtype): + img = torch.arange(16, device=device, dtype=dtype).view(1, 1, 4, 4) + m = kornia.contrib.CombineTensorPatches((4, 4), (2, 2), (2, 2)) + patches = kornia.contrib.extract_tensor_patches(img, window_size=(2, 2), stride=(2, 2)) + assert m(patches).shape == (1, 1, 4, 4) + self.assert_close(img, m(patches)) + + def test_error(self, device, dtype): + patches = kornia.contrib.extract_tensor_patches( + torch.arange(16, device=device, dtype=dtype).view(1, 1, 4, 4), window_size=(3, 2), stride=(2, 2), padding=1 + ) + with pytest.raises(RuntimeError): + kornia.contrib.combine_tensor_patches(patches, original_size=(4, 4), window_size=(2, 2), stride=(2, 2)) + + def test_rect_odd_dim(self, device, dtype): + img = torch.arange(12, device=device, dtype=dtype).view(1, 1, 4, 3) + patches = kornia.contrib.extract_tensor_patches(img, window_size=(2, 2), stride=(2, 2), padding=(0, 2)) + m = kornia.contrib.combine_tensor_patches( + patches, original_size=(4, 3), window_size=(2, 2), stride=(2, 2), unpadding=(0, 2) + ) + assert m.shape == (1, 1, 4, 3) + self.assert_close(img, m) + + def test_pad_triple_error(self, device, dtype): + patches = kornia.contrib.extract_tensor_patches( + torch.arange(36, device=device, dtype=dtype).view(1, 1, 6, 6), window_size=(4, 4), stride=(4, 4), padding=1 + ) + with pytest.raises(AssertionError): + kornia.contrib.combine_tensor_patches( + patches, original_size=(6, 6), window_size=(4, 4), stride=(4, 4), unpadding=(1, 1, 1) + ) + + def test_rectangle_array(self, device, dtype): + img = torch.arange(24, device=device, dtype=dtype).view(1, 1, 4, 6) + patches = kornia.contrib.extract_tensor_patches(img, window_size=(2, 2), stride=(2, 2), padding=1) + m = kornia.contrib.CombineTensorPatches((4, 6), (2, 2), (2, 2), unpadding=1) + assert m(patches).shape == (1, 1, 4, 6) + self.assert_close(img, m(patches)) + + def test_padding1(self, device, dtype): + img = torch.arange(16, device=device, dtype=dtype).view(1, 1, 4, 4) + padding = kornia.contrib.compute_padding((4, 4), (2, 2)) + patches = kornia.contrib.extract_tensor_patches(img, window_size=(2, 2), stride=(1, 1), padding=padding) + m = kornia.contrib.CombineTensorPatches((4, 4), (2, 2), stride=(1, 1), unpadding=padding) + assert m(patches).shape == (1, 1, 4, 4) + self.assert_close(img, m(patches)) + + def test_padding2(self, device, dtype): + img = torch.arange(64, device=device, dtype=dtype).view(1, 1, 8, 8) + patches = kornia.contrib.extract_tensor_patches(img, window_size=(2, 2), stride=(2, 2), padding=1) + m = kornia.contrib.CombineTensorPatches((8, 8), (2, 2), stride=(2, 2), unpadding=1) + assert m(patches).shape == (1, 1, 8, 8) + self.assert_close(img, m(patches)) + + def test_compute_padding(self, device, dtype): + img_shape = (8, 13) + rnge = img_shape[0] * img_shape[1] + img = torch.arange(rnge, device=device, dtype=dtype).view(1, 1, *img_shape) + window_size = (3, 3) + padding = kornia.contrib.compute_padding(img_shape, window_size) + patches = kornia.contrib.extract_tensor_patches( + img, window_size=window_size, stride=window_size, padding=padding + ) + m = kornia.contrib.CombineTensorPatches(img_shape, window_size, stride=window_size, unpadding=padding) + assert m(patches).shape == (1, 1, *img_shape) + self.assert_close(img, m(patches)) + + def test_stride_greater_than_window_size(self, device, dtype): + patches = kornia.contrib.extract_tensor_patches( + torch.arange(16, device=device, dtype=dtype).view(1, 1, 4, 4), window_size=(2, 2), stride=(2, 2), padding=1 + ) + with pytest.raises(AssertionError): + kornia.contrib.combine_tensor_patches(patches, original_size=(4, 4), window_size=(2, 2), stride=(3, 2)) + + def test_expl_autopadding(self, device, dtype): + img_shape = (8, 13) + img = torch.arange(img_shape[0] * img_shape[1], device=device, dtype=dtype).view( + 1, 1, img_shape[0], img_shape[1] + ) + window_size = (3, 3) + padding = kornia.contrib.compute_padding(img_shape, window_size) + patches = kornia.contrib.extract_tensor_patches( + img, window_size=window_size, stride=window_size, padding=padding + ) + m = kornia.contrib.CombineTensorPatches(img_shape, window_size, stride=window_size, unpadding=padding) + assert m(patches).shape == (1, 1, *img_shape) + + self.assert_close(img, m(patches)) + + def test_impl_autopadding(self, device, dtype): + img_shape = (11, 16) + img = torch.arange(img_shape[0] * img_shape[1], device=device, dtype=dtype).view(1, 1, *img_shape) + window_size = (3, 3) + patches = kornia.contrib.extract_tensor_patches( + img, window_size=window_size, stride=window_size, allow_auto_padding=True + ) + recomb = kornia.contrib.combine_tensor_patches( + patches, img_shape, window_size=window_size, stride=window_size, allow_auto_unpadding=True + ) + assert recomb.shape == img.shape + self.assert_close(img, recomb) + + def test_gradcheck(self, device): + patches = kornia.contrib.extract_tensor_patches( + torch.arange(16.0, device=device, dtype=torch.float64).view(1, 1, 4, 4), window_size=(2, 2), stride=(2, 2) + ) + self.gradcheck(kornia.contrib.combine_tensor_patches, (patches, (4, 4), (2, 2), (2, 2))) diff --git a/tests/contrib/test_connected_component.py b/tests/contrib/test_connected_component.py new file mode 100644 index 0000000..d0587a5 --- /dev/null +++ b/tests/contrib/test_connected_component.py @@ -0,0 +1,112 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestConnectedComponents(BaseTester): + def test_smoke(self, device, dtype): + img = torch.rand(1, 1, 3, 4, device=device, dtype=dtype) + out = kornia.contrib.connected_components(img, num_iterations=10) + assert out.shape == (1, 1, 3, 4) + + @pytest.mark.parametrize("shape", [(1, 3, 4), (2, 1, 3, 4)]) + def test_cardinality(self, device, dtype, shape): + img = torch.rand(shape, device=device, dtype=dtype) + out = kornia.contrib.connected_components(img, num_iterations=10) + assert out.shape == shape + + def test_exception(self, device, dtype): + img = torch.rand(1, 1, 3, 4, device=device, dtype=dtype) + + with pytest.raises(TypeError) as errinf: + assert kornia.contrib.connected_components(img, 1.0) + assert "Input num_iterations must be a positive integer." in str(errinf) + + with pytest.raises(TypeError) as errinf: + assert kornia.contrib.connected_components("not a tensor", 0) + assert "Input imagetype is not a torch.Tensor" in str(errinf) + + with pytest.raises(TypeError) as errinf: + assert kornia.contrib.connected_components(img, 0) + assert "Input num_iterations must be a positive integer." in str(errinf) + + with pytest.raises(ValueError) as errinf: + img = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + assert kornia.contrib.connected_components(img, 2) + assert "Input image shape must be (*,1,H,W). Got:" in str(errinf) + + def test_value(self, device, dtype): + img = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 0.0, 0.0, 1.0], + [0.0, 1.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 1.0, 1.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 15.0, 15.0, 0.0, 0.0, 12.0], + [0.0, 15.0, 15.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 35.0, 35.0, 0.0], + [0.0, 0.0, 0.0, 35.0, 35.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + out = kornia.contrib.connected_components(img, num_iterations=10) + self.assert_close(out, expected) + + def test_gradcheck(self, device): + B, C, H, W = 2, 1, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + self.gradcheck(kornia.contrib.connected_components, (img,)) + + def test_jit(self, device, dtype): + B, C, H, W = 2, 1, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.contrib.connected_components + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + +def test_compute_padding(): + assert kornia.contrib.compute_padding((6, 6), (2, 2)) == (0, 0, 0, 0) + assert kornia.contrib.compute_padding((7, 7), (2, 2)) == (0, 1, 0, 1) + assert kornia.contrib.compute_padding((8, 7), (4, 4)) == (0, 0, 0, 1) diff --git a/tests/contrib/test_conv_distance_transformer.py b/tests/contrib/test_conv_distance_transformer.py new file mode 100755 index 0000000..12bd14c --- /dev/null +++ b/tests/contrib/test_conv_distance_transformer.py @@ -0,0 +1,222 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia +from kornia.core.exceptions import BaseError, TypeCheckError +from kornia.geometry.grid import create_meshgrid, create_meshgrid3d + +from testing.base import BaseTester + + +class TestConvDistanceTransform(BaseTester): + @pytest.mark.parametrize("kernel_size", [3, 5, 7]) + @pytest.mark.parametrize("shape", [(1, 3, 100, 100), (2, 2, 10, 10, 10)]) + def test_smoke(self, kernel_size, shape, device, dtype): + sample = torch.rand(*shape, device=device, dtype=dtype) + distance_transformer = kornia.contrib.DistanceTransform(kernel_size) + + output1 = distance_transformer(sample) + output2 = kornia.contrib.distance_transform(sample, kernel_size) + + assert isinstance(output1, torch.Tensor) + assert isinstance(output2, torch.Tensor) + assert output1.shape == sample.shape + self.assert_close(output1, output2) + + def test_module(self, device, dtype): + B, C, H, W = 1, 1, 99, 100 + sample2d = torch.rand(B, C, H, W, device=device, dtype=dtype) + distance_transformer = kornia.contrib.DistanceTransform().to(device, dtype) + + output1 = distance_transformer(sample2d) + output2 = kornia.contrib.distance_transform(sample2d) + self.assert_close(output1, output2) + + B, C, D, H, W = 1, 1, 10, 10, 10 + sample3d = torch.rand(B, C, D, H, W, device=device, dtype=dtype) + output3 = distance_transformer(sample3d) + output4 = kornia.contrib.distance_transform(sample3d) + self.assert_close(output3, output4) + + def test_exception(self, device, dtype): + B, C, H, W = 1, 1, 32, 32 + sample2d = torch.rand(B, C, H, W, device=device, dtype=dtype) + sample_wrong_dims = torch.rand(C, H, W, device=device, dtype=dtype) + + # Non-odd kernel size -> BaseError from KORNIA_CHECK + with pytest.raises(BaseError) as excinfo: + ConvDT = kornia.contrib.DistanceTransform(6) + ConvDT.forward(sample2d) + assert "kernel_size must be an odd integer >= 3" in str(excinfo.value) + + with pytest.raises(BaseError): + kornia.contrib.distance_transform(sample2d, 4) + + # Kernel size too small + with pytest.raises(BaseError): + kornia.contrib.distance_transform(sample2d, 1) + + # Invalid input dimensions (3D tensor instead of 4D or 5D) + with pytest.raises(BaseError) as excinfo: + kornia.contrib.distance_transform(sample_wrong_dims) + assert "Invalid image shape" in str(excinfo.value) + + # Invalid input type (None) + with pytest.raises(TypeCheckError): + kornia.contrib.distance_transform(None) + + # Integer input not supported (dtype check) + sample_int = torch.randint(0, 2, (B, C, H, W), device=device) + with pytest.raises(BaseError): + kornia.contrib.distance_transform(sample_int) + + # invalid h + with pytest.raises(BaseError): + kornia.contrib.distance_transform(sample2d, kernel_size=3, h=0.0) + + def test_kernel_geometry(self, device, dtype): + kernel_size = 5 + k_half = kernel_size // 2 + h = 0.35 + + # 2D Kernel Check + grid2d = create_meshgrid(kernel_size, kernel_size, False, device, dtype) + grid2d = grid2d - k_half + dist2d = torch.norm(grid2d[0], p=2, dim=-1) + kernel2d = torch.exp(-dist2d / h) + + self.assert_close(kernel2d[k_half, k_half], torch.tensor(1.0, device=device, dtype=dtype)) + + self.assert_close(kernel2d[0, 0], kernel2d[-1, -1]) + + # 3D Kernel Check + grid3d = create_meshgrid3d(kernel_size, kernel_size, kernel_size, False, device, dtype) + grid3d = grid3d - k_half + dist3d = torch.norm(grid3d[0], p=2, dim=-1) + kernel3d = torch.exp(-dist3d / h) + + # Center must be exactly 1.0 + self.assert_close(kernel3d[k_half, k_half, k_half], torch.tensor(1.0, device=device, dtype=dtype)) + self.assert_close(kernel3d[0, 0, 0], kernel3d[-1, -1, -1]) + + def test_noncontiguous_multi_channel(self, device, dtype): + B, C, H, W = 1, 2, 4, 4 + sample = torch.rand(B, C, H, W, device=device, dtype=dtype) + sample = sample.transpose(2, 3) + op = kornia.contrib.DistanceTransform() + out = op(sample) + assert out.shape == sample.shape + + def test_value_2d(self, device, dtype): + B, C, H, W = 1, 1, 4, 4 + kernel_size = 7 + h = 0.35 + sample1 = torch.zeros(B, C, H, W, device=device, dtype=dtype) + sample1[:, :, 1, 1] = 1.0 + expected_output1 = torch.tensor( + [ + [ + [ + [1.4142135382, 1.0000000000, 1.4142135382, 2.2360680103], + [1.0000000000, 0.0000000000, 1.0000000000, 2.0000000000], + [1.4142135382, 1.0000000000, 1.4142135382, 2.2360680103], + [2.2360680103, 2.0000000000, 2.2360680103, 2.8284270763], + ] + ] + ], + device=device, + dtype=dtype, + ) + output1 = kornia.contrib.distance_transform(sample1, kernel_size, h) + self.assert_close(expected_output1, output1) + + def test_value_3d(self, device, dtype): + B, C, D, H, W = 1, 1, 3, 3, 3 + kernel_size = 3 + h = 0.35 + sample1 = torch.zeros(B, C, D, H, W, device=device, dtype=dtype) + + sample1[:, :, 1, 1, 1] = 1.0 + + output1 = kornia.contrib.distance_transform(sample1, kernel_size, h) + + self.assert_close(output1[0, 0, 1, 1, 1], torch.tensor(0.0, device=device, dtype=dtype)) + + self.assert_close(output1[0, 0, 0, 1, 1], torch.tensor(1.0, device=device, dtype=dtype)) + + expected_corner = torch.tensor(1.7320508, device=device, dtype=dtype) + self.assert_close(output1[0, 0, 0, 0, 0], expected_corner) + + def test_gradcheck(self, device): + B, C, H, W = 1, 1, 32, 32 + sample2d = torch.ones(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + self.gradcheck(kornia.contrib.distance_transform, (sample2d,)) + + B, C, D, H, W = 1, 1, 5, 5, 5 + sample3d = torch.ones(B, C, D, H, W, device=device, dtype=torch.float64, requires_grad=True) + self.gradcheck(kornia.contrib.distance_transform, (sample3d,)) + + def test_loss_grad(self, device, dtype): + B, C, H, W = 1, 1, 32, 32 + sample1 = torch.rand(B, C, H, W, device=device, dtype=dtype, requires_grad=True) + sample2 = torch.rand(B, C, H, W, device=device, dtype=dtype, requires_grad=True) + tiny_module = torch.nn.Conv2d(1, 1, (3, 3), (1, 1), (1, 1)).to(device=device, dtype=dtype) + out1 = kornia.contrib.distance_transform(tiny_module(sample1)) + out2 = kornia.contrib.distance_transform(sample2) + loss = torch.nn.functional.mse_loss(out1, out2) + loss.backward() + + B, C, D, H, W = 1, 1, 10, 10, 10 + sample3d_1 = torch.rand(B, C, D, H, W, device=device, dtype=dtype, requires_grad=True) + sample3d_2 = torch.rand(B, C, D, H, W, device=device, dtype=dtype, requires_grad=True) + out3 = kornia.contrib.distance_transform(sample3d_1) + out4 = kornia.contrib.distance_transform(sample3d_2) + loss_3d = torch.nn.functional.mse_loss(out3, out4) + loss_3d.backward() + + def test_offset_parenthesis_fix(self, device, dtype): + img = torch.zeros(1, 1, 8, 4, device=device, dtype=dtype) + img[0, 0, 1, :] = 1.0 + out = kornia.contrib.distance_transform(img, kernel_size=3, h=0.01) + expected = torch.tensor( + [ + [0.9998, 0.9998, 0.9998, 0.9998], + [0.0000, 0.0000, 0.0000, 0.0000], + [0.9998, 0.9998, 0.9998, 0.9998], + [1.9998, 1.9998, 1.9998, 1.9998], + [2.9998, 2.9998, 2.9998, 2.9998], + [3.9998, 3.9998, 3.9998, 3.9998], + [4.9998, 4.9998, 4.9998, 4.9998], + [5.9998, 5.9998, 5.9998, 5.9998], + ], + device=device, + dtype=dtype, + ) + + self.assert_close(out[0, 0], expected, rtol=1e-3, atol=1e-3) + + def test_dynamo(self, device, dtype, torch_optimizer): + input2d = torch.rand(1, 1, 16, 16, device=device, dtype=dtype) + op = kornia.contrib.distance_transform + op_optimized = torch_optimizer(op) + self.assert_close(op(input2d), op_optimized(input2d)) + + input3d = torch.rand(1, 1, 8, 8, 8, device=device, dtype=dtype) + self.assert_close(op(input3d), op_optimized(input3d)) diff --git a/tests/contrib/test_diamond_square.py b/tests/contrib/test_diamond_square.py new file mode 100644 index 0000000..3ea3e6f --- /dev/null +++ b/tests/contrib/test_diamond_square.py @@ -0,0 +1,48 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch + +import kornia + +from testing.base import BaseTester + + +class TestDiamondSquare(BaseTester): + def test_smoke(self, device, dtype): + torch.manual_seed(0) + output_size = (1, 1, 3, 4) + roughness = 0.5 + random_scale = 1.0 + out = kornia.contrib.diamond_square(output_size, roughness, random_scale, device=device, dtype=dtype) + assert out.shape == output_size + assert out.device == device + assert out.dtype == dtype + + def test_normalize(self, device, dtype): + torch.manual_seed(0) + output_size = (1, 1, 3, 4) + roughness = 0.5 + random_scale = 1.0 + normalize_range = (0.0, 1.0) + expected_min = torch.tensor(normalize_range[0], device=device, dtype=dtype) + expected_max = torch.tensor(normalize_range[1], device=device, dtype=dtype) + out = kornia.contrib.diamond_square( + output_size, roughness, random_scale, normalize_range=normalize_range, device=device, dtype=dtype + ) + self.assert_close(out.min(), expected_min) + self.assert_close(out.max(), expected_max) diff --git a/tests/contrib/test_edge_detector.py b/tests/contrib/test_edge_detector.py new file mode 100644 index 0000000..539d1de --- /dev/null +++ b/tests/contrib/test_edge_detector.py @@ -0,0 +1,44 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestEdgeDetector(BaseTester): + @pytest.mark.slow + def test_smoke(self, device, dtype): + img = torch.rand(2, 3, 64, 64, device=device, dtype=dtype) + net = kornia.contrib.EdgeDetectorBuilder.build(pretrained=False).to(device, dtype) + out = net(img) + # ResizePostProcessor returns a list, so we need to handle that + if isinstance(out, list): + assert len(out) == 2 + assert all(item.shape == (1, 1, 64, 64) for item in out) + else: + assert out.shape == (2, 1, 64, 64) + + @pytest.mark.slow + @pytest.mark.skip(reason="issue with `ClassVar[list[int]]`") + def test_jit(self, device, dtype): + op = kornia.contrib.EdgeDetectorBuilder.build(pretrained=False).to(device, dtype) + op_jit = torch.jit.script(op) + assert op_jit is not None diff --git a/tests/contrib/test_extract_tensor_patch.py b/tests/contrib/test_extract_tensor_patch.py new file mode 100644 index 0000000..9f3e7dd --- /dev/null +++ b/tests/contrib/test_extract_tensor_patch.py @@ -0,0 +1,152 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestExtractTensorPatches(BaseTester): + def test_smoke(self, device): + img = torch.arange(16.0, device=device).view(1, 1, 4, 4) + m = kornia.contrib.ExtractTensorPatches(3) + assert m(img).shape == (1, 4, 1, 3, 3) + + def test_b1_ch1_h4w4_ws3(self, device): + img = torch.arange(16.0, device=device).view(1, 1, 4, 4) + m = kornia.contrib.ExtractTensorPatches(3) + patches = m(img) + assert patches.shape == (1, 4, 1, 3, 3) + self.assert_close(img[0, :, :3, :3], patches[0, 0]) + self.assert_close(img[0, :, :3, 1:], patches[0, 1]) + self.assert_close(img[0, :, 1:, :3], patches[0, 2]) + self.assert_close(img[0, :, 1:, 1:], patches[0, 3]) + + def test_b1_ch2_h4w4_ws3(self, device): + img = torch.arange(16.0, device=device).view(1, 1, 4, 4) + img = img.expand(-1, 2, -1, -1) # copy all channels + m = kornia.contrib.ExtractTensorPatches(3) + patches = m(img) + assert patches.shape == (1, 4, 2, 3, 3) + self.assert_close(img[0, :, :3, :3], patches[0, 0]) + self.assert_close(img[0, :, :3, 1:], patches[0, 1]) + self.assert_close(img[0, :, 1:, :3], patches[0, 2]) + self.assert_close(img[0, :, 1:, 1:], patches[0, 3]) + + def test_b1_ch1_h4w4_ws2(self, device): + img = torch.arange(16.0, device=device).view(1, 1, 4, 4) + m = kornia.contrib.ExtractTensorPatches(2) + patches = m(img) + assert patches.shape == (1, 9, 1, 2, 2) + self.assert_close(img[0, :, 0:2, 1:3], patches[0, 1]) + self.assert_close(img[0, :, 0:2, 2:4], patches[0, 2]) + self.assert_close(img[0, :, 1:3, 1:3], patches[0, 4]) + self.assert_close(img[0, :, 2:4, 1:3], patches[0, 7]) + + def test_b1_ch1_h4w4_ws2_stride2(self, device): + img = torch.arange(16.0, device=device).view(1, 1, 4, 4) + m = kornia.contrib.ExtractTensorPatches(2, stride=2) + patches = m(img) + assert patches.shape == (1, 4, 1, 2, 2) + self.assert_close(img[0, :, 0:2, 0:2], patches[0, 0]) + self.assert_close(img[0, :, 0:2, 2:4], patches[0, 1]) + self.assert_close(img[0, :, 2:4, 0:2], patches[0, 2]) + self.assert_close(img[0, :, 2:4, 2:4], patches[0, 3]) + + def test_b1_ch1_h4w4_ws2_stride21(self, device): + img = torch.arange(16.0, device=device).view(1, 1, 4, 4) + m = kornia.contrib.ExtractTensorPatches(2, stride=(2, 1)) + patches = m(img) + assert patches.shape == (1, 6, 1, 2, 2) + self.assert_close(img[0, :, 0:2, 1:3], patches[0, 1]) + self.assert_close(img[0, :, 0:2, 2:4], patches[0, 2]) + self.assert_close(img[0, :, 2:4, 0:2], patches[0, 3]) + self.assert_close(img[0, :, 2:4, 2:4], patches[0, 5]) + + def test_b1_ch1_h3w3_ws2_stride1_padding1(self, device): + img = torch.arange(9.0).view(1, 1, 3, 3).to(device) + m = kornia.contrib.ExtractTensorPatches(2, stride=1, padding=1) + patches = m(img) + assert patches.shape == (1, 16, 1, 2, 2) + self.assert_close(img[0, :, 0:2, 0:2], patches[0, 5]) + self.assert_close(img[0, :, 0:2, 1:3], patches[0, 6]) + self.assert_close(img[0, :, 1:3, 0:2], patches[0, 9]) + self.assert_close(img[0, :, 1:3, 1:3], patches[0, 10]) + + def test_b2_ch1_h3w3_ws2_stride1_padding1(self, device): + batch_size = 2 + img = torch.arange(9.0).view(1, 1, 3, 3).to(device) + img = img.expand(batch_size, -1, -1, -1) + m = kornia.contrib.ExtractTensorPatches(2, stride=1, padding=1) + patches = m(img) + assert patches.shape == (batch_size, 16, 1, 2, 2) + for i in range(batch_size): + self.assert_close(img[i, :, 0:2, 0:2], patches[i, 5]) + self.assert_close(img[i, :, 0:2, 1:3], patches[i, 6]) + self.assert_close(img[i, :, 1:3, 0:2], patches[i, 9]) + self.assert_close(img[i, :, 1:3, 1:3], patches[i, 10]) + + def test_b1_ch1_h3w3_ws23(self, device): + img = torch.arange(9.0).view(1, 1, 3, 3).to(device) + m = kornia.contrib.ExtractTensorPatches((2, 3)) + patches = m(img) + assert patches.shape == (1, 2, 1, 2, 3) + self.assert_close(img[0, :, 0:2, 0:3], patches[0, 0]) + self.assert_close(img[0, :, 1:3, 0:3], patches[0, 1]) + + def test_b1_ch1_h3w4_ws23(self, device): + img = torch.arange(12.0).view(1, 1, 3, 4).to(device) + m = kornia.contrib.ExtractTensorPatches((2, 3)) + patches = m(img) + assert patches.shape == (1, 4, 1, 2, 3) + self.assert_close(img[0, :, 0:2, 0:3], patches[0, 0]) + self.assert_close(img[0, :, 0:2, 1:4], patches[0, 1]) + self.assert_close(img[0, :, 1:3, 0:3], patches[0, 2]) + self.assert_close(img[0, :, 1:3, 1:4], patches[0, 3]) + + @pytest.mark.skip(reason="turn off all jit for a while") + def test_jit(self, device): + @torch.jit.script + def op_script(img: torch.Tensor, height: int, width: int) -> torch.Tensor: + return kornia.geometry.denormalize_pixel_coordinates(img, height, width) + + height, width = 3, 4 + grid = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=True).to(device) + + actual = op_script(grid, height, width) + expected = kornia.denormalize_pixel_coordinates(grid, height, width) + + self.assert_close(actual, expected) + + def test_gradcheck(self, device): + img = torch.rand(2, 3, 4, 4, device=device, dtype=torch.float64) + self.gradcheck(kornia.contrib.extract_tensor_patches, (img, 3)) + + def test_auto_padding_stride(self, device, dtype): + img_shape = (11, 14) + window_size = (3, 3) + stride = 2 + rnge = img_shape[0] * img_shape[1] + img = torch.arange(rnge, device=device, dtype=dtype).view(1, 1, *img_shape) + patches = kornia.contrib.extract_tensor_patches( + img, window_size=window_size, stride=stride, allow_auto_padding=True + ) + # 5 patches vertical, 6 2/3 = 7 horizontal = 35 patches + assert patches.shape == (1, 35, 1, *window_size) diff --git a/tests/contrib/test_face_detection.py b/tests/contrib/test_face_detection.py new file mode 100644 index 0000000..c06138f --- /dev/null +++ b/tests/contrib/test_face_detection.py @@ -0,0 +1,130 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import warnings + +import pytest +import torch + +import kornia +from kornia.contrib.face_detection import FaceKeypoint + +from testing.base import BaseTester + + +class TestFaceDetection(BaseTester): + @pytest.fixture + def face_detector(self, device, dtype): + return kornia.contrib.FaceDetector().to(device, dtype) + + @pytest.mark.slow + def test_smoke(self, device, dtype): + assert kornia.contrib.FaceDetector().to(device, dtype) is not None + + @pytest.mark.slow + @pytest.mark.parametrize("batch_size", [1, 2, 4]) + def test_valid(self, batch_size, device, dtype): + torch.manual_seed(44) + img = torch.rand(batch_size, 3, 320, 320, device=device, dtype=dtype) + face_detection = kornia.contrib.FaceDetector().to(device, dtype) + dets = face_detection(img) + assert isinstance(dets, list) + assert len(dets) == batch_size # same as the number of images + assert isinstance(dets[0], torch.Tensor) + assert dets[0].shape[0] >= 0 # number of detections + assert dets[0].shape[1] == 15 # dims of each detection + + @pytest.mark.slow + def test_jit(self, device, dtype): + op = kornia.contrib.FaceDetector().to(device, dtype) + op_jit = torch.jit.script(op) + assert op_jit is not None + + @pytest.mark.slow + def test_results(self, device, dtype): + data = torch.tensor( + [0.0, 0.0, 100.0, 200.0, 10.0, 10.0, 20.0, 10.0, 10.0, 50.0, 100.0, 50.0, 150.0, 10.0, 0.99], + device=device, + dtype=dtype, + ) + res = kornia.contrib.FaceDetectorResult(data) + assert res.xmin == 0.0 + assert res.ymin == 0.0 + assert res.xmax == 100.0 + assert res.ymax == 200.0 + assert res.score == 0.99 + assert res.width == 100.0 + assert res.height == 200.0 + assert res.top_left.tolist() == [0.0, 0.0] + assert res.top_right.tolist() == [100.0, 0.0] + assert res.bottom_right.tolist() == [100.0, 200.0] + assert res.bottom_left.tolist() == [0.0, 200.0] + assert res.get_keypoint(FaceKeypoint.EYE_LEFT).tolist() == [10.0, 10.0] + assert res.get_keypoint(FaceKeypoint.EYE_RIGHT).tolist() == [20.0, 10.0] + assert res.get_keypoint(FaceKeypoint.NOSE).tolist() == [10.0, 50.0] + assert res.get_keypoint(FaceKeypoint.MOUTH_LEFT).tolist() == [100.0, 50.0] + assert res.get_keypoint(FaceKeypoint.MOUTH_RIGHT).tolist() == [150.0, 10.0] + + @pytest.mark.slow + def test_results_raise(self, device, dtype): + data = torch.zeros(14, device=device, dtype=dtype) + with pytest.raises(ValueError): + _ = kornia.contrib.FaceDetectorResult(data) + + @pytest.mark.slow + def test_model_onnx(self, device, dtype, tmp_path): + torch.manual_seed(44) + img = torch.rand(1, 3, 320, 320, device=device, dtype=dtype) + face_detection = kornia.contrib.FaceDetector().to(device, dtype) + model = face_detection.model + + model_path = tmp_path / "facedetector_model.onnx" + + dynamic_axes = {"images": {0: "B"}} + # Suppress onnxscript deprecation warnings (Python 3.15 compatibility) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning, module="onnxscript.converter") + torch.onnx.export( + model, + img, + model_path, + input_names=["images"], + output_names=["loc", "conf", "iou"], + dynamic_axes=dynamic_axes, + ) + assert model_path.is_file() + + def test_exception(self, face_detector, device, dtype): + img = torch.rand(1, 4, 320, 320, device=device, dtype=dtype) + + with pytest.raises(RuntimeError): + face_detector(img) + + @pytest.mark.slow + def test_dynamo(self, face_detector, device, dtype, torch_optimizer): + torch.manual_seed(44) + img = torch.rand(1, 3, 320, 320, device=device, dtype=dtype) + + op_compiled = torch_optimizer(face_detector) + + out_ref = face_detector(img) + out_opt = op_compiled(img) + + assert len(out_ref) == len(out_opt) + + for r, o in zip(out_ref, out_opt): + torch.testing.assert_close(r, o, rtol=1e-4, atol=1e-4) diff --git a/tests/contrib/test_hist_match.py b/tests/contrib/test_hist_match.py new file mode 100644 index 0000000..3880525 --- /dev/null +++ b/tests/contrib/test_hist_match.py @@ -0,0 +1,61 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestHistMatch(BaseTester): + def test_interp(self, device, dtype): + xp = torch.tensor([1, 2, 3], device=device, dtype=dtype) + fp = torch.tensor([4, 2, 0], device=device, dtype=dtype) + x = torch.tensor([4, 5, 6], device=device, dtype=dtype) + x_hat_expected = torch.tensor([-2.0, -4.0, -6.0], device=device, dtype=dtype) + x_hat = kornia.contrib.interp(x, xp, fp) + self.assert_close(x_hat_expected, x_hat) + + def test_histmatch(self, device, dtype): + torch.manual_seed(44) + # generate random value by CPU. + src = torch.randn(1, 4, 4).to(device=device, dtype=dtype) + dst = torch.randn(1, 16, 16).to(device=device, dtype=dtype) + out = kornia.contrib.histogram_matching(src, dst) + exp = torch.tensor( + [ + [ + [1.5902, 0.9295, 2.9409, 0.1211], + [0.2472, 1.2137, -0.1098, -0.4272], + [-0.2644, -1.1983, -0.6065, -0.8091], + [-1.4999, 0.6370, -0.9800, 0.4474], + ] + ], + device=device, + dtype=dtype, + ) + assert exp.shape == out.shape + self.assert_close(out, exp, rtol=1e-4, atol=1e-4) + + @pytest.mark.skip(reason="not differentiable now.") + def test_grad(self, device): + B, C, H, W = 1, 3, 32, 32 + src = torch.rand(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + dst = torch.rand(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + self.gradcheck(kornia.contrib.histogram_matching, (src, dst)) diff --git a/tests/contrib/test_image_stitcher.py b/tests/contrib/test_image_stitcher.py new file mode 100644 index 0000000..9eb67db --- /dev/null +++ b/tests/contrib/test_image_stitcher.py @@ -0,0 +1,149 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from unittest.mock import PropertyMock, patch + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestImageStitcher(BaseTester): + @pytest.mark.parametrize("estimator", ["ransac", "vanilla"]) + def test_smoke(self, estimator, device, dtype): + B, C, H, W = 1, 3, 6, 6 + sample1 = torch.tensor( + [ + [0.5349, 0.1988, 0.6592, 0.6569, 0.2328, 0.4251], + [0.2071, 0.6297, 0.3653, 0.8513, 0.8549, 0.5509], + [0.2868, 0.2063, 0.4451, 0.3593, 0.7204, 0.0731], + [0.9699, 0.1078, 0.8829, 0.4132, 0.7572, 0.6948], + [0.5209, 0.5932, 0.8797, 0.6286, 0.7653, 0.1132], + [0.8559, 0.6721, 0.6267, 0.5691, 0.7437, 0.9592], + ], + dtype=dtype, + device=device, + ) + sample2 = torch.tensor( + [ + [0.3887, 0.2214, 0.3742, 0.1953, 0.7405, 0.2529], + [0.2332, 0.9314, 0.9575, 0.5575, 0.4134, 0.4355], + [0.7369, 0.0331, 0.0914, 0.8994, 0.9936, 0.4703], + [0.1049, 0.5137, 0.2674, 0.4990, 0.7447, 0.7213], + [0.4414, 0.5550, 0.6361, 0.1081, 0.3305, 0.5196], + [0.2147, 0.2816, 0.6679, 0.7878, 0.5070, 0.3055], + ], + dtype=dtype, + device=device, + ) + sample1 = sample1.expand((B, C, H, W)) + sample2 = sample2.expand((B, C, H, W)) + return_value = { + "keypoints0": torch.tensor( + [ + [0.1546, 0.9391], + [0.8077, 0.1051], + [0.6768, 0.5596], + [0.5092, 0.7195], + [0.2856, 0.8889], + [0.4342, 0.0203], + [0.6701, 0.0585], + [0.3828, 0.9038], + [0.7301, 0.0762], + [0.7864, 0.4490], + [0.3509, 0.0756], + [0.6782, 0.9297], + [0.4132, 0.3664], + [0.3134, 0.5039], + [0.2073, 0.2552], + ], + device=device, + dtype=dtype, + ), + "keypoints1": torch.tensor( + [ + [0.2076, 0.2669], + [0.9679, 0.8137], + [0.9536, 0.8317], + [0.3718, 0.2456], + [0.3875, 0.8450], + [0.7592, 0.1687], + [0.5173, 0.6760], + [0.9446, 0.4570], + [0.6164, 0.1867], + [0.4732, 0.1786], + [0.4090, 0.8089], + [0.9742, 0.8943], + [0.5996, 0.7427], + [0.7038, 0.9210], + [0.6272, 0.0796], + ], + device=device, + dtype=dtype, + ), + "confidence": torch.tensor( + [ + 0.9314, + 0.5951, + 0.4187, + 0.0318, + 0.1434, + 0.7952, + 0.8306, + 0.7511, + 0.6407, + 0.7379, + 0.4363, + 0.9220, + 0.8453, + 0.5075, + 0.8141, + ], + device=device, + dtype=dtype, + ), + "batch_indexes": torch.zeros((15,), device=device, dtype=dtype), + } + with patch( + "kornia.contrib.ImageStitcher.on_matcher", new_callable=PropertyMock, return_value=lambda x: return_value + ): + # NOTE: This will need to download the pretrained weights. + # To avoid that, we mock as below + matcher = kornia.feature.LoFTR(None) + stitcher = kornia.contrib.ImageStitcher(matcher, estimator=estimator).to(device=device, dtype=dtype) + torch.manual_seed(1) # issue kornia#2027 + out = stitcher(sample1, sample2) + assert out.shape[:-1] == torch.Size([1, 3, 6]) + assert out.shape[-1] <= 12 + + @pytest.mark.slow + def test_exception(self, device, dtype): + B, C, H, W = 1, 3, 224, 224 + sample1 = torch.rand(B, C, H, W, device=device, dtype=dtype) + sample2 = torch.rand(B, C, H, W, device=device, dtype=dtype) + # NOTE: This will need to download the pretrained weights. + matcher = kornia.feature.LoFTR(None) + + with pytest.raises(NotImplementedError): + stitcher = kornia.contrib.ImageStitcher(matcher, estimator="random").to(device=device, dtype=dtype) + + stitcher = kornia.contrib.ImageStitcher(matcher).to(device=device, dtype=dtype) + with pytest.raises(RuntimeError): + stitcher(sample1, sample2) diff --git a/tests/contrib/test_kmeans.py b/tests/contrib/test_kmeans.py new file mode 100644 index 0000000..0d72600 --- /dev/null +++ b/tests/contrib/test_kmeans.py @@ -0,0 +1,152 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestKMeans(BaseTester): + @pytest.mark.parametrize("num_clusters", [3, 10, 1]) + @pytest.mark.parametrize("tolerance", [10e-4, 10e-5, 10e-1]) + @pytest.mark.parametrize("max_iterations", [10, 1000]) + def test_smoke(self, device, dtype, num_clusters, tolerance, max_iterations): + N = 1000 + D = 2 + + kmeans = kornia.contrib.KMeans(num_clusters, None, tolerance, max_iterations, 0) + kmeans.fit(torch.rand((N, D), dtype=dtype, device=device)) + + out1 = kmeans.cluster_assignments + out2 = kmeans.cluster_centers + + # output is of type tensor + assert isinstance(out1, torch.Tensor) + assert isinstance(out2, torch.Tensor) + + # output is of same dtype + assert out1.dtype == torch.int64 + assert out2.dtype == dtype + + @pytest.mark.parametrize("num_clusters", [3]) + @pytest.mark.parametrize("tolerance", [10e-4]) + @pytest.mark.parametrize("max_iterations", [100]) + def test_cardinality(self, device, dtype, num_clusters, tolerance, max_iterations): + N = 1000 + D = 2 + + kmeans = kornia.contrib.KMeans(num_clusters, None, tolerance, max_iterations, 0) + kmeans.fit(torch.rand((N, D), device=device, dtype=dtype)) + + out1 = kmeans.cluster_assignments + out2 = kmeans.cluster_centers + + # output is of correct shape + assert out1.shape == (N,) + assert out2.shape == (num_clusters, D) + + def test_exception(self, device, dtype): + from kornia.core.exceptions import BaseError, ShapeError + + # case: cluster_center = 0: + with pytest.raises(BaseError) as errinfo: + kornia.contrib.KMeans(0, None, 10e-4, 10, 0) + assert "num_clusters can't be 0" in str(errinfo.value) + + # case: cluster centers is not a 2D tensor + with pytest.raises(ShapeError) as errinfo: + starting_centers = torch.rand((2, 3, 5), device=device, dtype=dtype) + kmeans = kornia.contrib.KMeans(None, starting_centers, 10e-4, 100, 0) + assert "Shape dimension mismatch" in str(errinfo.value) + + # case: input data is not a 2D tensor + with pytest.raises(ShapeError) as errinfo: + kmeans = kornia.contrib.KMeans(3, None, 10e-4, 100, 0) + kmeans.fit(torch.rand((1000, 5, 60), dtype=dtype, device=device)) + assert "Shape dimension mismatch" in str(errinfo.value) or "Expected shape" in str(errinfo.value) + + # case: column dimensions of cluster centers and data to be predicted do not match + with pytest.raises(Exception) as errinfo: + kmeans = kornia.contrib.KMeans(3, None, 10e-4, 100, 0) + kmeans.fit(torch.rand((1000, 5), dtype=dtype)) + kmeans.predict(torch.rand((10, 7), dtype=dtype)) + assert "7 != 5" in str(errinfo) + + @staticmethod + def _create_data(device, dtype): + # create example dataset + torch.manual_seed(2023) + x = 5 * torch.randn((500, 2), dtype=dtype, device=device) + torch.tensor((-13, 17), dtype=dtype, device=device) + x = torch.vstack( + [x, torch.randn((500, 2), dtype=dtype, device=device) + torch.tensor((15, -12), dtype=dtype, device=device)] + ) + x = torch.vstack( + [ + x, + 13 * torch.randn((500, 2), dtype=dtype, device=device) + + torch.tensor((35, 15), dtype=dtype, device=device), + ] + ) + return x + + def test_module(self, device, dtype): + x = TestKMeans._create_data(device, dtype) + + kmeans = kornia.contrib.KMeans(3, None, 10e-4, 10000, 2023) + kmeans.fit(x) + + centers = kmeans.cluster_centers + prediction = kmeans.predict(torch.tensor([[-14, 16], [45, 12]], dtype=dtype, device=device)).tolist() + + expected_centers = torch.tensor([[-13, 17], [15, -12], [35, 15]], dtype=dtype, device=device) + expected_prediction = [0, 2] + + # sorting centers using dimension 0 as key so that they can be checked for equalness + order = torch.argsort(centers[:, 0]).tolist() + new_classes = {old_class: new_class for new_class, old_class in enumerate(order)} + + ordered_centers = centers[order] + oredered_prediction = [new_classes[predicted_class] for predicted_class in prediction] + + self.assert_close(ordered_centers, expected_centers, atol=2, rtol=0.1) + assert oredered_prediction == expected_prediction + + def test_dynamo(self, device, dtype, torch_optimizer): + x = TestKMeans._create_data(device, dtype) + kmeans_params = (3, None, 10e-4, 10000, 2023) + predict_param = torch.tensor([[-14, 16], [45, 12]], dtype=dtype, device=device) + + kmeans = kornia.contrib.KMeans(*kmeans_params) + kmeans.fit(x) + + centers = kmeans.cluster_centers + prediction = kmeans.predict(predict_param) + + kmeans_op = kornia.contrib.KMeans(*kmeans_params) + kmeans_op.fit = torch_optimizer(kmeans_op.fit) + kmeans_op.predict = torch_optimizer(kmeans_op.predict) + + kmeans_op.fit(x) + + centers_op = kmeans_op.cluster_centers + prediction_op = kmeans_op.predict(predict_param) + + self.assert_close(centers, centers_op) + self.assert_close(prediction, prediction_op) diff --git a/tests/contrib/test_lambda_module.py b/tests/contrib/test_lambda_module.py new file mode 100644 index 0000000..4b66f10 --- /dev/null +++ b/tests/contrib/test_lambda_module.py @@ -0,0 +1,63 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestLambdaModule(BaseTester): + def add_2_layer(self, tensor): + return tensor + 2 + + def add_x_mul_y(self, tensor, x, y=2): + return torch.mul(tensor + x, y) + + def test_smoke(self, device, dtype): + B, C, H, W = 1, 3, 4, 5 + img = torch.rand(B, C, H, W, device=device, dtype=dtype) + func = self.add_2_layer + if not callable(func): + raise TypeError(f"Argument lambd should be callable, got {type(func).__name__!r}") + assert isinstance(kornia.contrib.Lambda(func)(img), torch.Tensor) + + @pytest.mark.parametrize("x", [3, 2, 5]) + def test_lambda_with_arguments(self, x, device, dtype): + B, C, H, W = 2, 3, 5, 7 + img = torch.rand(B, C, H, W, device=device, dtype=dtype) + func = self.add_x_mul_y + lambda_module = kornia.contrib.Lambda(func) + out = lambda_module(img, x) + assert isinstance(out, torch.Tensor) + + @pytest.mark.parametrize("shape", [(1, 3, 2, 3), (2, 3, 5, 7)]) + def test_lambda(self, shape, device, dtype): + B, C, H, W = shape + img = torch.rand(B, C, H, W, device=device, dtype=dtype) + func = kornia.color.bgr_to_grayscale + lambda_module = kornia.contrib.Lambda(func) + out = lambda_module(img) + assert isinstance(out, torch.Tensor) + + def test_gradcheck(self, device): + B, C, H, W = 1, 3, 4, 5 + img = torch.rand(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + func = kornia.color.bgr_to_grayscale + self.gradcheck(kornia.contrib.Lambda(func), (img,)) diff --git a/tests/contrib/test_visual_prompter.py b/tests/contrib/test_visual_prompter.py new file mode 100644 index 0000000..189e3c7 --- /dev/null +++ b/tests/contrib/test_visual_prompter.py @@ -0,0 +1,74 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.contrib.visual_prompter import VisualPrompter + +from testing.base import BaseTester + + +class TestVisualPrompter(BaseTester): + @pytest.mark.slow + def test_smoke(self, device, dtype): + if dtype not in (torch.float32, torch.float16): + pytest.skip("VisualPrompter (SAM) primarily supports float32 and float16") + + prompter = VisualPrompter(device=device, dtype=dtype) + assert prompter is not None + assert not prompter.is_image_set + + @pytest.mark.slow + def test_batching_pipeline(self, device, dtype): + if dtype not in (torch.float32, torch.float16): + pytest.skip("VisualPrompter (SAM) primarily supports float32 and float16") + prompter = VisualPrompter(device=device, dtype=dtype) + + batch_size = 2 + image = torch.rand(batch_size, 3, 256, 256).to(device=device, dtype=dtype) + + prompter.set_image(image) + assert prompter.is_image_set + assert prompter.image_embeddings.shape[0] == batch_size + + boxes_tensor = torch.tensor( + [[[10.0, 10.0, 50.0, 50.0]], [[20.0, 20.0, 80.0, 80.0]]], + device=device, + dtype=dtype, + ) + + results = prompter.predict(boxes=boxes_tensor) + + assert results.logits.shape == (batch_size, 3, 256, 256) + + def test_exception(self, device, dtype): + + prompter = VisualPrompter(device=device, dtype=dtype) + + image = torch.rand(1, 2, 3, 256, 256).to(device=device, dtype=dtype) + with pytest.raises(Exception): + prompter.set_image(image) + + def test_gradcheck(self, device): + pytest.skip("Gradcheck is not currently applicable for VisualPrompter inference.") + + def test_cardinality(self, device, dtype): + pytest.skip("Cardinality is covered by the test_batching_pipeline.") + + def test_dynamo(self, device, dtype, torch_optimizer): + pytest.skip("Dynamo compilation currently broken for VisualPrompter. See #FIXME in source.") diff --git a/tests/core/test_check.py b/tests/core/test_check.py new file mode 100644 index 0000000..0dd6b77 --- /dev/null +++ b/tests/core/test_check.py @@ -0,0 +1,396 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.core.check import ( + KORNIA_CHECK, + KORNIA_CHECK_DM_DESC, + KORNIA_CHECK_IS_COLOR, + KORNIA_CHECK_IS_COLOR_OR_GRAY, + KORNIA_CHECK_IS_GRAY, + KORNIA_CHECK_IS_IMAGE, + KORNIA_CHECK_IS_LIST_OF_TENSOR, + KORNIA_CHECK_IS_TENSOR, + KORNIA_CHECK_LAF, + KORNIA_CHECK_SAME_DEVICE, + KORNIA_CHECK_SAME_DEVICES, + KORNIA_CHECK_SAME_SHAPE, + KORNIA_CHECK_SHAPE, + KORNIA_CHECK_TYPE, + are_checks_enabled, + disable_checks, + enable_checks, +) +from kornia.core.exceptions import ( + BaseError, + DeviceError, + ImageError, + ShapeError, + TypeCheckError, + ValueCheckError, +) + + +class TestCheck: + def test_valid(self): + assert KORNIA_CHECK(True, "This is a test") is True + + def test_invalid(self): + with pytest.raises(BaseError): + KORNIA_CHECK(False, "This is a test") + + def test_invalid_raises_false(self): + assert KORNIA_CHECK(False, "This should not raise", raises=False) is False + + def test_jit(self): + op_jit = torch.jit.script(KORNIA_CHECK) + assert op_jit is not None + assert op_jit(True, "Testing") is True + + +class TestCheckShape: + @pytest.mark.parametrize( + "data,shape", + [ + (torch.rand(2, 3), ["*", "H", "W"]), + (torch.rand(3, 2, 3), ["3", "H", "W"]), + (torch.rand(1, 1, 2, 3), ["1", "1", "H", "W"]), + (torch.rand(2, 3, 2, 3), ["2", "3", "H", "W"]), + ], + ) + def test_valid(self, data, shape): + assert KORNIA_CHECK_SHAPE(data, shape) is True + + @pytest.mark.parametrize( + "data,shape", + [ + (torch.rand(2, 3), ["1", "H", "W"]), + (torch.rand(3, 2, 3), ["H", "W"]), + (torch.rand(1, 2, 3), ["3", "H", "W"]), + (torch.rand(1, 3, 2, 3), ["2", "C", "H", "W"]), + ], + ) + def test_invalid(self, data, shape): + with pytest.raises(ShapeError): + KORNIA_CHECK_SHAPE(data, shape) + + def test_invalid_raises_false(self): + assert KORNIA_CHECK_SHAPE(torch.rand(2, 3), ["1", "H", "W"], raises=False) is False + + def test_jit(self): + op_jit = torch.jit.script(KORNIA_CHECK_SHAPE) + assert op_jit is not None + assert op_jit(torch.rand(2, 3, 2, 3), ["2", "3", "H", "W"]) is True + + +class TestCheckSameShape: + def test_valid(self): + assert KORNIA_CHECK_SAME_SHAPE(torch.rand(2, 3), torch.rand(2, 3)) is True + assert KORNIA_CHECK_SAME_SHAPE(torch.rand(1, 2, 3), torch.rand(1, 2, 3)) is True + assert KORNIA_CHECK_SAME_SHAPE(torch.rand(2, 3, 3), torch.rand(2, 3, 3)) is True + + def test_jit(self): + op_jit = torch.jit.script(KORNIA_CHECK_SAME_SHAPE) + assert op_jit is not None + assert op_jit(torch.rand(2, 3), torch.rand(2, 3)) is True + + def test_invalid(self): + with pytest.raises(ShapeError): + KORNIA_CHECK_SAME_SHAPE(torch.rand(2, 3), torch.rand(2, 2, 3)) + with pytest.raises(ShapeError): + KORNIA_CHECK_SAME_SHAPE(torch.rand(2, 3), torch.rand(1, 2, 3)) + with pytest.raises(ShapeError): + KORNIA_CHECK_SAME_SHAPE(torch.rand(2, 3), torch.rand(2, 3, 3)) + + def test_invalid_raises_false(self): + assert KORNIA_CHECK_SAME_SHAPE(torch.rand(2, 3), torch.rand(2, 2, 3), raises=False) is False + + +class TestCheckType: + def test_valid(self): + assert KORNIA_CHECK_TYPE("hello", str) is True + assert KORNIA_CHECK_TYPE(23, int) is True + assert KORNIA_CHECK_TYPE(torch.rand(1), torch.Tensor) is True + assert KORNIA_CHECK_TYPE(torch.rand(1), (int, torch.Tensor)) is True + + def test_invalid(self): + with pytest.raises(TypeCheckError): + KORNIA_CHECK_TYPE("world", int) + with pytest.raises(TypeCheckError): + KORNIA_CHECK_TYPE(23, float) + with pytest.raises(TypeCheckError): + KORNIA_CHECK_TYPE(23, (float, str)) + + def test_invalid_raises_false(self): + assert KORNIA_CHECK_TYPE("world", int, raises=False) is False + + +class TestCheckIsTensor: + def test_valid(self): + assert KORNIA_CHECK_IS_TENSOR(torch.rand(1)) is True + + def test_invalid(self): + with pytest.raises(TypeCheckError): + KORNIA_CHECK_IS_TENSOR([1, 2, 3]) + + def test_invalid_raises_false(self): + assert KORNIA_CHECK_IS_TENSOR([1, 2, 3], raises=False) is False + + +class TestCheckIsListOfTensor: + def test_valid(self): + assert KORNIA_CHECK_IS_LIST_OF_TENSOR([torch.rand(1), torch.rand(1), torch.rand(1)]) is True + + def test_invalid(self): + with pytest.raises(TypeCheckError): + KORNIA_CHECK_IS_LIST_OF_TENSOR([torch.rand(1), [2, 3], torch.rand(1)]) + with pytest.raises(TypeCheckError): + KORNIA_CHECK_IS_LIST_OF_TENSOR([1, 2, 3]) + + def test_invalid_raises_false(self): + assert KORNIA_CHECK_IS_LIST_OF_TENSOR([torch.rand(1), [2, 3], torch.rand(1)], raises=False) is False + + +class TestCheckSameDevice: + def test_valid(self, device): + assert KORNIA_CHECK_SAME_DEVICE(torch.rand(1, device=device), torch.rand(1, device=device)) is True + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="Skip if no GPU.") + def test_invalid(self): + with pytest.raises(DeviceError): + KORNIA_CHECK_SAME_DEVICE(torch.rand(1, device="cpu"), torch.rand(1, device="cuda")) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="Skip if no GPU.") + def test_invalid_raises_false(self): + assert ( + KORNIA_CHECK_SAME_DEVICE(torch.rand(1, device="cpu"), torch.rand(1, device="cuda"), raises=False) is False + ) + + +class TestCheckSameDevices: + def test_valid(self, device): + assert KORNIA_CHECK_SAME_DEVICES([torch.rand(1, device=device), torch.rand(1, device=device)]) is True + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="Skip if no GPU.") + def test_invalid(self): + with pytest.raises(DeviceError): + KORNIA_CHECK_SAME_DEVICES([torch.rand(1, device="cpu"), torch.rand(1, device="cuda")]) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="Skip if no GPU.") + def test_invalid_raises_false(self): + assert ( + KORNIA_CHECK_SAME_DEVICES([torch.rand(1, device="cpu"), torch.rand(1, device="cuda")], raises=False) + is False + ) + + +class TestCheckIsColor: + def test_valid(self): + assert KORNIA_CHECK_IS_COLOR(torch.rand(3, 4, 4)) is True + assert KORNIA_CHECK_IS_COLOR(torch.rand(1, 3, 4, 4)) is True + assert KORNIA_CHECK_IS_COLOR(torch.rand(2, 3, 4, 4)) is True + + def test_invalid(self): + with pytest.raises(ImageError): + KORNIA_CHECK_IS_COLOR(torch.rand(1, 4, 4)) + with pytest.raises(ImageError): + KORNIA_CHECK_IS_COLOR(torch.rand(2, 4, 4)) + with pytest.raises(ImageError): + KORNIA_CHECK_IS_COLOR(torch.rand(3, 4, 4, 4)) + with pytest.raises(ImageError): + KORNIA_CHECK_IS_COLOR(torch.rand(1, 3, 4, 4, 4)) + + def test_invalid_raises_false(self): + assert KORNIA_CHECK_IS_COLOR(torch.rand(1, 4, 4), raises=False) is False + + +class TestCheckIsGray: + def test_valid(self): + assert KORNIA_CHECK_IS_GRAY(torch.rand(1, 4, 4)) is True + assert KORNIA_CHECK_IS_GRAY(torch.rand(2, 1, 4, 4)) is True + assert KORNIA_CHECK_IS_GRAY(torch.rand(3, 1, 4, 4)) is True + + def test_invalid(self): + with pytest.raises(ImageError): + KORNIA_CHECK_IS_GRAY(torch.rand(3, 4, 4)) + with pytest.raises(ImageError): + KORNIA_CHECK_IS_GRAY(torch.rand(1, 4, 4, 4)) + with pytest.raises(ImageError): + KORNIA_CHECK_IS_GRAY(torch.rand(1, 3, 4, 4)) + with pytest.raises(ImageError): + KORNIA_CHECK_IS_GRAY(torch.rand(1, 3, 4, 4, 4)) + + def test_invalid_raises_false(self): + assert KORNIA_CHECK_IS_GRAY(torch.rand(1, 3, 4, 4, 4), raises=False) is False + + +class TestCheckIsColorOrGray: + def test_valid(self): + assert KORNIA_CHECK_IS_COLOR_OR_GRAY(torch.rand(3, 4, 4)) is True + assert KORNIA_CHECK_IS_COLOR_OR_GRAY(torch.rand(1, 3, 4, 4)) is True + assert KORNIA_CHECK_IS_COLOR_OR_GRAY(torch.rand(2, 3, 4, 4)) is True + assert KORNIA_CHECK_IS_COLOR_OR_GRAY(torch.rand(1, 4, 4)) is True + assert KORNIA_CHECK_IS_COLOR_OR_GRAY(torch.rand(2, 1, 4, 4)) is True + assert KORNIA_CHECK_IS_COLOR_OR_GRAY(torch.rand(3, 1, 4, 4)) is True + + def test_invalid(self): + with pytest.raises(ImageError): + KORNIA_CHECK_IS_COLOR_OR_GRAY(torch.rand(1, 4, 4, 4)) + with pytest.raises(ImageError): + KORNIA_CHECK_IS_COLOR_OR_GRAY(torch.rand(1, 3, 4, 4, 4)) + + def test_invalid_raises_false(self): + assert KORNIA_CHECK_IS_COLOR_OR_GRAY(torch.rand(1, 4, 4, 4), raises=False) is False + + +class TestCheckDmDesc: + def test_valid(self): + assert KORNIA_CHECK_DM_DESC(torch.rand(4), torch.rand(8), torch.rand(4, 8)) is True + + def test_invalid(self): + with pytest.raises(ShapeError): + KORNIA_CHECK_DM_DESC(torch.rand(4), torch.rand(8), torch.rand(4, 7)) + with pytest.raises(ShapeError): + KORNIA_CHECK_DM_DESC(torch.rand(4), torch.rand(8), torch.rand(3, 8)) + with pytest.raises(ShapeError): + KORNIA_CHECK_DM_DESC(torch.rand(4), torch.rand(8), torch.rand(3, 7)) + with pytest.raises(ShapeError): + KORNIA_CHECK_DM_DESC(torch.rand(4), torch.rand(8), torch.rand(4, 3, 8)) + + def test_invalid_raises_false(self): + assert KORNIA_CHECK_DM_DESC(torch.rand(4), torch.rand(8), torch.rand(4, 7), raises=False) is False + + +class TestCheckLaf: + def test_valid(self): + assert KORNIA_CHECK_LAF(torch.rand(4, 2, 2, 3)) is True + + def test_invalid(self): + with pytest.raises(ShapeError): + KORNIA_CHECK_LAF(torch.rand(4, 2, 2)) + with pytest.raises(ShapeError): + KORNIA_CHECK_LAF(torch.rand(4, 2, 3, 2)) + with pytest.raises(ShapeError): + KORNIA_CHECK_LAF(torch.rand(4, 2, 2, 2)) + with pytest.raises(ShapeError): + KORNIA_CHECK_LAF(torch.rand(4, 2, 3, 3, 3)) + + def test_invalid_raises_false(self): + assert KORNIA_CHECK_LAF(torch.rand(4, 2, 2), raises=False) is False + + +class TestCheckIsImage: + def test_valid_float(self): + assert KORNIA_CHECK_IS_IMAGE(torch.rand(3, 4, 4)) is True + assert KORNIA_CHECK_IS_IMAGE(torch.rand(2, 3, 4, 4)) is True + assert KORNIA_CHECK_IS_IMAGE(torch.rand(1, 1, 4, 4)) is True + + def test_valid_int(self): + x = torch.randint(0, 256, (3, 4, 4), dtype=torch.uint8) + assert KORNIA_CHECK_IS_IMAGE(x) is True + y = torch.randint(0, 256, (2, 3, 4, 4), dtype=torch.uint8) + assert KORNIA_CHECK_IS_IMAGE(y) is True + + def test_invalid_float_range(self): + with pytest.raises(ValueCheckError): + KORNIA_CHECK_IS_IMAGE(torch.tensor([[[-0.5, 1.2]]], dtype=torch.float32)) + + def test_invalid_int_range(self): + bad = torch.tensor([[[300]]], dtype=torch.int32) + with pytest.raises(ValueCheckError): + KORNIA_CHECK_IS_IMAGE(bad) + + def test_invalid_shape(self): + x = torch.rand(1, 4, 4, 4) + assert KORNIA_CHECK_IS_IMAGE(x) is True + + def test_invalid_range_no_raise(self): + bad = torch.tensor([[[-0.1, 2.0]]], dtype=torch.float32) + assert KORNIA_CHECK_IS_IMAGE(bad, raises=False) is False + + def test_invalid_shape_no_raise(self): + # When raises=False, shape check is actually enforced + bad = torch.rand(1, 4, 4, 4) + assert KORNIA_CHECK_IS_IMAGE(bad, raises=False) is False + + +class TestChecksEnableDisable: + """Tests for the runtime enable/disable check control API.""" + + def setup_method(self): + # Always restore checks after each test to avoid side effects + enable_checks() + + def teardown_method(self): + enable_checks() + + def test_are_checks_enabled_default(self): + enable_checks() + assert are_checks_enabled() is True + + def test_disable_checks(self): + disable_checks() + assert are_checks_enabled() is False + + def test_enable_checks(self): + disable_checks() + enable_checks() + assert are_checks_enabled() is True + + def test_disabled_checks_bypass_kornia_check(self): + disable_checks() + # With checks disabled, KORNIA_CHECK should return True even for False condition + result = KORNIA_CHECK(False, "should be bypassed") + assert result is True + + def test_disabled_checks_bypass_shape_check(self): + disable_checks() + # With checks disabled, wrong shape should not raise + result = KORNIA_CHECK_SHAPE(torch.rand(2, 3), ["1", "H", "W"]) + assert result is True + + def test_enabled_checks_enforce_shape_check(self): + enable_checks() + with pytest.raises(ShapeError): + KORNIA_CHECK_SHAPE(torch.rand(2, 3), ["1", "H", "W"]) + + def test_env_var_disables_checks(self, monkeypatch): + monkeypatch.setenv("KORNIA_CHECKS", "0") + from kornia.core.check import _should_enable_checks + + assert _should_enable_checks() is False + + def test_env_var_enables_checks(self, monkeypatch): + from kornia.core.check import _should_enable_checks + + monkeypatch.setenv("KORNIA_CHECKS", "1") + assert _should_enable_checks() is True + + def test_env_var_true_string_variants(self, monkeypatch): + from kornia.core.check import _should_enable_checks + + for val in ("true", "yes", "on", "1"): + monkeypatch.setenv("KORNIA_CHECKS", val) + assert _should_enable_checks() is True + + def test_env_var_false_string(self, monkeypatch): + from kornia.core.check import _should_enable_checks + + monkeypatch.setenv("KORNIA_CHECKS", "false") + assert _should_enable_checks() is False diff --git a/tests/core/test_helpers.py b/tests/core/test_helpers.py new file mode 100644 index 0000000..60aa8e8 --- /dev/null +++ b/tests/core/test_helpers.py @@ -0,0 +1,173 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.core.exceptions import DeviceError +from kornia.core.utils import ( + _extract_device_dtype, + _torch_histc_cast, + _torch_inverse_cast, + _torch_solve_cast, + _torch_svd_cast, + safe_inverse_with_mask, + safe_solve_with_mask, +) + +from testing.base import assert_close + + +@pytest.mark.parametrize( + "tensor_list,out_device,out_dtype,will_throw_error", + [ + ([], torch.device("cpu"), torch.get_default_dtype(), False), + ([None, None], torch.device("cpu"), torch.get_default_dtype(), False), + ([torch.tensor(0, device="cpu", dtype=torch.float16), None], torch.device("cpu"), torch.float16, False), + ([torch.tensor(0, device="cpu", dtype=torch.float32), None], torch.device("cpu"), torch.float32, False), + ([torch.tensor(0, device="cpu", dtype=torch.float64), None], torch.device("cpu"), torch.float64, False), + ([torch.tensor(0, device="cpu", dtype=torch.float16)] * 2, torch.device("cpu"), torch.float16, False), + ([torch.tensor(0, device="cpu", dtype=torch.float32)] * 2, torch.device("cpu"), torch.float32, False), + ([torch.tensor(0, device="cpu", dtype=torch.float64)] * 2, torch.device("cpu"), torch.float64, False), + ( + [torch.tensor(0, device="cpu", dtype=torch.float16), torch.tensor(0, device="cpu", dtype=torch.float64)], + None, + None, + True, + ), + ( + [torch.tensor(0, device="cpu", dtype=torch.float32), torch.tensor(0, device="cpu", dtype=torch.float64)], + None, + None, + True, + ), + ( + [torch.tensor(0, device="cpu", dtype=torch.float16), torch.tensor(0, device="cpu", dtype=torch.float32)], + None, + None, + True, + ), + ], +) +def test_extract_device_dtype(tensor_list, out_device, out_dtype, will_throw_error): + if will_throw_error: + with pytest.raises(DeviceError): + _extract_device_dtype(tensor_list) + else: + device, dtype = _extract_device_dtype(tensor_list) + assert device == out_device + assert dtype == out_dtype + + +class TestInverseCast: + @pytest.mark.parametrize("input_shape", [(4, 4), (1, 3, 4, 4), (2, 4, 5, 5)]) + def test_smoke(self, device, dtype, input_shape): + x = torch.rand(input_shape, device=device, dtype=dtype) + y = _torch_inverse_cast(x) + assert y.shape == x.shape + + def test_values(self, device, dtype): + x = torch.tensor([[4.0, 7.0], [2.0, 6.0]], device=device, dtype=dtype) + + y_expected = torch.tensor([[0.6, -0.7], [-0.2, 0.4]], device=device, dtype=dtype) + + y = _torch_inverse_cast(x) + + assert_close(y, y_expected) + + def test_jit(self, device, dtype): + x = torch.rand(1, 3, 4, 4, device=device, dtype=dtype) + op = _torch_inverse_cast + op_jit = torch.jit.script(op) + assert_close(op(x), op_jit(x)) + + def test_not_invertible(self, device, dtype): + with pytest.raises(RuntimeError): + x = torch.tensor([[0.0, 0.0], [0.0, 0.0]], device=device, dtype=dtype) + _ = _torch_inverse_cast(x) + + +class TestHistcCast: + def test_smoke(self, device, dtype): + x = torch.tensor([1.0, 2.0, 1.0], device=device, dtype=dtype) + y_expected = torch.tensor([0.0, 2.0, 1.0, 0.0], device=device, dtype=dtype) + + y = _torch_histc_cast(x, bins=4, min=0, max=3) + + assert_close(y, y_expected) + + +class TestSvdCast: + def test_smoke(self, device, dtype): + a = torch.randn(5, 3, 3, device=device, dtype=dtype) + u, s, v = _torch_svd_cast(a) + + tol_val: float = 1e-1 if dtype == torch.float16 else 1e-3 + assert_close(a, u @ torch.diag_embed(s) @ v.transpose(-2, -1), atol=tol_val, rtol=tol_val) + + +class TestSolveCast: + def test_smoke(self, device, dtype): + torch.manual_seed(0) + A = torch.randn(2, 3, 1, 4, 4, device=device, dtype=dtype) + B = torch.randn(2, 3, 1, 4, 6, device=device, dtype=dtype) + + X = _torch_solve_cast(A, B) + error = torch.dist(B, A.matmul(X)) + tol_val: float = 1e-1 if dtype == torch.float16 else 1e-4 + assert_close(error, torch.zeros_like(error), atol=tol_val, rtol=tol_val) + + +class TestSolveWithMask: + def test_smoke(self, device, dtype): + torch.manual_seed(0) # issue kornia#2027 + A = torch.randn(2, 3, 1, 4, 4, device=device, dtype=dtype) + B = torch.randn(2, 3, 1, 4, 6, device=device, dtype=dtype) + + X, _, mask = safe_solve_with_mask(B, A) + X2 = _torch_solve_cast(A, B) + tol_val: float = 1e-1 if dtype == torch.float16 else 1e-4 + if mask.sum() > 0: + assert_close(X[mask], X2[mask], atol=tol_val, rtol=tol_val) + + @pytest.mark.skipif( + (int(torch.__version__.split(".")[0]) == 1) and (int(torch.__version__.split(".")[1]) < 10), + reason="<1.10.0 not supporting", + ) + def test_all_bad(self, device, dtype): + A = torch.ones(10, 3, 3, device=device, dtype=dtype) + B = torch.ones(10, 3, device=device, dtype=dtype) + + _X, _, mask = safe_solve_with_mask(B, A) + assert torch.equal(mask, torch.zeros_like(mask)) + + +class TestInverseWithMask: + def test_smoke(self, device, dtype): + x = torch.tensor([[4.0, 7.0], [2.0, 6.0]], device=device, dtype=dtype) + + y_expected = torch.tensor([[0.6, -0.7], [-0.2, 0.4]], device=device, dtype=dtype) + + y, mask = safe_inverse_with_mask(x) + + assert_close(y, y_expected) + assert torch.equal(mask, torch.ones_like(mask)) + + def test_all_bad(self, device, dtype): + A = torch.ones(10, 3, 3, device=device, dtype=dtype) + _X, mask = safe_inverse_with_mask(A) + assert torch.equal(mask, torch.zeros_like(mask)) diff --git a/tests/core/test_lazyloader.py b/tests/core/test_lazyloader.py new file mode 100644 index 0000000..dc025a1 --- /dev/null +++ b/tests/core/test_lazyloader.py @@ -0,0 +1,76 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from io import StringIO + +import pytest + +from kornia.core.external import LazyLoader + + +class TestLazyLoader: + def test_lazy_loader_initialization(self): + # Test that the LazyLoader initializes with the correct module name and None module + loader = LazyLoader("math") + assert loader.module_name == "math" + assert loader.module is None + + def test_lazy_loader_loading_module(self): + # Test that the LazyLoader correctly loads the module upon attribute access + loader = LazyLoader("math") + assert loader.module is None # Should be None before any attribute access + + # Access an attribute to trigger module loading + assert loader.sqrt(4) == 2.0 + assert loader.module is not None # Should be loaded now + + def test_lazy_loader_invalid_module(self, monkeypatch): + monkeypatch.setattr("sys.stdin", StringIO("n")) + # Test that LazyLoader raises an ImportError for an invalid module + loader = LazyLoader("non_existent_module") + with pytest.raises(ImportError) as excinfo: + _ = loader.non_existent_attribute # Accessing any attribute should raise the error + + assert "Optional dependency 'non_existent_module' is not installed" in str(excinfo.value) + + def test_lazy_loader_getattr(self): + # Test that __getattr__ works correctly for a valid module + loader = LazyLoader("math") + assert loader.sqrt(16) == 4.0 + assert loader.pi == 3.141592653589793 + + def test_lazy_loader_dir(self): + # Test that dir() returns the correct list of attributes for the module + loader = LazyLoader("math") + attributes = dir(loader) + assert "sqrt" in attributes + assert "pi" in attributes + assert loader.module is not None + + def test_lazy_loader_multiple_attributes(self): + # Test accessing multiple attributes to ensure the module is loaded only once + loader = LazyLoader("math") + assert loader.sqrt(25) == 5.0 + assert loader.pi == 3.141592653589793 + assert loader.pow(2, 3) == 8.0 + assert loader.module is not None + + def test_lazy_loader_non_existing_attribute(self): + # Test that accessing a non-existing attribute raises an AttributeError after loading + loader = LazyLoader("math") + with pytest.raises(AttributeError): + _ = loader.non_existent_attribute diff --git a/tests/core/test_module.py b/tests/core/test_module.py new file mode 100644 index 0000000..d37714d --- /dev/null +++ b/tests/core/test_module.py @@ -0,0 +1,197 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from unittest.mock import MagicMock + +import numpy as np +import pytest +import torch +from PIL import Image as PILImage + +from kornia.core.module import ImageModule, ImageModuleMixIn + + +class TestImageModuleMixIn: + @pytest.fixture + def img_module(self): + class DummyModule(ImageModuleMixIn): + pass + + return DummyModule() + + @pytest.fixture + def sample_image(self): + # Create a sample PIL image for testing + return PILImage.fromarray(torch.randint(0, 255, (100, 100, 3)).numpy().astype(np.uint8)) + + @pytest.fixture + def sample_tensor(self): + # Create a sample tensor for testing + return torch.rand((3, 100, 100)) + + @pytest.fixture + def sample_numpy(self): + # Create a sample numpy array for testing + return torch.rand(100, 100, 3).numpy() + + def test_to_tensor_pil(self, img_module, sample_image): + tensor = img_module.to_tensor(sample_image) + assert isinstance(tensor, (torch.Tensor,)) + assert tensor.shape == (3, 100, 100) + + def test_to_tensor_numpy(self, img_module, sample_numpy): + tensor = img_module.to_tensor(sample_numpy) + assert isinstance(tensor, (torch.Tensor,)) + assert tensor.shape == (3, 100, 100) + + def test_to_tensor_tensor(self, img_module, sample_tensor): + tensor = img_module.to_tensor(sample_tensor) + assert tensor is sample_tensor + + def test_to_numpy_tensor(self, img_module, sample_tensor): + array = img_module.to_numpy(sample_tensor) + assert isinstance(array, (np.ndarray,)) + assert array.shape == (3, 100, 100) + + def test_to_numpy_numpy(self, img_module, sample_numpy): + array = img_module.to_numpy(sample_numpy) + assert array is sample_numpy + + def test_to_pil_tensor(self, img_module, sample_tensor): + pil_image = img_module.to_pil(sample_tensor) + assert isinstance(pil_image, (PILImage.Image,)) + + def test_to_pil_pil(self, img_module, sample_image): + pil_image = img_module.to_pil(sample_image) + assert pil_image is sample_image + + def test_convert_input_output(self, img_module, sample_image, sample_numpy, sample_tensor): + @img_module.convert_input_output(output_type="numpy") + def dummy_func(tensor): + return tensor + + output = dummy_func(sample_image) + assert isinstance(output, (np.ndarray,)) + + def test_show(self, img_module, sample_tensor): + img_module._output_image = sample_tensor + pil_image = img_module.show(display=False) + assert isinstance(pil_image, (PILImage.Image,)) + + def test_save(self, img_module, sample_tensor, tmpdir): + img_module._output_image = sample_tensor + save_path = tmpdir.join("test_image.jpg") + img_module.save(name=save_path) + assert os.path.exists(save_path) + + def test_to_pil_4d_tensor_returns_list(self, img_module): + # 4D (B, C, H, W) tensor -> list of PIL Images + t = torch.rand(3, 3, 16, 16) + result = img_module.to_pil(t) + assert isinstance(result, list) + assert len(result) == 3 + assert all(isinstance(im, PILImage.Image) for im in result) + + def test_to_pil_numpy_raises(self, img_module, sample_numpy): + with pytest.raises(NotImplementedError): + img_module.to_pil(sample_numpy) + + def test_to_pil_1d_tensor_raises(self, img_module): + with pytest.raises(NotImplementedError): + img_module.to_pil(torch.rand(8)) + + def test_to_numpy_pil(self, img_module, sample_image): + arr = img_module.to_numpy(sample_image) + assert isinstance(arr, np.ndarray) + assert arr.shape == (100, 100, 3) + + def test_convert_input_output_invalid_type_raises(self, img_module, sample_tensor): + with pytest.raises(ValueError, match="Invalid output_type"): + + @img_module.convert_input_output(output_type="invalid") + def dummy_func(tensor): + return tensor + + def test_convert_input_output_pil_output(self, img_module, sample_tensor): + @img_module.convert_input_output(output_type="pil") + def dummy_func(tensor): + return tensor + + result = dummy_func(sample_tensor) + assert isinstance(result, PILImage.Image) + + def test_convert_input_output_selective_input_names(self, img_module, sample_image): + # Only convert arguments named "image", leave others unchanged + @img_module.convert_input_output(input_names_to_handle=["image"], output_type="pt") + def dummy_func(image, other): + return image + + result = dummy_func(sample_image, "not_an_image") + assert isinstance(result, torch.Tensor) + + def test_show_4d_tensor(self, img_module): + img_module._output_image = torch.rand(4, 3, 16, 16) + result = img_module.show(display=False) + assert isinstance(result, PILImage.Image) + + def test_show_unsupported_backend_raises(self, img_module, sample_tensor): + img_module._output_image = sample_tensor + with pytest.raises(ValueError, match="Unsupported backend"): + img_module.show(backend="matplotlib", display=False) + + def test_detach_tensor_to_cpu_tensor(self, img_module, sample_tensor): + result = img_module._detach_tensor_to_cpu(sample_tensor) + assert isinstance(result, torch.Tensor) + assert result.device.type == "cpu" + + def test_detach_tensor_to_cpu_list(self, img_module): + tensors = [torch.rand(3, 4, 4), torch.rand(3, 4, 4)] + result = img_module._detach_tensor_to_cpu(tensors) + assert isinstance(result, list) + assert all(t.device.type == "cpu" for t in result) + + def test_detach_tensor_to_cpu_tuple(self, img_module): + tensors = (torch.rand(3, 4, 4), torch.rand(3, 4, 4)) + result = img_module._detach_tensor_to_cpu(tensors) + assert isinstance(result, tuple) + + +class TestImageModule: + @pytest.fixture + def image_module(self): + return ImageModule() + + @pytest.fixture + def sample_tensor(self): + return torch.rand((3, 100, 100)) + + def test_call_with_features_disabled(self, image_module, sample_tensor): + image_module.disable_features = True + mock_forward = MagicMock(return_value=sample_tensor) + image_module.forward = mock_forward + output = image_module(sample_tensor) + assert output is sample_tensor + mock_forward.assert_called_once() + + def test_call_with_features_enabled(self, image_module, sample_tensor): + image_module.disable_features = False + mock_forward = MagicMock(return_value=sample_tensor) + image_module.forward = mock_forward + output = image_module(sample_tensor) + assert output is sample_tensor + mock_forward.assert_called_once() diff --git a/tests/core/test_tensor_wrapper.py b/tests/core/test_tensor_wrapper.py new file mode 100644 index 0000000..f9e983e --- /dev/null +++ b/tests/core/test_tensor_wrapper.py @@ -0,0 +1,283 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.core.exceptions import TypeCheckError +from kornia.core.tensor_wrapper import TensorWrapper, _unwrap, _wrap + +from testing.base import BaseTester + + +class TestTensorWrapper(BaseTester): + def test_smoke(self, device, dtype): + data = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + tensor = _wrap(data, TensorWrapper) + assert isinstance(tensor, TensorWrapper) + assert isinstance(tensor.data, torch.Tensor) + assert tensor.shape == (1, 2, 3, 4) + assert tensor.device == device + assert tensor.dtype == dtype + self.assert_close(data, tensor.unwrap()) + + def test_init_validation(self): + """Test that TensorWrapper validates input type.""" + with pytest.raises(TypeCheckError, match="Tensor"): + TensorWrapper([1, 2, 3]) # type: ignore + + with pytest.raises(TypeCheckError, match="Tensor"): + TensorWrapper("not a tensor") # type: ignore + + def test_repr(self, device, dtype): + """Test string representation.""" + data = torch.rand(2, 3, device=device, dtype=dtype) + tensor = TensorWrapper(data) + repr_str = repr(tensor) + assert "TensorWrapper" in repr_str + assert str(data) in repr_str or repr(data) in repr_str + + def test_data_property(self, device, dtype): + """Test data property access.""" + data = torch.rand(2, 3, device=device, dtype=dtype) + tensor = TensorWrapper(data) + assert tensor.data is data + assert isinstance(tensor.data, torch.Tensor) + + def test_serialization(self, device, dtype, tmp_path): + data = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + tensor: TensorWrapper = _wrap(data, TensorWrapper) + + file_path = tmp_path / "tensor.pt" + torch.save(tensor, file_path) + assert file_path.is_file() + + loaded_tensor: TensorWrapper = torch.load(file_path, weights_only=False) + assert isinstance(loaded_tensor, TensorWrapper) + + self.assert_close(loaded_tensor.unwrap(), tensor.unwrap()) + # Check that used_attrs and used_calls are preserved + assert hasattr(loaded_tensor, "used_attrs") + assert hasattr(loaded_tensor, "used_calls") + + def test_wrap_list(self, device, dtype): + data_list = [ + torch.rand(2, device=device, dtype=dtype), + torch.rand(3, device=device, dtype=dtype), + TensorWrapper(torch.rand(3, device=device, dtype=dtype)), + 1, + 0.5, + ] + tensor_list = _wrap(data_list, TensorWrapper) + assert isinstance(tensor_list, list) + assert len(tensor_list) == 5 + + tensor_list_data = _unwrap(tensor_list) + assert len(tensor_list_data) == 5 + + self.assert_close(tensor_list_data[0], data_list[0]) + self.assert_close(tensor_list_data[1], data_list[1]) + self.assert_close(tensor_list_data[2], data_list[2]) + assert tensor_list_data[3] == data_list[3] + assert tensor_list_data[4] == data_list[4] + + for i in range(len(tensor_list_data[:3])): + self.assert_close(tensor_list[i].unwrap(), data_list[i]) + + def test_wrap_tuple(self, device, dtype): + """Test wrapping tuples.""" + data_tuple = ( + torch.rand(2, device=device, dtype=dtype), + torch.rand(3, device=device, dtype=dtype), + ) + tensor_tuple = _wrap(data_tuple, TensorWrapper) + assert isinstance(tensor_tuple, tuple) + assert len(tensor_tuple) == 2 + assert isinstance(tensor_tuple[0], TensorWrapper) + assert isinstance(tensor_tuple[1], TensorWrapper) + + def test_accessors(self, device, dtype): + data = torch.tensor([0.0, 1.0, 0.0], device=device, dtype=dtype) + x = _wrap(data, TensorWrapper) + self.assert_close(x[1].unwrap(), torch.tensor(1.0, device=device, dtype=dtype)) + y = x[0] + self.assert_close(y.unwrap(), torch.tensor(0.0, device=device, dtype=dtype)) + x[1] = 0.0 + self.assert_close(x.unwrap(), torch.zeros_like(data)) + + def test_unary_ops(self, device, dtype): + data = torch.rand(2, device=device, dtype=dtype) + x = TensorWrapper(data) + x1 = TensorWrapper(data) + x2 = TensorWrapper(data) + + # Methods like x.add() return unwrapped tensors, operators return wrapped tensors + self.assert_close(x.add(x), (x + x).unwrap()) + self.assert_close(x.add(1), (1 + x).unwrap()) + self.assert_close(x.add(1), (x + 1).unwrap()) + self.assert_close(x.mul(x), (x * x).unwrap()) + self.assert_close(x.mul(1), (1 * x).unwrap()) + self.assert_close(x.mul(1), (x * 1).unwrap()) + self.assert_close(x.sub(x), (x - x).unwrap()) + self.assert_close(x.sub(1), (x - 1).unwrap()) + self.assert_close(x.div(x), (x / x).unwrap()) + self.assert_close(x.true_divide(x), (x / x).unwrap()) + self.assert_close(x.floor_divide(x), (x // x).unwrap()) + self.assert_close(x1.ge(x2), (x1 >= x2).unwrap()) + self.assert_close(x1.gt(x2), (x1 > x2).unwrap()) + self.assert_close(x1.lt(x2), (x1 < x2).unwrap()) + self.assert_close(x1.le(x2), (x1 <= x2).unwrap()) + self.assert_close(x1.eq(x2), (x1 == x2).unwrap()) + self.assert_close(x1.ne(x2), (x1 != x2).unwrap()) + self.assert_close((-x).unwrap(), -data) + + def test_bool_int_conversion(self, device): + """Test boolean and integer conversion.""" + # Test __bool__ + data_true = torch.tensor([1.0], device=device) + tensor_true = TensorWrapper(data_true) + assert bool(tensor_true) is True + + data_false = torch.tensor([0.0], device=device) + tensor_false = TensorWrapper(data_false) + assert bool(tensor_false) is False + + # Test __int__ + data_int = torch.tensor([42], device=device, dtype=torch.int32) + tensor_int = TensorWrapper(data_int) + assert int(tensor_int) == 42 + assert isinstance(int(tensor_int), int) + + def test_right_side_operations(self, device, dtype): + """Test right-side operations (radd, rsub, rmul).""" + data = torch.tensor([2.0, 3.0], device=device, dtype=dtype) + x = TensorWrapper(data) + scalar = 5.0 + + # Test __radd__: scalar + tensor + result_radd = scalar + x + expected_radd = scalar + data + self.assert_close(result_radd.unwrap(), expected_radd) + + # Test __rsub__: scalar - tensor + result_rsub = scalar - x + expected_rsub = scalar - data + self.assert_close(result_rsub.unwrap(), expected_rsub) + + # Test __rmul__: scalar * tensor + result_rmul = scalar * x + expected_rmul = scalar * data + self.assert_close(result_rmul.unwrap(), expected_rmul) + + def test_used_attrs_tracking(self, device, dtype): + """Test that attribute usage is tracked.""" + data = torch.rand(2, 3, device=device, dtype=dtype) + tensor = TensorWrapper(data) + + # Initially empty + assert len(tensor.used_attrs) == 0 + + # Access some attributes + _ = tensor.shape + _ = tensor.device + _ = tensor.dtype + + # Check that they're tracked + assert "shape" in tensor.used_attrs + assert "device" in tensor.used_attrs + assert "dtype" in tensor.used_attrs + + def test_used_calls_tracking(self, device, dtype): + """Test that function calls are tracked.""" + data = torch.rand(2, 3, device=device, dtype=dtype) + tensor = TensorWrapper(data) + + # Initially empty + assert len(tensor.used_calls) == 0 + + # Call some operations + _ = tensor + tensor + _ = tensor * 2 + _ = torch.sum(tensor) + + # Check that they're tracked + assert len(tensor.used_calls) > 0 + assert torch.add in tensor.used_calls or torch.mul in tensor.used_calls + + def test_setattr_tracking(self, device, dtype): + """Test that setattr doesn't track internal attributes.""" + data = torch.rand(2, 3, device=device, dtype=dtype) + tensor = TensorWrapper(data) + + # Setting a new attribute on the underlying tensor should be tracked + tensor.some_new_attr = 42 + assert "some_new_attr" in tensor.used_attrs + + def test_callable(self, device, dtype): + data = torch.ones(2, device=device, dtype=dtype) + x = TensorWrapper(data) + y = (x * x).sum(-1, True) + # sum() method returns unwrapped tensor, so compare directly + expected = torch.ones_like(y) * 2 + self.assert_close(y, expected) + + def test_len(self, device, dtype): + """Test __len__ method.""" + data = torch.rand(5, device=device, dtype=dtype) + tensor = TensorWrapper(data) + assert len(tensor) == 5 + assert len(tensor) == len(data) + + def test_nested_wrapping(self, device, dtype): + """Test that wrapping already-wrapped tensors works correctly.""" + data = torch.rand(2, 3, device=device, dtype=dtype) + wrapped_once = TensorWrapper(data) + wrapped_twice = _wrap(wrapped_once, TensorWrapper) + + # Should still unwrap to original data + self.assert_close(_unwrap(wrapped_twice), data) + + def test_string_bytes_not_wrapped(self): + """Test that strings and bytes are not treated as sequences in __torch_function__.""" + # This tests the fix for not treating strings/bytes as sequences + data = torch.rand(2, 3) + tensor = TensorWrapper(data) + + # Should not raise errors when strings/bytes are in args + result = torch.cat([tensor, tensor]) + assert isinstance(result, TensorWrapper) + + @pytest.mark.skip(reason="not implemented yet") + def test_cardinality(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_jit(self, device, dtype): + pass + + def test_exception(self): + """Test exception handling for invalid inputs.""" + with pytest.raises(TypeCheckError): + TensorWrapper([1, 2, 3]) # type: ignore + + @pytest.mark.skip(reason="not implemented yet") + def test_module(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_gradcheck(self, device): + pass diff --git a/tests/enhance/test_adjust.py b/tests/enhance/test_adjust.py new file mode 100644 index 0000000..bdc9775 --- /dev/null +++ b/tests/enhance/test_adjust.py @@ -0,0 +1,1202 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +from torch import Tensor + +import kornia +from kornia.constants import pi + +from testing.base import BaseTester + + +class TestInvert(BaseTester): + def test_smoke(self, device, dtype): + img = torch.rand(1, 3, 4, 4, device=device, dtype=dtype) + assert kornia.enhance.invert(img) is not None + + @pytest.mark.parametrize("shape", [(3, 4, 4), (2, 4, 3, 3), (1, 3, 2, 1, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.rand(shape, device=device, dtype=dtype) + out = kornia.enhance.invert(img, torch.tensor(1.0)) + assert out.shape == shape + + def test_max_val_1(self, device, dtype): + img = torch.ones(1, 3, 4, 4, device=device, dtype=dtype) + out = kornia.enhance.invert(img, torch.tensor(1.0)) + self.assert_close(out, torch.zeros_like(out)) + + def test_max_val_255(self, device, dtype): + img = 255.0 * torch.ones(1, 3, 4, 4, device=device, dtype=dtype) + out = kornia.enhance.invert(img, torch.tensor(255.0)) + self.assert_close(out, torch.zeros_like(out)) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + B, C, H, W = 1, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=torch.float64, requires_grad=True) + max_val = torch.tensor(1.0, device=device, dtype=torch.float64, requires_grad=True) + self.gradcheck(kornia.enhance.invert, (img, max_val)) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.enhance.invert + op_jit = torch.jit.script(op) + self.assert_close(op(img), op_jit(img)) + + def test_module(self, device, dtype): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.enhance.invert + op_mod = kornia.enhance.Invert() + self.assert_close(op(img), op_mod(img)) + + +class TestAdjustSaturation(BaseTester): + @pytest.mark.parametrize("shape", [(3, 4, 4), (2, 3, 3, 3), (4, 3, 3, 1, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.rand(shape, device=device, dtype=dtype) + out = kornia.enhance.adjust_saturation(img, 1.0) + assert out.shape == shape + + @pytest.mark.parametrize("shape", [(3, 4, 4), (2, 3, 3, 3), (4, 3, 3, 1, 1)]) + def test_cardinality_with_gray_subtraction(self, device, dtype, shape): + img = torch.rand(shape, device=device, dtype=dtype) + out = kornia.enhance.adjust_saturation_with_gray_subtraction(img, 1.0) + assert out.shape == shape + + def test_saturation_one(self, device, dtype): + data = torch.tensor( + [[[0.5, 0.5], [0.5, 0.5]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + device=device, + dtype=dtype, + ) # 3x2x2 + + expected = data.clone() + + f = kornia.enhance.AdjustSaturation(1.0) + self.assert_close(f(data), expected) + + def test_saturation_with_gray_subtraction_one(self, device, dtype): + data = torch.tensor( + [[[0.5, 0.5], [0.5, 0.5]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + device=device, + dtype=dtype, + ) # 3x2x2 + + expected = data.clone() + + f = kornia.enhance.AdjustSaturationWithGraySubtraction(1.0) + self.assert_close(f(data), expected) + + def test_saturation_one_batch(self, device, dtype): + data = torch.tensor( + [ + [[[0.5, 0.5], [0.5, 0.5]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + [[[0.5, 0.5], [0.5, 0.5]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + ], + device=device, + dtype=dtype, + ) # 2x3x2x2 + + expected = data + f = kornia.enhance.AdjustSaturation(torch.ones(2)) + self.assert_close(f(data), expected) + + def test_saturation_with_gray_subtraction_one_batch(self, device, dtype): + data = torch.tensor( + [ + [[[0.5, 0.5], [0.5, 0.5]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + [[[0.5, 0.5], [0.5, 0.5]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + ], + device=device, + dtype=dtype, + ) # 2x3x2x2 + + expected = data + f = kornia.enhance.AdjustSaturationWithGraySubtraction(torch.ones(2)) + self.assert_close(f(data), expected) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 2, 3, 4, 5 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.enhance.adjust_saturation, (img, 2.0)) + + def test_gradcheck_with_gray_subtraction(self, device): + batch_size, channels, height, width = 2, 3, 4, 5 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.enhance.adjust_saturation_with_gray_subtraction, (img, 2.0)) + + +class TestAdjustHue(BaseTester): + @pytest.mark.parametrize("shape", [(3, 4, 4), (2, 3, 3, 3), (4, 3, 3, 1, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.rand(shape, device=device, dtype=dtype) + out = kornia.enhance.adjust_hue(img, 3.141516) + assert out.shape == shape + + def test_hue_one(self, device, dtype): + data = torch.tensor( + [[[0.5, 0.5], [0.5, 0.5]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + device=device, + dtype=dtype, + ) # 3x2x2 + + expected = data.clone() + + f = kornia.enhance.AdjustHue(0.0) + self.assert_close(f(data), expected) + + def test_hue_one_batch(self, device, dtype): + data = torch.tensor( + [ + [[[0.5, 0.5], [0.5, 0.5]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + [[[0.5, 0.5], [0.5, 0.5]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + ], + device=device, + dtype=dtype, + ) # 2x3x2x2 + + expected = data + f = kornia.enhance.AdjustHue(torch.tensor([0, 0])) + self.assert_close(f(data), expected) + + def test_hue_flip_batch(self, device, dtype): + data = torch.tensor( + [ + [[[0.5, 0.5], [0.5, 0.5]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + [[[0.5, 0.5], [0.5, 0.5]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + ], + device=device, + dtype=dtype, + ) # 2x3x2x2 + + pi_t = torch.tensor([-pi, pi], device=device, dtype=dtype) + f = kornia.enhance.AdjustHue(pi_t) + + result = f(data) + self.assert_close(result, result.flip(0)) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 2, 3, 4, 5 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.enhance.adjust_hue, (img, 2.0)) + + +class TestAdjustGamma(BaseTester): + @pytest.mark.parametrize("shape", [(3, 4, 4), (2, 3, 3, 3), (4, 3, 3, 1, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.rand(shape, device=device, dtype=dtype) + out = kornia.enhance.adjust_gamma(img, 1.0) + assert out.shape == shape + + def test_gamma_zero(self, device, dtype): + data = torch.tensor( + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + device=device, + dtype=dtype, + ) # 3x2x2 + + expected = torch.ones_like(data) + + f = kornia.enhance.AdjustGamma(0.0) + self.assert_close(f(data), expected) + + def test_gamma_one(self, device, dtype): + data = torch.tensor( + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + device=device, + dtype=dtype, + ) # 3x2x2 + + expected = data.clone() + + f = kornia.enhance.AdjustGamma(1.0) + self.assert_close(f(data), expected) + + def test_gamma_one_gain_two(self, device, dtype): + data = torch.tensor( + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + device=device, + dtype=dtype, + ) # 3x2x2 + + expected = torch.tensor( + [[[1.0, 1.0], [1.0, 1.0]], [[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]]], device=device, dtype=dtype + ) # 3x2x2 + + f = kornia.enhance.AdjustGamma(1.0, 2.0) + self.assert_close(f(data), expected) + + def test_gamma_two(self, device, dtype): + data = torch.tensor( + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + device=device, + dtype=dtype, + ) # 3x2x2 + + expected = torch.tensor( + [[[1.0, 1.0], [1.0, 1.0]], [[0.25, 0.25], [0.25, 0.25]], [[0.0625, 0.0625], [0.0625, 0.0625]]], + device=device, + dtype=dtype, + ) # 3x2x2 + + f = kornia.enhance.AdjustGamma(2.0) + self.assert_close(f(data), expected) + + def test_gamma_two_batch(self, device, dtype): + data = torch.tensor( + [ + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + ], + device=device, + dtype=dtype, + ) # 2x3x2x2 + + expected = torch.tensor( + [ + [[[1.0, 1.0], [1.0, 1.0]], [[0.25, 0.25], [0.25, 0.25]], [[0.0625, 0.0625], [0.0625, 0.0625]]], + [[[1.0, 1.0], [1.0, 1.0]], [[0.25, 0.25], [0.25, 0.25]], [[0.0625, 0.0625], [0.0625, 0.0625]]], + ], + device=device, + dtype=dtype, + ) # 2x3x2x2 + + p1 = torch.tensor([2.0, 2.0], device=device, dtype=dtype) + p2 = torch.ones(2, device=device, dtype=dtype) + + f = kornia.enhance.AdjustGamma(p1, gain=p2) + self.assert_close(f(data), expected) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 2, 3, 4, 5 + img = torch.ones(batch_size, channels, height, width, device=device, dtype=torch.float) + self.gradcheck(kornia.enhance.adjust_gamma, (img, 1.0, 2.0)) + + +class TestAdjustContrast(BaseTester): + @pytest.mark.parametrize("shape", [(3, 4, 4), (2, 3, 3, 3), (4, 3, 3, 1, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.rand(shape, device=device, dtype=dtype) + out = kornia.enhance.adjust_contrast(img, 0.5) + assert out.shape == shape + + @pytest.mark.parametrize("shape", [(3, 4, 4), (2, 3, 3, 3), (4, 3, 3, 1, 1)]) + def test_cardinality_with_mean_subtraction(self, device, dtype, shape): + img = torch.rand(shape, device=device, dtype=dtype) + out = kornia.enhance.adjust_contrast_with_mean_subtraction(img, 0.5) + assert out.shape == shape + + def test_factor_zero(self, device, dtype): + # prepare input data + data = torch.tensor( + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + device=device, + dtype=dtype, + ) # 3x2x2 + + expected = torch.zeros_like(data) + + f = kornia.enhance.AdjustContrast(0.0) + self.assert_close(f(data), expected) + + def test_factor_zero_with_mean_subtraction(self, device, dtype): + # prepare input data + data = torch.tensor( + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + device=device, + dtype=dtype, + ) # 3x2x2 + + expected = torch.tensor( + [ + [[0.6210, 0.6210], [0.6210, 0.6210]], + [[0.6210, 0.6210], [0.6210, 0.6210]], + [[0.6210, 0.6210], [0.6210, 0.6210]], + ], + device=device, + dtype=dtype, + ) # 3x2x2 + + f = kornia.enhance.AdjustContrastWithMeanSubtraction(0.0) + self.assert_close(f(data), expected) + + def test_factor_one_acumulative(self, device, dtype): + # prepare input data + data = torch.tensor( + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + device=device, + dtype=dtype, + ) # 3x2x2 + + expected = data.clone() + + f = kornia.enhance.AdjustContrast(1.0) + self.assert_close(f(data), expected) + + def test_factor_one_with_mean_subtraction(self, device, dtype): + # prepare input data + data = torch.tensor( + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + device=device, + dtype=dtype, + ) # 3x2x2 + + expected = data.clone() + + f = kornia.enhance.AdjustContrastWithMeanSubtraction(1.0) + self.assert_close(f(data), expected) + + def test_factor_two(self, device, dtype): + # prepare input data + data = torch.tensor( + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + device=device, + dtype=dtype, + ) # 3x2x2 + + expected = torch.tensor( + [[[1.0, 1.0], [1.0, 1.0]], [[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]]], device=device, dtype=dtype + ) # 3x2x2 + + f = kornia.enhance.AdjustContrast(2.0) + self.assert_close(f(data), expected) + + def test_factor_two_with_mean_subtraction(self, device, dtype): + # prepare input data + data = torch.tensor( + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + device=device, + dtype=dtype, + ) # 3x2x2 + + expected = torch.tensor( + [ + [[1.0000, 1.0000], [1.0000, 1.0000]], + [[0.3790, 0.3790], [0.3790, 0.3790]], + [[0.0000, 0.0000], [0.0000, 0.0000]], + ], + device=device, + dtype=dtype, + ) # 3x2x2 + + f = kornia.enhance.AdjustContrastWithMeanSubtraction(2.0) + self.assert_close(f(data), expected) + + def test_factor_tensor(self, device, dtype): + # prepare input data + data = torch.tensor( + [ + [[1.0, 1.0], [1.0, 1.0]], + [[0.5, 0.5], [0.5, 0.5]], + [[0.25, 0.25], [0.25, 0.25]], + [[0.5, 0.5], [0.5, 0.5]], + ], + device=device, + dtype=dtype, + ) # 4x2x2 + + expected = torch.tensor( + [ + [[0.0, 0.0], [0.0, 0.0]], + [[0.5, 0.5], [0.5, 0.5]], + [[0.375, 0.375], [0.375, 0.375]], + [[1.0, 1.0], [1.0, 1.0]], + ], + device=device, + dtype=dtype, + ) # 4x2x2 + + factor = torch.tensor([0, 1, 1.5, 2], device=device, dtype=dtype) + + f = kornia.enhance.AdjustContrast(factor) + self.assert_close(f(data), expected) + + def test_factor_tensor_color(self, device, dtype): + # prepare input data + data = torch.tensor( + [ + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + [[[0.0, 0.0], [0.0, 0.0]], [[0.3, 0.3], [0.3, 0.3]], [[0.6, 0.6], [0.6, 0.6]]], + ], + device=device, + dtype=dtype, + ) # 2x3x2x2 + + expected = torch.tensor( + [ + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + [[[0.0, 0.0], [0.0, 0.0]], [[0.6, 0.6], [0.6, 0.6]], [[1.0, 1.0], [1.0, 1.0]]], + ], + device=device, + dtype=dtype, + ) # 2x3x2x2 + + factor = torch.tensor([1, 2], device=device, dtype=dtype) + + f = kornia.enhance.AdjustContrast(factor) + self.assert_close(f(data), expected) + + def test_factor_tensor_color_with_mean_subtraction(self, device, dtype): + # prepare input data + data = torch.tensor( + [ + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + [[[0.0, 0.0], [0.0, 0.0]], [[0.3, 0.3], [0.3, 0.3]], [[0.6, 0.6], [0.6, 0.6]]], + ], + device=device, + dtype=dtype, + ) # 2x3x2x2 + + expected = torch.tensor( + [ + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + [[[0.0, 0.0], [0.0, 0.0]], [[0.3555, 0.3555], [0.3555, 0.3555]], [[0.9555, 0.9555], [0.9555, 0.9555]]], + ], + device=device, + dtype=dtype, + ) # 2x3x2x2 + + factor = torch.tensor([1, 2], device=device, dtype=dtype) + + f = kornia.enhance.AdjustContrastWithMeanSubtraction(factor) + self.assert_close(f(data), expected) + + def test_factor_tensor_shape(self, device, dtype): + # prepare input data + data = torch.tensor( + [ + [ + [[1.0, 1.0, 0.5], [1.0, 1.0, 0.5]], + [[0.5, 0.5, 0.25], [0.5, 0.5, 0.25]], + [[0.25, 0.25, 0.25], [0.6, 0.6, 0.3]], + ], + [ + [[0.0, 0.0, 1.0], [0.0, 0.0, 0.25]], + [[0.3, 0.3, 0.4], [0.3, 0.3, 0.4]], + [[0.6, 0.6, 0.0], [0.3, 0.2, 0.1]], + ], + ], + device=device, + dtype=dtype, + ) # 2x3x2x3 + + expected = torch.tensor( + [ + [ + [[1.0, 1.0, 0.75], [1.0, 1.0, 0.75]], + [[0.75, 0.75, 0.375], [0.75, 0.75, 0.375]], + [[0.375, 0.375, 0.375], [0.9, 0.9, 0.45]], + ], + [ + [[0.0, 0.0, 1.0], [0.0, 0.0, 0.5]], + [[0.6, 0.6, 0.8], [0.6, 0.6, 0.8]], + [[1.0, 1.0, 0.0], [0.6, 0.4, 0.2]], + ], + ], + device=device, + dtype=dtype, + ) # 2x3x2x3 + + factor = torch.tensor([1.5, 2.0], device=device, dtype=dtype) + + f = kornia.enhance.AdjustContrast(factor) + self.assert_close(f(data), expected) + + def test_factor_tensor_shape_with_mean_subtraction(self, device, dtype): + # prepare input data + data = torch.tensor( + [ + [ + [[1.0, 1.0, 0.5], [1.0, 1.0, 0.5]], + [[0.5, 0.5, 0.25], [0.5, 0.5, 0.25]], + [[0.25, 0.25, 0.25], [0.6, 0.6, 0.3]], + ], + [ + [[0.0, 0.0, 1.0], [0.0, 0.0, 0.25]], + [[0.3, 0.3, 0.4], [0.3, 0.3, 0.4]], + [[0.6, 0.6, 0.0], [0.3, 0.2, 0.1]], + ], + ], + device=device, + dtype=dtype, + ) # 2x3x2x3 + + expected = torch.tensor( + [ + [ + [[1.0000, 1.0000, 0.4818], [1.0000, 1.0000, 0.4818]], + [[0.4818, 0.4818, 0.1068], [0.4818, 0.4818, 0.1068]], + [[0.1068, 0.1068, 0.1068], [0.6318, 0.6318, 0.1818]], + ], + [ + [[0.0000, 0.0000, 1.0000], [0.0000, 0.0000, 0.2079]], + [[0.3079, 0.3079, 0.5079], [0.3079, 0.3079, 0.5079]], + [[0.9079, 0.9079, 0.0000], [0.3079, 0.1079, 0.0000]], + ], + ], + device=device, + dtype=dtype, + ) # 2x3x2x3 + + factor = torch.tensor([1.5, 2.0], device=device, dtype=dtype) + + f = kornia.enhance.AdjustContrastWithMeanSubtraction(factor) + + self.assert_close(f(data), expected, low_tolerance=True) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 2, 3, 4, 5 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.enhance.adjust_contrast, (img, 2.0)) + + def test_gradcheck_with_mean_subtraction(self, device): + batch_size, channels, height, width = 2, 3, 4, 5 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.enhance.adjust_contrast_with_mean_subtraction, (img, 2.0)) + + +class TestAdjustBrightness(BaseTester): + @pytest.mark.parametrize("shape", [(3, 4, 4), (2, 3, 3, 3), (4, 3, 3, 1, 1)]) + def test_cardinality(self, device, dtype, shape): + img = torch.rand(shape, device=device, dtype=dtype) + out = kornia.enhance.adjust_brightness(img, 1.0) + assert out.shape == shape + + def test_factor_zero(self, device, dtype): + # prepare input data + data = torch.tensor( + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + device=device, + dtype=dtype, + ) # 3x2x2 + + f = kornia.enhance.AdjustBrightness(0.0) + self.assert_close(f(data), data) + + def test_factor_saturat(self, device, dtype): + # prepare input data + data = 0.5 * torch.ones(1, 4, 3, 2, device=device, dtype=dtype) + ones = torch.ones_like(data) + + f = kornia.enhance.AdjustBrightness(0.6) + self.assert_close(f(data), ones) + + @pytest.mark.parametrize("channels", [1, 4, 5]) + def test_factor_tensor(self, device, dtype, channels): + # prepare input data + data = torch.ones(channels, 2, 3, device=device, dtype=dtype) # Cx2x3 + factor = torch.arange(0, 1, channels, device=device, dtype=dtype) + out = kornia.enhance.adjust_brightness(data, factor) + assert out.shape == (channels, 2, 3) + + def test_factor_tensor_color_accumulative(self, device, dtype): + # prepare input data + data = torch.tensor( + [ + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + [[[0.0, 0.0], [0.0, 0.0]], [[0.3, 0.3], [0.3, 0.3]], [[0.6, 0.6], [0.6, 0.6]]], + ], + device=device, + dtype=dtype, + ) # 2x3x2x2 + + expected = torch.tensor( + [ + [ + [[0.2500, 0.2500], [0.2500, 0.2500]], + [[0.1250, 0.1250], [0.1250, 0.1250]], + [[0.0625, 0.0625], [0.0625, 0.0625]], + ], + [ + [[0.0000, 0.0000], [0.0000, 0.0000]], + [[0.0300, 0.0300], [0.0300, 0.0300]], + [[0.0600, 0.0600], [0.0600, 0.0600]], + ], + ], + device=device, + dtype=dtype, + ) # 2x3x2x2 + + factor = torch.tensor([0.25, 0.1], device=device, dtype=dtype) + + f = kornia.enhance.AdjustBrightnessAccumulative(factor) + self.assert_close(f(data), expected) + + def test_gradcheck_additive(self, device): + batch_size, channels, height, width = 2, 3, 4, 5 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.enhance.adjust_brightness, (img, 1.0)) + + def test_gradcheck_accumulative(self, device): + batch_size, channels, height, width = 2, 3, 4, 5 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.enhance.adjust_brightness_accumulative, (img, 2.0)) + + +class TestAdjustSigmoid(BaseTester): + @pytest.mark.parametrize("shape", [(3, 4, 4), (2, 3, 4, 4)]) + def test_shape_sigmoid(self, shape, device): + inputs = torch.ones(*shape, device=device) + f = kornia.enhance.adjust_sigmoid + assert f(inputs).shape == torch.Size(shape) + + def test_sigmoid(self, device, dtype): + data = torch.tensor( + [ + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + [[[0.0, 0.0], [0.0, 0.0]], [[0.3, 0.3], [0.3, 0.3]], [[0.6, 0.6], [0.6, 0.6]]], + ], + device=device, + dtype=dtype, + ) # 2x3x2x2 + + expected = torch.tensor( + [ + [ + [[0.99330715, 0.99330715], [0.99330715, 0.99330715]], + [[0.5, 0.5], [0.5, 0.5]], + [[0.07585818, 0.07585818], [0.07585818, 0.07585818]], + ], + [ + [[0.00669285, 0.00669285], [0.00669285, 0.00669285]], + [[0.11920292, 0.11920292], [0.11920292, 0.11920292]], + [[0.73105858, 0.73105858], [0.73105858, 0.73105858]], + ], + ], + device=device, + dtype=dtype, + ) # 2x3x2x2 + + f = kornia.enhance.AdjustSigmoid() + self.assert_close(f(data), expected) + + def test_dynamo(self, device, dtype, torch_optimizer): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.enhance.adjust_sigmoid + op_optimized = torch_optimizer(op) + self.assert_close(op(img), op_optimized(img)) + + @pytest.mark.grad() + def test_gradcheck(self, device): + bs, channels, height, width = 1, 2, 3, 3 + inputs = torch.ones(bs, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.enhance.adjust_sigmoid, inputs) + + +class TestAdjustLog(BaseTester): + @pytest.mark.parametrize("shape", [(3, 4, 4), (2, 3, 4, 4)]) + def test_shape_sigmoid(self, shape, device): + inputs = torch.ones(*shape, device=device) + f = kornia.enhance.adjust_log + + assert f(inputs).shape == torch.Size(shape) + + def test_log(self, device, dtype): + data = torch.tensor( + [ + [[[1.0, 1.0], [1.0, 1.0]], [[0.5, 0.5], [0.5, 0.5]], [[0.25, 0.25], [0.25, 0.25]]], + [[[0.0, 0.0], [0.0, 0.0]], [[0.3, 0.3], [0.3, 0.3]], [[0.6, 0.6], [0.6, 0.6]]], + ], + device=device, + dtype=dtype, + ) # 2x3x2x2 + + expected = torch.tensor( + [ + [ + [[1, 1], [1, 1]], + [[0.5849625, 0.5849625], [0.5849625, 0.5849625]], + [[0.32192809, 0.32192809], [0.32192809, 0.32192809]], + ], + [ + [[0, 0], [0, 0]], + [[0.37851162, 0.37851162], [0.37851162, 0.37851162]], + [[0.67807191, 0.67807191], [0.67807191, 0.67807191]], + ], + ], + device=device, + dtype=dtype, + ) # 2x3x2x2 + + f = kornia.enhance.AdjustLog() + self.assert_close(f(data), expected) + + @pytest.mark.slow + def test_dynamo(self, device, dtype, torch_optimizer): + B, C, H, W = 2, 3, 4, 4 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + op = kornia.enhance.adjust_log + op_optimized = torch_optimizer(op) + self.assert_close(op(img), op_optimized(img)) + + @pytest.mark.grad() + def test_gradcheck(self, device): + bs, channels, height, width = 1, 2, 3, 3 + inputs = torch.ones(bs, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.enhance.adjust_log, (inputs, 0.1)) + + +class TestEqualize(BaseTester): + @pytest.mark.parametrize("shape", [(3, 4, 4), (2, 3, 4, 4), (3, 2, 3, 3, 4, 4)]) + def test_shape_equalize(self, shape, device, dtype): + inputs = torch.ones(*shape, device=device, dtype=dtype) + f = kornia.enhance.equalize + + assert f(inputs).shape == torch.Size(shape) + + def test_shape_equalize_batch(self, device, dtype): + bs, channels, height, width = 2, 3, 4, 5 + + inputs = torch.ones(bs, channels, height, width, device=device, dtype=dtype) + f = kornia.enhance.equalize + + assert f(inputs).shape == torch.Size([bs, channels, height, width]) + + def test_equalize(self, device, dtype): + bs, channels, height, width = 1, 3, 20, 20 + + inputs = self.build_input(bs, channels, height, width, device=device, dtype=dtype) + + row_expected = torch.tensor( + [ + 0.0000, + 0.07843, + 0.15686, + 0.2353, + 0.3137, + 0.3922, + 0.4706, + 0.5490, + 0.6275, + 0.7059, + 0.7843, + 0.8627, + 0.9412, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + ], + device=device, + dtype=dtype, + ) + + expected = self.build_input(bs, channels, height, width, device=device, dtype=dtype, row=row_expected) + + f = kornia.enhance.equalize + + self.assert_close(f(inputs), expected, low_tolerance=True) + + def test_equalize_batch(self, device, dtype): + bs, channels, height, width = 2, 3, 20, 20 + + inputs = self.build_input(bs, channels, height, width, device=device, dtype=dtype) + + row_expected = torch.tensor( + [ + 0.0000, + 0.07843, + 0.15686, + 0.2353, + 0.3137, + 0.3922, + 0.4706, + 0.5490, + 0.6275, + 0.7059, + 0.7843, + 0.8627, + 0.9412, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + 1.0000, + ], + device=device, + dtype=dtype, + ) + + expected = self.build_input(bs, channels, height, width, device=device, dtype=dtype, row=row_expected) + + f = kornia.enhance.equalize + + self.assert_close(f(inputs), expected, low_tolerance=True) + + def test_gradcheck(self, device): + bs, channels, height, width = 1, 2, 3, 3 + inputs = torch.ones(bs, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.enhance.equalize, (inputs,), fast_mode=False) + + @pytest.mark.skip(reason="args and kwargs in decorator") + def test_jit(self, device, dtype): + batch_size, channels, height, width = 1, 2, 3, 3 + inp = torch.ones(batch_size, channels, height, width, device=device, dtype=dtype) + + op = kornia.enhance.equalize + op_script = torch.jit.script(op) + + self.assert_close(op(inp), op_script(inp)) + + @staticmethod + def build_input(batch_size, channels, height, width, device, dtype, row=None): + if row is None: + row = torch.arange(width) / float(width) + + channel = torch.stack([row] * height).to(device, dtype) + image = torch.stack([channel] * channels).to(device, dtype) + batch = torch.stack([image] * batch_size).to(device, dtype) + + return batch + + +class TestEqualize3D(BaseTester): + @pytest.mark.parametrize("shape", [(3, 6, 10, 10), (2, 3, 6, 10, 10), (3, 2, 3, 6, 10, 10)]) + def test_shape_equalize3d(self, shape, device, dtype): + inputs3d = torch.ones(*shape, device=device, dtype=dtype) + f = kornia.enhance.equalize3d + + assert f(inputs3d).shape == torch.Size(shape) + + def test_shape_equalize3d_batch(self, device, dtype): + bs, channels, depth, height, width = 2, 3, 6, 10, 10 + + inputs3d = torch.ones(bs, channels, depth, height, width, device=device, dtype=dtype) + f = kornia.enhance.equalize3d + + assert f(inputs3d).shape == torch.Size([bs, channels, depth, height, width]) + + def test_equalize3d(self, device, dtype): + bs, channels, depth, height, width = 1, 3, 6, 10, 10 + + inputs3d = self.build_input(bs, channels, depth, height, width, device, dtype) + + row_expected = torch.tensor( + [0.0000, 0.11764, 0.2353, 0.3529, 0.4706, 0.5882, 0.7059, 0.8235, 0.9412, 1.0000], + device=device, + dtype=dtype, + ) + + expected = self.build_input(bs, channels, depth, height, width, device, dtype, row=row_expected) + + f = kornia.enhance.equalize3d + + self.assert_close(f(inputs3d), expected, low_tolerance=True) + + def test_equalize3d_batch(self, device, dtype): + bs, channels, depth, height, width = 2, 3, 6, 10, 10 + + inputs3d = self.build_input(bs, channels, depth, height, width, device, dtype) + + row_expected = torch.tensor( + [0.0000, 0.11764, 0.2353, 0.3529, 0.4706, 0.5882, 0.7059, 0.8235, 0.9412, 1.0000], + device=device, + dtype=dtype, + ) + + expected = self.build_input(bs, channels, depth, height, width, device, dtype, row=row_expected) + + f = kornia.enhance.equalize3d + + self.assert_close(f(inputs3d), expected, low_tolerance=True) + + def test_gradcheck(self, device): + bs, channels, depth, height, width = 1, 2, 3, 4, 5 + inputs3d = torch.ones(bs, channels, depth, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.enhance.equalize3d, (inputs3d,), fast_mode=False) + + @pytest.mark.skip(reason="args and kwargs in decorator") + def test_jit(self, device, dtype): + batch_size, channels, depth, height, width = 1, 2, 1, 3, 3 + inp = torch.ones(batch_size, channels, depth, height, width, device=device, dtype=dtype) + + op = kornia.enhance.equalize3d + op_script = torch.jit.script(op) + + self.assert_close(op(inp), op_script(inp)) + + @staticmethod + def build_input(batch_size, channels, depth, height, width, device, dtype, row=None): + if row is None: + row = torch.arange(width) / float(width) + + channel = torch.stack([row] * height).to(device, dtype) + image = torch.stack([channel] * channels).to(device, dtype) + image3d = torch.stack([image] * depth).transpose(0, 1).to(device, dtype) + batch = torch.stack([image3d] * batch_size).to(device, dtype) + + return batch + + +class TestSharpness(BaseTester): + f = kornia.enhance.sharpness + + def test_smoke(self, device, dtype): + B, C, H, W = 2, 3, 4, 5 + img = torch.rand(B, C, H, W, device=device, dtype=dtype) + assert isinstance(TestSharpness.f(img, 0.8), Tensor) + + @pytest.mark.parametrize("shape", [(1, 1, 4, 5), (2, 3, 4, 5), (2, 5, 4, 5), (4, 5), (5, 4, 5), (2, 3, 2, 3, 4, 5)]) + @pytest.mark.parametrize("factor", [0.7, 0.8]) + def test_cardinality(self, shape, factor, device, dtype): + inputs = torch.ones(*shape, device=device, dtype=dtype) + assert TestSharpness.f(inputs, factor).shape == torch.Size(shape) + + def test_exception(self, device, dtype): + img = torch.ones(2, 3, 4, 5, device=device, dtype=dtype) + with pytest.raises(AssertionError): + assert TestSharpness.f(img, [0.8, 0.9, 0.6]) + with pytest.raises(AssertionError): + assert TestSharpness.f(img, torch.tensor([0.8, 0.9, 0.6])) + with pytest.raises(AssertionError): + assert TestSharpness.f(img, torch.tensor([0.8])) + + def test_value(self, device, dtype): + torch.manual_seed(0) + + inputs = torch.rand(1, 1, 3, 3).to(device=device, dtype=dtype) + + # Output generated is similar (1e-2 due to the uint8 conversions) to the below output: + # img = PIL.Image.fromarray(arr) + # en = ImageEnhance.Sharpness(img).enhance(0.8) + # np.array(en) / 255. + expected = torch.tensor( + [[[[0.4963, 0.7682, 0.0885], [0.1320, 0.3305, 0.6341], [0.4901, 0.8964, 0.4556]]]], + device=device, + dtype=dtype, + ) + + # If factor == 1, shall return original + # TODO(jian): add test for this case + # assert_close(TestSharpness.f(inputs, 0.), inputs, rtol=1e-4, atol=1e-4) + self.assert_close(TestSharpness.f(inputs, 1.0), inputs, low_tolerance=True) + self.assert_close(TestSharpness.f(inputs, 0.8), expected, low_tolerance=True) + + def test_value_batch(self, device, dtype): + torch.manual_seed(0) + + inputs = torch.rand(2, 1, 3, 3).to(device=device, dtype=dtype) + + # Output generated is similar (1e-2 due to the uint8 conversions) to the below output: + # img = PIL.Image.fromarray(arr) + # en = ImageEnhance.Sharpness(img).enhance(0.8) + # np.array(en) / 255. + expected_08 = torch.tensor( + [ + [[[0.4963, 0.7682, 0.0885], [0.1320, 0.3305, 0.6341], [0.4901, 0.8964, 0.4556]]], + [[[0.6323, 0.3489, 0.4017], [0.0223, 0.2052, 0.2939], [0.5185, 0.6977, 0.8000]]], + ], + device=device, + dtype=dtype, + ) + + expected_08_13 = torch.tensor( + [ + [[[0.4963, 0.7682, 0.0885], [0.1320, 0.3305, 0.6341], [0.4901, 0.8964, 0.4556]]], + [[[0.6323, 0.3489, 0.4017], [0.0223, 0.1143, 0.2939], [0.5185, 0.6977, 0.8000]]], + ], + device=device, + dtype=dtype, + ) + + # If factor == 1, shall return original + # tol_val: float = utils._get_precision(device, dtype) + self.assert_close(TestSharpness.f(inputs, 1), inputs, low_tolerance=True) + self.assert_close(TestSharpness.f(inputs, torch.tensor([1.0, 1.0])), inputs, low_tolerance=True) + self.assert_close(TestSharpness.f(inputs, 0.8), expected_08, low_tolerance=True) + self.assert_close(TestSharpness.f(inputs, torch.tensor([0.8, 1.3])), expected_08_13, low_tolerance=True) + + @pytest.mark.grad() + def test_gradcheck(self, device): + bs, channels, height, width = 2, 3, 4, 5 + inputs = torch.rand(bs, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(TestSharpness.f, (inputs, 0.8)) + + @pytest.mark.skip(reason="union type input") + @pytest.mark.jit() + def test_jit(self, device, dtype): + op = TestSharpness.f + op_script = torch.jit.script(TestSharpness.f) + img = torch.rand(2, 1, 3, 3).to(device=device, dtype=dtype) + expected = op(img, 0.8) + actual = op_script(img, 0.8) + self.assert_close(actual, expected) + + +@pytest.mark.skipif(kornia.core.utils.xla_is_available(), reason="issues with xla device") +class TestSolarize(BaseTester): + f = kornia.enhance.solarize + + def test_smoke(self, device, dtype): + B, C, H, W = 2, 3, 4, 5 + img = torch.rand(B, C, H, W, device=device, dtype=dtype) + assert isinstance(TestSolarize.f(img, 0.8), Tensor) + + @pytest.mark.parametrize( + "shape, thresholds, additions", + [ + ((1, 1, 4, 5), 0.8, 0.4), + ((4, 5), 0.8, 0.4), + ((2, 4, 5), 0.8, None), + ((2, 3, 2, 3, 4, 5), torch.tensor(0.8), None), + ((2, 5, 4, 5), torch.tensor([0.8, 0.7]), None), + ((2, 3, 4, 5), torch.tensor([0.8, 0.7]), torch.tensor([0.0, 0.4])), + ], + ) + def test_cardinality(self, shape, thresholds, additions, device, dtype): + inputs = torch.ones(*shape, device=device, dtype=dtype) + assert TestSolarize.f(inputs, thresholds, additions).shape == torch.Size(shape) + + # TODO(jian): add better assertions + def test_exception(self, device, dtype): + img = torch.ones(2, 3, 4, 5, device=device, dtype=dtype) + + with pytest.raises(TypeError): + assert TestSolarize.f([1.0], 0.0) + + with pytest.raises(TypeError): + assert TestSolarize.f(img, 1) + + with pytest.raises(TypeError): + assert TestSolarize.f(img, 0.8, 1) + + # TODO: add better cases + def test_value(self, device, dtype): + torch.manual_seed(0) + + inputs = torch.rand(1, 1, 3, 3).to(device=device, dtype=dtype) + + # Output generated is similar (1e-2 due to the uint8 conversions) to the below output: + # img = PIL.Image.fromarray((255*inputs[0,0]).byte().numpy()) + # en = ImageOps.Solarize(img, 128) + # np.array(en) / 255. + expected = torch.tensor( + [ + [ + [ + [0.49411765, 0.23529412, 0.08627451], + [0.12941176, 0.30588235, 0.36862745], + [0.48627451, 0.10588235, 0.45490196], + ] + ] + ], + device=device, + dtype=dtype, + ) + + # TODO(jian): precision is very bad compared to PIL + self.assert_close(TestSolarize.f(inputs, 0.5), expected, rtol=1e-2, atol=1e-2) + + @pytest.mark.grad() + def test_gradcheck(self, device): + bs, channels, height, width = 2, 3, 4, 5 + inputs = torch.rand(bs, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(TestSolarize.f, (inputs, 0.8)) + + # TODO: implement me + @pytest.mark.skip(reason="union type input") + @pytest.mark.jit() + def test_jit(self, device, dtype): + op = TestSolarize.f + op_script = torch.jit.script(op) + img = torch.rand(2, 1, 3, 3).to(device=device, dtype=dtype) + expected = op(img, 0.8) + actual = op_script(img, 0.8) + self.assert_close(actual, expected) + + +class TestPosterize(BaseTester): + f = kornia.enhance.posterize + + def test_smoke(self, device, dtype): + B, C, H, W = 2, 3, 4, 5 + img = torch.rand(B, C, H, W, device=device, dtype=dtype) + assert isinstance(TestPosterize.f(img, 8), Tensor) + + @pytest.mark.parametrize( + "shape, bits", + [ + ((1, 4, 5), 8), + ((2, 3, 4, 5), 1), + ((2, 3, 4, 5, 4, 5), 0), + ((1, 4, 5), torch.tensor(8)), + ((3, 4, 5), torch.tensor(8)), + ((2, 5, 4, 5), torch.tensor([0, 8])), + ((3, 3, 4, 5), torch.tensor([0, 1, 8])), + ], + ) + def test_cardinality(self, shape, bits, device, dtype): + inputs = torch.ones(*shape, device=device, dtype=dtype) + assert TestPosterize.f(inputs, bits).shape == torch.Size(shape) + + # TODO(jian): add better assertions + def test_exception(self, device, dtype): + img = torch.ones(2, 3, 4, 5, device=device, dtype=dtype) + + with pytest.raises(TypeError): + assert TestPosterize.f([1.0], 0.0) + + with pytest.raises(TypeError): + assert TestPosterize.f(img, 1.0) + + # TODO(jian): add better cases + @pytest.mark.skipif(kornia.core.utils.xla_is_available(), reason="issues with xla device") + def test_value(self, device, dtype): + torch.manual_seed(0) + + inputs = torch.rand(1, 1, 3, 3).to(device=device, dtype=dtype) + + # Output generated is similar (1e-2 due to the uint8 conversions) to the below output: + # img = PIL.Image.fromarray((255*inputs[0,0]).byte().numpy()) + # en = ImageOps.posterize(img, 1) + # np.array(en) / 255. + expected = torch.tensor( + [[[[0.0, 0.50196078, 0.0], [0.0, 0.0, 0.50196078], [0.0, 0.50196078, 0.0]]]], device=device, dtype=dtype + ) + + self.assert_close(TestPosterize.f(inputs, 1), expected) + self.assert_close(TestPosterize.f(inputs, 0), torch.zeros_like(inputs)) + self.assert_close(TestPosterize.f(inputs, 8), inputs) + + @pytest.mark.skip(reason="IndexError: tuple index out of range") + @pytest.mark.grad() + def test_gradcheck(self, device): + bs, channels, height, width = 2, 3, 4, 5 + inputs = torch.rand(bs, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(TestPosterize.f, (inputs, 0)) + + # TODO: implement me + @pytest.mark.skip(reason="union type input") + @pytest.mark.jit() + def test_jit(self, device, dtype): + op = TestPosterize.f + op_script = torch.jit.script(op) + img = torch.rand(2, 1, 3, 3).to(device=device, dtype=dtype) + expected = op(img, 8) + actual = op_script(img, 8) + self.assert_close(actual, expected) diff --git a/tests/enhance/test_core.py b/tests/enhance/test_core.py new file mode 100644 index 0000000..117da3d --- /dev/null +++ b/tests/enhance/test_core.py @@ -0,0 +1,94 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import random + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +def random_shape(dim, min_elem=1, max_elem=10): + return tuple(random.randint(min_elem, max_elem) for _ in range(dim)) + + +class TestAddWeighted(BaseTester): + fcn = kornia.enhance.add_weighted + + def get_input(self, device, dtype, size, max_elem=10): + shape = random_shape(size, max_elem) + src1 = torch.randn(shape, device=device, dtype=dtype) + src2 = torch.randn(shape, device=device, dtype=dtype) + alpha = random.random() + beta = random.random() + gamma = random.random() + return src1, src2, alpha, beta, gamma + + @pytest.mark.parametrize("size", [2, 3, 4, 5]) + def test_smoke(self, device, dtype, size): + src1, src2, alpha, beta, gamma = self.get_input(device, dtype, size=3) + self.assert_close(TestAddWeighted.fcn(src1, alpha, src2, beta, gamma), src1 * alpha + src2 * beta + gamma) + + @pytest.mark.parametrize("size1, size2", [((2, 5, 5), (4, 5, 5)), ((2, 5, 5), (2, 3, 5, 5))]) + def test_shape_mismatch(self, device, dtype, size1, size2): + src1 = torch.randn(size1, device=device, dtype=dtype) + src2 = torch.randn(size2, device=device, dtype=dtype) + with pytest.raises(Exception): + TestAddWeighted.fcn(src1, 1.0, src2, 1.0, 0.0) + + @pytest.mark.parametrize("size1, size2", [((2, 3, 5, 5), (2, 3, 5, 5)), ((2, 3, 5, 5), (2, 3, 5, 5))]) + @pytest.mark.parametrize("alpha", [torch.randn(2, 3, 5, 5), 1.0]) + @pytest.mark.parametrize("beta", [torch.randn(2, 3, 5, 5), 1.0]) + @pytest.mark.parametrize("gamma", [torch.randn(2, 3, 5, 5), 1.0]) + def test_shape(self, device, dtype, size1, size2, alpha, beta, gamma): + src1 = torch.randn(size1, device=device, dtype=dtype) + src2 = torch.randn(size2, device=device, dtype=dtype) + if isinstance(alpha, torch.Tensor): + alpha = alpha.to(src1) + if isinstance(beta, torch.Tensor): + beta = beta.to(src2) + if isinstance(gamma, torch.Tensor): + gamma = gamma.to(src1) + self.assert_close(TestAddWeighted.fcn(src1, alpha, src2, beta, gamma), src1 * alpha + src2 * beta + gamma) + + def test_dynamo(self, device, dtype, torch_optimizer): + src1, src2, alpha, beta, gamma = self.get_input(device, dtype, size=3) + inputs = (src1, alpha, src2, beta, gamma) + + op = TestAddWeighted.fcn + op_optimized = torch_optimizer(op) + + self.assert_close(op(*inputs), op_optimized(*inputs), atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("size", [2, 3]) + def test_gradcheck(self, size, device): + src1, src2, alpha, beta, gamma = self.get_input( + device, torch.float64, size=3, max_elem=5 + ) # to shave time on gradcheck + self.gradcheck(kornia.enhance.AddWeighted(alpha, beta, gamma), (src1, src2)) + + def test_module(self, device, dtype): + src1, src2, alpha, beta, gamma = self.get_input(device, dtype, size=3) + inputs = (src1, alpha, src2, beta, gamma) + + op = TestAddWeighted.fcn + op_module = kornia.enhance.AddWeighted(alpha, beta, gamma) + + self.assert_close(op(*inputs), op_module(src1, src2)) diff --git a/tests/enhance/test_equalization.py b/tests/enhance/test_equalization.py new file mode 100644 index 0000000..2ea2ae7 --- /dev/null +++ b/tests/enhance/test_equalization.py @@ -0,0 +1,271 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Tuple + +import pytest +import torch + +from kornia import enhance +from kornia.geometry import rotate + +from testing.base import BaseTester + + +class TestEqualization(BaseTester): + def test_smoke(self, device, dtype): + C, H, W = 1, 10, 20 + img = torch.rand(C, H, W, device=device, dtype=dtype) + res = enhance.equalize_clahe(img) + assert isinstance(res, torch.Tensor) + assert res.shape == img.shape + assert res.device == img.device + assert res.dtype == img.dtype + + @pytest.mark.parametrize("B, C", [(None, 1), (None, 3), (1, 1), (1, 3), (4, 1), (4, 3)]) + def test_cardinality(self, B, C, device, dtype): + H, W = 10, 20 + if B is None: + img = torch.rand(C, H, W, device=device, dtype=dtype) + else: + img = torch.rand(B, C, H, W, device=device, dtype=dtype) + res = enhance.equalize_clahe(img) + assert res.shape == img.shape + + @pytest.mark.parametrize("clip, grid", [(0.0, None), (None, (2, 2)), (2.0, (2, 2))]) + def test_optional_params(self, clip, grid, device, dtype): + C, H, W = 1, 10, 20 + img = torch.rand(C, H, W, device=device, dtype=dtype) + if clip is None: + res = enhance.equalize_clahe(img, grid_size=grid) + elif grid is None: + res = enhance.equalize_clahe(img, clip_limit=clip) + else: + res = enhance.equalize_clahe(img, clip, grid) + assert isinstance(res, torch.Tensor) + assert res.shape == img.shape + + @pytest.mark.parametrize( + "B, clip, grid, exception_type, expected_error_msg", + [ + (0, 1.0, (2, 2), ValueError, "Invalid input tensor, it is empty."), # from perform_keep_shape_image + (1, 1, (2, 2), TypeError, "Input clip_limit type is not float. Got"), + (1, 2.0, 2, TypeError, "Input grid_size type is not Tuple. Got"), + (1, 2.0, (2, 2, 2), TypeError, "Input grid_size is not a Tuple with 2 elements. Got 3"), + (1, 2.0, (2, 2.0), TypeError, "Input grid_size type is not valid, must be a Tuple[int, int]"), + (1, 2.0, (2, 0), ValueError, "Input grid_size elements must be positive. Got"), + ], + ) + def test_exception(self, B, clip, grid, exception_type, expected_error_msg): + C, H, W = 1, 10, 20 + img = torch.rand(B, C, H, W) + with pytest.raises(exception_type) as errinfo: + enhance.equalize_clahe(img, clip, grid) + assert expected_error_msg in str(errinfo) + + @pytest.mark.parametrize("dims", [(1, 1, 1, 1, 1), (1, 1)]) + def test_exception_tensor_dims(self, dims): + img = torch.rand(dims) + with pytest.raises(ValueError): + enhance.equalize_clahe(img) + + def test_exception_tensor_type(self): + with pytest.raises(TypeError): + enhance.equalize_clahe([1, 2, 3]) + + def test_gradcheck(self, device): + torch.random.manual_seed(4) + bs, channels, height, width = 1, 1, 11, 11 + inputs = torch.rand(bs, channels, height, width, device=device, dtype=torch.float64) + + def grad_rot(data, a, b, c): + rot = rotate(data, torch.tensor(30.0, dtype=data.dtype, device=device)) + return enhance.equalize_clahe(rot, a, b, c) + + self.gradcheck(grad_rot, (inputs, 40.0, (2, 2), True), nondet_tol=1e-4) + + @pytest.mark.skip(reason="args and kwargs in decorator") + def test_jit(self, device, dtype): + batch_size, channels, height, width = 1, 2, 10, 20 + inp = torch.rand(batch_size, channels, height, width, device=device, dtype=dtype) + op = enhance.equalize_clahe + op_script = torch.jit.script(op) + self.assert_close(op(inp), op_script(inp)) + + def test_module(self): + # equalize_clahe is only a function + pass + + @pytest.fixture() + def img(self, device, dtype): + height, width = 20, 20 + # TODO: test with a more realistic pattern + img = torch.arange(width, device=device).div(float(width - 1))[None].expand(height, width)[None][None] + return img + + def test_he(self, img): + # should be similar to enhance.equalize but slower. Similar because the lut is computed in a different way. + clip_limit: float = 0.0 + grid_size: Tuple = (1, 1) + res = enhance.equalize_clahe(img, clip_limit=clip_limit, grid_size=grid_size) + # NOTE: for next versions we need to improve the computation of the LUT + # and test with a better image + self.assert_close( + res[..., 0, :], + torch.tensor( + [ + [ + [ + 0.0471, + 0.0980, + 0.1490, + 0.2000, + 0.2471, + 0.2980, + 0.3490, + 0.3490, + 0.4471, + 0.4471, + 0.5490, + 0.5490, + 0.6471, + 0.6471, + 0.6980, + 0.7490, + 0.8000, + 0.8471, + 0.8980, + 1.0000, + ] + ] + ], + dtype=res.dtype, + device=res.device, + ), + low_tolerance=True, + ) + + def test_ahe(self, img): + clip_limit: float = 0.0 + grid_size: Tuple = (8, 8) + res = enhance.equalize_clahe(img, clip_limit=clip_limit, grid_size=grid_size) + # NOTE: for next versions we need to improve the computation of the LUT + # and test with a better image + self.assert_close( + res[..., 0, :], + torch.tensor( + [ + [ + [ + 0.2471, + 0.4980, + 0.7490, + 0.6667, + 0.4980, + 0.4980, + 0.7490, + 0.4993, + 0.4980, + 0.2471, + 0.7490, + 0.4993, + 0.4980, + 0.2471, + 0.4980, + 0.4993, + 0.3333, + 0.2471, + 0.4980, + 1.0000, + ] + ] + ], + dtype=res.dtype, + device=res.device, + ), + low_tolerance=True, + ) + + def test_clahe(self, img): + clip_limit: float = 2.0 + grid_size: Tuple = (8, 8) + res = enhance.equalize_clahe(img, clip_limit=clip_limit, grid_size=grid_size) + res_diff = enhance.equalize_clahe(img, clip_limit=clip_limit, grid_size=grid_size, slow_and_differentiable=True) + # NOTE: for next versions we need to improve the computation of the LUT + # and test with a better image + expected = torch.tensor( + [ + [ + [ + 0.1216, + 0.8745, + 0.9373, + 0.9163, + 0.8745, + 0.8745, + 0.9373, + 0.8745, + 0.8745, + 0.8118, + 0.9373, + 0.8745, + 0.8745, + 0.8118, + 0.8745, + 0.8745, + 0.8327, + 0.8118, + 0.8745, + 1.0000, + ] + ] + ], + dtype=res.dtype, + device=res.device, + ) + exp_diff = torch.tensor( + [ + [ + [ + 0.1250, + 0.8752, + 0.9042, + 0.9167, + 0.8401, + 0.8852, + 0.9302, + 0.9120, + 0.8750, + 0.8370, + 0.9620, + 0.9077, + 0.8750, + 0.8754, + 0.9204, + 0.9167, + 0.8370, + 0.8806, + 0.9096, + 1.0000, + ] + ] + ], + dtype=res.dtype, + device=res.device, + ) + self.assert_close(res[..., 0, :], expected, low_tolerance=True) + self.assert_close(res_diff[..., 0, :], exp_diff, low_tolerance=True) diff --git a/tests/enhance/test_histogram.py b/tests/enhance/test_histogram.py new file mode 100644 index 0000000..e321bcf --- /dev/null +++ b/tests/enhance/test_histogram.py @@ -0,0 +1,194 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +from packaging import version + +import kornia + +from testing.base import BaseTester + + +class TestImageHistogram2d(BaseTester): + fcn = kornia.enhance.image_histogram2d + + @pytest.mark.parametrize("kernel", ["triangular", "gaussian", "uniform", "epanechnikov"]) + def test_shape(self, device, dtype, kernel): + sample = torch.ones(32, 32, device=device, dtype=dtype) + hist, pdf = TestImageHistogram2d.fcn(sample, 0.0, 1.0, 256, kernel=kernel) + assert hist.shape == (256,) + assert pdf.shape == (256,) + + @pytest.mark.parametrize("kernel", ["triangular", "gaussian", "uniform", "epanechnikov"]) + def test_shape_channels(self, device, dtype, kernel): + sample = torch.ones(3, 32, 32, device=device, dtype=dtype) + hist, pdf = TestImageHistogram2d.fcn(sample, 0.0, 1.0, 256, kernel=kernel) + assert hist.shape == (3, 256) + assert pdf.shape == (3, 256) + + @pytest.mark.parametrize("kernel", ["triangular", "gaussian", "uniform", "epanechnikov"]) + def test_shape_batch(self, device, dtype, kernel): + sample = torch.ones(8, 3, 32, 32, device=device, dtype=dtype) + hist, pdf = TestImageHistogram2d.fcn(sample, 0.0, 1.0, 256, kernel=kernel) + assert hist.shape == (8, 3, 256) + assert pdf.shape == (8, 3, 256) + + @pytest.mark.parametrize("kernel", ["triangular", "gaussian", "uniform", "epanechnikov"]) + def test_gradcheck(self, device, kernel): + sample = torch.ones(32, 32, device=device, dtype=torch.float64) + centers = torch.linspace(0, 255, 8, device=device, dtype=torch.float64) + self.gradcheck(TestImageHistogram2d.fcn, (sample, 0.0, 255.0, 256, None, centers, True, kernel)) + + @pytest.mark.skipif( + version.parse(torch.__version__) < version.parse("1.9"), reason="Tuple cannot be jitted with PyTorch < v1.9" + ) + @pytest.mark.parametrize("kernel", ["triangular", "gaussian", "uniform", "epanechnikov"]) + def test_jit(self, device, dtype, kernel): + sample = torch.linspace(0, 255, 10, device=device, dtype=dtype) + sample_x, _ = torch.meshgrid(sample, sample, indexing="ij") + samples = (sample_x, 0.0, 255.0, 10, None, None, False, kernel) + + op = TestImageHistogram2d.fcn + op_script = torch.jit.script(op) + + out, out_script = op(*samples), op_script(*samples) + self.assert_close(out[0], out_script[0]) + self.assert_close(out[1], out_script[1]) + + @pytest.mark.parametrize("kernel", ["triangular", "gaussian", "uniform", "epanechnikov"]) + @pytest.mark.parametrize("size", [(1, 1), (3, 1, 1), (8, 3, 1, 1)]) + def test_uniform_hist(self, device, dtype, kernel, size): + sample = torch.linspace(0, 255, 10, device=device, dtype=dtype) + sample_x, _ = torch.meshgrid(sample, sample, indexing="ij") + sample_x = sample_x.repeat(*size) + if kernel == "gaussian": + bandwidth = 2 * 0.4**2 + else: + bandwidth = None + hist, _ = TestImageHistogram2d.fcn(sample_x, 0.0, 255.0, 10, bandwidth=bandwidth, centers=sample, kernel=kernel) + ans = 10 * torch.ones_like(hist) + self.assert_close(ans, hist) + + @pytest.mark.parametrize("kernel", ["triangular", "gaussian", "uniform", "epanechnikov"]) + @pytest.mark.parametrize("size", [(1, 1), (3, 1, 1), (8, 3, 1, 1)]) + def test_uniform_dist(self, device, dtype, kernel, size): + sample = torch.linspace(0, 255, 10, device=device, dtype=dtype) + sample_x, _ = torch.meshgrid(sample, sample, indexing="ij") + sample_x = sample_x.repeat(*size) + if kernel == "gaussian": + bandwidth = 2 * 0.4**2 + else: + bandwidth = None + hist, pdf = TestImageHistogram2d.fcn( + sample_x, 0.0, 255.0, 10, bandwidth=bandwidth, centers=sample, kernel=kernel, return_pdf=True + ) + ans = 0.1 * torch.ones_like(hist) + self.assert_close(ans, pdf) + + +class TestHistogram2d(BaseTester): + fcn = kornia.enhance.histogram2d + + def test_shape(self, device, dtype): + inp1 = torch.ones(1, 32, device=device, dtype=dtype) + inp2 = torch.ones(1, 32, device=device, dtype=dtype) + bins = torch.linspace(0, 255, 128, device=device, dtype=dtype) + bandwidth = torch.tensor(0.9, device=device, dtype=dtype) + pdf = TestHistogram2d.fcn(inp1, inp2, bins, bandwidth) + assert pdf.shape == (1, 128, 128) + + def test_shape_batch(self, device, dtype): + inp1 = torch.ones(8, 32, device=device, dtype=dtype) + inp2 = torch.ones(8, 32, device=device, dtype=dtype) + bins = torch.linspace(0, 255, 128, device=device, dtype=dtype) + bandwidth = torch.tensor(0.9, device=device, dtype=dtype) + pdf = TestHistogram2d.fcn(inp1, inp2, bins, bandwidth) + assert pdf.shape == (8, 128, 128) + + def test_gradcheck(self, device): + inp1 = torch.ones(1, 8, device=device, dtype=torch.float64) + inp2 = torch.ones(1, 8, device=device, dtype=torch.float64) + bins = torch.linspace(0, 255, 8, device=device, dtype=torch.float64) + bandwidth = torch.tensor(0.9, device=device, dtype=torch.float64) + self.gradcheck(TestHistogram2d.fcn, (inp1, inp2, bins, bandwidth)) + + def test_jit(self, device, dtype): + sample1 = torch.linspace(0, 255, 10, device=device, dtype=dtype).unsqueeze(0) + sample2 = torch.linspace(0, 255, 10, device=device, dtype=dtype).unsqueeze(0) + bins = torch.linspace(0, 255, 10, device=device, dtype=dtype) + bandwidth = torch.tensor(2 * 0.4**2, device=device, dtype=dtype) + samples = (sample1, sample2, bins, bandwidth) + + op = TestHistogram2d.fcn + op_script = torch.jit.script(op) + + self.assert_close(op(*samples), op_script(*samples)) + + def test_uniform_dist(self, device, dtype): + sample1 = torch.linspace(0, 255, 10, device=device, dtype=dtype).unsqueeze(0) + sample2 = torch.linspace(0, 255, 10, device=device, dtype=dtype).unsqueeze(0) + bins = torch.linspace(0, 255, 10, device=device, dtype=dtype) + bandwidth = torch.tensor(2 * 0.4**2, device=device, dtype=dtype) + + pdf = TestHistogram2d.fcn(sample1, sample2, bins, bandwidth) + ans = 0.1 * kornia.core.ops.eye_like(10, pdf) + self.assert_close(ans, pdf) + + +class TestHistogram(BaseTester): + fcn = kornia.enhance.histogram + + def test_shape(self, device, dtype): + inp = torch.ones(1, 32, device=device, dtype=dtype) + bins = torch.linspace(0, 255, 128, device=device, dtype=dtype) + bandwidth = torch.tensor(0.9, device=device, dtype=dtype) + pdf = TestHistogram.fcn(inp, bins, bandwidth) + assert pdf.shape == (1, 128) + + def test_shape_batch(self, device, dtype): + inp = torch.ones(8, 32, device=device, dtype=dtype) + bins = torch.linspace(0, 255, 128, device=device, dtype=dtype) + bandwidth = torch.tensor(0.9, device=device, dtype=dtype) + pdf = TestHistogram.fcn(inp, bins, bandwidth) + assert pdf.shape == (8, 128) + + def test_gradcheck(self, device): + inp = torch.ones(1, 8, device=device, dtype=torch.float64) + bins = torch.linspace(0, 255, 8, device=device, dtype=torch.float64) + bandwidth = torch.tensor(0.9, device=device, dtype=torch.float64) + self.gradcheck(TestHistogram.fcn, (inp, bins, bandwidth)) + + def test_jit(self, device, dtype): + input1 = torch.linspace(0, 255, 10, device=device, dtype=dtype).unsqueeze(0) + bins = torch.linspace(0, 255, 10, device=device, dtype=dtype) + bandwidth = torch.tensor(2 * 0.4**2, device=device, dtype=dtype) + inputs = (input1, bins, bandwidth) + + op = TestHistogram.fcn + op_script = torch.jit.script(op) + + self.assert_close(op(*inputs), op_script(*inputs)) + + def test_uniform_dist(self, device, dtype): + input1 = torch.linspace(0, 255, 10, device=device, dtype=dtype).unsqueeze(0) + input2 = torch.linspace(0, 255, 10, device=device, dtype=dtype) + bandwidth = torch.tensor(2 * 0.4**2, device=device, dtype=dtype) + + pdf = TestHistogram.fcn(input1, input2, bandwidth) + ans = 0.1 * torch.ones(1, 10, device=device, dtype=dtype) + self.assert_close(ans, pdf) diff --git a/tests/enhance/test_integral.py b/tests/enhance/test_integral.py new file mode 100644 index 0000000..5c91b95 --- /dev/null +++ b/tests/enhance/test_integral.py @@ -0,0 +1,104 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.enhance.integral import IntegralImage, IntegralTensor, integral_image, integral_tensor + +from testing.base import BaseTester + + +class TestIntegralTensor(BaseTester): + def test_smoke(self, device, dtype): + tensor = torch.rand(1, 1, 4, 4, device=device, dtype=dtype) + output = integral_tensor(tensor) + assert output.shape == (1, 1, 4, 4) + + @pytest.mark.parametrize("shape", [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1)]) + def test_cardinality(self, device, dtype, shape): + tensor = torch.rand(*shape, device=device, dtype=dtype) + output = integral_tensor(tensor) + assert output.shape == shape + + def test_exception(self, device, dtype): + tensor = torch.rand(1, 1, 4, 4, device=device, dtype=dtype) + with pytest.raises(Exception): + dim = (0, 1, 2, 3, 4) + integral_tensor(tensor, dim) + with pytest.raises(Exception): + dim = (4, 5) + integral_tensor(tensor, dim) + with pytest.raises(Exception) as errinfo: + integral_tensor(tensor, ()) + assert "dim must be a non-empty tuple." in str(errinfo) + + def test_module(self, device, dtype): + mod = IntegralTensor() + op = integral_tensor + tensor = torch.rand(1, 1, 4, 4, device=device, dtype=dtype) + self.assert_close(mod(tensor), op(tensor)) + + def test_gradcheck(self, device): + tensor = torch.rand(1, 1, 4, 4, device=device, dtype=torch.float64, requires_grad=True) + self.gradcheck(integral_tensor, (tensor,)) + + def test_value(self, device, dtype): + tensor = torch.tensor([[[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]]]], device=device, dtype=dtype) + output = integral_tensor(tensor) + expected = torch.tensor([[[[1.0, 3.0, 6.0], [4.0, 9.0, 15.0], [7.0, 15.0, 24.0]]]], device=device, dtype=dtype) + self.assert_close(output, expected) + + +class TestIntegralImage(BaseTester): + def test_smoke(self, device, dtype): + tensor = torch.rand(1, 1, 4, 4, device=device, dtype=dtype) + output = integral_image(tensor) + assert output.shape == (1, 1, 4, 4) + + @pytest.mark.parametrize("shape", [(1, 3, 4, 4), (2, 3, 2, 4), (3, 3, 4, 1)]) + def test_cardinality(self, device, dtype, shape): + tensor = torch.rand(*shape, device=device, dtype=dtype) + output = integral_image(tensor) + assert output.shape == shape + + def test_exception(self, device, dtype): + tensor = torch.rand(4, device=device, dtype=dtype) + with pytest.raises(Exception): + integral_image(tensor) + + def test_module(self, device, dtype): + mod = IntegralImage() + op = integral_image + tensor = torch.rand(1, 1, 4, 4, device=device, dtype=dtype) + self.assert_close(mod(tensor), op(tensor)) + + def test_gradcheck(self, device): + tensor = torch.rand(1, 1, 4, 4, device=device, dtype=torch.float64, requires_grad=True) + self.gradcheck(integral_image, (tensor,)) + + def test_values(self, device, dtype): + tensor = torch.tensor( + [[[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]]]], device=device, dtype=dtype + ) + output = integral_image(tensor) + expected = torch.tensor( + [[[[1, 3, 6, 10], [6.0, 14.0, 24.0, 36.0], [15.0, 33.0, 54.0, 78.0], [28.0, 60.0, 96.0, 136.0]]]], + device=device, + dtype=dtype, + ) + self.assert_close(output, expected) diff --git a/tests/enhance/test_jpeg.py b/tests/enhance/test_jpeg.py new file mode 100644 index 0000000..8af5ac3 --- /dev/null +++ b/tests/enhance/test_jpeg.py @@ -0,0 +1,1112 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestDiffJPEG(BaseTester): + def test_smoke(self, device, dtype) -> None: + """This test standard usage.""" + B, H, W = 2, 32, 32 + img = torch.rand(B, 3, H, W, device=device, dtype=dtype) + jpeg_quality = torch.randint(low=0, high=100, size=(B,), device=device, dtype=dtype) + img_jpeg = kornia.enhance.jpeg_codec_differentiable(img, jpeg_quality) + assert img_jpeg is not None + assert img_jpeg.shape == img.shape + + def test_keyword_argument(self, device, dtype) -> None: + """Regression test for #3745: the image must be passable via the ``input=`` keyword.""" + B, H, W = 2, 32, 32 + img = torch.rand(B, 3, H, W, device=device, dtype=dtype) + jpeg_quality = torch.randint(low=0, high=100, size=(B,), device=device, dtype=dtype) + img_jpeg = kornia.enhance.jpeg_codec_differentiable(input=img, jpeg_quality=jpeg_quality) + self.assert_close(img_jpeg, kornia.enhance.jpeg_codec_differentiable(img, jpeg_quality)) + + def test_smoke_not_div_by_16(self, device, dtype) -> None: + """This test standard usage.""" + B, H, W = 2, 33, 33 + img = torch.rand(B, 3, H, W, device=device, dtype=dtype) + jpeg_quality = torch.randint(low=0, high=100, size=(B,), device=device, dtype=dtype) + img_jpeg = kornia.enhance.jpeg_codec_differentiable(img, jpeg_quality) + assert img_jpeg is not None + assert img_jpeg.shape == img.shape + + def test_multi_batch(self, device, dtype) -> None: + """Here we test two batch dimensions.""" + B, H, W = 4, 32, 32 + img = torch.rand(B, B, 3, H, W, device=device, dtype=dtype) + jpeg_quality = torch.randint(low=0, high=100, size=(1,), device=device, dtype=dtype) + qt_y = torch.randint(low=1, high=255, size=(B * B, 8, 8), device=device, dtype=dtype) + qt_c = torch.randint(low=1, high=255, size=(B * B, 8, 8), device=device, dtype=dtype) + img_jpeg = kornia.enhance.jpeg_codec_differentiable(img, jpeg_quality, qt_y, qt_c) + assert img_jpeg is not None + assert img_jpeg.shape == img.shape + + def test_custom_qt(self, device, dtype) -> None: + """Here we test if we can handle custom quantization tables.""" + B, H, W = 4, 32, 32 + img = torch.rand(B, 3, H, W, device=device, dtype=dtype) + jpeg_quality = torch.randint(low=0, high=100, size=(B,), device=device, dtype=dtype) + qt_y = torch.randint(low=1, high=255, size=(B, 8, 8), device=device, dtype=dtype) + qt_c = torch.randint(low=1, high=255, size=(B, 8, 8), device=device, dtype=dtype) + img_jpeg = kornia.enhance.jpeg_codec_differentiable(img, jpeg_quality, qt_y, qt_c) + assert img_jpeg is not None + assert img_jpeg.shape == img.shape + + def test_non_batch_param(self, device, dtype) -> None: + """Here we test if we can handle non-batched JPEG parameters (JPEG quality and QT's).""" + B, H, W = 3, 32, 32 + img = torch.rand(B, 3, H, W, device=device, dtype=dtype) + jpeg_quality = torch.randint(low=0, high=100, size=(1,), device=device, dtype=dtype) + qt_y = torch.randint(low=1, high=255, size=(1, 8, 8), device=device, dtype=dtype) + qt_c = torch.randint(low=1, high=255, size=(1, 8, 8), device=device, dtype=dtype) + img_jpeg = kornia.enhance.jpeg_codec_differentiable(img, jpeg_quality, qt_y, qt_c) + assert img_jpeg is not None + assert img_jpeg.shape == img.shape + + def test_non_batch_inp(self, device, dtype) -> None: + """Here we test if we can handle non-batched inputs (input image, JPEG quality, and QT's).""" + H, W = 32, 32 + img = torch.rand(3, H, W, device=device, dtype=dtype) + jpeg_quality = torch.randint(low=0, high=100, size=(1,), device=device, dtype=dtype) + qt_y = torch.randint(low=1, high=255, size=(8, 8), device=device, dtype=dtype) + qt_c = torch.randint(low=1, high=255, size=(8, 8), device=device, dtype=dtype) + img_jpeg = kornia.enhance.jpeg_codec_differentiable(img, jpeg_quality, qt_y, qt_c) + assert img_jpeg is not None + assert img_jpeg.shape == img.shape + + def test_exception(self, device, dtype) -> None: + """Test exceptions (non-tensor input, wrong JPEG quality shape, wrong img shape, and wrong QT shape.)""" + with pytest.raises(TypeError) as errinfo: + B = 2 + jpeg_quality = torch.randint(low=0, high=100, size=(B,), device=device, dtype=dtype) + kornia.enhance.jpeg_codec_differentiable(1904.0, jpeg_quality) + assert "Input input type is not a torch.Tensor" in str(errinfo.value) + + from kornia.core.exceptions import TypeCheckError + + with pytest.raises(TypeCheckError) as errinfo: + B, H, W = 2, 32, 32 + img = torch.rand(B, 3, H, W, device=device, dtype=dtype) + kornia.enhance.jpeg_codec_differentiable(img, None) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + from kornia.core.exceptions import BaseError + + with pytest.raises(BaseError) as errinfo: + B, H, W = 2, 32, 32 + img = torch.rand(B, 3, H, W, device=device, dtype=dtype) + jpeg_quality = torch.randint(low=0, high=100, size=(B, 3, 2, 1), device=device, dtype=dtype) + kornia.enhance.jpeg_codec_differentiable(img, jpeg_quality) + assert "Shape dimension mismatch" in str(errinfo.value) or "Expected shape" in str(errinfo.value) + + from kornia.core.exceptions import BaseError + + with pytest.raises(BaseError) as errinfo: + B, H, W = 4, 32, 32 + img = torch.rand(B, 3, H, W, device=device, dtype=dtype) + jpeg_quality = torch.randint(low=0, high=100, size=(B,), device=device, dtype=dtype) + qt_y = torch.randint(low=1, high=255, size=(B, 7, 8), device=device, dtype=dtype) + qt_c = torch.randint(low=1, high=255, size=(B, 8, 8), device=device, dtype=dtype) + kornia.enhance.jpeg_codec_differentiable(img, jpeg_quality, qt_y, qt_c) + assert ( + "Shape dimension mismatch" in str(errinfo.value) + or "Expected shape" in str(errinfo.value) + or "shape must be" in str(errinfo.value) + ) + + from kornia.core.exceptions import BaseError + + with pytest.raises(BaseError) as errinfo: + B, H, W = 4, 32, 32 + img = torch.rand(B, 3, H, W, device=device, dtype=dtype) + jpeg_quality = torch.randint(low=0, high=100, size=(B,), device=device, dtype=dtype) + qt_y = torch.randint(low=1, high=255, size=(B, 8, 8), device=device, dtype=dtype) + qt_c = torch.randint(low=1, high=255, size=(B, 8, 7), device=device, dtype=dtype) + kornia.enhance.jpeg_codec_differentiable(img, jpeg_quality, qt_y, qt_c) + assert ( + "Shape dimension mismatch" in str(errinfo.value) + or "Expected shape" in str(errinfo.value) + or "shape must be" in str(errinfo.value) + ) + + from kornia.core.exceptions import BaseError + + with pytest.raises(BaseError) as errinfo: + B, H, W = 4, 32, 32 + img = torch.rand(B, B, 3, H, W, device=device, dtype=dtype) + jpeg_quality = torch.randint(low=0, high=100, size=(B * B,), device=device, dtype=dtype) + qt_y = torch.randint(low=1, high=255, size=(B * B, 8, 8), device=device, dtype=dtype) + qt_c = torch.randint(low=1, high=255, size=(B * 2, 8, 8), device=device, dtype=dtype) + kornia.enhance.jpeg_codec_differentiable(img, jpeg_quality, qt_y, qt_c) + assert "Batch dimensions do not match" in str(errinfo.value) + + from kornia.core.exceptions import BaseError + + with pytest.raises(BaseError) as errinfo: + B, H, W = 4, 32, 32 + img = torch.rand(B, B, 3, H, W, device=device, dtype=dtype) + jpeg_quality = torch.randint(low=0, high=100, size=(B * B,), device=device, dtype=dtype) + qt_y = torch.randint(low=1, high=255, size=(B * 2, 8, 8), device=device, dtype=dtype) + qt_c = torch.randint(low=1, high=255, size=(B * B, 8, 8), device=device, dtype=dtype) + kornia.enhance.jpeg_codec_differentiable(img, jpeg_quality, qt_y, qt_c) + assert "Batch dimensions do not match" in str(errinfo.value) + + from kornia.core.exceptions import BaseError + + with pytest.raises(BaseError) as errinfo: + B, H, W = 4, 32, 32 + img = torch.rand(B, B, 3, H, W, device=device, dtype=dtype) + jpeg_quality = torch.randint(low=0, high=100, size=(B * 2,), device=device, dtype=dtype) + qt_y = torch.randint(low=1, high=255, size=(B * B, 8, 8), device=device, dtype=dtype) + qt_c = torch.randint(low=1, high=255, size=(B * B, 8, 8), device=device, dtype=dtype) + kornia.enhance.jpeg_codec_differentiable(img, jpeg_quality, qt_y, qt_c) + assert "Batch dimensions do not match" in str(errinfo.value) + + def test_cardinality(self, device, dtype) -> None: + B, H, W = 1, 16, 16 + img = torch.zeros(B, 3, H, W, device=device, dtype=dtype) + img[..., 4:-4, 4:-4] = 1.0 + jpeg_quality = torch.tensor([2.0], device=device, dtype=dtype) + img_jpeg = kornia.enhance.jpeg_codec_differentiable(img, jpeg_quality) + # Numbers generated based on reference implementation + img_jpeg_ref = torch.tensor( + [ + [ + [ + [ + -0.000, + 0.002, + 0.063, + 0.060, + 0.020, + 0.017, + 0.078, + 0.146, + 0.146, + 0.078, + 0.017, + 0.020, + 0.060, + 0.063, + 0.002, + -0.000, + ], + [ + 0.002, + 0.015, + 0.008, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + 0.008, + 0.015, + 0.002, + ], + [ + 0.063, + 0.009, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + 0.009, + 0.063, + ], + [ + 0.060, + -0.000, + -0.000, + -0.000, + 0.173, + 0.246, + 0.178, + 0.080, + 0.080, + 0.178, + 0.246, + 0.173, + -0.000, + -0.000, + -0.000, + 0.060, + ], + [ + 0.020, + -0.000, + -0.000, + 0.173, + 0.694, + 0.971, + 0.960, + 0.847, + 0.847, + 0.960, + 0.971, + 0.694, + 0.173, + -0.000, + -0.000, + 0.020, + ], + [ + 0.017, + -0.000, + -0.000, + 0.246, + 0.971, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 0.971, + 0.246, + -0.000, + -0.000, + 0.017, + ], + [ + 0.078, + -0.000, + -0.000, + 0.178, + 0.960, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 0.960, + 0.178, + -0.000, + -0.000, + 0.078, + ], + [ + 0.146, + -0.000, + -0.000, + 0.080, + 0.847, + 1.000, + 1.000, + 0.781, + 0.781, + 1.000, + 1.000, + 0.847, + 0.080, + -0.000, + -0.000, + 0.146, + ], + [ + 0.146, + -0.000, + -0.000, + 0.080, + 0.847, + 1.000, + 1.000, + 0.781, + 0.781, + 1.000, + 1.000, + 0.847, + 0.080, + -0.000, + -0.000, + 0.146, + ], + [ + 0.078, + -0.000, + -0.000, + 0.178, + 0.960, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 0.960, + 0.178, + -0.000, + -0.000, + 0.078, + ], + [ + 0.017, + -0.000, + -0.000, + 0.246, + 0.971, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 0.971, + 0.246, + -0.000, + -0.000, + 0.017, + ], + [ + 0.020, + -0.000, + -0.000, + 0.173, + 0.694, + 0.971, + 0.960, + 0.847, + 0.847, + 0.960, + 0.971, + 0.694, + 0.173, + -0.000, + -0.000, + 0.020, + ], + [ + 0.060, + -0.000, + -0.000, + -0.000, + 0.173, + 0.246, + 0.178, + 0.080, + 0.080, + 0.178, + 0.246, + 0.173, + -0.000, + -0.000, + -0.000, + 0.060, + ], + [ + 0.063, + 0.009, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + 0.009, + 0.063, + ], + [ + 0.002, + 0.015, + 0.008, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + 0.008, + 0.015, + 0.002, + ], + [ + -0.000, + 0.002, + 0.063, + 0.060, + 0.020, + 0.017, + 0.078, + 0.146, + 0.146, + 0.078, + 0.017, + 0.020, + 0.060, + 0.063, + 0.002, + -0.000, + ], + ], + [ + [ + -0.000, + 0.002, + 0.063, + 0.060, + 0.020, + 0.017, + 0.078, + 0.146, + 0.146, + 0.078, + 0.017, + 0.020, + 0.060, + 0.063, + 0.002, + -0.000, + ], + [ + 0.002, + 0.015, + 0.008, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + 0.008, + 0.015, + 0.002, + ], + [ + 0.063, + 0.009, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + 0.009, + 0.063, + ], + [ + 0.060, + -0.000, + -0.000, + -0.000, + 0.173, + 0.246, + 0.178, + 0.080, + 0.080, + 0.178, + 0.246, + 0.173, + -0.000, + -0.000, + -0.000, + 0.060, + ], + [ + 0.020, + -0.000, + -0.000, + 0.173, + 0.694, + 0.971, + 0.960, + 0.847, + 0.847, + 0.960, + 0.971, + 0.694, + 0.173, + -0.000, + -0.000, + 0.020, + ], + [ + 0.017, + -0.000, + -0.000, + 0.246, + 0.971, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 0.971, + 0.246, + -0.000, + -0.000, + 0.017, + ], + [ + 0.078, + -0.000, + -0.000, + 0.178, + 0.960, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 0.960, + 0.178, + -0.000, + -0.000, + 0.078, + ], + [ + 0.146, + -0.000, + -0.000, + 0.080, + 0.847, + 1.000, + 1.000, + 0.781, + 0.781, + 1.000, + 1.000, + 0.847, + 0.080, + -0.000, + -0.000, + 0.146, + ], + [ + 0.146, + -0.000, + -0.000, + 0.080, + 0.847, + 1.000, + 1.000, + 0.781, + 0.781, + 1.000, + 1.000, + 0.847, + 0.080, + -0.000, + -0.000, + 0.146, + ], + [ + 0.078, + -0.000, + -0.000, + 0.178, + 0.960, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 0.960, + 0.178, + -0.000, + -0.000, + 0.078, + ], + [ + 0.017, + -0.000, + -0.000, + 0.246, + 0.971, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 0.971, + 0.246, + -0.000, + -0.000, + 0.017, + ], + [ + 0.020, + -0.000, + -0.000, + 0.173, + 0.694, + 0.971, + 0.960, + 0.847, + 0.847, + 0.960, + 0.971, + 0.694, + 0.173, + -0.000, + -0.000, + 0.020, + ], + [ + 0.060, + -0.000, + -0.000, + -0.000, + 0.173, + 0.246, + 0.178, + 0.080, + 0.080, + 0.178, + 0.246, + 0.173, + -0.000, + -0.000, + -0.000, + 0.060, + ], + [ + 0.063, + 0.009, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + 0.009, + 0.063, + ], + [ + 0.002, + 0.015, + 0.008, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + 0.008, + 0.015, + 0.002, + ], + [ + -0.000, + 0.002, + 0.063, + 0.060, + 0.020, + 0.017, + 0.078, + 0.146, + 0.146, + 0.078, + 0.017, + 0.020, + 0.060, + 0.063, + 0.002, + -0.000, + ], + ], + [ + [ + -0.000, + 0.002, + 0.063, + 0.060, + 0.020, + 0.017, + 0.078, + 0.146, + 0.146, + 0.078, + 0.017, + 0.020, + 0.060, + 0.063, + 0.002, + -0.000, + ], + [ + 0.002, + 0.015, + 0.008, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + 0.008, + 0.015, + 0.002, + ], + [ + 0.063, + 0.009, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + 0.009, + 0.063, + ], + [ + 0.060, + -0.000, + -0.000, + -0.000, + 0.173, + 0.246, + 0.178, + 0.080, + 0.080, + 0.178, + 0.246, + 0.173, + -0.000, + -0.000, + -0.000, + 0.060, + ], + [ + 0.020, + -0.000, + -0.000, + 0.173, + 0.694, + 0.971, + 0.960, + 0.847, + 0.847, + 0.960, + 0.971, + 0.694, + 0.173, + -0.000, + -0.000, + 0.020, + ], + [ + 0.017, + -0.000, + -0.000, + 0.246, + 0.971, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 0.971, + 0.246, + -0.000, + -0.000, + 0.017, + ], + [ + 0.078, + -0.000, + -0.000, + 0.178, + 0.960, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 0.960, + 0.178, + -0.000, + -0.000, + 0.078, + ], + [ + 0.146, + -0.000, + -0.000, + 0.080, + 0.847, + 1.000, + 1.000, + 0.781, + 0.781, + 1.000, + 1.000, + 0.847, + 0.080, + -0.000, + -0.000, + 0.146, + ], + [ + 0.146, + -0.000, + -0.000, + 0.080, + 0.847, + 1.000, + 1.000, + 0.781, + 0.781, + 1.000, + 1.000, + 0.847, + 0.080, + -0.000, + -0.000, + 0.146, + ], + [ + 0.078, + -0.000, + -0.000, + 0.178, + 0.960, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 0.960, + 0.178, + -0.000, + -0.000, + 0.078, + ], + [ + 0.017, + -0.000, + -0.000, + 0.246, + 0.971, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 1.000, + 0.971, + 0.246, + -0.000, + -0.000, + 0.017, + ], + [ + 0.020, + -0.000, + -0.000, + 0.173, + 0.694, + 0.971, + 0.960, + 0.847, + 0.847, + 0.960, + 0.971, + 0.694, + 0.173, + -0.000, + -0.000, + 0.020, + ], + [ + 0.060, + -0.000, + -0.000, + -0.000, + 0.173, + 0.246, + 0.178, + 0.080, + 0.080, + 0.178, + 0.246, + 0.173, + -0.000, + -0.000, + -0.000, + 0.060, + ], + [ + 0.063, + 0.009, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + 0.009, + 0.063, + ], + [ + 0.002, + 0.015, + 0.008, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + -0.000, + 0.008, + 0.015, + 0.002, + ], + [ + -0.000, + 0.002, + 0.063, + 0.060, + 0.020, + 0.017, + 0.078, + 0.146, + 0.146, + 0.078, + 0.017, + 0.020, + 0.060, + 0.063, + 0.002, + -0.000, + ], + ], + ] + ], + device=device, + dtype=dtype, + ) + # We use a slightly higher tolerance since our implementation varies from the reference implementation + self.assert_close(img_jpeg, img_jpeg_ref, rtol=0.01, atol=0.01) + + def test_module(self, device, dtype) -> None: + B, H, W = 4, 16, 16 + img = torch.rand(B, 3, H, W, device=device, dtype=dtype) + jpeg_quality = torch.randint(low=0, high=100, size=(B,), device=device, dtype=dtype) + qt_y = torch.randint(low=1, high=255, size=(B, 8, 8), device=device, dtype=dtype) + qt_c = torch.randint(low=1, high=255, size=(B, 8, 8), device=device, dtype=dtype) + diff_jpeg_module = kornia.enhance.JPEGCodecDifferentiable(qt_y, qt_c) + img_jpeg = diff_jpeg_module(img, jpeg_quality) + assert img_jpeg is not None + assert img_jpeg.shape == img.shape + + def test_module_with_param(self, device, dtype) -> None: + B, H, W = 4, 16, 16 + img = torch.rand(B, 3, H, W, device=device, dtype=dtype) + jpeg_quality = torch.randint(low=0, high=100, size=(B,), device=device, dtype=dtype) + qt_y = torch.nn.Parameter(torch.randint(low=1, high=255, size=(B, 8, 8), device=device, dtype=dtype)) + qt_c = torch.nn.Parameter(torch.randint(low=1, high=255, size=(B, 8, 8), device=device, dtype=dtype)) + diff_jpeg_module = kornia.enhance.JPEGCodecDifferentiable(qt_y, qt_c) + img_jpeg = diff_jpeg_module(img, jpeg_quality) + assert img_jpeg is not None + assert img_jpeg.shape == img.shape + + # @pytest.mark.slow + def test_gradcheck(self, device) -> None: + """We test that the gradient matches the gradient of the reference implementation.""" + B, H, W = 1, 16, 16 + img = torch.zeros(B, 3, H, W, device=device, dtype=torch.float) + img[..., 0, 4:-4, 4:-4] = 1.0 + img[..., 1, 4:-4, 4:-4] = 0.5 + img[..., 2, 4:-4, 4:-4] = 0.5 + img.requires_grad = True + jpeg_quality = torch.tensor([10.0], device=device, dtype=torch.float, requires_grad=True) + img_jpeg = kornia.enhance.jpeg_codec_differentiable(img, jpeg_quality) + (img_jpeg - torch.zeros_like(img_jpeg)).abs().sum().backward() + # Numbers generated based on reference implementation + img_jpeg_mean_grad_ref = torch.tensor([0.1919], device=device) + jpeg_quality_grad_ref = torch.tensor([0.0042], device=device) + # We use a slightly higher tolerance since our implementation varies from the reference implementation + self.assert_close(img.grad.mean().view(-1), img_jpeg_mean_grad_ref, rtol=0.01, atol=0.01) + self.assert_close(jpeg_quality.grad, jpeg_quality_grad_ref, rtol=0.01, atol=0.01) diff --git a/tests/enhance/test_jpeg_misc.py b/tests/enhance/test_jpeg_misc.py new file mode 100644 index 0000000..ba80a10 --- /dev/null +++ b/tests/enhance/test_jpeg_misc.py @@ -0,0 +1,69 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch + +from kornia.enhance.jpeg import ( + _differentiable_clipping as differentiable_clipping, +) +from kornia.enhance.jpeg import ( + _differentiable_polynomial_floor as differentiable_polynomial_floor, +) +from kornia.enhance.jpeg import ( + _differentiable_polynomial_rounding as differentiable_polynomial_rounding, +) + +from testing.base import BaseTester + + +class TestDifferentiableClipping(BaseTester): + def test_differentiable_clipping(self, device): + x = torch.tensor([1.0, 6.0, 10.0, 12.0], device=device) + y = differentiable_clipping(x, min_val=5.0, max_val=10.0) + y_expected = torch.tensor([4.9804, 6.0, 10.0, 10.0173], device=device) + + self.assert_close(y, y_expected) + + def test_gradcheck(self, device): + x = torch.tensor([1.0, 6.0, 11.0, 12.0], device=device, dtype=torch.float64) + self.gradcheck(differentiable_clipping, (x, 5.0, 10.0)) + + +class TestDifferentiablePolynomialRounding(BaseTester): + def test_differentiable_polynomial_rounding(self, device): + x = torch.tensor([1.5], device=device) + y = differentiable_polynomial_rounding(x) + y_expected = torch.tensor([1.875], device=device) + + self.assert_close(y, y_expected) + + def test_gradcheck(self, device): + x = torch.tensor([1.0, 6.0, 10.0, 12.0], device=device, dtype=torch.float64) + self.gradcheck(differentiable_polynomial_rounding, (x)) + + +class TestDifferentiablePolynomialFloor(BaseTester): + def test_differentiable_polynomial_floor(self, device): + x = torch.tensor([1.5], device=device) + y = differentiable_polynomial_floor(x) + y_expected = torch.tensor([1.0], device=device) + + self.assert_close(y, y_expected) + + def test_gradcheck(self, device): + x = torch.tensor([1.5, 3.1, 5.9, 6.6], device=device, dtype=torch.float64) + self.gradcheck(differentiable_polynomial_floor, (x)) diff --git a/tests/enhance/test_normalize.py b/tests/enhance/test_normalize.py new file mode 100644 index 0000000..5bb174e --- /dev/null +++ b/tests/enhance/test_normalize.py @@ -0,0 +1,387 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestNormalize(BaseTester): + def test_smoke(self, device, dtype): + mean = [0.5] + std = [0.1] + repr = "Normalize(mean=tensor([[0.5000]]), std=tensor([[0.1000]]))" + assert str(kornia.enhance.Normalize(mean, std)) == repr + + def test_normalize(self, device, dtype): + # prepare input data + data = torch.ones(1, 2, 2, device=device, dtype=dtype) + mean = torch.tensor([0.5], device=device, dtype=dtype) + std = torch.tensor([2.0], device=device, dtype=dtype) + + # expected output + expected = torch.tensor([0.25], device=device, dtype=dtype).repeat(1, 2, 2).view_as(data) + + f = kornia.enhance.Normalize(mean, std) + self.assert_close(f(data), expected) + + def test_broadcast_normalize(self, device, dtype): + # prepare input data + data = torch.ones(2, 3, 1, 1, device=device, dtype=dtype) + data += 2 + + mean = torch.tensor([2.0], device=device, dtype=dtype) + std = torch.tensor([0.5], device=device, dtype=dtype) + + # expected output + expected = torch.ones_like(data) + 1 + + f = kornia.enhance.Normalize(mean, std) + self.assert_close(f(data), expected) + + def test_int_input(self, device, dtype): + data = torch.ones(2, 3, 1, 1, device=device, dtype=dtype) + data += 2 + + mean: int = 2 + std: int = 1 + + # expected output + expected = torch.ones_like(data) + + f = kornia.enhance.Normalize(mean, std) + self.assert_close(f(data), expected) + + def test_float_input(self, device, dtype): + data = torch.ones(2, 3, 1, 1, device=device, dtype=dtype) + data += 2 + + mean: float = 2.0 + std: float = 0.5 + + # expected output + expected = torch.ones_like(data) + 1 + + f = kornia.enhance.Normalize(mean, std) + self.assert_close(f(data), expected) + + def test_batch_normalize(self, device, dtype): + # prepare input data + data = torch.ones(2, 3, 1, 1, device=device, dtype=dtype) + data += 2 + + mean = torch.tensor([0.5, 1.0, 2.0], device=device, dtype=dtype).repeat(2, 1) + std = torch.tensor([2.0, 2.0, 2.0], device=device, dtype=dtype).repeat(2, 1) + + # expected output + expected = torch.tensor([1.25, 1, 0.5], device=device, dtype=dtype).repeat(2, 1, 1).view_as(data) + + f = kornia.enhance.Normalize(mean, std) + self.assert_close(f(data), expected) + + @pytest.mark.skip(reason="union type not supported") + def test_jit(self, device, dtype): + data = torch.ones(2, 3, 1, 1, device=device, dtype=dtype) + mean = torch.tensor([0.5, 1.0, 2.0], device=device, dtype=dtype).repeat(2, 1) + std = torch.tensor([2.0, 2.0, 2.0], device=device, dtype=dtype).repeat(2, 1) + inputs = (data, mean, std) + + op = kornia.enhance.normalize + op_script = torch.jit.script(op) + + self.assert_close(op(*inputs), op_script(*inputs)) + + def test_gradcheck(self, device): + # prepare input data + data = torch.ones(2, 3, 1, 1, device=device, dtype=torch.float64) + mean = torch.tensor([0.5, 1.0, 2.0], device=device, dtype=torch.float64).repeat(2, 1) + std = torch.tensor([2.0, 2.0, 2.0], device=device, dtype=torch.float64).repeat(2, 1) + + self.gradcheck(kornia.enhance.Normalize(mean, std), (data,)) + + def test_single_value(self, device, dtype): + # prepare input data + mean = torch.tensor(2, device=device, dtype=dtype) + std = torch.tensor(3, device=device, dtype=dtype) + data = torch.ones(2, 3, 256, 313, device=device, dtype=dtype) + + # expected output + expected = (data - mean) / std + + self.assert_close(kornia.enhance.normalize(data, mean, std), expected) + + def test_module(self, device, dtype): + data = torch.ones(2, 3, 1, 1, device=device, dtype=dtype) + mean = torch.tensor([0.5, 1.0, 2.0], device=device, dtype=dtype).repeat(2, 1) + std = torch.tensor([2.0, 2.0, 2.0], device=device, dtype=dtype).repeat(2, 1) + inputs = (data, mean, std) + + op = kornia.enhance.normalize + op_module = kornia.enhance.Normalize(mean, std) + + self.assert_close(op(*inputs), op_module(data)) + + @pytest.mark.parametrize( + "mean, std", [((1.0, 1.0, 1.0), (0.5, 0.5, 0.5)), (1.0, 0.5), (torch.tensor([1.0]), torch.tensor([0.5]))] + ) + def test_random_normalize_different_parameter_types(self, mean, std): + f = kornia.enhance.Normalize(mean=mean, std=std) + data = torch.ones(2, 3, 256, 313) + if isinstance(mean, float): + expected = (data - torch.as_tensor(mean)) / torch.as_tensor(std) + else: + expected = (data - torch.as_tensor(mean[0])) / torch.as_tensor(std[0]) + self.assert_close(f(data), expected) + + @pytest.mark.parametrize("mean, std", [((1.0, 1.0, 1.0, 1.0), (0.5, 0.5, 0.5, 0.5)), ((1.0, 1.0), (0.5, 0.5))]) + def test_random_normalize_invalid_parameter_shape(self, mean, std): + f = kornia.enhance.Normalize(mean=mean, std=std) + inputs = torch.arange(0.0, 16.0, step=1).reshape(1, 4, 4).unsqueeze(0) + with pytest.raises((ValueError, RuntimeError)): + f(inputs) + + @pytest.mark.skip(reason="not implemented yet") + def test_cardinality(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_exception(self, device, dtype): + pass + + +class TestDenormalize(BaseTester): + def test_smoke(self, device, dtype): + mean = [0.5] + std = [0.1] + repr = "Denormalize(mean=[0.5], std=[0.1])" + assert str(kornia.enhance.Denormalize(mean, std)) == repr + + def test_denormalize(self, device, dtype): + # prepare input data + data = torch.ones(1, 2, 2, device=device, dtype=dtype) + mean = torch.tensor([0.5]) + std = torch.tensor([2.0]) + + # expected output + expected = torch.tensor([2.5], device=device, dtype=dtype).repeat(1, 2, 2).view_as(data) + + f = kornia.enhance.Denormalize(mean, std) + self.assert_close(f(data), expected) + + def test_broadcast_denormalize(self, device, dtype): + # prepare input data + data = torch.ones(2, 3, 1, 1, device=device, dtype=dtype) + data += 2 + + mean = torch.tensor([2.0], device=device, dtype=dtype) + std = torch.tensor([0.5], device=device, dtype=dtype) + + # expected output + expected = torch.ones_like(data) + 2.5 + + f = kornia.enhance.Denormalize(mean, std) + self.assert_close(f(data), expected) + + def test_float_input(self, device, dtype): + data = torch.ones(2, 3, 1, 1, device=device, dtype=dtype) + data += 2 + + mean: float = 2.0 + std: float = 0.5 + + # expected output + expected = torch.ones_like(data) + 2.5 + + f = kornia.enhance.Denormalize(mean, std) + self.assert_close(f(data), expected) + + def test_batch_denormalize(self, device, dtype): + # prepare input data + data = torch.ones(2, 3, 1, 1, device=device, dtype=dtype) + data += 2 + + mean = torch.tensor([0.5, 1.0, 2.0], device=device, dtype=dtype).repeat(2, 1) + std = torch.tensor([2.0, 2.0, 2.0], device=device, dtype=dtype).repeat(2, 1) + + # expected output + expected = torch.tensor([6.5, 7, 8], device=device, dtype=dtype).repeat(2, 1, 1).view_as(data) + + f = kornia.enhance.Denormalize(mean, std) + self.assert_close(f(data), expected) + + @pytest.mark.skip(reason="union type not supported") + def test_jit(self, device, dtype): + data = torch.ones(2, 3, 1, 1, device=device, dtype=dtype) + mean = torch.tensor([0.5, 1.0, 2.0], device=device, dtype=dtype).repeat(2, 1) + std = torch.tensor([2.0, 2.0, 2.0], device=device, dtype=dtype).repeat(2, 1) + inputs = (data, mean, std) + + op = kornia.enhance.denormalize + op_script = torch.jit.script(op) + + self.assert_close(op(*inputs), op_script(*inputs)) + + def test_gradcheck(self, device): + # prepare input data + data = torch.ones(2, 3, 1, 1, device=device, dtype=torch.float64) + data += 2 + mean = torch.tensor([0.5, 1.0, 2.0], device=device, dtype=torch.float64) + std = torch.tensor([2.0, 2.0, 2.0], device=device, dtype=torch.float64) + + self.gradcheck(kornia.enhance.Denormalize(mean, std), (data,)) + + def test_single_value(self, device, dtype): + # prepare input data + mean = torch.tensor(2, device=device, dtype=dtype) + std = torch.tensor(3, device=device, dtype=dtype) + data = torch.ones(2, 3, 256, 313, device=device, dtype=dtype) + + # expected output + expected = (data * std) + mean + + self.assert_close(kornia.enhance.denormalize(data, mean, std), expected) + + def test_module(self, device, dtype): + data = torch.ones(2, 3, 1, 1, device=device, dtype=dtype) + mean = torch.tensor([0.5, 1.0, 2.0], device=device, dtype=dtype).repeat(2, 1) + std = torch.tensor([2.0, 2.0, 2.0], device=device, dtype=dtype).repeat(2, 1) + inputs = (data, mean, std) + + op = kornia.enhance.denormalize + op_module = kornia.enhance.Denormalize(mean, std) + + self.assert_close(op(*inputs), op_module(data)) + + @pytest.mark.skip(reason="not implemented yet") + def test_cardinality(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_exception(self, device, dtype): + pass + + +class TestNormalizeMinMax(BaseTester): + def test_smoke(self, device, dtype): + x = torch.ones(1, 1, 1, 1, device=device, dtype=dtype) + assert kornia.enhance.normalize_min_max(x) is not None + assert kornia.enhance.normalize_min_max(x) is not None + + def test_exception(self, device, dtype): + x = torch.ones(1, 1, 3, 4, device=device, dtype=dtype) + with pytest.raises(TypeError): + assert kornia.enhance.normalize_min_max(0.0) + + with pytest.raises(TypeError): + assert kornia.enhance.normalize_min_max(x, "", "") + + with pytest.raises(TypeError): + assert kornia.enhance.normalize_min_max(x, 2.0, "") + + @pytest.mark.parametrize("input_shape", [(1, 2, 3, 4), (2, 1, 4, 3), (1, 3, 2, 1)]) + def test_cardinality(self, device, dtype, input_shape): + x = torch.rand(input_shape, device=device, dtype=dtype) + assert kornia.enhance.normalize_min_max(x).shape == input_shape + + @pytest.mark.parametrize("min_val, max_val", [(1.0, 2.0), (2.0, 3.0), (5.0, 20.0), (40.0, 1000.0)]) + def test_range(self, device, dtype, min_val, max_val): + x = torch.rand(1, 2, 4, 5, device=device, dtype=dtype) + out = kornia.enhance.normalize_min_max(x, min_val=min_val, max_val=max_val) + self.assert_close(out.min(), torch.tensor(min_val, device=device, dtype=dtype), low_tolerance=True) + self.assert_close(out.max(), torch.tensor(max_val, device=device, dtype=dtype), low_tolerance=True) + + def test_values(self, device, dtype): + x = torch.tensor([[[[0.0, 1.0, 3.0], [-1.0, 4.0, 3.0], [9.0, 5.0, 2.0]]]], device=device, dtype=dtype) + + expected = torch.tensor( + [[[[-0.8, -0.6, -0.2], [-1.0, 0.0, -0.2], [1.0, 0.2, -0.4]]]], device=device, dtype=dtype + ) + + actual = kornia.enhance.normalize_min_max(x, min_val=-1.0, max_val=1.0) + self.assert_close(actual, expected, low_tolerance=True) + + @pytest.mark.skip(reason="args and kwargs in decorator") + def test_jit(self, device, dtype): + x = torch.ones(1, 1, 1, 1, device=device, dtype=dtype) + op = kornia.enhance.normalize_min_max + op_jit = torch.jit.script(op) + self.assert_close(op(x), op_jit(x)) + + @pytest.mark.grad() + def test_gradcheck(self, device): + x = torch.ones(1, 1, 1, 1, device=device, dtype=torch.float64, requires_grad=True) + self.gradcheck(kornia.enhance.normalize_min_max, (x,)) + + def test_3d_tensor(self, device, dtype): + # Test with 3D tensor (C, H, W) - the main bug fix + x = torch.tensor([[[0.0, 1.0, 3.0], [-1.0, 4.0, 3.0], [9.0, 5.0, 2.0]]], device=device, dtype=dtype) + + # Expected: normalized to [-1, 1] range + expected = torch.tensor([[[-0.8, -0.6, -0.2], [-1.0, 0.0, -0.2], [1.0, 0.2, -0.4]]], device=device, dtype=dtype) + + actual = kornia.enhance.normalize_min_max(x, min_val=-1.0, max_val=1.0) + + # Verify shape is preserved + assert actual.shape == x.shape + self.assert_close(actual, expected, low_tolerance=True) + + def test_3d_tensor_multiple_channels(self, device, dtype): + # Test with 3D tensor with multiple channels (C, H, W) + x = torch.rand(3, 4, 5, device=device, dtype=dtype) + out = kornia.enhance.normalize_min_max(x, min_val=0.0, max_val=1.0) + + # Verify shape is preserved + assert out.shape == x.shape + + # Verify per-channel normalization + for c in range(x.shape[0]): + channel_out = out[c] + self.assert_close(channel_out.min(), torch.tensor(0.0, device=device, dtype=dtype), low_tolerance=True) + self.assert_close(channel_out.max(), torch.tensor(1.0, device=device, dtype=dtype), low_tolerance=True) + + def test_2d_tensor(self, device, dtype): + # Test with 2D tensor (H, W) + x = torch.tensor([[0.0, 5.0], [10.0, 15.0]], device=device, dtype=dtype) + out = kornia.enhance.normalize_min_max(x, min_val=0.0, max_val=1.0) + + # Verify shape is preserved + assert out.shape == x.shape + + # Verify normalization + expected = torch.tensor([[0.0, 1.0 / 3.0], [2.0 / 3.0, 1.0]], device=device, dtype=dtype) + self.assert_close(out, expected, low_tolerance=True) + + @pytest.mark.parametrize("input_shape", [(3, 4, 5), (1, 32, 32), (4, 8, 8)]) + def test_3d_shapes(self, device, dtype, input_shape): + # Test various 3D tensor shapes + x = torch.rand(input_shape, device=device, dtype=dtype) + out = kornia.enhance.normalize_min_max(x, min_val=-1.0, max_val=1.0) + + # Verify shape is preserved + assert out.shape == input_shape + + def test_keyword_argument(self, device, dtype): + # Regression test for #3745: the perform_keep_shape_image wrapper binds the + # first argument as `input`, so passing the image by keyword must use `input=` + # and match the documented signature. + x = torch.rand(1, 2, 4, 5, device=device, dtype=dtype) + out_kwarg = kornia.enhance.normalize_min_max(input=x, min_val=-1.0, max_val=1.0) + out_positional = kornia.enhance.normalize_min_max(x, min_val=-1.0, max_val=1.0) + self.assert_close(out_kwarg, out_positional) diff --git a/tests/enhance/test_rescale.py b/tests/enhance/test_rescale.py new file mode 100644 index 0000000..78989a3 --- /dev/null +++ b/tests/enhance/test_rescale.py @@ -0,0 +1,78 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.enhance.rescale import Rescale + +from testing.base import BaseTester + + +class TestRescale(BaseTester): + def test_smoke(self, device, dtype): + r = Rescale(0.5).to(device, dtype) + x = torch.ones(1, 3, 4, 4, device=device, dtype=dtype) + assert isinstance(r(x), torch.Tensor) + + def test_cardinality(self, device, dtype): + r = Rescale(2.0).to(device, dtype) + x = torch.ones(2, 3, 5, 5, device=device, dtype=dtype) + assert r(x).shape == x.shape + + def test_float_factor(self, device, dtype): + r = Rescale(0.5).to(device, dtype) + x = torch.ones(1, 1, 2, 2, device=device, dtype=dtype) * 4.0 + expected = torch.ones(1, 1, 2, 2, device=device, dtype=dtype) * 2.0 + self.assert_close(r(x), expected) + + def test_tensor_factor(self, device, dtype): + factor = torch.tensor(3.0, device=device, dtype=dtype) + r = Rescale(factor) + x = torch.ones(1, 1, 2, 2, device=device, dtype=dtype) * 2.0 + expected = torch.ones(1, 1, 2, 2, device=device, dtype=dtype) * 6.0 + self.assert_close(r(x), expected) + + def test_zero_factor(self, device, dtype): + r = Rescale(0.0).to(device, dtype) + x = torch.rand(1, 3, 4, 4, device=device, dtype=dtype) + expected = torch.zeros_like(x) + self.assert_close(r(x), expected) + + def test_exception(self, device, dtype): + with pytest.raises(TypeError): + Rescale(torch.ones(2)) # not a 0-d tensor + + with pytest.raises(TypeError): + Rescale([0.5]) # list is not valid either + + def test_gradcheck(self, device): + factor = torch.tensor(2.0) + r = Rescale(factor) + x = torch.rand(1, 3, 4, 4, device=device, dtype=torch.float64, requires_grad=True) + self.gradcheck(r, (x,)) + + def test_module(self, device, dtype): + r1 = Rescale(2.0).to(device, dtype) + x = torch.rand(1, 3, 4, 4, device=device, dtype=dtype) + self.assert_close(r1(x), x * 2.0) + + def test_dynamo(self, device, dtype, torch_optimizer): + r = Rescale(0.5).to(device, dtype) + x = torch.rand(1, 3, 4, 4, device=device, dtype=dtype) + op = torch_optimizer(r) + self.assert_close(op(x), r(x)) diff --git a/tests/enhance/test_shift_rgb.py b/tests/enhance/test_shift_rgb.py new file mode 100644 index 0000000..7cd10d1 --- /dev/null +++ b/tests/enhance/test_shift_rgb.py @@ -0,0 +1,78 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestRGBShift(BaseTester): + def test_rgb_shift_no_shift(self, device, dtype): + r_shift, g_shift, b_shift = torch.Tensor([0]), torch.Tensor([0]), torch.Tensor([0]) + image = torch.rand(2, 3, 5, 5, device=device, dtype=dtype) + expected = image + shifted = kornia.enhance.shift_rgb(image, r_shift, g_shift, b_shift) + + self.assert_close(shifted, expected) + + def test_rgb_shift_all_zeros(self, device, dtype): + r_shift, g_shift, b_shift = torch.Tensor([-0.1]), torch.Tensor([-0.1]), torch.Tensor([-0.1]) + image = torch.zeros(2, 3, 5, 5, device=device, dtype=dtype) + expected = image + shifted = kornia.enhance.shift_rgb(image, r_shift, g_shift, b_shift) + + self.assert_close(shifted, expected) + + def test_rgb_shift_all_ones(self, device, dtype): + r_shift, g_shift, b_shift = torch.Tensor([1]), torch.Tensor([1]), torch.Tensor([1]) + image = torch.rand(2, 3, 5, 5, device=device, dtype=dtype) + expected = torch.ones(2, 3, 5, 5, device=device, dtype=dtype) + shifted = kornia.enhance.shift_rgb(image, r_shift, g_shift, b_shift) + + self.assert_close(shifted, expected) + + def test_rgb_shift_invalid_parameter_shape(self, device, dtype): + from kornia.core.exceptions import ImageError + + r_shift, g_shift, b_shift = torch.Tensor([0.5]), torch.Tensor([0.5]), torch.Tensor([0.5]) + image = torch.randn(3, 3, device=device, dtype=dtype) + with pytest.raises(ImageError): + kornia.enhance.shift_rgb(image, r_shift, g_shift, b_shift) + + def test_rgb_shift_gradcheck(self, device): + r_shift, g_shift, b_shift = torch.Tensor([0.4]), torch.Tensor([0.5]), torch.Tensor([0.2]) + image = torch.randn(2, 3, 5, 5, device=device, dtype=torch.float64) + self.gradcheck(kornia.enhance.shift_rgb, (image, r_shift, g_shift, b_shift)) + + def test_rgb_shift(self, device, dtype): + r_shift, g_shift, b_shift = torch.Tensor([0.1]), torch.Tensor([0.3]), torch.Tensor([-0.3]) + image = torch.tensor( + [[[[0.2, 0.0]], [[0.3, 0.5]], [[0.4, 0.7]]], [[[0.2, 0.7]], [[0.0, 0.8]], [[0.2, 0.3]]]], + device=device, + dtype=dtype, + ) + shifted = kornia.enhance.shift_rgb(image, r_shift, g_shift, b_shift) + expected = torch.tensor( + [[[[0.3, 0.1]], [[0.6, 0.8]], [[0.1, 0.4]]], [[[0.3, 0.8]], [[0.3, 1.0]], [[0.0, 0.0]]]], + device=device, + dtype=dtype, + ) + + self.assert_close(shifted, expected) diff --git a/tests/enhance/test_threshold.py b/tests/enhance/test_threshold.py new file mode 100644 index 0000000..8d21f5c --- /dev/null +++ b/tests/enhance/test_threshold.py @@ -0,0 +1,77 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.enhance.threshold import ThresholdType, threshold + + +class TestThreshold: + @pytest.mark.parametrize( + "ttype", + [ + ThresholdType.THRESH_BINARY, + ThresholdType.THRESH_BINARY_INV, + ThresholdType.THRESH_TRUNC, + ThresholdType.THRESH_TOZERO, + ThresholdType.THRESH_TOZERO_INV, + ], + ) + @pytest.mark.parametrize("shape", [(1, 1, 5, 7), (2, 3, 11, 9)]) + def test_output_properties(self, ttype, shape, device, dtype): + x = torch.rand(shape, device=device, dtype=dtype) + out = threshold(x, thresh=0.5, maxval=1.0, type=ttype) + + assert out.shape == x.shape + assert out.dtype == x.dtype + assert out.device == x.device + + def test_binary_rule_strict_greater(self, device, dtype): + x = torch.tensor([0.2, 0.5, 0.7], device=device, dtype=dtype) + out = threshold(x, thresh=0.5, maxval=9.0, type=ThresholdType.THRESH_BINARY) + expected = torch.tensor([0.0, 0.0, 9.0], device=device, dtype=dtype) + assert torch.allclose(out, expected) + + def test_binary_inv_rule_strict_greater(self, device, dtype): + x = torch.tensor([0.2, 0.5, 0.7], device=device, dtype=dtype) + out = threshold(x, thresh=0.5, maxval=9.0, type=ThresholdType.THRESH_BINARY_INV) + expected = torch.tensor([9.0, 9.0, 0.0], device=device, dtype=dtype) + assert torch.allclose(out, expected) + + def test_trunc(self, device, dtype): + x = torch.tensor([0.2, 0.5, 0.7], device=device, dtype=dtype) + out = threshold(x, thresh=0.5, maxval=9.0, type=ThresholdType.THRESH_TRUNC) + expected = torch.tensor([0.2, 0.5, 0.5], device=device, dtype=dtype) + assert torch.allclose(out, expected) + + def test_tozero(self, device, dtype): + x = torch.tensor([0.2, 0.5, 0.7], device=device, dtype=dtype) + out = threshold(x, thresh=0.5, maxval=9.0, type=ThresholdType.THRESH_TOZERO) + expected = torch.tensor([0.0, 0.0, 0.7], device=device, dtype=dtype) + assert torch.allclose(out, expected) + + def test_tozero_inv(self, device, dtype): + x = torch.tensor([0.2, 0.5, 0.7], device=device, dtype=dtype) + out = threshold(x, thresh=0.5, maxval=9.0, type=ThresholdType.THRESH_TOZERO_INV) + expected = torch.tensor([0.2, 0.5, 0.0], device=device, dtype=dtype) + assert torch.allclose(out, expected) + + def test_otsu_raises(self, device, dtype): + x = torch.rand(1, 1, 5, 5, device=device, dtype=dtype) + with pytest.raises(NotImplementedError): + threshold(x, thresh=0.0, maxval=1.0, type=ThresholdType.THRESH_OTSU) diff --git a/tests/enhance/test_zca.py b/tests/enhance/test_zca.py new file mode 100644 index 0000000..bcd9b4b --- /dev/null +++ b/tests/enhance/test_zca.py @@ -0,0 +1,163 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestZCA(BaseTester): + @pytest.mark.parametrize("unbiased", [True, False]) + def test_zca_unbiased(self, unbiased, device, dtype): + data = torch.tensor([[0, 1], [1, 0], [-1, 0], [0, -1]], device=device, dtype=dtype) + + if unbiased: + unbiased_val = 1.5 + else: + unbiased_val = 2.0 + + expected = torch.sqrt(unbiased_val * torch.abs(data)) * torch.sign(data) + + zca = kornia.enhance.ZCAWhitening(unbiased=unbiased).fit(data) + + actual = zca(data) + + self.assert_close(actual, expected, low_tolerance=True) + + @pytest.mark.parametrize("dim", [0, 1]) + def test_dim_args(self, dim, device, dtype): + if "xla" in device.type: + pytest.skip("buggy with XLA devices.") + + if dtype == torch.float16: + pytest.skip("not work for half-precision") + + data = torch.tensor([[0, 1], [1, 0], [-1, 0], [0, -1]], device=device, dtype=dtype) + + if dim == 1: + expected = torch.tensor( + [ + [-0.35360718, 0.35360718], + [0.35351562, -0.35351562], + [-0.35353088, 0.35353088], + [0.35353088, -0.35353088], + ], + device=device, + dtype=dtype, + ) + elif dim == 0: + expected = torch.tensor( + [[0.0, 1.2247448], [1.2247448, 0.0], [-1.2247448, 0.0], [0.0, -1.2247448]], device=device, dtype=dtype + ) + + zca = kornia.enhance.ZCAWhitening(dim=dim) + actual = zca(data, True) + + self.assert_close(actual, expected, low_tolerance=True) + + @pytest.mark.parametrize("input_shape,eps", [((15, 2, 2, 2), 1e-6), ((10, 4), 0.1), ((20, 3, 2, 2), 1e-3)]) + def test_identity(self, input_shape, eps, device, dtype): + """Assert that data can be recovered by the inverse transform.""" + data = torch.randn(*input_shape, device=device, dtype=dtype) + + zca = kornia.enhance.ZCAWhitening(compute_inv=True, eps=eps).fit(data) + + data_w = zca(data) + + data_hat = zca.inverse_transform(data_w) + + self.assert_close(data, data_hat, low_tolerance=True) + + def test_grad_zca_individual_transforms(self, device): + """Check if the gradients of the transforms are correct w.r.t to the input data.""" + if device.type == "mps": + pytest.skip("MPS does not support float64 required for gradcheck") + data = torch.tensor([[2, 0], [0, 1], [-2, 0], [0, -1]], device=device, dtype=torch.float64) + + def zca_T(x): + return kornia.enhance.zca_mean(x)[0] + + def zca_mu(x): + return kornia.enhance.zca_mean(x)[1] + + def zca_T_inv(x): + return kornia.enhance.zca_mean(x, return_inverse=True)[2] + + self.gradcheck(zca_T, (data,)) + self.gradcheck(zca_mu, (data,)) + self.gradcheck(zca_T_inv, (data,)) + + def test_grad_zca_with_fit(self, device): + if device.type == "mps": + pytest.skip("MPS does not support float64 required for gradcheck") + data = torch.tensor([[2, 0], [0, 1], [-2, 0], [0, -1]], device=device, dtype=torch.float64) + + def zca_fit(x): + zca = kornia.enhance.ZCAWhitening(detach_transforms=False) + return zca(x, include_fit=True) + + self.gradcheck(zca_fit, (data,)) + + def test_grad_detach_zca(self, device): + if device.type == "mps": + pytest.skip("MPS does not support float64 required for gradcheck") + data = torch.tensor([[1, 0], [0, 1], [-2, 0], [0, -1]], device=device, dtype=torch.float64) + + zca = kornia.enhance.ZCAWhitening() + + zca.fit(data) + + self.gradcheck(zca, (data,)) + + def test_not_fitted(self, device, dtype): + with pytest.raises(RuntimeError): + data = torch.rand(10, 2, device=device, dtype=dtype) + + zca = kornia.enhance.ZCAWhitening() + zca(data) + + def test_not_fitted_inv(self, device, dtype): + with pytest.raises(RuntimeError): + data = torch.rand(10, 2, device=device, dtype=dtype) + + zca = kornia.enhance.ZCAWhitening() + zca.inverse_transform(data) + + def test_jit(self, device, dtype): + data = torch.rand(10, 3, 1, 2, device=device, dtype=dtype) + zca = kornia.enhance.ZCAWhitening().fit(data) + zca_jit = kornia.enhance.ZCAWhitening().fit(data) + zca_jit = torch.jit.script(zca_jit) + self.assert_close(zca_jit(data), zca(data)) + + @pytest.mark.parametrize("unbiased", [True, False]) + def test_zca_whiten_func_unbiased(self, unbiased, device, dtype): + data = torch.tensor([[0, 1], [1, 0], [-1, 0], [0, -1]], device=device, dtype=dtype) + + if unbiased: + unbiased_val = 1.5 + else: + unbiased_val = 2.0 + + expected = torch.sqrt(unbiased_val * torch.abs(data)) * torch.sign(data) + + actual = kornia.enhance.zca_whiten(data, unbiased=unbiased) + + self.assert_close(actual, expected, low_tolerance=True) diff --git a/tests/feature/test_affine_shape_estimator.py b/tests/feature/test_affine_shape_estimator.py new file mode 100644 index 0000000..45e891e --- /dev/null +++ b/tests/feature/test_affine_shape_estimator.py @@ -0,0 +1,167 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.feature.affine_shape import LAFAffineShapeEstimator, LAFAffNetShapeEstimator, PatchAffineShapeEstimator + +from testing.base import BaseTester + + +class TestPatchAffineShapeEstimator(BaseTester): + def test_shape(self, device): + inp = torch.rand(1, 1, 32, 32, device=device) + ori = PatchAffineShapeEstimator(32).to(device) + ang = ori(inp) + assert ang.shape == torch.Size([1, 1, 3]) + + def test_shape_batch(self, device): + inp = torch.rand(2, 1, 32, 32, device=device) + ori = PatchAffineShapeEstimator(32).to(device) + ang = ori(inp) + assert ang.shape == torch.Size([2, 1, 3]) + + def test_print(self, device): + sift = PatchAffineShapeEstimator(32) + sift.__repr__() + + def test_toy(self, device): + aff = PatchAffineShapeEstimator(19).to(device) + inp = torch.zeros(1, 1, 19, 19, device=device) + inp[:, :, 5:-5, 1:-1] = 1 + abc = aff(inp) + expected = torch.tensor([[[0.4146, 0.0000, 1.0000]]], device=device) + self.assert_close(abc, expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 1, 13, 13 + ori = PatchAffineShapeEstimator(width).to(device) + patches = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(ori, (patches,), nondet_tol=1e-4) + + +class TestLAFAffineShapeEstimator(BaseTester): + def test_shape(self, device): + inp = torch.rand(1, 1, 32, 32, device=device) + laf = torch.rand(1, 1, 2, 3, device=device) + ori = LAFAffineShapeEstimator().to(device) + out = ori(laf, inp) + assert out.shape == laf.shape + + def test_shape_batch(self, device): + inp = torch.rand(2, 1, 32, 32, device=device) + laf = torch.rand(2, 34, 2, 3, device=device) + ori = LAFAffineShapeEstimator().to(device) + out = ori(laf, inp) + assert out.shape == laf.shape + + def test_print(self, device): + sift = LAFAffineShapeEstimator() + sift.__repr__() + + def test_toy(self, device, dtype): + aff = LAFAffineShapeEstimator(32, preserve_orientation=False).to(device, dtype) + inp = torch.zeros(1, 1, 32, 32, device=device, dtype=dtype) + inp[:, :, 15:-15, 9:-9] = 1 + laf = torch.tensor([[[[20.0, 0.0, 16.0], [0.0, 20.0, 16.0]]]], device=device, dtype=dtype) + new_laf = aff(laf, inp) + expected = torch.tensor([[[[35.078, 0.0, 16.0], [0.0, 11.403, 16.0]]]], device=device, dtype=dtype) + self.assert_close(new_laf, expected, atol=1e-4, rtol=1e-4) + + def test_toy_preserve(self, device, dtype): + aff = LAFAffineShapeEstimator(32, preserve_orientation=True).to(device, dtype) + inp = torch.zeros(1, 1, 32, 32, device=device, dtype=dtype) + inp[:, :, 15:-15, 9:-9] = 1 + laf = torch.tensor([[[[0.0, 20.0, 16.0], [-20.0, 0.0, 16.0]]]], device=device, dtype=dtype) + new_laf = aff(laf, inp) + expected = torch.tensor([[[[0.0, 35.078, 16.0], [-11.403, 0, 16.0]]]], device=device, dtype=dtype) + self.assert_close(new_laf, expected, atol=1e-4, rtol=1e-4) + + def test_toy_not_preserve(self, device): + aff = LAFAffineShapeEstimator(32, preserve_orientation=False).to(device) + inp = torch.zeros(1, 1, 32, 32, device=device) + inp[:, :, 15:-15, 9:-9] = 1 + laf = torch.tensor([[[[0.0, 20.0, 16.0], [-20.0, 0.0, 16.0]]]], device=device) + new_laf = aff(laf, inp) + expected = torch.tensor([[[[35.078, 0, 16.0], [0, 11.403, 16.0]]]], device=device) + self.assert_close(new_laf, expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 1, 40, 40 + patches = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + laf = torch.tensor([[[[5.0, 0.0, 26.0], [0.0, 5.0, 26.0]]]], device=device, dtype=torch.float64) + self.gradcheck( + LAFAffineShapeEstimator(11).to(device), + (laf, patches), + rtol=1e-3, + atol=1e-3, + nondet_tol=1e-4, + ) + + +class TestLAFAffNetShapeEstimator(BaseTester): + def test_shape(self, device): + inp = torch.rand(1, 1, 32, 32, device=device) + laf = torch.rand(1, 1, 2, 3, device=device) + ori = LAFAffNetShapeEstimator(False).to(device).eval() + out = ori(laf, inp) + assert out.shape == laf.shape + + @pytest.mark.slow + def test_pretrained(self, device): + inp = torch.rand(1, 1, 32, 32, device=device) + laf = torch.rand(1, 1, 2, 3, device=device) + ori = LAFAffNetShapeEstimator(True).to(device).eval() + out = ori(laf, inp) + assert out.shape == laf.shape + + def test_shape_batch(self, device): + inp = torch.rand(2, 1, 32, 32, device=device) + laf = torch.rand(2, 5, 2, 3, device=device) + ori = LAFAffNetShapeEstimator().to(device).eval() + out = ori(laf, inp) + assert out.shape == laf.shape + + def test_print(self, device): + sift = LAFAffNetShapeEstimator() + sift.__repr__() + + def test_toy(self, device, dtype): + aff = LAFAffNetShapeEstimator(True).to(device, dtype).eval() + inp = torch.zeros(1, 1, 32, 32, device=device, dtype=dtype) + inp[:, :, 15:-15, 9:-9] = 1 + laf = torch.tensor([[[[20.0, 0.0, 16.0], [0.0, 20.0, 16.0]]]], device=device, dtype=dtype) + new_laf = aff(laf, inp) + expected = torch.tensor([[[[33.2073, 0.0, 16.0], [-1.3766, 12.0456, 16.0]]]], device=device, dtype=dtype) + atol = 5e-3 if (device.type == "cuda" and dtype == torch.float32) else 1e-4 + self.assert_close(new_laf, expected, atol=atol, rtol=1e-4) + + @pytest.mark.slow + def test_gradcheck(self, device): + torch.manual_seed(0) + batch_size, channels, height, width = 1, 1, 35, 35 + patches = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + laf = torch.tensor([[[[8.0, 0.0, 16.0], [0.0, 8.0, 16.0]]]], device=device, dtype=torch.float64) + self.gradcheck( + LAFAffNetShapeEstimator(True).to(device, dtype=patches.dtype), + (laf, patches), + requires_grad=[False, True], + rtol=1e-3, + atol=1e-3, + nondet_tol=1e-3, + ) diff --git a/tests/feature/test_aliked.py b/tests/feature/test_aliked.py new file mode 100644 index 0000000..3e3d399 --- /dev/null +++ b/tests/feature/test_aliked.py @@ -0,0 +1,319 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.feature.aliked import ALIKED, ALIKEDFeatures +from kornia.feature.aliked.deform_conv2d import deform_conv2d + +from testing.base import BaseTester + + +def _has_torchvision() -> bool: + try: + import torchvision.ops # noqa: F401 + + return True + except ImportError: + return False + + +# --------------------------------------------------------------------------- +# deform_conv2d tests +# --------------------------------------------------------------------------- + + +class TestDeformConv2d: + """Tests for the pure-PyTorch deform_conv2d implementation.""" + + @pytest.mark.parametrize("dtype", [torch.float32, torch.float64]) + def test_zero_offset_matches_regular_conv(self, device, dtype): + """With all-zero offsets, deform_conv2d should equal regular conv2d.""" + if device.type == "mps" and dtype == torch.float64: + pytest.skip("MPS does not support float64") + B, C_in, H, W = 1, 4, 8, 8 + C_out, kH, kW = 8, 3, 3 + K = kH * kW + + x = torch.randn(B, C_in, H, W, device=device, dtype=dtype) + weight = torch.randn(C_out, C_in, kH, kW, device=device, dtype=dtype) + offset = torch.zeros(B, 2 * K, H, W, device=device, dtype=dtype) + + out_dcn = deform_conv2d(x, offset, weight, padding=1) + out_reg = torch.nn.functional.conv2d(x, weight, padding=1) + + assert out_dcn.shape == out_reg.shape + assert torch.allclose(out_dcn, out_reg, atol=1e-5) + + @pytest.mark.parametrize("dtype", [torch.float32]) + def test_output_shape(self, device, dtype): + """Check output spatial dimensions for various settings.""" + B, C_in, H, W = 2, 3, 16, 16 + C_out, kH, kW = 6, 3, 3 + K = kH * kW + padding, stride = 1, 1 + + H_out = (H + 2 * padding - kH) // stride + 1 + W_out = (W + 2 * padding - kW) // stride + 1 + + x = torch.randn(B, C_in, H, W, device=device, dtype=dtype) + weight = torch.randn(C_out, C_in, kH, kW, device=device, dtype=dtype) + offset = torch.zeros(B, 2 * K, H_out, W_out, device=device, dtype=dtype) + + out = deform_conv2d(x, offset, weight, padding=padding, stride=stride) + assert out.shape == (B, C_out, H_out, W_out) + + @pytest.mark.skipif(not _has_torchvision(), reason="torchvision not installed") + @pytest.mark.parametrize("dtype", [torch.float32]) + @pytest.mark.parametrize("use_mask", [False, True]) + def test_matches_torchvision(self, device, dtype, use_mask): + """Pure-PyTorch implementation should match torchvision reference.""" + import torchvision.ops as tvops + + B, C_in, H, W = 2, 4, 10, 10 + C_out, kH, kW = 8, 3, 3 + K = kH * kW + padding = 1 + + H_out = H # padding=1, stride=1, 3x3 kernel + W_out = W + + torch.manual_seed(0) + x = torch.randn(B, C_in, H, W, device=device, dtype=dtype) + weight = torch.randn(C_out, C_in, kH, kW, device=device, dtype=dtype) + bias = torch.randn(C_out, device=device, dtype=dtype) + offset = torch.randn(B, 2 * K, H_out, W_out, device=device, dtype=dtype) * 0.5 + mask = torch.rand(B, K, H_out, W_out, device=device, dtype=dtype) if use_mask else None + + out_ours = deform_conv2d(x, offset, weight, bias=bias, padding=padding, mask=mask) + out_tv = tvops.deform_conv2d(x, offset, weight, bias=bias, padding=padding, mask=mask) + + assert torch.allclose(out_ours, out_tv, atol=1e-4), f"Max diff: {(out_ours - out_tv).abs().max():.2e}" + + def test_gradients(self, device): + """Gradients should flow through the pure-PyTorch implementation.""" + B, C_in, H, W = 1, 2, 6, 6 + C_out, kH, kW = 4, 3, 3 + K = kH * kW + + x = torch.randn(B, C_in, H, W, device=device, requires_grad=True) + weight = torch.randn(C_out, C_in, kH, kW, device=device, requires_grad=True) + offset = (torch.randn(B, 2 * K, H, W, device=device) * 0.1).requires_grad_(True) + + out = deform_conv2d(x, offset, weight, padding=1) + out.sum().backward() + assert x.grad is not None + assert weight.grad is not None + assert offset.grad is not None + + +# --------------------------------------------------------------------------- +# ALIKED tests +# --------------------------------------------------------------------------- + + +class TestALIKED(BaseTester): + def test_smoke(self, dtype, device): + aliked = ALIKED().to(device, dtype) + inp = torch.rand(1, 3, 64, 64, device=device, dtype=dtype) + output = aliked(inp) + assert isinstance(output, list) + assert len(output) == 1 + assert isinstance(output[0], ALIKEDFeatures) + + def test_smoke_batch(self, dtype, device): + aliked = ALIKED().to(device, dtype) + inp = torch.rand(2, 3, 64, 64, device=device, dtype=dtype) + output = aliked(inp) + assert len(output) == 2 + assert all(isinstance(f, ALIKEDFeatures) for f in output) + + def test_grayscale_input(self, dtype, device): + """1-channel input should be broadcast to 3 channels automatically.""" + aliked = ALIKED().to(device, dtype) + inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype) + output = aliked(inp) + assert isinstance(output[0], ALIKEDFeatures) + + def test_output_shapes(self, dtype, device): + """Keypoints should be (N,2), descriptors (N,D), scores (N,).""" + aliked = ALIKED(model_name="aliked-t16").to(device, dtype) + inp = torch.rand(1, 3, 64, 64, device=device, dtype=dtype) + output = aliked(inp) + feat = output[0] + N = feat.n + assert feat.keypoints.shape == (N, 2) + assert feat.descriptors.shape[0] == N + assert feat.keypoint_scores.shape == (N,) + + def test_descriptor_normalized(self, dtype, device): + """Descriptors should be L2-normalised.""" + aliked = ALIKED(model_name="aliked-t16").to(device, dtype) + inp = torch.rand(1, 3, 64, 64, device=device, dtype=dtype) + output = aliked(inp) + feat = output[0] + if feat.n > 0: + norms = feat.descriptors.norm(dim=-1) + assert torch.allclose(norms, torch.ones_like(norms), atol=1e-4) + + def test_keypoints_in_image(self, dtype, device): + """Keypoints should lie within the image boundaries.""" + H, W = 64, 64 + aliked = ALIKED(model_name="aliked-t16").to(device, dtype) + inp = torch.rand(1, 3, H, W, device=device, dtype=dtype) + output = aliked(inp) + feat = output[0] + if feat.n > 0: + assert (feat.keypoints[:, 0] >= 0).all() + assert (feat.keypoints[:, 0] <= W - 1).all() + assert (feat.keypoints[:, 1] >= 0).all() + assert (feat.keypoints[:, 1] <= H - 1).all() + + @pytest.mark.parametrize("model_name", ["aliked-t16", "aliked-n16", "aliked-n32"]) + def test_model_configs(self, device, model_name): + """All model configurations should run without error.""" + aliked = ALIKED(model_name=model_name).to(device) + inp = torch.rand(1, 3, 64, 64, device=device) + output = aliked(inp) + assert len(output) == 1 + + def test_aliked_features_to(self, device): + """ALIKEDFeatures.to() should move all tensors.""" + feat = ALIKEDFeatures( + keypoints=torch.rand(10, 2), + descriptors=torch.rand(10, 64), + keypoint_scores=torch.rand(10), + ) + feat_dev = feat.to(device) + assert feat_dev.keypoints.device.type == device.type + assert feat_dev.descriptors.device.type == device.type + assert feat_dev.keypoint_scores.device.type == device.type + + @pytest.mark.slow + @pytest.mark.parametrize("model_name", ["aliked-n16"]) + def test_pretrained(self, device, model_name): + """Pretrained model should load and produce reasonable detections.""" + aliked = ALIKED.from_pretrained(model_name=model_name, device=device) + inp = torch.rand(1, 3, 256, 256, device=device) + with torch.no_grad(): + output = aliked(inp) + assert len(output) == 1 + assert output[0].n > 0 + + def test_gradcheck(self, device): + """gradcheck on the fully-differentiable extract_dense_map sub-graph. + + The full pipeline includes discrete NMS/argmax steps that are not + differentiable, so gradcheck is run on extract_dense_map which covers + the backbone, feature pyramid, and score head. The model is placed + in eval mode so BatchNorm uses fixed running statistics. + """ + aliked = ALIKED(model_name="aliked-t16").to(device, torch.float64).eval() + inp = torch.rand(1, 3, 32, 32, device=device, dtype=torch.float64, requires_grad=True) + + def fn(x: torch.Tensor) -> torch.Tensor: + fm, sm = aliked.extract_dense_map(x) + return fm, sm + + assert torch.autograd.gradcheck(fn, [inp], eps=1e-4, atol=1e-3, rtol=1e-3, fast_mode=True, nondet_tol=1e-3) + + +# --------------------------------------------------------------------------- +# forward_laf tests +# --------------------------------------------------------------------------- + + +class TestALIKEDForwardLAF(BaseTester): + """Tests for ALIKED.forward_laf returning kornia LAF-format outputs.""" + + def test_smoke(self, device, dtype): + """forward_laf should return the three expected tensors.""" + aliked = ALIKED(model_name="aliked-t16", max_num_keypoints=32).to(device, dtype) + inp = torch.rand(1, 3, 64, 64, device=device, dtype=dtype) + lafs, responses, descs = aliked.forward_laf(inp, compute_affine=False) + assert lafs.ndim == 4 # (B, N, 2, 3) + assert responses.ndim == 3 # (B, N, 1) + assert descs.ndim == 3 # (B, N, D) + + def test_output_shapes(self, device, dtype): + """Shapes should be (B, N, 2, 3), (B, N, 1), (B, N, D).""" + B, H, W = 2, 64, 64 + top_k = 20 + aliked = ALIKED(model_name="aliked-t16", max_num_keypoints=top_k).to(device, dtype) + inp = torch.rand(B, 3, H, W, device=device, dtype=dtype) + lafs, responses, descs = aliked.forward_laf(inp, compute_affine=False) + N = lafs.shape[1] + assert lafs.shape == (B, N, 2, 3) + assert responses.shape == (B, N, 1) + assert descs.shape[0] == B + assert descs.shape[1] == N + + def test_laf_center_in_image(self, device, dtype): + """LAF centres (col 2) should lie within image bounds.""" + H, W = 64, 64 + top_k = 30 + aliked = ALIKED(model_name="aliked-t16", max_num_keypoints=top_k).to(device, dtype) + inp = torch.rand(1, 3, H, W, device=device, dtype=dtype) + lafs, responses, _ = aliked.forward_laf(inp, compute_affine=False) + # Only check keypoints with positive response (non-padding). + valid = responses[0, :, 0] > 0 + if valid.any(): + centers = lafs[0, valid, :, 2] # (N_valid, 2) — [x, y] + assert (centers[:, 0] >= 0).all() + assert (centers[:, 0] <= W - 1).all() + assert (centers[:, 1] >= 0).all() + assert (centers[:, 1] <= H - 1).all() + + def test_batch_padding(self, device, dtype): + """All images in the batch should share the same N (padded to max).""" + B = 3 + aliked = ALIKED(model_name="aliked-t16", max_num_keypoints=25).to(device, dtype) + inp = torch.rand(B, 3, 64, 64, device=device, dtype=dtype) + lafs, responses, descs = aliked.forward_laf(inp, compute_affine=False) + assert lafs.shape[0] == B + assert responses.shape[0] == B + assert descs.shape[0] == B + + def test_laf_with_mask(self, device, dtype): + """Providing a mask should suppress keypoints in the masked region.""" + H, W = 64, 64 + aliked = ALIKED(model_name="aliked-t16", max_num_keypoints=40).to(device, dtype) + inp = torch.rand(1, 3, H, W, device=device, dtype=dtype) + # Suppress the entire image except the bottom-right quarter. + mask = torch.zeros(1, 1, H, W, device=device, dtype=dtype) + mask[:, :, H // 2 :, W // 2 :] = 1.0 + lafs_masked, _, _ = aliked.forward_laf(inp, mask=mask, compute_affine=False) + assert lafs_masked.shape[0] == 1 + + @pytest.mark.slow + def test_laf_affine_shape(self, device, dtype): + """The 2x2 affine part of each LAF should be non-degenerate for valid kpts. + + Requires ``--runslow``: uses ``torch.linalg.eigh`` (cast to float32 for + half-precision dtypes) which is slower than the identity-affine path. + """ + top_k = 20 + aliked = ALIKED(model_name="aliked-t16", max_num_keypoints=top_k).to(device, dtype) + inp = torch.rand(1, 3, 64, 64, device=device, dtype=dtype) + lafs, responses, _ = aliked.forward_laf(inp, compute_affine=True) + valid = responses[0, :, 0] > 0 + if valid.any(): + A = lafs[0, valid, :, :2] # (N_valid, 2, 2) + # det(A) should be non-zero (non-degenerate) + dets = torch.det(A) + assert (dets.abs() > 1e-6).all() diff --git a/tests/feature/test_dedode.py b/tests/feature/test_dedode.py new file mode 100644 index 0000000..1519dfc --- /dev/null +++ b/tests/feature/test_dedode.py @@ -0,0 +1,41 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.feature.dedode import DeDoDe + + +@pytest.mark.skip(reason="DeDoDe is ummaintained") +class TestDeDoDe: + @pytest.mark.slow + @pytest.mark.parametrize("descriptor_model", ["B", "G"]) + @pytest.mark.parametrize("detector_model", ["L"]) + def test_smoke(self, dtype, device, descriptor_model, detector_model): + if "G" in descriptor_model and device.type != "cuda" and dtype == torch.float16: + pytest.skip('G descriptors do not support no cuda device. "LayerNormKernelImpl" not implemented for `Half`') + dedode = DeDoDe(descriptor_model=descriptor_model, detector_model=detector_model, amp_dtype=dtype).to( + device, dtype + ) + shape = (2, 3, 128, 128) + n = 1000 + inp = torch.randn(*shape, device=device, dtype=dtype) + keypoints, scores, descriptions = dedode(inp, n=n) + assert keypoints.shape == (shape[0], n, 2) + assert scores.shape == (shape[0], n) + assert descriptions.shape == (shape[0], n, 256) diff --git a/tests/feature/test_defmo.py b/tests/feature/test_defmo.py new file mode 100644 index 0000000..a9da67b --- /dev/null +++ b/tests/feature/test_defmo.py @@ -0,0 +1,58 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.feature import DeFMO + +from testing.base import BaseTester + + +class TestDeFMO(BaseTester): + @pytest.mark.slow + def test_shape(self, device, dtype): + inp = torch.ones(1, 6, 128, 160, device=device, dtype=dtype) + defmo = DeFMO().to(device, dtype) + defmo.eval() # batchnorm with size 1 is not allowed in train mode + out = defmo(inp) + assert out.shape == (1, 24, 4, 128, 160) + + @pytest.mark.slow + def test_shape_batch(self, device, dtype): + inp = torch.ones(2, 6, 128, 160, device=device, dtype=dtype) + defmo = DeFMO().to(device, dtype) + out = defmo(inp) + with torch.no_grad(): + assert out.shape == (2, 24, 4, 128, 160) + + @pytest.mark.slow + @pytest.mark.grad + def test_gradcheck(self, device): + patches = torch.rand(2, 6, 64, 64, device=device, dtype=torch.float64) + defmo = DeFMO().to(patches.device, patches.dtype) + self.gradcheck(defmo, (patches,), eps=1e-4, atol=1e-4, nondet_tol=1e-8) + + @pytest.mark.slow + @pytest.mark.jit + def test_jit(self, device, dtype): + B, C, H, W = 1, 6, 128, 160 + patches = torch.rand(B, C, H, W, device=device, dtype=dtype) + model = DeFMO(True).to(patches.device, patches.dtype).eval() + model_jit = torch.jit.script(DeFMO(True).to(patches.device, patches.dtype).eval()) + with torch.no_grad(): + self.assert_close(model(patches), model_jit(patches)) diff --git a/tests/feature/test_disk.py b/tests/feature/test_disk.py new file mode 100644 index 0000000..06341ca --- /dev/null +++ b/tests/feature/test_disk.py @@ -0,0 +1,106 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys + +import pytest +import torch + +from kornia.feature.disk import DISK, DISKFeatures + +from testing.base import BaseTester +from testing.casts import dict_to + + +class TestDisk(BaseTester): + def test_smoke(self, dtype, device): + disk = DISK().to(device, dtype) + inp = torch.ones(1, 3, 64, 64, device=device, dtype=dtype) + output = disk(inp) + assert isinstance(output, list) + assert len(output) == 1 + assert all(isinstance(e, DISKFeatures) for e in output) + + def test_smoke_n_detections(self, dtype, device): + """Unless we give it an actual image and use pretrained weights, we can't expect the number of detections + to really match the limit. + """ + disk = DISK().to(device, dtype) + inp = torch.ones(1, 3, 64, 64, device=device, dtype=dtype) + output = disk(inp, n=100) + assert isinstance(output, list) + assert len(output) == 1 + assert all(isinstance(e, DISKFeatures) for e in output) + + @pytest.mark.slow + def test_smoke_pretrained(self, device): + disk = DISK.from_pretrained(checkpoint="depth", device=device) + inp = torch.ones(1, 3, 64, 64, device=device) + output = disk(inp) + assert isinstance(output, list) + assert len(output) == 1 + assert all(isinstance(e, DISKFeatures) for e in output) + + @pytest.mark.slow + @pytest.mark.skipif(sys.platform == "win32", reason="this test takes so much memory in the CI with Windows") + @pytest.mark.parametrize("data", ["disk_outdoor"], indirect=True) + def test_pretrained_outdoor(self, device, dtype, data): + disk = DISK.from_pretrained(checkpoint="depth", device=device).to(dtype) + data_dev = dict_to(data, device, dtype) + num_feat = 256 + with torch.no_grad(): + out = disk(data_dev["img1"], num_feat) + if device.type != "cpu": + pytest.skip("Reference keypoints were computed on CPU; NMS outcomes differ on non-CPU devices") + self.assert_close(out[0].keypoints, data_dev["disk1"][0].keypoints.to(device=device, dtype=dtype)) + self.assert_close(out[0].descriptors, data_dev["disk1"][0].descriptors.to(device=device, dtype=dtype)) + + def test_heatmap_and_dense_descriptors(self, dtype, device): + disk = DISK().to(device, dtype) + inp = torch.ones(1, 3, 64, 64, device=device, dtype=dtype) + heatmaps, descriptors = disk.heatmap_and_dense_descriptors(inp) + + assert heatmaps.shape == (1, 1, 64, 64) + assert descriptors.shape == (1, 128, 64, 64) + assert heatmaps.dtype == dtype + assert descriptors.dtype == dtype + + def test_not_divisible_by_16(self, device): + disk = DISK().to(device) + inp = torch.ones(1, 3, 72, 64, device=device) + with pytest.raises(ValueError): + _ = disk(inp) + + _ = disk(inp, pad_if_not_divisible=True) + + inp = torch.ones(1, 3, 64, 72, device=device) + with pytest.raises(ValueError): + _ = disk(inp) + + _ = disk(inp, pad_if_not_divisible=True) + + inp = torch.ones(1, 3, 72, 72, device=device) + with pytest.raises(ValueError): + _ = disk(inp) + + _ = disk(inp, pad_if_not_divisible=True) + + def test_wrong_n_channels(self, device): + disk = DISK().to(device) + inp = torch.ones(1, 1, 64, 64, device=device) + with pytest.raises(ValueError): + _ = disk(inp) diff --git a/tests/feature/test_hardnet.py b/tests/feature/test_hardnet.py new file mode 100644 index 0000000..dbc4657 --- /dev/null +++ b/tests/feature/test_hardnet.py @@ -0,0 +1,82 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.feature import HardNet, HardNet8 + +from testing.base import BaseTester + + +class TestHardNet(BaseTester): + @pytest.mark.slow + def test_shape(self, device): + inp = torch.ones(1, 1, 32, 32, device=device) + hardnet = HardNet().to(device) + hardnet.eval() # batchnorm with size 1 is not allowed in train mode + out = hardnet(inp) + assert out.shape == (1, 128) + + @pytest.mark.slow + def test_shape_batch(self, device): + inp = torch.ones(16, 1, 32, 32, device=device) + hardnet = HardNet().to(device) + out = hardnet(inp) + assert out.shape == (16, 128) + + def test_gradcheck(self, device): + patches = torch.rand(2, 1, 32, 32, device=device, dtype=torch.float64) + hardnet = HardNet().to(patches.device, patches.dtype) + self.gradcheck(hardnet, (patches,), eps=1e-4, atol=1e-4, nondet_tol=1e-8) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 1, 32, 32 + patches = torch.ones(B, C, H, W, device=device, dtype=dtype) + model = HardNet().to(patches.device, patches.dtype).eval() + model_jit = torch.jit.script(HardNet().to(patches.device, patches.dtype).eval()) + self.assert_close(model(patches), model_jit(patches)) + + +class TestHardNet8(BaseTester): + def test_shape(self, device): + inp = torch.ones(1, 1, 32, 32, device=device) + hardnet = HardNet8().to(device) + hardnet.eval() # batchnorm with size 1 is not allowed in train mode + out = hardnet(inp) + assert out.shape == (1, 128) + + def test_shape_batch(self, device): + inp = torch.ones(16, 1, 32, 32, device=device) + hardnet = HardNet8().to(device) + out = hardnet(inp) + assert out.shape == (16, 128) + + @pytest.mark.skip("jacobian not well computed") + def test_gradcheck(self, device): + patches = torch.rand(2, 1, 32, 32, device=device, dtype=torch.float32) + hardnet = HardNet8().to(patches.device, patches.dtype) + self.gradcheck(hardnet, (patches,), eps=1e-4, atol=1e-4) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 1, 32, 32 + patches = torch.ones(B, C, H, W, device=device, dtype=dtype) + model = HardNet8().to(patches.device, patches.dtype).eval() + model_jit = torch.jit.script(HardNet8().to(patches.device, patches.dtype).eval()) + self.assert_close(model(patches), model_jit(patches)) diff --git a/tests/feature/test_hynet.py b/tests/feature/test_hynet.py new file mode 100644 index 0000000..5c4b5b7 --- /dev/null +++ b/tests/feature/test_hynet.py @@ -0,0 +1,50 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.feature import HyNet + +from testing.base import BaseTester + + +class TestHyNet(BaseTester): + def test_shape(self, device): + inp = torch.ones(1, 1, 32, 32, device=device) + hynet = HyNet().to(device) + out = hynet(inp) + assert out.shape == (1, 128) + + def test_shape_batch(self, device): + inp = torch.ones(16, 1, 32, 32, device=device) + hynet = HyNet().to(device) + out = hynet(inp) + assert out.shape == (16, 128) + + def test_gradcheck(self, device): + patches = torch.rand(2, 1, 32, 32, device=device, dtype=torch.float64) + hynet = HyNet().to(patches.device, patches.dtype) + self.gradcheck(hynet, (patches,), eps=1e-4, atol=1e-4, nondet_tol=1e-8) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 1, 32, 32 + patches = torch.rand(B, C, H, W, device=device, dtype=dtype) + model = HyNet().to(patches.device, patches.dtype).eval() + model_jit = torch.jit.script(model) + self.assert_close(model(patches), model_jit(patches)) diff --git a/tests/feature/test_integrated.py b/tests/feature/test_integrated.py new file mode 100644 index 0000000..351fa7d --- /dev/null +++ b/tests/feature/test_integrated.py @@ -0,0 +1,354 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys + +import pytest +import torch +from torch import nn + +import kornia +from kornia.core._compat import torch_version_le +from kornia.feature import ( + DescriptorMatcher, + GFTTAffNetHardNet, + KeyNetHardNet, + LAFDescriptor, + LocalFeature, + ScaleSpaceDetector, + SIFTDescriptor, + SIFTFeature, + extract_patches_from_pyramid, + get_laf_center, + get_laf_descriptors, + get_laf_orientation, + get_laf_scale, +) +from kornia.feature.integrated import LocalFeatureMatcher +from kornia.geometry import RANSAC, resize, transform_points + +from testing.base import BaseTester +from testing.casts import dict_to + + +class TestGetLAFDescriptors(BaseTester): + def test_same(self, device, dtype): + B, C, H, W = 1, 3, 64, 64 + PS = 16 + img = torch.rand(B, C, H, W, device=device, dtype=dtype) + img_gray = kornia.color.rgb_to_grayscale(img) + centers = torch.tensor([[H / 3.0, W / 3.0], [2.0 * H / 3.0, W / 2.0]], device=device, dtype=dtype).view(1, 2, 2) + scales = torch.tensor([(H + W) / 4.0, (H + W) / 8.0], device=device, dtype=dtype).view(1, 2, 1, 1) + ori = torch.tensor([0.0, 30.0], device=device, dtype=dtype).view(1, 2, 1) + lafs = kornia.feature.laf_from_center_scale_ori(centers, scales, ori) + sift = SIFTDescriptor(PS).to(device, dtype) + descs_test_from_rgb = get_laf_descriptors(img, lafs, sift, PS, True) + descs_test_from_gray = get_laf_descriptors(img_gray, lafs, sift, PS, True) + + patches = extract_patches_from_pyramid(img_gray, lafs, PS) + B1, N1, CH1, H1, W1 = patches.size() + # Descriptor accepts standard tensor [B, CH, H, W], while patches are [B, N, CH, H, W] shape + # So we need to reshape a bit :) + descs_reference = sift(patches.view(B1 * N1, CH1, H1, W1)).view(B1, N1, -1) + self.assert_close(descs_test_from_rgb, descs_reference) + self.assert_close(descs_test_from_gray, descs_reference) + + def test_gradcheck(self, device): + dtype = torch.float64 + B, C, H, W = 1, 1, 32, 32 + PS = 16 + img = torch.rand(B, C, H, W, device=device) + centers = torch.tensor([[H / 2.0, W / 2.0], [2.0 * H / 3.0, W / 2.0]], device=device, dtype=dtype).view(1, 2, 2) + scales = torch.tensor([(H + W) / 5.0, (H + W) / 6.0], device=device, dtype=dtype).view(1, 2, 1, 1) + ori = torch.tensor([0.0, 30.0], device=device, dtype=dtype).view(1, 2, 1) + lafs = kornia.feature.laf_from_center_scale_ori(centers, scales, ori) + + class _MeanPatch(nn.Module): + def forward(self, inputs): + return inputs.mean(dim=(2, 3)) + + desc = _MeanPatch() + self.gradcheck( + get_laf_descriptors, + (img, lafs, desc, PS, True), + eps=1e-3, + atol=1e-3, + nondet_tol=1e-3, + ) + + +class TestLAFDescriptor(BaseTester): + def test_same(self, device, dtype): + B, C, H, W = 1, 3, 64, 64 + PS = 16 + img = torch.rand(B, C, H, W, device=device, dtype=dtype) + img_gray = kornia.color.rgb_to_grayscale(img) + centers = torch.tensor([[H / 3.0, W / 3.0], [2.0 * H / 3.0, W / 2.0]], device=device, dtype=dtype).view(1, 2, 2) + scales = torch.tensor([(H + W) / 4.0, (H + W) / 8.0], device=device, dtype=dtype).view(1, 2, 1, 1) + ori = torch.tensor([0.0, 30.0], device=device, dtype=dtype).view(1, 2, 1) + lafs = kornia.feature.laf_from_center_scale_ori(centers, scales, ori) + sift = SIFTDescriptor(PS).to(device, dtype) + lafsift = LAFDescriptor(sift, PS) + descs_test = lafsift(img, lafs) + patches = extract_patches_from_pyramid(img_gray, lafs, PS) + B1, N1, CH1, H1, W1 = patches.size() + # Descriptor accepts standard tensor [B, CH, H, W], while patches are [B, N, CH, H, W] shape + # So we need to reshape a bit :) + descs_reference = sift(patches.view(B1 * N1, CH1, H1, W1)).view(B1, N1, -1) + self.assert_close(descs_test, descs_reference) + + def test_empty(self, device): + B, C, H, W = 1, 1, 32, 32 + PS = 16 + img = torch.rand(B, C, H, W, device=device) + lafs = torch.zeros(B, 0, 2, 3, device=device) + sift = SIFTDescriptor(PS).to(device) + lafsift = LAFDescriptor(sift, PS) + descs_test = lafsift(img, lafs) + assert descs_test.shape == (B, 0, 128) + + def test_gradcheck(self, device): + B, C, H, W = 1, 1, 32, 32 + PS = 16 + img = torch.rand(B, C, H, W, device=device) + centers = torch.tensor([[H / 2.0, W / 2.0], [2.0 * H / 3.0, W / 2.0]], device=device).view(1, 2, 2) + scales = torch.tensor([(H + W) / 5.0, (H + W) / 6.0], device=device).view(1, 2, 1, 1) + ori = torch.tensor([0.0, 30.0], device=device).view(1, 2, 1) + lafs = kornia.feature.laf_from_center_scale_ori(centers, scales, ori) + + class _MeanPatch(nn.Module): + def forward(self, inputs): + return inputs.mean(dim=(2, 3)) + + lafdesc = LAFDescriptor(_MeanPatch(), PS) + self.gradcheck(lafdesc, (img, lafs), eps=1e-3, atol=1e-3, nondet_tol=1e-3) + + +class TestLocalFeature(BaseTester): + def test_smoke(self, device, dtype): + det = ScaleSpaceDetector(10) + desc = SIFTDescriptor(32) + local_feature = LocalFeature(det, desc).to(device, dtype) + assert local_feature is not None + + def test_same(self, device, dtype): + B, C, H, W = 1, 1, 64, 64 + PS = 16 + img = torch.rand(B, C, H, W, device=device, dtype=dtype) + det = ScaleSpaceDetector(10) + desc = SIFTDescriptor(PS) + local_feature = LocalFeature(det, LAFDescriptor(desc, PS)).to(device, dtype) + lafs, responses, descs = local_feature(img) + lafs1, responses1 = det(img) + self.assert_close(lafs, lafs1) + self.assert_close(responses, responses1) + patches = extract_patches_from_pyramid(img, lafs1, PS) + B1, N1, CH1, H1, W1 = patches.size() + # Descriptor accepts standard tensor [B, CH, H, W], while patches are [B, N, CH, H, W] shape + # So we need to reshape a bit :) + descs1 = desc(patches.view(B1 * N1, CH1, H1, W1)).view(B1, N1, -1) + self.assert_close(descs, descs1) + + def test_scale(self, device, dtype): + B, C, H, W = 1, 1, 64, 64 + PS = 16 + img = torch.rand(B, C, H, W, device=device, dtype=dtype) + det = ScaleSpaceDetector(10) + desc = SIFTDescriptor(PS) + local_feature = LocalFeature(det, LAFDescriptor(desc, PS), 1.0).to(device, dtype) + local_feature2 = LocalFeature(det, LAFDescriptor(desc, PS), 2.0).to(device, dtype) + lafs, _responses, _descs = local_feature(img) + lafs2, _responses2, _descs2 = local_feature2(img) + self.assert_close(get_laf_center(lafs), get_laf_center(lafs2)) + self.assert_close(get_laf_orientation(lafs), get_laf_orientation(lafs2)) + self.assert_close(2.0 * get_laf_scale(lafs), get_laf_scale(lafs2)) + + def test_gradcheck(self, device): + B, C, H, W = 1, 1, 32, 32 + PS = 16 + img = torch.rand(B, C, H, W, device=device, dtype=torch.float64) + local_feature = LocalFeature(ScaleSpaceDetector(2), LAFDescriptor(SIFTDescriptor(PS), PS)).to(device, img.dtype) + self.gradcheck(local_feature, img, eps=1e-4, atol=1e-4, nondet_tol=1e-8) + + +class TestSIFTFeature(BaseTester): + # The real test is in TestLocalFeatureMatcher + def test_smoke(self, device, dtype): + sift = SIFTFeature() + assert sift is not None + + @pytest.mark.skip("jacobian not well computed") + def test_gradcheck(self, device): + B, C, H, W = 1, 1, 32, 32 + img = torch.rand(B, C, H, W, device=device, dtype=torch.float64) + local_feature = SIFTFeature(2, True).to(device) + self.gradcheck(local_feature, img, eps=1e-4, atol=1e-4, fast_mode=False) + + +class TestKeyNetHardNetFeature(BaseTester): + # The real test is in TestLocalFeatureMatcher + def test_smoke(self, device, dtype): + sift = KeyNetHardNet(2).to(device, dtype) + B, C, H, W = 1, 1, 32, 32 + img = torch.rand(B, C, H, W, device=device, dtype=dtype) + out = sift(img) + assert out is not None + + @pytest.mark.skip("jacobian not well computed") + def test_gradcheck(self, device): + B, C, H, W = 1, 1, 32, 32 + img = torch.rand(B, C, H, W, device=device, dtype=torch.float64) + local_feature = KeyNetHardNet(2, True).to(device).to(device) + self.gradcheck(local_feature, img, eps=1e-4, atol=1e-4, fast_mode=False) + + +class TestGFTTAffNetHardNet(BaseTester): + # The real test is in TestLocalFeatureMatcher + def test_smoke(self, device, dtype): + feat = GFTTAffNetHardNet().to(device, dtype) + assert feat is not None + + @pytest.mark.skip("jacobian not well computed") + def test_gradcheck(self, device): + B, C, H, W = 1, 1, 32, 32 + img = torch.rand(B, C, H, W, device=device, dtype=torch.float64) + local_feature = GFTTAffNetHardNet(2, True).to(device, img.dtype) + self.gradcheck(local_feature, img, eps=1e-4, atol=1e-4, fast_mode=False) + + +class TestLocalFeatureMatcher(BaseTester): + def test_smoke(self, device): + matcher = LocalFeatureMatcher(SIFTFeature(5), DescriptorMatcher("snn", 0.8)).to(device) + assert matcher is not None + + @pytest.mark.slow + @pytest.mark.parametrize("data", ["loftr_homo"], indirect=True) + def test_nomatch(self, device, dtype, data): + matcher = LocalFeatureMatcher(GFTTAffNetHardNet(100), DescriptorMatcher("snn", 0.8)).to(device, dtype) + data_dev = dict_to(data, device, dtype) + with torch.no_grad(): + out = matcher({"image0": data_dev["image0"], "image1": 0 * data_dev["image0"]}) + assert len(out["keypoints0"]) == 0 + + @pytest.mark.skip("Takes too long time (but works)") + def test_gradcheck(self, device): + matcher = LocalFeatureMatcher(SIFTFeature(5), DescriptorMatcher("nn", 1.0)).to(device) + patches = torch.rand(1, 1, 32, 32, device=device, dtype=torch.float64) + patches05 = resize(patches, (48, 48)) + + def proxy_forward(x, y): + return matcher({"image0": x, "image1": y})["keypoints0"] + + self.gradcheck(proxy_forward, (patches, patches05), eps=1e-4, atol=1e-4) + + @pytest.mark.slow + @pytest.mark.skipif(torch_version_le(1, 9, 1), reason="Fails for bached torch.linalg.solve") + @pytest.mark.parametrize("data", ["loftr_homo"], indirect=True) + def test_real_sift(self, device, dtype, data): + torch.random.manual_seed(0) + # This is not unit test, but that is quite good integration test + matcher = LocalFeatureMatcher(SIFTFeature(1000), DescriptorMatcher("snn", 0.8)).to(device, dtype) + ransac = RANSAC("homography", 1.0, 1024, 5).to(device, dtype) + data_dev = dict_to(data, device, dtype) + pts_src = data_dev["pts0"] + pts_dst = data_dev["pts1"] + with torch.no_grad(): + out = matcher(data_dev) + homography, inliers = ransac(out["keypoints0"], out["keypoints1"]) + assert inliers.sum().item() > 50 # we have enough inliers + # Reprojection error of 5px is OK + self.assert_close(transform_points(homography[None], pts_src[None]), pts_dst[None], rtol=5e-2, atol=5) + + @pytest.mark.slow + @pytest.mark.skipif(torch_version_le(1, 9, 1), reason="Fails for bached torch.linalg.solve") + @pytest.mark.parametrize("data", ["loftr_homo"], indirect=True) + def test_real_sift_preextract(self, device, dtype, data): + torch.random.manual_seed(0) + # This is not unit test, but that is quite good integration test + feat = SIFTFeature(1000).to(device, dtype) + matcher = LocalFeatureMatcher(feat, DescriptorMatcher("snn", 0.8)).to(device) + ransac = RANSAC("homography", 1.0, 1024, 5).to(device, dtype) + data_dev = dict_to(data, device, dtype) + pts_src = data_dev["pts0"] + pts_dst = data_dev["pts1"] + + lafs, _, descs = feat(data_dev["image0"]) + data_dev["lafs0"] = lafs + data_dev["descriptors0"] = descs + + lafs2, _, descs2 = feat(data_dev["image1"]) + data_dev["lafs1"] = lafs2 + data_dev["descriptors1"] = descs2 + + with torch.no_grad(): + out = matcher(data_dev) + homography, inliers = ransac(out["keypoints0"], out["keypoints1"]) + assert inliers.sum().item() > 50 # we have enough inliers + # Reprojection error of 5px is OK + self.assert_close(transform_points(homography[None], pts_src[None]), pts_dst[None], rtol=5e-2, atol=5) + + @pytest.mark.slow + @pytest.mark.skipif(torch_version_le(1, 9, 1), reason="Fails for bached torch.linalg.solve") + @pytest.mark.skipif(sys.platform == "win32", reason="this test takes so much memory in the CI with Windows") + @pytest.mark.parametrize("data", ["loftr_homo"], indirect=True) + def test_real_gftt(self, device, dtype, data): + # This is not unit test, but that is quite good integration test + matcher = LocalFeatureMatcher(GFTTAffNetHardNet(1000), DescriptorMatcher("snn", 0.8)).to(device, dtype) + ransac = RANSAC("homography", 1.0, 1024, 5).to(device, dtype) + data_dev = dict_to(data, device, dtype) + pts_src = data_dev["pts0"] + pts_dst = data_dev["pts1"] + with torch.no_grad(): + torch.manual_seed(0) + out = matcher(data_dev) + homography, inliers = ransac(out["keypoints0"], out["keypoints1"]) + assert inliers.sum().item() > 50 # we have enough inliers + # Reprojection error of 5px is OK + self.assert_close(transform_points(homography[None], pts_src[None]), pts_dst[None], rtol=5e-2, atol=5) + + @pytest.mark.slow + @pytest.mark.skipif(torch_version_le(1, 9, 1), reason="Fails for bached torch.linalg.solve") + @pytest.mark.skipif(sys.platform == "win32", reason="this test takes so much memory in the CI with Windows") + @pytest.mark.parametrize("data", ["loftr_homo"], indirect=True) + def test_real_keynet(self, device, dtype, data): + torch.random.manual_seed(0) + # This is not unit test, but that is quite good integration test + matcher = LocalFeatureMatcher(KeyNetHardNet(500), DescriptorMatcher("snn", 0.9)).to(device, dtype) + ransac = RANSAC("homography", 1.0, 1024, 5).to(device, dtype) + data_dev = dict_to(data, device, dtype) + pts_src = data_dev["pts0"] + pts_dst = data_dev["pts1"] + with torch.no_grad(): + out = matcher(data_dev) + homography, inliers = ransac(out["keypoints0"], out["keypoints1"]) + assert inliers.sum().item() > 50 # we have enough inliers + # Reprojection error of 5px is OK + self.assert_close(transform_points(homography[None], pts_src[None]), pts_dst[None], rtol=5e-2, atol=5) + + @pytest.mark.skip("ScaleSpaceDetector now is not jittable") + def test_jit(self, device, dtype): + B, C, H, W = 1, 1, 32, 32 + patches = torch.rand(B, C, H, W, device=device, dtype=dtype) + patches2x = resize(patches, (48, 48)) + inputs = {"image0": patches, "image1": patches2x} + model = LocalFeatureMatcher(SIFTDescriptor(32), DescriptorMatcher("snn", 0.8)).to(device).eval() + model_jit = torch.jit.script(model) + + out = model(inputs) + out_jit = model_jit(inputs) + for k, v in out.items(): + self.assert_close(v, out_jit[k]) diff --git a/tests/feature/test_keynet.py b/tests/feature/test_keynet.py new file mode 100644 index 0000000..acadb95 --- /dev/null +++ b/tests/feature/test_keynet.py @@ -0,0 +1,41 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch + +from kornia.feature import KeyNet + +from testing.base import BaseTester + + +class TestKeyNet(BaseTester): + def test_shape(self, device, dtype): + inp = torch.rand(1, 1, 16, 16, device=device, dtype=dtype) + keynet = KeyNet().to(device, dtype) + out = keynet(inp) + assert out.shape == inp.shape + + def test_shape_batch(self, device, dtype): + inp = torch.ones(16, 1, 16, 16, device=device, dtype=dtype) + keynet = KeyNet().to(device, dtype) + out = keynet(inp) + assert out.shape == inp.shape + + def test_gradcheck(self, device): + patches = torch.rand(2, 1, 16, 16, device=device, dtype=torch.float64) + keynet = KeyNet().to(patches.device, patches.dtype) + self.gradcheck(keynet, (patches,), eps=1e-4, atol=1e-4, nondet_tol=1e-8) diff --git a/tests/feature/test_laf.py b/tests/feature/test_laf.py new file mode 100644 index 0000000..6acbbb2 --- /dev/null +++ b/tests/feature/test_laf.py @@ -0,0 +1,687 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia +import kornia.geometry.transform.imgwarp + +from testing.base import BaseTester +from testing.geometry.create import create_random_homography + + +class TestAngleToRotationMatrix(BaseTester): + def test_shape(self, device): + inp = torch.ones(1, 3, 4, 4).to(device) + rotmat = kornia.geometry.transform.imgwarp.angle_to_rotation_matrix(inp) + assert rotmat.shape == (1, 3, 4, 4, 2, 2) + + def test_angles(self, device): + ang_deg = torch.tensor([0, 90.0], device=device) + expected = torch.tensor([[[1.0, 0.0], [0.0, 1.0]], [[0, 1.0], [-1.0, 0]]], device=device) + rotmat = kornia.geometry.transform.imgwarp.angle_to_rotation_matrix(ang_deg) + self.assert_close(rotmat, expected) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 2, 5, 4 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.transform.imgwarp.angle_to_rotation_matrix, (img,)) + + @pytest.mark.jit() + @pytest.mark.skip("Problems with kornia.pi") + def test_jit(self, device, dtype): + B, C, H, W = 2, 1, 32, 32 + patches = torch.rand(B, C, H, W, device=device, dtype=dtype) + model = kornia.geometry.transform.imgwarp.angle_to_rotation_matrix + model_jit = torch.jit.script(kornia.geometry.transform.imgwarp.angle_to_rotation_matrix) + self.assert_close(model(patches), model_jit(patches)) + + +class TestGetLAFScale(BaseTester): + def test_shape(self, device): + inp = torch.ones(1, 3, 2, 3, device=device) + rotmat = kornia.feature.get_laf_scale(inp) + assert rotmat.shape == (1, 3, 1, 1) + + def test_scale(self, device): + inp = torch.tensor([[5.0, 1, 0], [1, 1, 0]], device=device).float() + inp = inp.view(1, 1, 2, 3) + expected = torch.tensor([[[[2]]]], device=device).float() + rotmat = kornia.feature.get_laf_scale(inp) + self.assert_close(rotmat, expected) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 2, 2, 3 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.feature.get_laf_scale, (img,)) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + batch_size, channels, height, width = 1, 2, 2, 3 + img = torch.rand(batch_size, channels, height, width, device=device) + model = kornia.feature.get_laf_scale + model_jit = torch.jit.script(kornia.feature.get_laf_scale) + self.assert_close(model(img), model_jit(img)) + + +class TestGetLAFCenter(BaseTester): + def test_shape(self, device): + inp = torch.ones(1, 3, 2, 3, device=device) + xy = kornia.feature.get_laf_center(inp) + assert xy.shape == (1, 3, 2) + + def test_center(self, device): + inp = torch.tensor([[5.0, 1, 2], [1, 1, 3]], device=device).float() + inp = inp.view(1, 1, 2, 3) + expected = torch.tensor([[[2, 3]]], device=device).float() + xy = kornia.feature.get_laf_center(inp) + self.assert_close(xy, expected) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 2, 2, 3 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.feature.get_laf_center, (img,)) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + batch_size, channels, height, width = 1, 2, 2, 3 + img = torch.rand(batch_size, channels, height, width, device=device) + model = kornia.feature.get_laf_center + model_jit = torch.jit.script(kornia.feature.get_laf_center) + self.assert_close(model(img), model_jit(img)) + + +class TestGetLAFOri(BaseTester): + def test_shape(self, device): + inp = torch.ones(1, 3, 2, 3, device=device) + ori = kornia.feature.get_laf_orientation(inp) + assert ori.shape == (1, 3, 1) + + def test_ori(self, device): + inp = torch.tensor([[1, 1, 2], [1, 1, 3]], device=device).float() + inp = inp.view(1, 1, 2, 3) + expected = torch.tensor([[[45.0]]], device=device).float() + angle = kornia.feature.get_laf_orientation(inp) + self.assert_close(angle, expected) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 2, 2, 3 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.feature.get_laf_orientation, (img,)) + + @pytest.mark.jit() + @pytest.mark.skip("Union") + def test_jit(self, device, dtype): + batch_size, channels, height, width = 1, 2, 2, 3 + img = torch.rand(batch_size, channels, height, width, device=device) + model = kornia.feature.get_laf_orientation + model_jit = torch.jit.script(kornia.feature.get_laf_orientation) + self.assert_close(model(img), model_jit(img)) + + +class TestScaleLAF(BaseTester): + def test_shape_float(self, device): + inp = torch.ones(7, 3, 2, 3, device=device).float() + scale = 23.0 + assert kornia.feature.scale_laf(inp, scale).shape == inp.shape + + def test_shape_tensor(self, device): + inp = torch.ones(7, 3, 2, 3, device=device).float() + scale = torch.zeros(7, 1, 1, 1, device=device).float() + assert kornia.feature.scale_laf(inp, scale).shape == inp.shape + + def test_scale(self, device): + inp = torch.tensor([[5.0, 1, 0.8], [1, 1, -4.0]], device=device).float() + inp = inp.view(1, 1, 2, 3) + scale = torch.tensor([[[[2.0]]]], device=device).float() + out = kornia.feature.scale_laf(inp, scale) + expected = torch.tensor([[[[10.0, 2, 0.8], [2, 2, -4.0]]]], device=device).float() + self.assert_close(out, expected) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 2, 2, 3 + laf = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + scale = torch.rand(batch_size, device=device, dtype=torch.float64) + self.gradcheck(kornia.feature.scale_laf, (laf, scale), atol=1e-4) + + @pytest.mark.jit() + @pytest.mark.skip("Union") + def test_jit(self, device, dtype): + batch_size, channels, height, width = 1, 2, 2, 3 + laf = torch.rand(batch_size, channels, height, width, device=device) + scale = torch.rand(batch_size, device=device) + model = kornia.feature.scale_laf + model_jit = torch.jit.script(kornia.feature.scale_laf) + self.assert_close(model(laf, scale), model_jit(laf, scale)) + + +class TestSetLAFOri(BaseTester): + def test_shape_tensor(self, device): + inp = torch.ones(7, 3, 2, 3, device=device).float() + ori = torch.ones(7, 3, 1, 1, device=device).float() + assert kornia.feature.set_laf_orientation(inp, ori).shape == inp.shape + + def test_ori(self, device): + inp = torch.tensor([[0.0, 5.0, 0.8], [-5.0, 0, -4.0]], device=device).float() + inp = inp.view(1, 1, 2, 3) + ori = torch.zeros(1, 1, 1, 1, device=device).float() + out = kornia.feature.set_laf_orientation(inp, ori) + expected = torch.tensor([[[[5.0, 0.0, 0.8], [0.0, 5.0, -4.0]]]], device=device).float() + self.assert_close(out, expected) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 2, 2, 3 + laf = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + ori = torch.rand(batch_size, channels, 1, 1, device=device, dtype=torch.float64) + self.gradcheck(kornia.feature.set_laf_orientation, (laf, ori), atol=1e-4) + + @pytest.mark.jit() + @pytest.mark.skip("Union") + def test_jit(self, device, dtype): + batch_size, channels, height, width = 1, 2, 2, 3 + laf = torch.rand(batch_size, channels, height, width, device=device) + ori = torch.rand(batch_size, channels, 1, 1, device=device) + model = kornia.feature.set_laf_orientation + model_jit = torch.jit.script(kornia.feature.set_laf_orientation) + self.assert_close(model(laf, ori), model_jit(laf, ori)) + + +class TestMakeUpright(BaseTester): + def test_shape(self, device): + inp = torch.ones(5, 3, 2, 3, device=device) + rotmat = kornia.feature.make_upright(inp) + assert rotmat.shape == (5, 3, 2, 3) + + def test_do_nothing(self, device): + inp = torch.tensor([[1, 0, 0], [0, 1, 0]], device=device).float() + inp = inp.view(1, 1, 2, 3) + expected = torch.tensor([[[[1, 0, 0], [0, 1, 0]]]], device=device).float() + laf = kornia.feature.make_upright(inp) + self.assert_close(laf, expected) + + def test_do_nothing_with_scalea(self, device): + inp = torch.tensor([[2, 0, 0], [0, 2, 0]], device=device).float() + inp = inp.view(1, 1, 2, 3) + expected = torch.tensor([[[[2, 0, 0], [0, 2, 0]]]], device=device).float() + laf = kornia.feature.make_upright(inp) + self.assert_close(laf, expected) + + def test_check_zeros(self, device): + inp = torch.rand(4, 5, 2, 3, device=device) + laf = kornia.feature.make_upright(inp) + must_be_zeros = laf[:, :, 0, 1] + self.assert_close(must_be_zeros, torch.zeros_like(must_be_zeros)) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 14, 2, 2, 3 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.feature.make_upright, (img,)) + + @pytest.mark.jit() + @pytest.mark.skip("Union") + def test_jit(self, device, dtype): + batch_size, channels, height, width = 1, 2, 2, 3 + img = torch.rand(batch_size, channels, height, width, device=device) + model = kornia.feature.make_upright + model_jit = torch.jit.script(kornia.feature.make_upright) + self.assert_close(model(img), model_jit(img)) + + +class TestELL2LAF(BaseTester): + def test_shape(self, device): + inp = torch.ones(5, 3, 5, device=device) + inp[:, :, 3] = 0 + rotmat = kornia.feature.ellipse_to_laf(inp) + assert rotmat.shape == (5, 3, 2, 3) + + def test_conversion(self, device): + inp = torch.tensor([[10, -20, 0.01, 0, 0.01]], device=device).float() + inp = inp.view(1, 1, 5) + expected = torch.tensor([[10, 0, 10.0], [0, 10, -20]], device=device).float() + expected = expected.view(1, 1, 2, 3) + laf = kornia.feature.ellipse_to_laf(inp) + self.assert_close(laf, expected) + + def test_gradcheck(self, device): + batch_size, channels, height = 1, 2, 5 + img = torch.rand(batch_size, channels, height, device=device, dtype=torch.float64).abs() + img[:, :, 2] = img[:, :, 3].abs() + 0.3 + img[:, :, 4] += 1.0 + # assure it is positive definite + self.gradcheck(kornia.feature.ellipse_to_laf, (img,)) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + batch_size, channels, height = 1, 2, 5 + img = torch.rand(batch_size, channels, height, device=device).abs() + img[:, :, 2] = img[:, :, 3].abs() + 0.3 + img[:, :, 4] += 1.0 + model = kornia.feature.ellipse_to_laf + model_jit = torch.jit.script(kornia.feature.ellipse_to_laf) + self.assert_close(model(img), model_jit(img)) + + +class TestNormalizeLAF(BaseTester): + def test_shape(self, device): + inp = torch.rand(5, 3, 2, 3) + img = torch.rand(5, 3, 10, 10) + assert inp.shape == kornia.feature.normalize_laf(inp, img).shape + + def test_roundtrip_non_square_wide(self, device, dtype): + # Wide image (W >> H): x/y coords are normalized differently from scale components. + # The normalize→denormalize round-trip must be exact. + laf = torch.tensor([[10.0, 0.0, 160.0], [0.0, 10.0, 60.0]], device=device, dtype=dtype).view(1, 1, 2, 3) + img = torch.zeros(1, 1, 120, 320, device=device, dtype=dtype) + laf_norm = kornia.feature.normalize_laf(laf, img) + laf_back = kornia.feature.denormalize_laf(laf_norm, img) + self.assert_close(laf_back, laf) + + def test_roundtrip_non_square_tall(self, device, dtype): + # Tall image (H >> W): verify round-trip in the opposite aspect ratio. + laf = torch.tensor([[5.0, 0.0, 40.0], [0.0, 5.0, 100.0]], device=device, dtype=dtype).view(1, 1, 2, 3) + img = torch.zeros(1, 1, 240, 80, device=device, dtype=dtype) + laf_norm = kornia.feature.normalize_laf(laf, img) + laf_back = kornia.feature.denormalize_laf(laf_norm, img) + self.assert_close(laf_back, laf) + + def test_conversion(self, device): + w, h = 9, 5 + laf = torch.tensor([[1, 0, 1], [0, 1, 1]]).float() + laf = laf.view(1, 1, 2, 3) + img = torch.rand(1, 3, h, w) + expected = torch.tensor([[[[0.25, 0, 0.125], [0, 0.25, 0.25]]]]).float() + lafn = kornia.feature.normalize_laf(laf, img) + self.assert_close(lafn, expected) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 2, 2, 3 + + laf = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + img = torch.rand(batch_size, 3, 10, 32, device=device, dtype=torch.float64) + self.gradcheck(kornia.feature.normalize_laf, (laf, img)) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + batch_size, channels, height, width = 1, 2, 2, 3 + + laf = torch.rand(batch_size, channels, height, width) + img = torch.rand(batch_size, 3, 10, 32) + model = kornia.feature.normalize_laf + model_jit = torch.jit.script(kornia.feature.normalize_laf) + self.assert_close(model(laf, img), model_jit(laf, img)) + + +class TestLAF2pts(BaseTester): + def test_shape(self, device): + inp = torch.rand(5, 3, 2, 3, device=device) + n_pts = 13 + assert kornia.feature.laf_to_boundary_points(inp, n_pts).shape == (5, 3, n_pts, 2) + + def test_conversion(self, device): + laf = torch.tensor([[1, 0, 1], [0, 1, 1]], device=device).float() + laf = laf.view(1, 1, 2, 3) + n_pts = 6 + expected = torch.tensor([[[[1, 1], [1, 2], [2, 1], [1, 0], [0, 1], [1, 2]]]], device=device).float() + pts = kornia.feature.laf_to_boundary_points(laf, n_pts) + self.assert_close(pts, expected) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 3, 2, 2, 3 + laf = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.feature.laf_to_boundary_points, (laf)) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + batch_size, channels, height, width = 3, 2, 2, 3 + laf = torch.rand(batch_size, channels, height, width, device=device) + model = kornia.feature.laf_to_boundary_points + model_jit = torch.jit.script(kornia.feature.laf_to_boundary_points) + self.assert_close(model(laf), model_jit(laf)) + + +class TestDenormalizeLAF(BaseTester): + def test_shape(self, device): + inp = torch.rand(5, 3, 2, 3, device=device) + img = torch.rand(5, 3, 10, 10, device=device) + assert inp.shape == kornia.feature.denormalize_laf(inp, img).shape + + def test_conversion(self, device): + w, h = 9, 5 + expected = torch.tensor([[1, 0, 1], [0, 1, 1]], device=device).float() + expected = expected.view(1, 1, 2, 3) + img = torch.rand(1, 3, h, w, device=device) + lafn = torch.tensor([[0.25, 0, 0.125], [0, 0.25, 0.25]], device=device).float() + laf = kornia.feature.denormalize_laf(lafn.view(1, 1, 2, 3), img) + self.assert_close(laf, expected) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 2, 2, 3 + + laf = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + img = torch.rand(batch_size, 3, 10, 32, device=device, dtype=torch.float64) + self.gradcheck(kornia.feature.denormalize_laf, (laf, img)) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + batch_size, channels, height, width = 1, 2, 2, 3 + + laf = torch.rand(batch_size, channels, height, width) + img = torch.rand(batch_size, 3, 10, 32) + model = kornia.feature.denormalize_laf + model_jit = torch.jit.script(kornia.feature.denormalize_laf) + self.assert_close(model(laf, img), model_jit(laf, img)) + + +class TestGenPatchGrid(BaseTester): + def test_shape(self, device): + laf = torch.rand(5, 3, 2, 3, device=device) + img = torch.rand(5, 3, 10, 10, device=device) + PS = 3 + from kornia.feature.laf import generate_patch_grid_from_normalized_LAF + + grid = generate_patch_grid_from_normalized_LAF(img, laf, PS) + assert grid.shape == (15, 3, 3, 2) + + def test_gradcheck(self, device): + laf = torch.rand(5, 3, 2, 3, device=device, dtype=torch.float64) + img = torch.rand(5, 3, 10, 10, device=device, dtype=torch.float64) + PS = 3 + from kornia.feature.laf import generate_patch_grid_from_normalized_LAF + + self.gradcheck(generate_patch_grid_from_normalized_LAF, (img, laf, PS)) + + +class TestExtractPatchesSimple(BaseTester): + def test_shape(self, device): + laf = torch.rand(5, 4, 2, 3, device=device) + img = torch.rand(5, 3, 100, 30, device=device) + PS = 10 + patches = kornia.feature.extract_patches_simple(img, laf, PS) + assert patches.shape == (5, 4, 3, PS, PS) + + def test_non_zero(self, device): + img = torch.zeros(1, 1, 24, 24, device=device) + img[:, :, 10:, 20:] = 1.0 + laf = torch.tensor([[8.0, 0, 14.0], [0, 8.0, 8.0]], device=device).reshape(1, 1, 2, 3) + + PS = 32 + patches = kornia.feature.extract_patches_simple(img, laf, PS) + assert patches.mean().item() > 0.01 + assert patches.shape == (1, 1, 1, PS, PS) + + def test_same_odd(self, device, dtype): + img = torch.arange(5)[None].repeat(5, 1)[None, None].to(device, dtype) + laf = torch.tensor([[2.0, 0, 2.0], [0, 2.0, 2.0]]).reshape(1, 1, 2, 3).to(device, dtype) + + patch = kornia.feature.extract_patches_simple(img, laf, 5, 1.0) + self.assert_close(img, patch[0]) + + def test_same_even(self, device, dtype): + img = torch.arange(4)[None].repeat(4, 1)[None, None].to(device, dtype) + laf = torch.tensor([[1.5, 0, 1.5], [0, 1.5, 1.5]]).reshape(1, 1, 2, 3).to(device, dtype) + + patch = kornia.feature.extract_patches_simple(img, laf, 4, 1.0) + self.assert_close(img, patch[0]) + + def test_gradcheck(self, device): + nlaf = torch.tensor([[0.1, 0.001, 0.5], [0, 0.1, 0.5]], device=device, dtype=torch.float64) + nlaf = nlaf.view(1, 1, 2, 3) + img = torch.rand(1, 3, 20, 30, device=device, dtype=torch.float64) + PS = 11 + self.gradcheck(kornia.feature.extract_patches_simple, (img, nlaf, PS, False), fast_mode=False) + + +class TestExtractPatchesPyr(BaseTester): + def test_shape(self, device): + laf = torch.rand(5, 4, 2, 3, device=device) + img = torch.rand(5, 3, 100, 30, device=device) + PS = 10 + patches = kornia.feature.extract_patches_from_pyramid(img, laf, PS) + assert patches.shape == (5, 4, 3, PS, PS) + + def test_non_zero(self, device): + img = torch.zeros(1, 1, 24, 24, device=device) + img[:, :, 10:, 20:] = 1.0 + laf = torch.tensor([[8.0, 0, 14.0], [0, 8.0, 8.0]], device=device).reshape(1, 1, 2, 3) + + PS = 32 + patches = kornia.feature.extract_patches_from_pyramid(img, laf, PS) + assert patches.mean().item() > 0.01 + assert patches.shape == (1, 1, 1, PS, PS) + + def test_same_odd(self, device, dtype): + img = torch.arange(5)[None].repeat(5, 1)[None, None].to(device, dtype) + laf = torch.tensor([[2.0, 0, 2.0], [0, 2.0, 2.0]]).reshape(1, 1, 2, 3).to(device, dtype) + + patch = kornia.feature.extract_patches_from_pyramid(img, laf, 5, 1.0) + self.assert_close(img, patch[0]) + + def test_same_even(self, device, dtype): + img = torch.arange(4)[None].repeat(4, 1)[None, None].to(device, dtype) + laf = torch.tensor([[1.5, 0, 1.5], [0, 1.5, 1.5]]).reshape(1, 1, 2, 3).to(device, dtype) + + patch = kornia.feature.extract_patches_from_pyramid(img, laf, 4, 1.0) + self.assert_close(img, patch[0]) + + def test_small_image_single_level(self, device, dtype): + # When min(H, W) < 2 * PS, the pyramid cannot descend beyond level 0. + # All patches must still have the correct shape and non-zero content. + PS = 16 + img = torch.rand(1, 1, 24, 24, device=device, dtype=dtype) # 24 < 2*16=32 → only level 0 + laf = torch.tensor([[6.0, 0.0, 12.0], [0.0, 6.0, 12.0]], device=device, dtype=dtype).view(1, 1, 2, 3) + patches = kornia.feature.extract_patches_from_pyramid(img, laf, PS) + assert patches.shape == (1, 1, 1, PS, PS) + assert patches.abs().sum().item() > 0 + + def test_multi_level_uses_correct_pyramid_level(self, device, dtype): + # Two LAFs with very different scales should be extracted from different pyramid levels. + # We verify the output shape and that the function runs without errors. + PS = 8 + img = torch.rand(1, 1, 128, 128, device=device, dtype=dtype) + # Small-scale LAF (extracted at level 0) and large-scale LAF (extracted at higher level). + laf_small = torch.tensor([[2.0, 0.0, 64.0], [0.0, 2.0, 64.0]], device=device, dtype=dtype).view(1, 1, 2, 3) + laf_large = torch.tensor([[32.0, 0.0, 64.0], [0.0, 32.0, 64.0]], device=device, dtype=dtype).view(1, 1, 2, 3) + laf_both = torch.cat([laf_small, laf_large], dim=1) + patches = kornia.feature.extract_patches_from_pyramid(img, laf_both, PS) + assert patches.shape == (1, 2, 1, PS, PS) + + def test_gradcheck(self, device): + nlaf = torch.tensor([[0.1, 0.001, 0.5], [0, 0.1, 0.5]], device=device, dtype=torch.float64) + nlaf = nlaf.view(1, 1, 2, 3) + img = torch.rand(1, 3, 20, 30, device=device, dtype=torch.float64) + PS = 11 + self.gradcheck( + kornia.feature.extract_patches_from_pyramid, + (img, nlaf, PS, False), + nondet_tol=1e-8, + ) + + +class TestLAFIsTouchingBoundary(BaseTester): + def test_shape(self, device): + inp = torch.rand(5, 3, 2, 3, device=device) + img = torch.rand(5, 3, 10, 10, device=device) + assert (5, 3) == kornia.feature.laf_is_inside_image(inp, img).shape + + def test_touch(self, device): + w, h = 10, 5 + img = torch.rand(1, 3, h, w, device=device) + laf = torch.tensor([[[[10, 0, 3], [0, 10, 3]], [[1, 0, 5], [0, 1, 2]]]], device=device).float() + expected = torch.tensor([[False, True]], device=device) + assert torch.all(kornia.feature.laf_is_inside_image(laf, img) == expected).item() + + @pytest.mark.jit() + def test_jit(self, device, dtype): + w, h = 10, 5 + img = torch.rand(1, 3, h, w, device=device) + laf = torch.tensor([[[[10, 0, 3], [0, 10, 3]], [[1, 0, 5], [0, 1, 2]]]], device=device).float() + model = kornia.feature.laf_is_inside_image + model_jit = torch.jit.script(kornia.feature.laf_is_inside_image) + self.assert_close(model(laf, img), model_jit(laf, img)) + + +class TestGetCreateLAF(BaseTester): + def test_shape(self, device): + xy = torch.ones(1, 3, 2, device=device) + ori = torch.ones(1, 3, 1, device=device) + scale = torch.ones(1, 3, 1, 1, device=device) + laf = kornia.feature.laf_from_center_scale_ori(xy, scale, ori) + assert laf.shape == (1, 3, 2, 3) + + def test_laf(self, device): + xy = torch.ones(1, 1, 2, device=device) + ori = torch.zeros(1, 1, 1, device=device) + scale = 5 * torch.ones(1, 1, 1, 1, device=device) + expected = torch.tensor([[[[5, 0, 1], [0, 5, 1]]]], device=device).float() + laf = kornia.feature.laf_from_center_scale_ori(xy, scale, ori) + self.assert_close(laf, expected) + + def test_laf_def(self, device): + xy = torch.ones(1, 1, 2, device=device) + expected = torch.tensor([[[[1, 0, 1], [0, 1, 1]]]], device=device).float() + laf = kornia.feature.laf_from_center_scale_ori(xy) + self.assert_close(laf, expected) + + def test_cross_consistency(self, device): + batch_size, channels = 3, 2 + xy = torch.rand(batch_size, channels, 2, device=device) + ori = torch.rand(batch_size, channels, 1, device=device) + scale = torch.abs(torch.rand(batch_size, channels, 1, 1, device=device)) + laf = kornia.feature.laf_from_center_scale_ori(xy, scale, ori) + scale2 = kornia.feature.get_laf_scale(laf) + self.assert_close(scale, scale2) + xy2 = kornia.feature.get_laf_center(laf) + self.assert_close(xy2, xy) + ori2 = kornia.feature.get_laf_orientation(laf) + self.assert_close(ori2, ori) + + def test_gradcheck(self, device): + batch_size, channels = 3, 2 + xy = torch.rand(batch_size, channels, 2, device=device, dtype=torch.float64) + ori = torch.rand(batch_size, channels, 1, device=device, dtype=torch.float64) + scale = torch.abs(torch.rand(batch_size, channels, 1, 1, device=device, dtype=torch.float64)) + self.gradcheck(kornia.feature.laf_from_center_scale_ori, (xy, scale, ori)) + + @pytest.mark.skip("Depends on angle-to-rotation-matric") + @pytest.mark.jit() + def test_jit(self, device, dtype): + batch_size, channels = 3, 2 + xy = torch.rand(batch_size, channels, 2, device=device) + ori = torch.rand(batch_size, channels, 1, device=device) + scale = torch.abs(torch.rand(batch_size, channels, 1, 1, device=device)) + model = kornia.feature.laf_from_center_scale_ori + model_jit = torch.jit.script(kornia.feature.laf_from_center_scale_ori) + self.assert_close(model(xy, scale, ori), model_jit(xy, scale, ori)) + + +class TestGetLAF3pts(BaseTester): + def test_shape(self, device): + inp = torch.ones(1, 3, 2, 3, device=device) + out = kornia.feature.laf_to_three_points(inp) + assert out.shape == inp.shape + + def test_batch_shape(self, device): + inp = torch.ones(5, 3, 2, 3, device=device) + out = kornia.feature.laf_to_three_points(inp) + assert out.shape == inp.shape + + def test_conversion(self, device): + inp = torch.tensor([[1, 0, 2], [0, 1, 3]], device=device).float().view(1, 1, 2, 3) + expected = torch.tensor([[3, 2, 2], [3, 4, 3]], device=device).float().view(1, 1, 2, 3) + threepts = kornia.feature.laf_to_three_points(inp) + self.assert_close(threepts, expected) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 3, 2, 2, 3 + inp = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.feature.laf_to_three_points, (inp,)) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + batch_size, channels, height, width = 3, 2, 2, 3 + inp = torch.rand(batch_size, channels, height, width, device=device) + model = kornia.feature.laf_to_three_points + model_jit = torch.jit.script(kornia.feature.laf_to_three_points) + self.assert_close(model(inp), model_jit(inp)) + + +class TestGetLAFFrom3pts(BaseTester): + def test_shape(self, device): + inp = torch.ones(1, 3, 2, 3, device=device) + out = kornia.feature.laf_from_three_points(inp) + assert out.shape == inp.shape + + def test_batch_shape(self, device): + inp = torch.ones(5, 3, 2, 3, device=device) + out = kornia.feature.laf_from_three_points(inp) + assert out.shape == inp.shape + + def test_conversion(self, device): + expected = torch.tensor([[1, 0, 2], [0, 1, 3]], device=device).float().view(1, 1, 2, 3) + inp = torch.tensor([[3, 2, 2], [3, 4, 3]], device=device).float().view(1, 1, 2, 3) + threepts = kornia.feature.laf_from_three_points(inp) + self.assert_close(threepts, expected) + + def test_cross_consistency(self, device): + batch_size, channels, height, width = 3, 2, 2, 3 + inp = torch.rand(batch_size, channels, height, width, device=device) + inp_2 = kornia.feature.laf_from_three_points(inp) + inp_2 = kornia.feature.laf_to_three_points(inp_2) + self.assert_close(inp_2, inp) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 3, 2, 2, 3 + inp = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.feature.laf_from_three_points, (inp,)) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + batch_size, channels, height, width = 3, 2, 2, 3 + inp = torch.rand(batch_size, channels, height, width, device=device) + model = kornia.feature.laf_from_three_points + model_jit = torch.jit.script(kornia.feature.laf_from_three_points) + self.assert_close(model(inp), model_jit(inp)) + + +class TestTransformLAFs(BaseTester): + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + @pytest.mark.parametrize("num_points", [2, 3, 5]) + def test_transform_points(self, batch_size, num_points, device, dtype): + # generate input data + eye_size = 3 + lafs_src = torch.rand(batch_size, num_points, 2, 3, device=device, dtype=dtype) + + dst_homo_src = create_random_homography(lafs_src, eye_size) + # transform the points from dst to ref + lafs_dst = kornia.feature.perspective_transform_lafs(dst_homo_src, lafs_src) + + # transform the points from ref to dst + src_homo_dst = torch.inverse(dst_homo_src) + lafs_dst_to_src = kornia.feature.perspective_transform_lafs(src_homo_dst, lafs_dst) + + # projected should be equal as initial + self.assert_close(lafs_src, lafs_dst_to_src) + + def test_gradcheck(self, device): + # generate input data + batch_size, num_points = 2, 3 + eye_size = 3 + points_src = torch.rand(batch_size, num_points, 2, 3, device=device, dtype=torch.float64) + dst_homo_src = create_random_homography(points_src, eye_size) + # evaluate function gradient + self.gradcheck(kornia.feature.perspective_transform_lafs, (dst_homo_src, points_src)) diff --git a/tests/feature/test_lightglue.py b/tests/feature/test_lightglue.py new file mode 100644 index 0000000..31c9600 --- /dev/null +++ b/tests/feature/test_lightglue.py @@ -0,0 +1,382 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.feature.lightglue import ( + LearnableFourierPositionalEncoding, + LightGlue, + TokenConfidence, + apply_cached_rotary_emb, + normalize_keypoints, + pad_to_length, + rotate_half, +) + +from testing.base import BaseTester + +# --------------------------------------------------------------------------- +# Pure function tests +# --------------------------------------------------------------------------- + + +def test_lightglue_empty_after_pruning(): + model = LightGlue(features="superpoint", width_confidence=0.99) + model.eval() + + data = { + "image0": { + "keypoints": torch.empty(1, 0, 2), + "descriptors": torch.empty(1, 0, 256), + "image_size": torch.tensor([[640, 480]]), + }, + "image1": { + "keypoints": torch.empty(1, 0, 2), + "descriptors": torch.empty(1, 0, 256), + "image_size": torch.tensor([[640, 480]]), + }, + } + + with torch.no_grad(): + out = model(data) + + assert out["matches0"].shape == (1, 0) + assert out["matches1"].shape == (1, 0) + assert out["matching_scores0"].shape == (1, 0) + assert out["matching_scores1"].shape == (1, 0) + + +def test_lightglue_pruning_removes_all(): + model = LightGlue(features="superpoint", width_confidence=0.0) + model.eval() + + B, M, D = 1, 8, 256 + + data = { + "image0": { + "keypoints": torch.rand(B, M, 2), + "descriptors": torch.rand(B, M, D), + "image_size": torch.tensor([[640, 480]]), + }, + "image1": { + "keypoints": torch.rand(B, M, 2), + "descriptors": torch.rand(B, M, D), + "image_size": torch.tensor([[640, 480]]), + }, + } + + with torch.no_grad(): + out = model(data) + + assert "matches0" in out + assert "matches1" in out + assert out["matches0"].shape == (B, M) + assert out["matches1"].shape == (B, M) + + +def test_normalize_keypoints_smoke(): + kpts = torch.zeros(1, 5, 2) + size = torch.tensor([[100, 200]]) + out = normalize_keypoints(kpts, size) + assert out.shape == kpts.shape + + +def test_normalize_keypoints_center(): + size = torch.tensor([[100, 100]]) + kpts = size.float().unsqueeze(0) / 2 # center pixel + out = normalize_keypoints(kpts, size) + assert torch.allclose(out, torch.zeros_like(out), atol=1e-6) + + +def test_normalize_keypoints_range(): + size = torch.tensor([[128, 128]]) + kpts = torch.rand(1, 20, 2) * 128 + out = normalize_keypoints(kpts, size) + assert out.min() >= -1.0 - 1e-3 + assert out.max() <= 1.0 + 1e-3 + + +def test_pad_to_length_no_pad(): + x = torch.rand(1, 10, 64) + y, mask = pad_to_length(x, 5) + assert y is x + assert mask.shape[-2] == 10 + + +def test_pad_to_length_pads(): + x = torch.rand(1, 5, 64) + y, mask = pad_to_length(x, 10) + assert y.shape[-2] == 10 + assert mask.shape[-2] == 10 + assert mask[..., :5, :].all() + assert not mask[..., 5:, :].any() + + +def test_rotate_half_shape(): + x = torch.rand(2, 8, 4) + out = rotate_half(x) + assert out.shape == x.shape + + +def test_rotate_half_double_gives_negation(): + x = torch.rand(1, 4, 8) + assert torch.allclose(rotate_half(rotate_half(x)), -x, atol=1e-6) + + +def test_apply_cached_rotary_emb_shape(): + B, N, D = 1, 10, 16 + freqs = torch.rand(2, B, 1, N, D) + t = torch.rand(B, 1, N, D) + out = apply_cached_rotary_emb(freqs, t) + assert out.shape == t.shape + + +# --------------------------------------------------------------------------- +# Module tests +# --------------------------------------------------------------------------- + + +class TestLearnableFourierPositionalEncoding(BaseTester): + def test_smoke(self, device, dtype): + enc = LearnableFourierPositionalEncoding(2, 64).to(device, dtype) + x = torch.rand(1, 1, 10, 2, device=device, dtype=dtype) + out = enc(x) + assert out.shape[0] == 2 # cosines/sines stack + + def test_cardinality(self, device, dtype): + M, dim = 2, 32 + enc = LearnableFourierPositionalEncoding(M, dim).to(device, dtype) + B, N = 2, 15 + x = torch.rand(B, 1, N, M, device=device, dtype=dtype) + out = enc(x) + assert out.shape[-1] == dim + + def test_gradcheck(self, device): + enc = LearnableFourierPositionalEncoding(2, 32).to(device, torch.float64) + x = torch.rand(1, 1, 5, 2, device=device, dtype=torch.float64, requires_grad=True) + self.gradcheck(enc, (x,)) + + def test_dynamo(self, device, dtype, torch_optimizer): + enc = LearnableFourierPositionalEncoding(2, 32).to(device, dtype) + x = torch.rand(1, 1, 5, 2, device=device, dtype=dtype) + op = torch_optimizer(enc) + self.assert_close(op(x), enc(x)) + + def test_exception(self, device, dtype): + pass + + def test_module(self, device, dtype): + enc = LearnableFourierPositionalEncoding(2, 32).to(device, dtype) + x = torch.rand(1, 1, 5, 2, device=device, dtype=dtype) + assert enc(x).shape[0] == 2 + + +class TestTokenConfidence(BaseTester): + def test_smoke(self, device, dtype): + tc = TokenConfidence(64).to(device, dtype) + d0 = torch.rand(1, 10, 64, device=device, dtype=dtype) + d1 = torch.rand(1, 8, 64, device=device, dtype=dtype) + s0, s1 = tc(d0, d1) + assert s0.shape == (1, 10) + assert s1.shape == (1, 8) + + def test_cardinality(self, device, dtype): + tc = TokenConfidence(32).to(device, dtype) + B, M, N = 2, 12, 15 + d0 = torch.rand(B, M, 32, device=device, dtype=dtype) + d1 = torch.rand(B, N, 32, device=device, dtype=dtype) + s0, s1 = tc(d0, d1) + assert s0.shape == (B, M) + assert s1.shape == (B, N) + + def test_scores_in_range(self, device, dtype): + tc = TokenConfidence(32).to(device, dtype) + d0 = torch.rand(1, 20, 32, device=device, dtype=dtype) + d1 = torch.rand(1, 20, 32, device=device, dtype=dtype) + s0, s1 = tc(d0, d1) + assert (s0 >= 0).all() and (s0 <= 1).all() + assert (s1 >= 0).all() and (s1 <= 1).all() + + def test_gradcheck(self, device): + pass # TokenConfidence uses detach() on inputs; not differentiable w.r.t. inputs + + def test_dynamo(self, device, dtype, torch_optimizer): + tc = TokenConfidence(32).to(device, dtype) + d0 = torch.rand(1, 10, 32, device=device, dtype=dtype) + d1 = torch.rand(1, 8, 32, device=device, dtype=dtype) + op = torch_optimizer(tc) + s0_jit, s1_jit = op(d0, d1) + s0, s1 = tc(d0, d1) + self.assert_close(s0_jit, s0) + self.assert_close(s1_jit, s1) + + def test_exception(self, device, dtype): + pass + + def test_module(self, device, dtype): + tc = TokenConfidence(32).to(device, dtype) + d0 = torch.rand(1, 5, 32, device=device, dtype=dtype) + d1 = torch.rand(1, 5, 32, device=device, dtype=dtype) + s0, _ = tc(d0, d1) + assert s0.shape == (1, 5) + + +# --------------------------------------------------------------------------- +# LightGlue (no pretrained weights) tests +# --------------------------------------------------------------------------- + + +def _make_lightglue(device, dtype, input_dim=64, n_layers=2): + """Instantiate a small LightGlue with random weights (features=None).""" + return ( + LightGlue( + features=None, + input_dim=input_dim, + descriptor_dim=64, + n_layers=n_layers, + num_heads=4, + depth_confidence=-1, + width_confidence=-1, + flash=False, + ) + .to(device, dtype) + .eval() + ) + + +def _make_data(device, dtype, B=1, M=20, N=15, H=64, W=64, D=64): + """Build a minimal data dict for LightGlue forward.""" + return { + "image0": { + "keypoints": torch.rand(B, M, 2, device=device, dtype=dtype) + * torch.tensor([W, H], device=device, dtype=dtype), + "descriptors": torch.rand(B, M, D, device=device, dtype=dtype), + "image_size": torch.tensor([[W, H]], device=device, dtype=dtype).expand(B, -1), + }, + "image1": { + "keypoints": torch.rand(B, N, 2, device=device, dtype=dtype) + * torch.tensor([W, H], device=device, dtype=dtype), + "descriptors": torch.rand(B, N, D, device=device, dtype=dtype), + "image_size": torch.tensor([[W, H]], device=device, dtype=dtype).expand(B, -1), + }, + } + + +class TestLightGlue(BaseTester): + def test_smoke(self, device, dtype): + if dtype == torch.float16: + pytest.skip("LightGlue requires float32 or float64") + lg = _make_lightglue(device, dtype) + data = _make_data(device, dtype) + with torch.no_grad(): + out = lg(data) + assert "matches0" in out + assert "matching_scores0" in out + + def test_cardinality(self, device, dtype): + if dtype == torch.float16: + pytest.skip("LightGlue requires float32 or float64") + B, M, N = 1, 20, 15 + lg = _make_lightglue(device, dtype) + data = _make_data(device, dtype, B=B, M=M, N=N) + with torch.no_grad(): + out = lg(data) + assert out["matches0"].shape == (B, M) + assert out["matches1"].shape == (B, N) + assert out["matching_scores0"].shape == (B, M) + + def test_image_tensor_input(self, device, dtype): + """Accept image tensor instead of image_size.""" + if dtype == torch.float16: + pytest.skip("LightGlue requires float32 or float64") + B, M, N, H, W, D = 1, 10, 8, 32, 32, 64 + lg = _make_lightglue(device, dtype) + data = { + "image0": { + "keypoints": torch.rand(B, M, 2, device=device, dtype=dtype) * W, + "descriptors": torch.rand(B, M, D, device=device, dtype=dtype), + "image": torch.rand(B, 3, H, W, device=device, dtype=dtype), + }, + "image1": { + "keypoints": torch.rand(B, N, 2, device=device, dtype=dtype) * W, + "descriptors": torch.rand(B, N, D, device=device, dtype=dtype), + "image": torch.rand(B, 3, H, W, device=device, dtype=dtype), + }, + } + with torch.no_grad(): + out = lg(data) + assert "matches0" in out + + def test_matches_are_valid_indices(self, device, dtype): + if dtype == torch.float16: + pytest.skip("LightGlue requires float32 or float64") + B, M, N = 1, 30, 25 + lg = _make_lightglue(device, dtype) + data = _make_data(device, dtype, B=B, M=M, N=N) + with torch.no_grad(): + out = lg(data) + m0 = out["matches0"] # (B, M), values in [-1, N-1] + assert (m0 >= -1).all() + assert (m0 < N).all() + + def test_scores_in_range(self, device, dtype): + if dtype == torch.float16: + pytest.skip("LightGlue requires float32 or float64") + lg = _make_lightglue(device, dtype) + data = _make_data(device, dtype) + with torch.no_grad(): + out = lg(data) + scores = out["matching_scores0"] + assert (scores >= 0).all() + assert (scores <= 1).all() + + def test_exception_missing_key(self, device, dtype): + if dtype == torch.float16: + pytest.skip("LightGlue requires float32 or float64") + lg = _make_lightglue(device, dtype) + with pytest.raises(Exception): + lg({"image0": {}}) + + def test_exception_wrong_features(self, device, dtype): + with pytest.raises(Exception): + LightGlue(features="nonexistent_feature_type") + + def test_gradcheck(self, device): + pass # LightGlue uses detach() on descriptors; not differentiable end-to-end + + def test_dynamo(self, device, dtype, torch_optimizer): + pass # LightGlue uses dynamic control flow; not torch.compile compatible + + def test_module(self, device, dtype): + if dtype == torch.float16: + pytest.skip("LightGlue requires float32 or float64") + lg = _make_lightglue(device, dtype) + data = _make_data(device, dtype) + with torch.no_grad(): + out = lg(data) + assert isinstance(out, dict) + + @pytest.mark.slow + def test_pretrained_smoke(self, device): + """Instantiating with real feature type downloads and loads weights.""" + lg = LightGlue(features="disk", depth_confidence=-1, width_confidence=-1).to(device).eval() + B, M, N, D = 1, 50, 50, 128 + data = _make_data(device, torch.float32, B=B, M=M, N=N, D=D) + with torch.no_grad(): + out = lg(data) + assert "matches0" in out diff --git a/tests/feature/test_lightglue_onnx.py b/tests/feature/test_lightglue_onnx.py new file mode 100644 index 0000000..4da71c1 --- /dev/null +++ b/tests/feature/test_lightglue_onnx.py @@ -0,0 +1,156 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest + +pytest.importorskip("onnx") +import torch + +from kornia.core._compat import torch_version_le +from kornia.feature import OnnxLightGlue +from kornia.feature.lightglue_onnx.utils import normalize_keypoints + +try: + import onnxruntime as ort +except ImportError: + ort = None + + +@pytest.mark.skipif(ort is None, reason="OnnxLightGlue requires onnxruntime-gpu") +@pytest.mark.skipif(torch_version_le(1, 9, 1), reason="Needs dlpack") +class TestOnnxLightGlue: + @pytest.mark.slow + @pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires CUDA") + @pytest.mark.parametrize("weights", ["disk", "superpoint", "disk_fp16", "superpoint_fp16"]) + def test_pretrained_weights_cuda(self, weights): + model = OnnxLightGlue(weights, "cuda") + assert model is not None + + @pytest.mark.slow + @pytest.mark.parametrize("weights", ["disk", "superpoint"]) + def test_pretrained_weights_cpu(self, weights): + model = OnnxLightGlue(weights, "cpu") + assert model is not None + + @pytest.mark.slow + @pytest.mark.parametrize("dtype", [torch.float32]) + def test_forward(self, dtype, device): + if device.type == "cuda" and "CUDAExecutionProvider" not in ort.get_available_providers(): + pytest.skip("OnnxLightGlue on CUDA requires onnxruntime-gpu with CUDAExecutionProvider") + model = OnnxLightGlue(device=device) + + kpts = torch.zeros(1, 5, 2, dtype=dtype, device=device) + desc = torch.zeros(1, 5, 128, dtype=dtype, device=device) + image = torch.zeros(1, 3, 10, 10, dtype=dtype) + outputs = model( + { + "image0": {"keypoints": kpts, "descriptors": desc, "image": image}, + "image1": {"keypoints": kpts, "descriptors": desc, "image": image}, + } + ) + + assert "matches" in outputs + assert "scores" in outputs + assert outputs["matches"].device.type == model.device.type + assert outputs["scores"].device.type == model.device.type + assert outputs["matches"].dtype == torch.int64 + assert outputs["scores"].dtype == torch.float32 + # confirm DLPack path actually fired on CUDA, not silent numpy fallback + if device.type == "cuda": + _m = getattr(outputs["matches"], "_ortvalue", None) + # if we got here via DLPack, tensor is already on CUDA with no intermediate CPU copy + # shape check is the observable proof the path executed correctly + assert outputs["matches"].ndim == 2 + assert outputs["matches"].shape[-1] == 2 + assert outputs["scores"].ndim == 1 + assert outputs["matches"].shape[0] == outputs["scores"].shape[0] + + def test_normalize_keypoints(self): + kpts = torch.randint(0, 100, (1, 5, 2)) + size = torch.tensor([[100, 100]]) + kpts = normalize_keypoints(kpts, size) + assert torch.all(torch.abs(kpts) <= 1).item() + + @pytest.mark.slow + @pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires CUDA") + def test_exception(self, device): + with pytest.raises(RuntimeError) as e: + OnnxLightGlue(device="invalid device") + assert "Invalid device string: 'invalid device'" in str(e) + + with pytest.raises(Exception) as e: + OnnxLightGlue("disk_fp16", "cpu") + assert "FP16 requires CUDA." in str(e) + + model = OnnxLightGlue(device=device) + + kpts = torch.zeros(1, 5, 2, device=device) + desc = torch.zeros(1, 5, 128, device=device) + image = torch.zeros(1, 3, 10, 10) + + # Missing input + with pytest.raises(Exception) as e: + model({"image0": {"keypoints": kpts, "descriptors": desc, "image": image}}) + assert "Missing key image1 in data" in str(e) + + # Wrong dtype + with pytest.raises(Exception) as e: + model( + { + "image0": { + "keypoints": torch.zeros(1, 5, 2, dtype=torch.int32, device=device), + "descriptors": desc, + "image": image, + }, + "image1": {"keypoints": kpts, "descriptors": desc, "image": image}, + } + ) + assert "Wrong dtype" in str(e) + + # Wrong device + wrong_device = torch.device("cpu" if device.type == "cuda" else "cuda") + with pytest.raises(Exception) as e: + model( + { + "image0": { + "keypoints": torch.zeros(1, 5, 2, device=wrong_device), + "descriptors": desc, + "image": image, + }, + "image1": {"keypoints": kpts, "descriptors": desc, "image": image}, + } + ) + assert "Wrong device" in str(e) + + # Wrong shapes + with pytest.raises(Exception) as e: + model( + { + "image0": {"keypoints": torch.zeros(1, 4, 2, device=device), "descriptors": desc, "image": image}, + "image1": {"keypoints": kpts, "descriptors": desc, "image": image}, + } + ) + assert "Number of keypoints does not match number of descriptors" in str(e) + + with pytest.raises(Exception) as e: + model( + { + "image0": {"keypoints": kpts, "descriptors": torch.zeros(1, 5, 127, device=device), "image": image}, + "image1": {"keypoints": kpts, "descriptors": desc, "image": image}, + } + ) + assert "Descriptors' dimensions do not match" in str(e) diff --git a/tests/feature/test_local_features_orientation.py b/tests/feature/test_local_features_orientation.py new file mode 100644 index 0000000..995636a --- /dev/null +++ b/tests/feature/test_local_features_orientation.py @@ -0,0 +1,182 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.feature.orientation import LAFOrienter, OriNet, PassLAF, PatchDominantGradientOrientation +from kornia.geometry.conversions import rad2deg + +from testing.base import BaseTester + + +class TestPassLAF(BaseTester): + def test_shape(self, device): + inp = torch.rand(1, 1, 32, 32, device=device) + laf = torch.rand(1, 1, 2, 3, device=device) + ori = PassLAF().to(device) + out = ori(laf, inp) + assert out.shape == laf.shape + + def test_shape_batch(self, device): + inp = torch.rand(2, 1, 32, 32, device=device) + laf = torch.rand(2, 34, 2, 3, device=device) + ori = PassLAF().to(device) + out = ori(laf, inp) + assert out.shape == laf.shape + + def test_print(self, device): + sift = PassLAF() + sift.__repr__() + + def test_pass(self, device): + inp = torch.rand(1, 1, 32, 32, device=device) + laf = torch.rand(1, 1, 2, 3, device=device) + ori = PassLAF().to(device) + out = ori(laf, inp) + self.assert_close(out, laf) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 1, 21, 21 + patches = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + laf = torch.rand(batch_size, 4, 2, 3, device=device, dtype=torch.float64) + self.gradcheck(PassLAF().to(device), (patches, laf)) + + +class TestPatchDominantGradientOrientation(BaseTester): + def test_shape(self, device): + inp = torch.rand(1, 1, 32, 32, device=device) + ori = PatchDominantGradientOrientation(32).to(device) + ang = ori(inp) + assert ang.shape == torch.Size([1]) + + def test_shape_batch(self, device): + inp = torch.rand(10, 1, 32, 32, device=device) + ori = PatchDominantGradientOrientation(32).to(device) + ang = ori(inp) + assert ang.shape == torch.Size([10]) + + def test_print(self, device): + sift = PatchDominantGradientOrientation(32) + sift.__repr__() + + def test_toy(self, device): + ori = PatchDominantGradientOrientation(19).to(device) + inp = torch.zeros(1, 1, 19, 19, device=device) + inp[:, :, :10, :] = 1 + ang = ori(inp) + expected = torch.tensor([90.0], device=device) + self.assert_close(rad2deg(ang), expected) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 1, 13, 13 + ori = PatchDominantGradientOrientation(width).to(device) + patches = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(ori, (patches,)) + + @pytest.mark.jit() + @pytest.mark.skip(" Compiled functions can't take variable number") + def test_jit(self, device, dtype): + B, C, H, W = 2, 1, 13, 13 + patches = torch.ones(B, C, H, W, device=device, dtype=dtype) + model = PatchDominantGradientOrientation(13).to(patches.device, patches.dtype).eval() + model_jit = torch.jit.script(PatchDominantGradientOrientation(13).to(patches.device, patches.dtype).eval()) + self.assert_close(model(patches), model_jit(patches)) + + +class TestOriNet(BaseTester): + def test_shape(self, device): + inp = torch.rand(1, 1, 32, 32, device=device) + ori = OriNet().to(device=device, dtype=inp.dtype).eval() + ang = ori(inp) + assert ang.shape == torch.Size([1]) + + def test_pretrained(self, device): + inp = torch.rand(1, 1, 32, 32, device=device) + ori = OriNet(True).to(device=device, dtype=inp.dtype).eval() + ang = ori(inp) + assert ang.shape == torch.Size([1]) + + def test_shape_batch(self, device): + inp = torch.rand(2, 1, 32, 32, device=device) + ori = OriNet(True).to(device=device, dtype=inp.dtype).eval() + ang = ori(inp) + assert ang.shape == torch.Size([2]) + + def test_print(self, device): + sift = OriNet(32) + sift.__repr__() + + def test_toy(self, device): + inp = torch.zeros(1, 1, 32, 32, device=device) + inp[:, :, :16, :] = 1 + ori = OriNet(True).to(device=device, dtype=inp.dtype).eval() + ang = ori(inp) + expected = torch.tensor([70.58], device=device) + self.assert_close(rad2deg(ang), expected, atol=1e-2, rtol=1e-3) + + @pytest.mark.skip("jacobian not well computed") + def test_gradcheck(self, device): + batch_size, channels, height, width = 2, 1, 32, 32 + patches = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + ori = OriNet().to(device=device, dtype=patches.dtype) + self.gradcheck(ori, (patches,), fast_mode=False) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 1, 32, 32 + patches = torch.ones(B, C, H, W, device=device, dtype=dtype) + tfeat = OriNet(True).to(patches.device, patches.dtype).eval() + tfeat_jit = torch.jit.script(OriNet(True).to(patches.device, patches.dtype).eval()) + self.assert_close(tfeat_jit(patches), tfeat(patches)) + + +class TestLAFOrienter(BaseTester): + def test_shape(self, device): + inp = torch.rand(1, 1, 32, 32, device=device) + laf = torch.rand(1, 1, 2, 3, device=device) + ori = LAFOrienter().to(device) + out = ori(laf, inp) + assert out.shape == laf.shape + + def test_shape_batch(self, device): + inp = torch.rand(2, 1, 32, 32, device=device) + laf = torch.rand(2, 34, 2, 3, device=device) + ori = LAFOrienter().to(device) + out = ori(laf, inp) + assert out.shape == laf.shape + + def test_print(self, device): + sift = LAFOrienter() + sift.__repr__() + + def test_toy(self, device): + ori = LAFOrienter(32).to(device) + inp = torch.zeros(1, 1, 19, 19, device=device) + inp[:, :, :, :10] = 1 + laf = torch.tensor([[[[5.0, 0.0, 8.0], [0.0, 5.0, 8.0]]]], device=device) + new_laf = ori(laf, inp) + expected = torch.tensor([[[[-5.0, 0.0, 8.0], [0.0, -5.0, 8.0]]]], device=device) + self.assert_close(new_laf, expected) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 1, 21, 21 + patches = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + laf = torch.ones(batch_size, 2, 2, 3, device=device, dtype=torch.float64) + laf[:, :, 0, 1] = 0 + laf[:, :, 1, 0] = 0 + self.gradcheck(LAFOrienter(8).to(device), (laf, patches), rtol=1e-3, atol=1e-3, nondet_tol=1e-3) diff --git a/tests/feature/test_loftr.py b/tests/feature/test_loftr.py new file mode 100644 index 0000000..44c9a9c --- /dev/null +++ b/tests/feature/test_loftr.py @@ -0,0 +1,98 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys + +import pytest +import torch + +from kornia.core._compat import torch_version_ge +from kornia.feature import LoFTR +from kornia.geometry import resize + +from testing.base import BaseTester +from testing.casts import dict_to + + +class TestLoFTR(BaseTester): + @pytest.mark.slow + def test_pretrained_outdoor_smoke(self, device, dtype): + loftr = LoFTR("outdoor").to(device, dtype) + assert loftr is not None + + @pytest.mark.slow + def test_pretrained_indoor_smoke(self, device, dtype): + loftr = LoFTR("indoor").to(device, dtype) + assert loftr is not None + + @pytest.mark.slow + @pytest.mark.skipif(torch_version_ge(1, 10), reason="RuntimeError: CUDA out of memory with pytorch>=1.10") + @pytest.mark.skipif(sys.platform == "win32", reason="this test takes so much memory in the CI with Windows") + @pytest.mark.parametrize("data", ["loftr_fund"], indirect=True) + def test_pretrained_indoor(self, device, dtype, data): + loftr = LoFTR("indoor").to(device, dtype) + data_dev = dict_to(data, device, dtype) + with torch.no_grad(): + out = loftr(data_dev) + self.assert_close(out["keypoints0"], data_dev["loftr_indoor_tentatives0"]) + self.assert_close(out["keypoints1"], data_dev["loftr_indoor_tentatives1"]) + + @pytest.mark.slow + @pytest.mark.skipif(torch_version_ge(1, 10), reason="RuntimeError: CUDA out of memory with pytorch>=1.10") + @pytest.mark.skipif(sys.platform == "win32", reason="this test takes so much memory in the CI with Windows") + @pytest.mark.parametrize("data", ["loftr_homo"], indirect=True) + def test_pretrained_outdoor(self, device, dtype, data): + loftr = LoFTR("outdoor").to(device, dtype) + data_dev = dict_to(data, device, dtype) + with torch.no_grad(): + out = loftr(data_dev) + self.assert_close(out["keypoints0"], data_dev["loftr_outdoor_tentatives0"]) + self.assert_close(out["keypoints1"], data_dev["loftr_outdoor_tentatives1"]) + + @pytest.mark.slow + def test_mask(self, device): + patches = torch.rand(1, 1, 32, 32, device=device) + mask = torch.rand(1, 32, 32, device=device) + loftr = LoFTR().to(patches.device, patches.dtype) + sample = {"image0": patches, "image1": patches, "mask0": mask, "mask1": mask} + with torch.no_grad(): + out = loftr(sample) + assert out is not None + + @pytest.mark.slow + def test_gradcheck(self, device): + patches = torch.rand(1, 1, 32, 32, device=device, dtype=torch.float64) + patches05 = resize(patches, (48, 48)) + loftr = LoFTR().to(patches.device, patches.dtype) + + def proxy_forward(x, y): + return loftr.forward({"image0": x, "image1": y})["keypoints0"] + + self.gradcheck(proxy_forward, (patches, patches05), eps=1e-4, atol=1e-4) + + @pytest.mark.skip("does not like transformer.py:L99, zip iteration") + def test_jit(self, device, dtype): + B, C, H, W = 1, 1, 32, 32 + patches = torch.rand(B, C, H, W, device=device, dtype=dtype) + patches2x = resize(patches, (48, 48)) + sample = {"image0": patches, "image1": patches2x} + model = LoFTR().to(patches.device, patches.dtype).eval() + model_jit = torch.jit.script(model) + out = model(sample) + out_jit = model_jit(sample) + for k, v in out.items(): + self.assert_close(v, out_jit[k]) diff --git a/tests/feature/test_matching.py b/tests/feature/test_matching.py new file mode 100644 index 0000000..4b67686 --- /dev/null +++ b/tests/feature/test_matching.py @@ -0,0 +1,611 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.core._compat import torch_version_le +from kornia.feature.integrated import LightGlueMatcher +from kornia.feature.laf import laf_from_center_scale_ori +from kornia.feature.matching import ( + DescriptorMatcher, + DescriptorMatcherWithSteerer, + GeometryAwareDescriptorMatcher, + match_adalam, + match_fginn, + match_mnn, + match_nn, + match_smnn, + match_snn, +) +from kornia.feature.steerers import DiscreteSteerer + +from testing.base import BaseTester +from testing.casts import dict_to + + +class TestMatchNN(BaseTester): + @pytest.mark.parametrize("num_desc1, num_desc2, dim", [(1, 4, 4), (2, 5, 128), (6, 2, 32)]) + def test_shape(self, num_desc1, num_desc2, dim, device): + desc1 = torch.rand(num_desc1, dim, device=device) + desc2 = torch.rand(num_desc2, dim, device=device) + + dists, idxs = match_nn(desc1, desc2) + assert idxs.shape == (num_desc1, 2) + assert dists.shape == (num_desc1, 1) + + def test_matching(self, device): + desc1 = torch.tensor([[0, 0.0], [1, 1], [2, 2], [3, 3.0], [5, 5.0]], device=device) + desc2 = torch.tensor([[5, 5.0], [3, 3.0], [2.3, 2.4], [1, 1], [0, 0.0]], device=device) + + dists, idxs = match_nn(desc1, desc2) + expected_dists = torch.tensor([0, 0, 0.5, 0, 0], device=device).view(-1, 1) + expected_idx = torch.tensor([[0, 4], [1, 3], [2, 2], [3, 1], [4, 0]], device=device) + self.assert_close(dists, expected_dists) + self.assert_close(idxs, expected_idx) + + dists1, idxs1 = match_nn(desc1, desc2) + self.assert_close(dists1, expected_dists) + self.assert_close(idxs1, expected_idx) + + def test_gradcheck(self, device): + desc1 = torch.rand(5, 8, device=device, dtype=torch.float64) + desc2 = torch.rand(7, 8, device=device, dtype=torch.float64) + self.gradcheck(match_mnn, (desc1, desc2), nondet_tol=1e-4) + + +class TestMatchMNN(BaseTester): + @pytest.mark.parametrize("num_desc1, num_desc2, dim", [(1, 4, 4), (2, 5, 128), (6, 2, 32)]) + def test_shape(self, num_desc1, num_desc2, dim, device): + desc1 = torch.rand(num_desc1, dim, device=device) + desc2 = torch.rand(num_desc2, dim, device=device) + + dists, idxs = match_mnn(desc1, desc2) + assert idxs.shape[1] == 2 + assert dists.shape[1] == 1 + assert idxs.shape[0] == dists.shape[0] + assert dists.shape[0] <= num_desc1 + + def test_matching(self, device): + desc1 = torch.tensor([[0, 0.0], [1, 1], [2, 2], [3, 3.0], [5, 5.0]], device=device) + desc2 = torch.tensor([[5, 5.0], [3, 3.0], [2.3, 2.4], [1, 1], [0, 0.0]], device=device) + + dists, idxs = match_mnn(desc1, desc2) + expected_dists = torch.tensor([0, 0, 0.5, 0, 0], device=device).view(-1, 1) + expected_idx = torch.tensor([[0, 4], [1, 3], [2, 2], [3, 1], [4, 0]], device=device) + self.assert_close(dists, expected_dists) + self.assert_close(idxs, expected_idx) + matcher = DescriptorMatcher("mnn").to(device) + dists1, idxs1 = matcher(desc1, desc2) + self.assert_close(dists1, expected_dists) + self.assert_close(idxs1, expected_idx) + + def test_gradcheck(self, device): + desc1 = torch.rand(5, 8, device=device, dtype=torch.float64) + desc2 = torch.rand(7, 8, device=device, dtype=torch.float64) + self.gradcheck(match_mnn, (desc1, desc2), nondet_tol=1e-4) + + +class TestMatchSNN(BaseTester): + @pytest.mark.parametrize("num_desc1, num_desc2, dim", [(2, 4, 4), (2, 5, 128), (6, 2, 32)]) + def test_shape(self, num_desc1, num_desc2, dim, device): + desc1 = torch.rand(num_desc1, dim, device=device) + desc2 = torch.rand(num_desc2, dim, device=device) + + dists, idxs = match_snn(desc1, desc2) + assert idxs.shape[1] == 2 + assert dists.shape[1] == 1 + assert idxs.shape[0] == dists.shape[0] + assert dists.shape[0] <= num_desc1 + + def test_nomatch(self, device): + desc1 = torch.tensor([[0, 0.0], [1, 1], [2, 2], [3, 3.0], [5, 5.0]], device=device) + desc2 = torch.tensor([[5, 5.0]], device=device) + + dists, idxs = match_snn(desc1, desc2, 0.8) + assert len(dists) == 0 + assert len(idxs) == 0 + + def test_matching1(self, device): + desc1 = torch.tensor([[0, 0.0], [1, 1], [2, 2], [3, 3.0], [5, 5.0]], device=device) + desc2 = torch.tensor([[5, 5.0], [3, 3.0], [2.3, 2.4], [1, 1], [0, 0.0]], device=device) + + dists, idxs = match_snn(desc1, desc2, 0.8) + expected_dists = torch.tensor([0, 0, 0.35355339059327373, 0, 0], device=device).view(-1, 1) + expected_idx = torch.tensor([[0, 4], [1, 3], [2, 2], [3, 1], [4, 0]], device=device) + self.assert_close(dists, expected_dists) + self.assert_close(idxs, expected_idx) + matcher = DescriptorMatcher("snn", 0.8).to(device) + dists1, idxs1 = matcher(desc1, desc2) + self.assert_close(dists1, expected_dists) + self.assert_close(idxs1, expected_idx) + + def test_matching2(self, device): + desc1 = torch.tensor([[0, 0.0], [1, 1], [2, 2], [3, 3.0], [5, 5.0]], device=device) + desc2 = torch.tensor([[5, 5.0], [3, 3.0], [2.3, 2.4], [1, 1], [0, 0.0]], device=device) + + dists, idxs = match_snn(desc1, desc2, 0.1) + expected_dists = torch.tensor([0.0, 0, 0, 0], device=device).view(-1, 1) + expected_idx = torch.tensor([[0, 4], [1, 3], [3, 1], [4, 0]], device=device) + self.assert_close(dists, expected_dists) + self.assert_close(idxs, expected_idx) + matcher = DescriptorMatcher("snn", 0.1).to(device) + dists1, idxs1 = matcher(desc1, desc2) + self.assert_close(dists1, expected_dists) + self.assert_close(idxs1, expected_idx) + + def test_gradcheck(self, device): + desc1 = torch.rand(5, 8, device=device, dtype=torch.float64) + desc2 = torch.rand(7, 8, device=device, dtype=torch.float64) + self.gradcheck(match_snn, (desc1, desc2, 0.8), nondet_tol=1e-4) + + +class TestMatchSMNN(BaseTester): + @pytest.mark.parametrize("num_desc1, num_desc2, dim", [(2, 4, 4), (2, 5, 128), (6, 2, 32)]) + def test_shape(self, num_desc1, num_desc2, dim, device): + desc1 = torch.rand(num_desc1, dim, device=device) + desc2 = torch.rand(num_desc2, dim, device=device) + + dists, idxs = match_smnn(desc1, desc2, 0.8) + assert idxs.shape[1] == 2 + assert dists.shape[1] == 1 + assert idxs.shape[0] == dists.shape[0] + assert dists.shape[0] <= num_desc1 + assert dists.shape[0] <= num_desc2 + + def test_matching1(self, device): + desc1 = torch.tensor([[0, 0.0], [1, 1], [2, 2], [3, 3.0], [5, 5.0]], device=device) + desc2 = torch.tensor([[5, 5.0], [3, 3.0], [2.3, 2.4], [1, 1], [0, 0.0]], device=device) + + dists, idxs = match_smnn(desc1, desc2, 0.8) + expected_dists = torch.tensor([0, 0, 0.5423, 0, 0], device=device).view(-1, 1) + expected_idx = torch.tensor([[0, 4], [1, 3], [2, 2], [3, 1], [4, 0]], device=device) + self.assert_close(dists, expected_dists) + self.assert_close(idxs, expected_idx) + matcher = DescriptorMatcher("smnn", 0.8).to(device) + dists1, idxs1 = matcher(desc1, desc2) + self.assert_close(dists1, expected_dists) + self.assert_close(idxs1, expected_idx) + + def test_nomatch(self, device): + desc1 = torch.tensor([[0, 0.0]], device=device) + desc2 = torch.tensor([[5, 5.0]], device=device) + + dists, idxs = match_smnn(desc1, desc2, 0.8) + assert len(dists) == 0 + assert len(idxs) == 0 + + def test_matching2(self, device): + desc1 = torch.tensor([[0, 0.0], [1, 1], [2, 2], [3, 3.0], [5, 5.0]], device=device) + desc2 = torch.tensor([[5, 5.0], [3, 3.0], [2.3, 2.4], [1, 1], [0, 0.0]], device=device) + + dists, idxs = match_smnn(desc1, desc2, 0.1) + expected_dists = torch.tensor([0.0, 0, 0, 0], device=device).view(-1, 1) + expected_idx = torch.tensor([[0, 4], [1, 3], [3, 1], [4, 0]], device=device) + self.assert_close(dists, expected_dists) + self.assert_close(idxs, expected_idx) + matcher = DescriptorMatcher("smnn", 0.1).to(device) + dists1, idxs1 = matcher(desc1, desc2) + self.assert_close(dists1, expected_dists) + self.assert_close(idxs1, expected_idx) + + @pytest.mark.parametrize( + "match_type, d1, d2", + [ + ("nn", 0, 10), + ("nn", 10, 0), + ("nn", 0, 0), + ("snn", 0, 10), + ("snn", 10, 0), + ("snn", 0, 0), + ("mnn", 0, 10), + ("mnn", 10, 0), + ("mnn", 0, 0), + ("smnn", 0, 10), + ("smnn", 10, 0), + ("smnn", 0, 0), + ], + ) + def test_empty_nocrash(self, match_type, d1, d2, device, dtype): + desc1 = torch.empty(d1, 8, device=device, dtype=dtype) + desc2 = torch.empty(d2, 8, device=device, dtype=dtype) + matcher = DescriptorMatcher(match_type, 0.8).to(device) + dists, idxs = matcher(desc1, desc2) + assert dists is not None + assert idxs is not None + + def test_gradcheck(self, device): + desc1 = torch.rand(5, 8, device=device, dtype=torch.float64) + desc2 = torch.rand(7, 8, device=device, dtype=torch.float64) + matcher = DescriptorMatcher("smnn", 0.8).to(device) + self.gradcheck(match_smnn, (desc1, desc2, 0.8), nondet_tol=1e-4) + self.gradcheck(matcher, (desc1, desc2), nondet_tol=1e-4) + + @pytest.mark.jit() + @pytest.mark.parametrize("match_type", ["nn", "snn", "mnn", "smnn"]) + def test_jit(self, match_type, device, dtype): + desc1 = torch.rand(5, 8, device=device, dtype=dtype) + desc2 = torch.rand(7, 8, device=device, dtype=dtype) + matcher = DescriptorMatcher(match_type, 0.8).to(device) + matcher_jit = torch.jit.script(DescriptorMatcher(match_type, 0.8).to(device)) + self.assert_close(matcher(desc1, desc2)[0], matcher_jit(desc1, desc2)[0]) + self.assert_close(matcher(desc1, desc2)[1], matcher_jit(desc1, desc2)[1]) + + +class TestMatchFGINN(BaseTester): + @pytest.mark.parametrize("num_desc1, num_desc2, dim", [(2, 4, 4), (2, 5, 128), (6, 2, 32)]) + def test_shape_one_way(self, num_desc1, num_desc2, dim, device): + desc1 = torch.rand(num_desc1, dim, device=device) + desc2 = torch.rand(num_desc2, dim, device=device) + lafs1 = torch.rand(1, num_desc1, 2, 3, device=device) + lafs2 = torch.rand(1, num_desc2, 2, 3, device=device) + + dists, idxs = match_fginn(desc1, desc2, lafs1, lafs2, 0.9, 1000) + assert idxs.shape[1] == 2 + assert dists.shape[1] == 1 + assert idxs.shape[0] == dists.shape[0] + assert dists.shape[0] <= num_desc1 + + @pytest.mark.parametrize("num_desc1, num_desc2, dim", [(2, 4, 4), (2, 5, 128), (6, 2, 32)]) + def test_shape_two_way(self, num_desc1, num_desc2, dim, device): + desc1 = torch.rand(num_desc1, dim, device=device) + desc2 = torch.rand(num_desc2, dim, device=device) + lafs1 = torch.rand(1, num_desc1, 2, 3, device=device) + lafs2 = torch.rand(1, num_desc2, 2, 3, device=device) + + dists, idxs = match_fginn(desc1, desc2, lafs1, lafs2, 0.9, 1000, mutual=True) + assert idxs.shape[1] == 2 + assert dists.shape[1] == 1 + assert idxs.shape[0] == dists.shape[0] + assert dists.shape[0] <= num_desc1 + assert dists.shape[0] <= num_desc2 + + def test_matching1(self, device, dtype): + desc1 = torch.tensor([[0, 0.0], [1, 1.001], [2, 2], [3, 3.0], [5, 5.0]], dtype=dtype, device=device) + desc2 = torch.tensor([[5, 5.0], [3, 3.0], [2.3, 2.4], [1, 1.001], [0, 0.0]], dtype=dtype, device=device) + lafs1 = laf_from_center_scale_ori(desc1[None]) + lafs2 = laf_from_center_scale_ori(desc2[None]) + + dists, idxs = match_fginn(desc1, desc2, lafs1, lafs2, 0.8, 0.01) + expected_dists = torch.tensor([0, 0, 0.3536, 0, 0], dtype=dtype, device=device).view(-1, 1) + expected_idx = torch.tensor([[0, 4], [1, 3], [2, 2], [3, 1], [4, 0]], device=device) + self.assert_close(dists, expected_dists, rtol=0.001, atol=1e-3) + self.assert_close(idxs, expected_idx) + matcher = GeometryAwareDescriptorMatcher("fginn", {"spatial_th": 0.01}).to(device) + dists1, idxs1 = matcher(desc1, desc2, lafs1, lafs2) + self.assert_close(dists1, expected_dists, rtol=0.001, atol=1e-3) + self.assert_close(idxs1, expected_idx) + + def test_matching_mutual(self, device, dtype): + desc1 = torch.tensor([[0, 0.1], [1, 1.001], [2, 2], [3, 3.0], [5, 5.0], [0.0, 0]], dtype=dtype, device=device) + desc2 = torch.tensor([[5, 5.0], [3, 3.0], [2.3, 2.4], [1, 1.001], [0, 0.0]], dtype=dtype, device=device) + lafs1 = laf_from_center_scale_ori(desc1[None]) + lafs2 = laf_from_center_scale_ori(desc2[None]) + + dists, idxs = match_fginn(desc1, desc2, lafs1, lafs2, 0.8, 2.0, mutual=True) + expected_dists = torch.tensor([0, 0.1768, 0, 0, 0], dtype=dtype, device=device).view(-1, 1) + expected_idx = torch.tensor([[1, 3], [2, 2], [3, 1], [4, 0], [5, 4]], device=device) + self.assert_close(dists, expected_dists, rtol=0.001, atol=1e-3) + self.assert_close(idxs, expected_idx) + matcher = GeometryAwareDescriptorMatcher("fginn", {"spatial_th": 2.0, "mutual": True}).to(device) + dists1, idxs1 = matcher(desc1, desc2, lafs1, lafs2) + self.assert_close(dists1, expected_dists, rtol=0.001, atol=1e-3) + self.assert_close(idxs1, expected_idx) + + def test_nomatch(self, device, dtype): + desc1 = torch.tensor([[0, 0.0]], dtype=dtype, device=device) + desc2 = torch.tensor([[5, 5.0]], dtype=dtype, device=device) + lafs1 = laf_from_center_scale_ori(desc1[None]) + lafs2 = laf_from_center_scale_ori(desc2[None]) + + dists, idxs = match_fginn(desc1, desc2, lafs1, lafs2, 0.8) + assert len(dists) == 0 + assert len(idxs) == 0 + + def test_matching2(self, device, dtype): + desc1 = torch.tensor([[0, 0.0], [1, 1.001], [2, 2], [3, 3.0], [5, 5.0]], dtype=dtype, device=device) + desc2 = torch.tensor([[5, 5.0], [3, 3.0], [2.3, 2.4], [1, 1.001], [0, 0.0]], dtype=dtype, device=device) + lafs1 = laf_from_center_scale_ori(desc1[None]) + lafs2 = laf_from_center_scale_ori(desc2[None]) + + dists, idxs = match_fginn(desc1, desc2, lafs1, lafs2, 0.8, 2.0) + expected_dists = torch.tensor([0, 0, 0.1768, 0, 0], dtype=dtype, device=device).view(-1, 1) + expected_idx = torch.tensor([[0, 4], [1, 3], [2, 2], [3, 1], [4, 0]], device=device) + self.assert_close(dists, expected_dists, rtol=0.001, atol=1e-3) + self.assert_close(idxs, expected_idx) + matcher = GeometryAwareDescriptorMatcher("fginn", {"spatial_th": 2.0}).to(device) + dists1, idxs1 = matcher(desc1, desc2, lafs1, lafs2) + self.assert_close(dists1, expected_dists, rtol=0.001, atol=1e-3) + self.assert_close(idxs1, expected_idx) + + def test_gradcheck(self, device): + desc1 = torch.rand(5, 8, device=device, dtype=torch.float64) + desc2 = torch.rand(7, 8, device=device, dtype=torch.float64) + center1 = torch.rand(1, 5, 2, device=device, dtype=torch.float64) + center2 = torch.rand(1, 7, 2, device=device, dtype=torch.float64) + lafs1 = laf_from_center_scale_ori(center1) + lafs2 = laf_from_center_scale_ori(center2) + self.gradcheck(match_fginn, (desc1, desc2, lafs1, lafs2, 0.8, 0.05), nondet_tol=1e-4) + + @pytest.mark.jit() + @pytest.mark.skip("keyword-arg expansion is not supported") + def test_jit(self, device, dtype): + desc1 = torch.rand(5, 8, device=device, dtype=dtype) + desc2 = torch.rand(7, 8, device=device, dtype=dtype) + center1 = torch.rand(1, 5, 2, device=device) + center2 = torch.rand(1, 7, 2, device=device) + lafs1 = laf_from_center_scale_ori(center1) + lafs2 = laf_from_center_scale_ori(center2) + matcher = GeometryAwareDescriptorMatcher("fginn", 0.8).to(device) + matcher_jit = torch.jit.script(GeometryAwareDescriptorMatcher("fginn", 0.8).to(device)) + self.assert_close(matcher(desc1, desc2)[0], matcher_jit(desc1, desc2, lafs1, lafs2)[0]) + self.assert_close(matcher(desc1, desc2)[1], matcher_jit(desc1, desc2, lafs1, lafs2)[1]) + + +class TestAdalam(BaseTester): + @pytest.mark.slow + @pytest.mark.parametrize("data", ["adalam_idxs"], indirect=True) + def test_real(self, device, dtype, data): + torch.random.manual_seed(0) + # This is not unit test, but that is quite good integration test + data_dev = dict_to(data, device, dtype) + with torch.no_grad(): + dists, idxs = match_adalam(data_dev["descs1"], data_dev["descs2"], data_dev["lafs1"], data_dev["lafs2"]) + assert idxs.shape[1] == 2 + assert dists.shape[1] == 1 + assert idxs.shape[0] == dists.shape[0] + assert dists.shape[0] <= data_dev["descs1"].shape[0] + assert dists.shape[0] <= data_dev["descs2"].shape[0] + expected_idxs = data_dev["expected_idxs"].long() + self.assert_close(idxs, expected_idxs, rtol=1e-4, atol=1e-4) + + @pytest.mark.slow + @pytest.mark.parametrize("data", ["adalam_idxs"], indirect=True) + def test_single_nocrash(self, device, dtype, data): + torch.random.manual_seed(0) + # This is not unit test, but that is quite good integration test + data_dev = dict_to(data, device, dtype) + with torch.no_grad(): + _dists, _idxs = match_adalam( + data_dev["descs1"], data_dev["descs2"][:1], data_dev["lafs1"], data_dev["lafs2"][:, :1] + ) + _dists, _idxs = match_adalam( + data_dev["descs1"][:1], data_dev["descs2"], data_dev["lafs1"][:, :1], data_dev["lafs2"] + ) + + @pytest.mark.slow + @pytest.mark.parametrize("data", ["adalam_idxs"], indirect=True) + def test_small_user_conf(self, device, dtype, data): + torch.random.manual_seed(0) + # This is not unit test, but that is quite good integration test + data_dev = dict_to(data, device, dtype) + adalam_config = {"device": device} + with torch.no_grad(): + _dists, _idxs = match_adalam( + data_dev["descs1"], data_dev["descs2"][:1], data_dev["lafs1"], data_dev["lafs2"][:, :1] + ) + _dists, _idxs = match_adalam( + data_dev["descs1"], data_dev["descs2"], data_dev["lafs1"], data_dev["lafs2"], config=adalam_config + ) + + @pytest.mark.slow + @pytest.mark.parametrize("data", ["adalam_idxs"], indirect=True) + def test_empty_nocrash(self, device, dtype, data): + torch.random.manual_seed(0) + # This is not unit test, but that is quite good integration test + data_dev = dict_to(data, device, dtype) + with torch.no_grad(): + _dists, _idxs = match_adalam( + data_dev["descs1"], + torch.empty(0, 128, device=device, dtype=dtype), + data_dev["lafs1"], + torch.empty(0, 0, 2, 3, device=device, dtype=dtype), + ) + _dists, _idxs = match_adalam( + torch.empty(0, 128, device=device, dtype=dtype), + data_dev["descs2"], + torch.empty(0, 0, 2, 3, device=device, dtype=dtype), + data_dev["lafs2"], + ) + + @pytest.mark.slow + @pytest.mark.parametrize("data", ["adalam_idxs"], indirect=True) + def test_small(self, device, dtype, data): + torch.random.manual_seed(0) + # This is not unit test, but that is quite good integration test + data_dev = dict_to(data, device, dtype) + with torch.no_grad(): + _dists, _idxs = match_adalam( + data_dev["descs1"][:4], data_dev["descs2"][:4], data_dev["lafs1"][:, :4], data_dev["lafs2"][:, :4] + ) + + @pytest.mark.slow + @pytest.mark.parametrize("data", ["adalam_idxs"], indirect=True) + def test_seeds_fail(self, device, dtype, data): + torch.random.manual_seed(0) + # This is not unit test, but that is quite good integration test + data_dev = dict_to(data, device, dtype) + with torch.no_grad(): + _dists, _idxs = match_adalam( + data_dev["descs1"][:100], + data_dev["descs2"][:100], + data_dev["lafs1"][:, :100], + data_dev["lafs2"][:, :100], + ) + + @pytest.mark.slow + @pytest.mark.parametrize("data", ["adalam_idxs"], indirect=True) + def test_module(self, device, dtype, data): + torch.random.manual_seed(0) + # This is not unit test, but that is quite good integration test + data_dev = dict_to(data, device, dtype) + matcher = GeometryAwareDescriptorMatcher("adalam", {"device": device}).to(device, dtype) + with torch.no_grad(): + dists, idxs = matcher(data_dev["descs1"], data_dev["descs2"], data_dev["lafs1"], data_dev["lafs2"]) + assert idxs.shape[1] == 2 + assert dists.shape[1] == 1 + assert idxs.shape[0] == dists.shape[0] + assert dists.shape[0] <= data_dev["descs1"].shape[0] + assert dists.shape[0] <= data_dev["descs2"].shape[0] + expected_idxs = data_dev["expected_idxs"].long() + self.assert_close(idxs, expected_idxs, rtol=1e-4, atol=1e-4) + + +class TestLightGlueDISK(BaseTester): + @pytest.mark.slow + @pytest.mark.skipif(torch_version_le(1, 9, 1), reason="Needs autocast") + @pytest.mark.parametrize("data", ["lightglue_idxs"], indirect=True) + def test_real(self, device, dtype, data): + torch.random.manual_seed(0) + # This is not unit test, but that is quite good integration test + data_dev = dict_to(data, device, dtype) + config = {"depth_confidence": -1, "width_confidence": -1} + lg = LightGlueMatcher("disk", config).to(device=device, dtype=dtype).eval() + with torch.no_grad(): + dists, idxs = lg(data_dev["descs1"], data_dev["descs2"], data_dev["lafs1"], data_dev["lafs2"]) + assert idxs.shape[1] == 2 + assert dists.shape[1] == 1 + assert idxs.shape[0] == dists.shape[0] + assert dists.shape[0] <= data_dev["descs1"].shape[0] + assert dists.shape[0] <= data_dev["descs2"].shape[0] + if device.type == "cpu": + expected_idxs = data_dev["lightglue_disk_idxs"].long() + self.assert_close(idxs, expected_idxs, rtol=1e-4, atol=1e-4) + + @pytest.mark.slow + @pytest.mark.parametrize("data", ["lightglue_idxs"], indirect=True) + def test_single_nocrash(self, device, dtype, data): + torch.random.manual_seed(0) + # This is not unit test, but that is quite good integration test + data_dev = dict_to(data, device, dtype) + lg = LightGlueMatcher("disk").to(device, dtype).eval() + with torch.no_grad(): + _dists, _idxs = lg(data_dev["descs1"], data_dev["descs2"][:1], data_dev["lafs1"], data_dev["lafs2"][:, :1]) + _dists, _idxs = lg(data_dev["descs1"][:1], data_dev["descs2"], data_dev["lafs1"][:, :1], data_dev["lafs2"]) + + @pytest.mark.slow + @pytest.mark.parametrize("data", ["lightglue_idxs"], indirect=True) + def test_empty_nocrash(self, device, dtype, data): + torch.random.manual_seed(0) + # This is not unit test, but that is quite good integration test + data_dev = dict_to(data, device, dtype) + lg = LightGlueMatcher("disk").to(device, dtype).eval() + with torch.no_grad(): + _dists, _idxs = lg( + data_dev["descs1"], + torch.empty(0, 256, device=device, dtype=dtype), + data_dev["lafs1"], + torch.empty(0, 0, 2, 3, device=device, dtype=dtype), + ) + _dists, _idxs = lg( + torch.empty(0, 256, device=device, dtype=dtype), + data_dev["descs2"], + torch.empty(0, 0, 2, 3, device=device, dtype=dtype), + data_dev["lafs2"], + ) + + +class TestLightGlueHardNet(BaseTester): + def test_smoke(self): + lg = LightGlueMatcher("doghardnet") + assert isinstance(lg, LightGlueMatcher) + + +class TestMatchSteererGlobal(BaseTester): + @pytest.mark.parametrize("num_desc1, num_desc2, dim", [(1, 4, 4), (2, 5, 128), (6, 2, 32), (32, 32, 8)]) + @pytest.mark.parametrize("matching_mode", ["nn", "mnn", "snn", "smnn"]) + @pytest.mark.parametrize("fast", [False, True]) + def test_shape(self, num_desc1, num_desc2, dim, matching_mode, fast, device): + desc1 = torch.rand(num_desc1, dim, device=device) + generator = torch.rand(dim, dim, device=device) + steerer = DiscreteSteerer(generator) + desc2 = steerer(desc1) + + matcher = DescriptorMatcherWithSteerer( + steerer=steerer, steerer_order=3, steer_mode="global", match_mode=matching_mode + ) + + dists, idxs, _num_rot = matcher( + desc1, + desc2, + subset_size=max(1, min(num_desc1 // 2, num_desc2 // 2)) if fast else None, + ) + assert dists.shape[1] == 1 + assert dists.shape[0] <= num_desc1 + assert idxs.shape[1] == 2 + assert idxs.shape[0] == dists.shape[0] + + def test_matching(self, device): + desc1 = torch.tensor([[0, 0.0], [1, 1], [2, 2], [3, 3.0], [5, 5.0]], device=device) + desc2 = torch.tensor([[5, 5.0], [3, 3.0], [2.3, 2.4], [1, 1], [0, 0.0]], device=device) + + # rotate desc2 270 deg anti-clockwise + desc2 = desc2[:, [1, 0]] + desc2[:, 0] = -desc2[:, 0] + + generator = torch.tensor([[0.0, 1], [-1, 0]], device=device) + steerer = DiscreteSteerer(generator) + matcher = DescriptorMatcherWithSteerer(steerer=steerer, steerer_order=4, steer_mode="global", match_mode="mnn") + + dists, idxs, num_rot = matcher(desc1, desc2) + expected_dists = torch.tensor([0, 0, 0.5, 0, 0], device=device).view(-1, 1) + expected_idx = torch.tensor([[0, 4], [1, 3], [2, 2], [3, 1], [4, 0]], device=device) + self.assert_close(dists, expected_dists) + self.assert_close(idxs, expected_idx) + + assert num_rot == 3 + + +class TestMatchSteererLocal(BaseTester): + @pytest.mark.parametrize("num_desc1, num_desc2, dim", [(1, 4, 4), (2, 5, 128), (6, 2, 32)]) + def test_shape(self, num_desc1, num_desc2, dim, device): + desc1 = torch.rand(num_desc1, dim, device=device) + generator = torch.rand(dim, dim, device=device) + steerer = DiscreteSteerer(generator) + desc2 = steerer(desc1) + desc2[:1] = steerer(desc2[:1]) + + matcher = DescriptorMatcherWithSteerer(steerer=steerer, steerer_order=3, steer_mode="local", match_mode="mnn") + + dists, idxs, _num_rot = matcher(desc1, desc2) + assert dists.shape[1] == 1 + assert idxs.shape == (dists.shape[0], 2) + assert dists.shape[0] == num_desc1 + + def test_matching(self, device): + desc1 = torch.tensor([[0, 0.0], [1, 1], [2, 2], [3, 3.0], [5, 5.0]], device=device) + desc2 = torch.tensor([[5, 5.0], [3, 3.0], [2.3, 2.4], [1, 1], [0, 0.0]], device=device) + + # rotate second to last element of desc2 90 deg anti-clockwise + desc2[-2] = desc2[-2, [1, 0]] + desc2[-2, 1] = -desc2[-2, 1] + + # rotate first two elements of desc2 270 deg anti-clockwise + desc2[:2] = desc2[:2, [1, 0]] + desc2[:2, 0] = -desc2[:2, 0] + + generator = torch.tensor([[0.0, 1], [-1, 0]], device=device) + steerer = DiscreteSteerer(generator) + matcher = DescriptorMatcherWithSteerer(steerer=steerer, steerer_order=4, steer_mode="local", match_mode="mnn") + + dists, idxs, num_rot = matcher(desc1, desc2) + expected_dists = torch.tensor([0, 0, 0.5, 0, 0], device=device).view(-1, 1) + expected_idx = torch.tensor([[0, 4], [1, 3], [2, 2], [3, 1], [4, 0]], device=device) + self.assert_close(dists, expected_dists) + self.assert_close(idxs, expected_idx) + + assert num_rot is None diff --git a/tests/feature/test_mkd.py b/tests/feature/test_mkd.py new file mode 100644 index 0000000..0419d98 --- /dev/null +++ b/tests/feature/test_mkd.py @@ -0,0 +1,503 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from math import pi +from unittest.mock import patch + +import pytest +import torch + +from kornia.feature.mkd import ( + COEFFS, + EmbedGradients, + ExplicitSpacialEncoding, + MKDDescriptor, + MKDGradients, + SimpleKD, + VonMisesKernel, + Whitening, + get_grid_dict, + get_kron_order, + spatial_kernel_embedding, +) + +from testing.base import BaseTester + + +@pytest.fixture(autouse=True) +def mock_torch_hub_load_state_dict_from_url(): + with patch("torch.hub.load_state_dict_from_url") as mock_load: + + def side_effect(url, **kwargs): + # Create a dummy model structure depending on the kernel type in the URL + if "cart" in url: + in_dims = 63 + elif "polar" in url: + in_dims = 175 + elif "concat" in url: + in_dims = 238 + else: + in_dims = 256 + + dummy_algo = { + "mean": torch.zeros(in_dims), + "eigvecs": torch.eye(in_dims), + "eigvals": torch.ones(in_dims), + } + dummy_training_set = {"lw": dummy_algo, "pca": dummy_algo} + return { + "liberty": dummy_training_set, + "notredame": dummy_training_set, + "yosemite": dummy_training_set, + } + + mock_load.side_effect = side_effect + yield mock_load + + +@pytest.mark.parametrize("ps", [5, 13, 25]) +def test_get_grid_dict(ps): + grid_dict = get_grid_dict(ps) + param_keys = ["x", "y", "phi", "rho"] + assert set(grid_dict.keys()) == set(param_keys) + for k in param_keys: + assert grid_dict[k].shape == (ps, ps) + + +@pytest.mark.parametrize("d1,d2", [(1, 1), (1, 2), (2, 1), (5, 6)]) +def test_get_kron_order(d1, d2): + out = get_kron_order(d1, d2) + assert out.shape == (d1 * d2, 2) + + +def test_get_kron_order_values_and_dtype(): + d1, d2 = 2, 3 + expected_output = torch.tensor( + [ + [0, 0], + [0, 1], + [0, 2], + [1, 0], + [1, 1], + [1, 2], + ], + dtype=torch.int64, + ) + + actual_output = get_kron_order(d1, d2) + assert actual_output.dtype == torch.int64 + assert torch.equal(actual_output, expected_output) + + +class TestMKDGradients(BaseTester): + @pytest.mark.parametrize("ps", [5, 13, 25]) + def test_shape(self, ps, device): + inp = torch.ones(1, 1, ps, ps).to(device) + gradients = MKDGradients().to(device) + out = gradients(inp) + assert out.shape == (1, 2, ps, ps) + + @pytest.mark.parametrize("bs", [1, 5, 13]) + def test_batch_shape(self, bs, device): + inp = torch.ones(bs, 1, 15, 15).to(device) + gradients = MKDGradients().to(device) + out = gradients(inp) + assert out.shape == (bs, 2, 15, 15) + + def test_print(self, device): + gradients = MKDGradients().to(device) + gradients.__repr__() + + def test_toy(self, device): + patch = torch.ones(1, 1, 6, 6).to(device).float() + patch[0, 0, :, 3:] = 0 + gradients = MKDGradients().to(device) + out = gradients(patch) + expected_mags_1 = torch.Tensor([0, 0, 1.0, 1.0, 0, 0]).to(device) + expected_mags = expected_mags_1.unsqueeze(0).repeat(6, 1) + expected_oris_1 = torch.Tensor([-pi, -pi, 0, 0, -pi, -pi]).to(device) + expected_oris = expected_oris_1.unsqueeze(0).repeat(6, 1) + self.assert_close(out[0, 0, :, :], expected_mags, atol=1e-3, rtol=1e-3) + self.assert_close(out[0, 1, :, :], expected_oris, atol=1e-3, rtol=1e-3) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 1, 13, 13 + patches = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + + def grad_describe(patches): + mkd_grads = MKDGradients() + mkd_grads.to(device) + return mkd_grads(patches) + + self.gradcheck(grad_describe, (patches), nondet_tol=1e-4) + + +class TestVonMisesKernel(BaseTester): + @pytest.mark.parametrize("ps", [5, 13, 25]) + def test_shape(self, ps, device): + inp = torch.ones(1, 1, ps, ps).to(device) + vm = VonMisesKernel(patch_size=ps, coeffs=[0.38214156, 0.48090413]).to(device) + out = vm(inp) + assert out.shape == (1, 3, ps, ps) + + @pytest.mark.parametrize("bs", [1, 5, 13]) + def test_batch_shape(self, bs, device): + inp = torch.ones(bs, 1, 15, 15).to(device) + vm = VonMisesKernel(patch_size=15, coeffs=[0.38214156, 0.48090413]).to(device) + out = vm(inp) + assert out.shape == (bs, 3, 15, 15) + + @pytest.mark.parametrize("coeffs", COEFFS.values()) + def test_coeffs(self, coeffs, device): + inp = torch.ones(1, 1, 15, 15).to(device) + vm = VonMisesKernel(patch_size=15, coeffs=coeffs).to(device) + out = vm(inp) + assert out.shape == (1, 2 * len(coeffs) - 1, 15, 15) + + def test_print(self, device): + vm = VonMisesKernel(patch_size=32, coeffs=[0.38214156, 0.48090413]).to(device) + vm.__repr__() + + def test_toy(self, device): + patch = torch.ones(1, 1, 6, 6).float().to(device) + patch[0, 0, :, 3:] = 0 + vm = VonMisesKernel(patch_size=6, coeffs=[0.38214156, 0.48090413]).to(device) + out = vm(patch) + expected = torch.ones_like(out[0, 0, :, :]).to(device) + self.assert_close(out[0, 0, :, :], expected * 0.6182, atol=1e-3, rtol=1e-3) + + expected = torch.Tensor([0.3747, 0.3747, 0.3747, 0.6935, 0.6935, 0.6935]).to(device) + expected = expected.unsqueeze(0).repeat(6, 1) + self.assert_close(out[0, 1, :, :], expected, atol=1e-3, rtol=1e-3) + + expected = torch.Tensor([0.5835, 0.5835, 0.5835, 0.0000, 0.0000, 0.0000]).to(device) + expected = expected.unsqueeze(0).repeat(6, 1) + self.assert_close(out[0, 2, :, :], expected, atol=1e-3, rtol=1e-3) + + def test_gradcheck(self, device): + batch_size, channels, ps = 1, 1, 13 + patches = torch.rand(batch_size, channels, ps, ps, device=device, dtype=torch.float64) + + def vm_describe(patches, ps=13): + vmkernel = VonMisesKernel(patch_size=ps, coeffs=[0.38214156, 0.48090413]).double() + vmkernel.to(device) + return vmkernel(patches.double()) + + self.gradcheck(vm_describe, (patches, ps), nondet_tol=1e-4) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 1, 13, 13 + patches = torch.rand(B, C, H, W, device=device, dtype=dtype) + model = VonMisesKernel(patch_size=13, coeffs=[0.38214156, 0.48090413]).to(patches.device, patches.dtype).eval() + model_jit = torch.jit.script( + VonMisesKernel(patch_size=13, coeffs=[0.38214156, 0.48090413]).to(patches.device, patches.dtype).eval() + ) + self.assert_close(model(patches), model_jit(patches)) + + +class TestEmbedGradients(BaseTester): + @pytest.mark.parametrize("ps,relative", [(5, True), (13, True), (25, True), (5, False), (13, False), (25, False)]) + def test_shape(self, ps, relative, device): + inp = torch.ones(1, 2, ps, ps).to(device) + emb_grads = EmbedGradients(patch_size=ps, relative=relative).to(device) + out = emb_grads(inp) + assert out.shape == (1, 7, ps, ps) + + @pytest.mark.parametrize("bs", [1, 5, 13]) + def test_batch_shape(self, bs, device): + inp = torch.ones(bs, 2, 15, 15).to(device) + emb_grads = EmbedGradients(patch_size=15, relative=True).to(device) + out = emb_grads(inp) + assert out.shape == (bs, 7, 15, 15) + + def test_print(self, device): + emb_grads = EmbedGradients(patch_size=15, relative=True).to(device) + emb_grads.__repr__() + + def test_toy(self, device): + grads = torch.ones(1, 2, 6, 6).float().to(device) + grads[0, 0, :, 3:] = 0 + emb_grads = EmbedGradients(patch_size=6, relative=True).to(device) + out = emb_grads(grads) + expected = torch.ones_like(out[0, 0, :, :3]).to(device) + self.assert_close(out[0, 0, :, :3], expected * 0.3787, atol=1e-3, rtol=1e-3) + self.assert_close(out[0, 0, :, 3:], expected * 0, atol=1e-3, rtol=1e-3) + + # TODO: review this test implementation + # @pytest.mark.xfail(reason="RuntimeError: Jacobian mismatch for output 0 with respect to input 0,") + def test_gradcheck(self, device): + batch_size, channels, ps = 1, 2, 13 + patches = torch.rand(batch_size, channels, ps, ps, device=device, dtype=torch.float64) + + def emb_grads_describe(patches, ps=13): + emb_grads = EmbedGradients(patch_size=ps, relative=True).double() + emb_grads.to(device) + return emb_grads(patches.double()) + + self.gradcheck(emb_grads_describe, (patches, ps), nondet_tol=1e-4) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 2, 13, 13 + patches = torch.rand(B, C, H, W, device=device, dtype=dtype) + model = EmbedGradients(patch_size=W, relative=True).to(patches.device, patches.dtype).eval() + model_jit = torch.jit.script( + EmbedGradients(patch_size=W, relative=True).to(patches.device, patches.dtype).eval() + ) + self.assert_close(model(patches), model_jit(patches)) + + +@pytest.mark.parametrize("kernel_type,d,ps", [("cart", 9, 9), ("polar", 25, 9), ("cart", 9, 16), ("polar", 25, 16)]) +def test_spatial_kernel_embedding(kernel_type, ps, d): + grids = get_grid_dict(ps) + spatial_kernel = spatial_kernel_embedding(kernel_type, grids) + assert spatial_kernel.shape == (d, ps, ps) + + +class TestExplicitSpacialEncoding(BaseTester): + @pytest.mark.parametrize( + "kernel_type,ps,in_dims", [("cart", 9, 3), ("polar", 9, 3), ("cart", 13, 7), ("polar", 13, 7)] + ) + def test_shape(self, kernel_type, ps, in_dims, device): + inp = torch.ones(1, in_dims, ps, ps).to(device) + ese = ExplicitSpacialEncoding(kernel_type=kernel_type, fmap_size=ps, in_dims=in_dims).to(device) + out = ese(inp) + d_ = 9 if kernel_type == "cart" else 25 + assert out.shape == (1, d_ * in_dims) + + @pytest.mark.parametrize( + "kernel_type,bs", [("cart", 1), ("cart", 5), ("cart", 13), ("polar", 1), ("polar", 5), ("polar", 13)] + ) + def test_batch_shape(self, kernel_type, bs, device): + inp = torch.ones(bs, 7, 15, 15).to(device) + ese = ExplicitSpacialEncoding(kernel_type=kernel_type, fmap_size=15, in_dims=7).to(device) + out = ese(inp) + d_ = 9 if kernel_type == "cart" else 25 + assert out.shape == (bs, d_ * 7) + + @pytest.mark.parametrize("kernel_type", ["cart", "polar"]) + def test_print(self, kernel_type, device): + ese = ExplicitSpacialEncoding(kernel_type=kernel_type, fmap_size=15, in_dims=7).to(device) + ese.__repr__() + + def test_toy(self, device): + inp = torch.ones(1, 2, 6, 6).to(device).float() + inp[0, 0, :, :] = 0 + cart_ese = ExplicitSpacialEncoding(kernel_type="cart", fmap_size=6, in_dims=2).to(device) + out = cart_ese(inp) + out_part = out[:, :9] + expected = torch.zeros_like(out_part).to(device) + self.assert_close(out_part, expected, atol=1e-3, rtol=1e-3) + + polar_ese = ExplicitSpacialEncoding(kernel_type="polar", fmap_size=6, in_dims=2).to(device) + out = polar_ese(inp) + out_part = out[:, :25] + expected = torch.zeros_like(out_part).to(device) + self.assert_close(out_part, expected, atol=1e-3, rtol=1e-3) + + @pytest.mark.parametrize("kernel_type", ["cart", "polar"]) + def test_gradcheck(self, kernel_type, device): + batch_size, channels, ps = 1, 2, 13 + patches = torch.rand(batch_size, channels, ps, ps, device=device, dtype=torch.float64) + + def explicit_spatial_describe(patches, ps=13): + ese = ExplicitSpacialEncoding(kernel_type=kernel_type, fmap_size=ps, in_dims=2) + ese.to(device) + return ese(patches) + + self.gradcheck(explicit_spatial_describe, (patches, ps), nondet_tol=1e-4) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 2, 13, 13 + patches = torch.rand(B, C, H, W, device=device, dtype=dtype) + model = ( + ExplicitSpacialEncoding(kernel_type="cart", fmap_size=W, in_dims=2).to(patches.device, patches.dtype).eval() + ) + model_jit = torch.jit.script( + ExplicitSpacialEncoding(kernel_type="cart", fmap_size=W, in_dims=2).to(patches.device, patches.dtype).eval() + ) + self.assert_close(model(patches), model_jit(patches)) + + +class TestWhitening(BaseTester): + @pytest.mark.parametrize( + "kernel_type,xform,output_dims", + [ + ("cart", None, 3), + ("polar", None, 3), + ("cart", "lw", 7), + ("polar", "lw", 7), + ("cart", "pca", 9), + ("polar", "pca", 9), + ], + ) + def test_shape(self, kernel_type, xform, output_dims, device): + in_dims = 63 if kernel_type == "cart" else 175 + wh = Whitening(xform=xform, whitening_model=None, in_dims=in_dims, output_dims=output_dims).to(device) + inp = torch.ones(1, in_dims).to(device) + out = wh(inp) + assert out.shape == (1, output_dims) + + @pytest.mark.parametrize("bs", [1, 3, 7]) + def test_batch_shape(self, bs, device): + wh = Whitening(xform="lw", whitening_model=None, in_dims=175, output_dims=128).to(device) + inp = torch.ones(bs, 175).to(device) + out = wh(inp) + assert out.shape == (bs, 128) + + def test_print(self, device): + wh = Whitening(xform="lw", whitening_model=None, in_dims=175, output_dims=128).to(device) + wh.__repr__() + + def test_toy(self, device): + wh = Whitening(xform="lw", whitening_model=None, in_dims=175, output_dims=175).to(device) + inp = torch.ones(1, 175).to(device).float() + out = wh(inp) + expected = torch.ones_like(inp).to(device) * 0.0756 + self.assert_close(out, expected, atol=1e-3, rtol=1e-3) + + def test_gradcheck(self, device): + batch_size, in_dims = 1, 175 + patches = torch.rand(batch_size, in_dims, device=device, dtype=torch.float64) + + def whitening_describe(patches, in_dims=175): + wh = Whitening(xform="lw", whitening_model=None, in_dims=in_dims).double() + wh.to(device) + return wh(patches.double()) + + self.gradcheck(whitening_describe, (patches, in_dims), nondet_tol=1e-4) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + batch_size, in_dims = 1, 175 + patches = torch.rand(batch_size, in_dims).to(device) + model = Whitening(xform="lw", whitening_model=None, in_dims=in_dims).to(patches.device, patches.dtype).eval() + model_jit = torch.jit.script( + Whitening(xform="lw", whitening_model=None, in_dims=in_dims).to(patches.device, patches.dtype).eval() + ) + self.assert_close(model(patches), model_jit(patches)) + + +class TestMKDDescriptor(BaseTester): + dims = {"cart": 63, "polar": 175, "concat": 238} + + @pytest.mark.parametrize( + "ps,kernel_type", [(9, "concat"), (9, "cart"), (9, "polar"), (32, "concat"), (32, "cart"), (32, "polar")] + ) + def test_shape(self, ps, kernel_type, device): + mkd = MKDDescriptor(patch_size=ps, kernel_type=kernel_type, whitening=None).to(device) + inp = torch.ones(1, 1, ps, ps).to(device) + out = mkd(inp) + assert out.shape == (1, self.dims[kernel_type]) + + @pytest.mark.parametrize( + "ps,kernel_type,whitening", + [ + (9, "concat", "lw"), + (9, "cart", "lw"), + (9, "polar", "lw"), + (9, "concat", "pcawt"), + (9, "cart", "pcawt"), + (9, "polar", "pcawt"), + ], + ) + def test_whitened_shape(self, ps, kernel_type, whitening, device): + mkd = MKDDescriptor(patch_size=ps, kernel_type=kernel_type, whitening=whitening).to(device) + inp = torch.ones(1, 1, ps, ps).to(device) + out = mkd(inp) + output_dims = min(self.dims[kernel_type], 128) + assert out.shape == (1, output_dims) + + @pytest.mark.parametrize("bs", [1, 3, 7]) + def test_batch_shape(self, bs, device): + mkd = MKDDescriptor(patch_size=19, kernel_type="concat", whitening=None).to(device) + inp = torch.ones(bs, 1, 19, 19).to(device) + out = mkd(inp) + assert out.shape == (bs, 238) + + def test_print(self, device): + mkd = MKDDescriptor(patch_size=32, whitening="lw", training_set="liberty", output_dims=128).to(device) + mkd.__repr__() + + def test_toy(self, device): + inp = torch.ones(1, 1, 6, 6).to(device).float() + inp[0, 0, :, :] = 0 + mkd = MKDDescriptor(patch_size=6, kernel_type="concat", whitening=None).to(device) + out = mkd(inp) + out_part = out[0, -28:] + expected = torch.zeros_like(out_part).to(device) + self.assert_close(out_part, expected, atol=1e-3, rtol=1e-3) + + @pytest.mark.parametrize("whitening", [None, "lw", "pca"]) + def test_gradcheck(self, whitening, device): + batch_size, channels, ps = 1, 1, 19 + patches = torch.rand(batch_size, channels, ps, ps, device=device, dtype=torch.float64) + + def mkd_describe(patches, patch_size=19): + mkd = MKDDescriptor(patch_size=patch_size, kernel_type="concat", whitening=whitening).double() + mkd.to(device) + return mkd(patches.double()) + + self.gradcheck(mkd_describe, (patches, ps), nondet_tol=1e-4) + + @pytest.mark.skip("neither dict, nor nn.ModuleDict works") + @pytest.mark.jit() + def test_jit(self, device, dtype): + batch_size, channels, ps = 1, 1, 19 + patches = torch.rand(batch_size, channels, ps, ps).to(device) + kt = "concat" + wt = "lw" + model = MKDDescriptor(patch_size=ps, kernel_type=kt, whitening=wt).to(patches.device, patches.dtype).eval() + model_jit = torch.jit.script( + MKDDescriptor(patch_size=ps, kernel_type=kt, whitening=wt).to(patches.device, patches.dtype).eval() + ) + self.assert_close(model(patches), model_jit(patches)) + + +class TestSimpleKD(BaseTester): + dims = {"cart": 63, "polar": 175} + + @pytest.mark.parametrize("ps,kernel_type", [(9, "cart"), (9, "polar"), (32, "cart"), (32, "polar")]) + def test_shape(self, ps, kernel_type, device): + skd = SimpleKD(patch_size=ps, kernel_type=kernel_type).to(device) + inp = torch.ones(1, 1, ps, ps).to(device) + out = skd(inp) + assert out.shape == (1, min(128, self.dims[kernel_type])) + + @pytest.mark.parametrize("bs", [1, 3, 7]) + def test_batch_shape(self, bs, device): + skd = SimpleKD(patch_size=19, kernel_type="polar").to(device) + inp = torch.ones(bs, 1, 19, 19).to(device) + out = skd(inp) + assert out.shape == (bs, 128) + + def test_print(self, device): + skd = SimpleKD(patch_size=19, kernel_type="polar").to(device) + skd.__repr__() + + def test_gradcheck(self, device): + batch_size, channels, ps = 1, 1, 19 + patches = torch.rand(batch_size, channels, ps, ps, device=device, dtype=torch.float64) + + def skd_describe(patches, patch_size=19): + skd = SimpleKD(patch_size=ps, kernel_type="polar", whitening="lw").double() + skd.to(device) + return skd(patches.double()) + + self.gradcheck(skd_describe, (patches, ps), nondet_tol=1e-4) diff --git a/tests/feature/test_responces_local_features.py b/tests/feature/test_responces_local_features.py new file mode 100644 index 0000000..a951279 --- /dev/null +++ b/tests/feature/test_responces_local_features.py @@ -0,0 +1,387 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch + +import kornia + +from testing.base import BaseTester + + +class TestCornerHarris(BaseTester): + def test_shape(self, device): + inp = torch.ones(1, 3, 4, 4, device=device) + harris = kornia.feature.CornerHarris(k=0.04).to(device) + assert harris(inp).shape == (1, 3, 4, 4) + + def test_shape_batch(self, device): + inp = torch.zeros(2, 6, 4, 4, device=device) + harris = kornia.feature.CornerHarris(k=0.04).to(device) + assert harris(inp).shape == (2, 6, 4, 4) + + def test_corners(self, device, dtype): + inp = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ).float() + + expected = torch.tensor( + [ + [ + [ + [0.0042, 0.0054, 0.0035, 0.0006, 0.0035, 0.0054, 0.0042], + [0.0054, 0.0068, 0.0046, 0.0014, 0.0046, 0.0068, 0.0054], + [0.0035, 0.0046, 0.0034, 0.0014, 0.0034, 0.0046, 0.0035], + [0.0006, 0.0014, 0.0014, 0.0006, 0.0014, 0.0014, 0.0006], + [0.0035, 0.0046, 0.0034, 0.0014, 0.0034, 0.0046, 0.0035], + [0.0054, 0.0068, 0.0046, 0.0014, 0.0046, 0.0068, 0.0054], + [0.0042, 0.0054, 0.0035, 0.0006, 0.0035, 0.0054, 0.0042], + ] + ] + ], + device=device, + dtype=dtype, + ).float() + harris = kornia.feature.CornerHarris(k=0.04).to(device) + scores = harris(inp) + self.assert_close(scores, expected, atol=1e-4, rtol=1e-3) + + def test_corners_batch(self, device, dtype): + inp = torch.tensor( + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + ], + device=device, + dtype=dtype, + ).repeat(2, 1, 1, 1) + expected = ( + torch.tensor( + [ + [ + [0.0415, 0.0541, 0.0346, 0.0058, 0.0346, 0.0541, 0.0415], + [0.0541, 0.0678, 0.0457, 0.0145, 0.0457, 0.0678, 0.0541], + [0.0346, 0.0457, 0.0335, 0.0139, 0.0335, 0.0457, 0.0346], + [0.0058, 0.0145, 0.0139, 0.0064, 0.0139, 0.0145, 0.0058], + [0.0346, 0.0457, 0.0335, 0.0139, 0.0335, 0.0457, 0.0346], + [0.0541, 0.0678, 0.0457, 0.0145, 0.0457, 0.0678, 0.0541], + [0.0415, 0.0541, 0.0346, 0.0058, 0.0346, 0.0541, 0.0415], + ], + [ + [0.0415, 0.0547, 0.0447, 0.0440, 0.0490, 0.0182, 0.0053], + [0.0547, 0.0688, 0.0557, 0.0549, 0.0610, 0.0229, 0.0066], + [0.0447, 0.0557, 0.0444, 0.0437, 0.0489, 0.0168, 0.0035], + [0.0440, 0.0549, 0.0437, 0.0431, 0.0481, 0.0166, 0.0034], + [0.0490, 0.0610, 0.0489, 0.0481, 0.0541, 0.0205, 0.0060], + [0.0182, 0.0229, 0.0168, 0.0166, 0.0205, 0.0081, 0.0025], + [0.0053, 0.0066, 0.0035, 0.0034, 0.0060, 0.0025, 0.0008], + ], + ], + device=device, + dtype=dtype, + ).repeat(2, 1, 1, 1) + / 10.0 + ) + scores = kornia.feature.harris_response(inp, k=0.04) + self.assert_close(scores, expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + k = 0.04 + batch_size, channels, height, width = 1, 2, 5, 4 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.feature.harris_response, (img, k), nondet_tol=1e-4) + + +class TestCornerGFTT(BaseTester): + def test_shape(self, device): + inp = torch.ones(1, 3, 4, 4, device=device) + shi_tomasi = kornia.feature.CornerGFTT().to(device) + assert shi_tomasi(inp).shape == (1, 3, 4, 4) + + def test_shape_batch(self, device): + inp = torch.zeros(2, 6, 4, 4, device=device) + shi_tomasi = kornia.feature.CornerGFTT().to(device) + assert shi_tomasi(inp).shape == (2, 6, 4, 4) + + def test_corners(self, device, dtype): + inp = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ).float() + + expected = torch.tensor( + [ + [ + [ + [0.0379, 0.0456, 0.0283, 0.0121, 0.0283, 0.0456, 0.0379], + [0.0456, 0.0598, 0.0402, 0.0168, 0.0402, 0.0598, 0.0456], + [0.0283, 0.0402, 0.0545, 0.0245, 0.0545, 0.0402, 0.0283], + [0.0121, 0.0168, 0.0245, 0.0276, 0.0245, 0.0168, 0.0121], + [0.0283, 0.0402, 0.0545, 0.0245, 0.0545, 0.0402, 0.0283], + [0.0456, 0.0598, 0.0402, 0.0168, 0.0402, 0.0598, 0.0456], + [0.0379, 0.0456, 0.0283, 0.0121, 0.0283, 0.0456, 0.0379], + ] + ] + ], + device=device, + dtype=dtype, + ).float() + shi_tomasi = kornia.feature.CornerGFTT().to(device) + scores = shi_tomasi(inp) + self.assert_close(scores, expected, atol=1e-4, rtol=1e-3) + + def test_corners_batch(self, device, dtype): + inp = torch.tensor( + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + ], + device=device, + dtype=dtype, + ).repeat(2, 1, 1, 1) + expected = torch.tensor( + [ + [ + [0.0379, 0.0456, 0.0283, 0.0121, 0.0283, 0.0456, 0.0379], + [0.0456, 0.0598, 0.0402, 0.0168, 0.0402, 0.0598, 0.0456], + [0.0283, 0.0402, 0.0545, 0.0245, 0.0545, 0.0402, 0.0283], + [0.0121, 0.0168, 0.0245, 0.0276, 0.0245, 0.0168, 0.0121], + [0.0283, 0.0402, 0.0545, 0.0245, 0.0545, 0.0402, 0.0283], + [0.0456, 0.0598, 0.0402, 0.0168, 0.0402, 0.0598, 0.0456], + [0.0379, 0.0456, 0.0283, 0.0121, 0.0283, 0.0456, 0.0379], + ], + [ + [0.0379, 0.0462, 0.0349, 0.0345, 0.0443, 0.0248, 0.0112], + [0.0462, 0.0608, 0.0488, 0.0483, 0.0581, 0.0274, 0.0119], + [0.0349, 0.0488, 0.0669, 0.0664, 0.0460, 0.0191, 0.0084], + [0.0345, 0.0483, 0.0664, 0.0660, 0.0455, 0.0189, 0.0083], + [0.0443, 0.0581, 0.0460, 0.0455, 0.0555, 0.0262, 0.0114], + [0.0248, 0.0274, 0.0191, 0.0189, 0.0262, 0.0172, 0.0084], + [0.0112, 0.0119, 0.0084, 0.0083, 0.0114, 0.0084, 0.0046], + ], + ], + device=device, + dtype=dtype, + ).repeat(2, 1, 1, 1) + shi_tomasi = kornia.feature.CornerGFTT().to(device) + scores = shi_tomasi(inp) + self.assert_close(scores, expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 2, 5, 4 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.feature.gftt_response, (img), nondet_tol=1e-4) + + +class TestBlobHessian(BaseTester): + def test_shape(self, device): + inp = torch.ones(1, 3, 4, 4, device=device) + shi_tomasi = kornia.feature.BlobHessian().to(device) + assert shi_tomasi(inp).shape == (1, 3, 4, 4) + + def test_shape_batch(self, device): + inp = torch.zeros(2, 6, 4, 4, device=device) + shi_tomasi = kornia.feature.BlobHessian().to(device) + assert shi_tomasi(inp).shape == (2, 6, 4, 4) + + def test_blobs_batch(self, device, dtype): + inp = torch.tensor( + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + ], + device=device, + dtype=dtype, + ).repeat(2, 1, 1, 1) + expected = torch.tensor( + [ + [ + [-0.0564, -0.0759, -0.0342, -0.0759, -0.0564, -0.0057, 0.0000], + [-0.0759, -0.0330, 0.0752, -0.0330, -0.0759, -0.0096, 0.0000], + [-0.0342, 0.0752, 0.1914, 0.0752, -0.0342, -0.0068, 0.0000], + [-0.0759, -0.0330, 0.0752, -0.0330, -0.0759, -0.0096, 0.0000], + [-0.0564, -0.0759, -0.0342, -0.0759, -0.0564, -0.0057, 0.0000], + [-0.0057, -0.0096, -0.0068, -0.0096, -0.0057, -0.0005, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + ], + [ + [-0.0564, -0.0522, -0.0522, -0.0564, -0.0057, 0.0000, 0.0000], + [-0.0522, 0.0688, 0.0688, -0.0123, 0.0033, -0.0057, -0.0005], + [-0.0522, 0.0688, -0.0755, -0.1111, -0.0123, -0.0564, -0.0057], + [-0.0564, -0.0123, -0.1111, -0.0755, 0.0688, -0.0522, -0.0080], + [-0.0057, 0.0033, -0.0123, 0.0688, 0.0688, -0.0522, -0.0080], + [0.0000, -0.0057, -0.0564, -0.0522, -0.0522, -0.0564, -0.0057], + [0.0000, -0.0005, -0.0057, -0.0080, -0.0080, -0.0057, -0.0005], + ], + ], + device=device, + dtype=dtype, + ).repeat(2, 1, 1, 1) + shi_tomasi = kornia.feature.BlobHessian().to(device) + scores = shi_tomasi(inp) + self.assert_close(scores, expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 2, 5, 4 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.feature.hessian_response, (img), nondet_tol=1e-4) + + +class TestBlobDoGSingle(BaseTester): + def test_shape(self, device): + inp = torch.ones(1, 3, 9, 9, device=device) + shi_tomasi = kornia.feature.BlobDoGSingle().to(device) + assert shi_tomasi(inp).shape == (1, 3, 9, 9) + + def test_shape_batch(self, device): + inp = torch.zeros(2, 6, 9, 9, device=device) + shi_tomasi = kornia.feature.BlobHessian().to(device) + assert shi_tomasi(inp).shape == (2, 6, 9, 9) + + def test_blobs_batch(self, device, dtype): + inp = torch.tensor( + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + ], + device=device, + dtype=dtype, + ).expand(2, 2, 7, 7) + expected = torch.tensor( + [ + [ + [ + [0.1640, 0.0811, -0.0148, 0.0040, 0.0850, 0.0962, 0.0754], + [0.0811, -0.0249, -0.1416, -0.0950, 0.0437, 0.0894, 0.0754], + [-0.0148, -0.1416, -0.2758, -0.2005, -0.0044, 0.0777, 0.0718], + [0.0040, -0.0950, -0.2005, -0.1445, 0.0045, 0.0648, 0.0586], + [0.0850, 0.0437, -0.0044, 0.0045, 0.0441, 0.0488, 0.0382], + [0.0962, 0.0894, 0.0777, 0.0648, 0.0488, 0.0295, 0.0197], + [0.0754, 0.0754, 0.0718, 0.0586, 0.0382, 0.0197, 0.0124], + ], + [ + [0.0689, -0.0056, -0.0254, 0.0802, 0.1118, 0.0708, 0.0491], + [-0.0056, -0.0895, -0.1031, 0.0362, 0.0986, 0.0771, 0.0621], + [-0.0254, -0.1031, -0.1423, -0.0651, 0.0039, 0.0567, 0.0794], + [0.0802, 0.0362, -0.0651, -0.1795, -0.1617, -0.0048, 0.0771], + [0.1118, 0.0986, 0.0039, -0.1617, -0.1706, -0.0095, 0.0761], + [0.0708, 0.0771, 0.0567, -0.0048, -0.0095, 0.0521, 0.0836], + [0.0491, 0.0621, 0.0794, 0.0771, 0.0761, 0.0836, 0.0858], + ], + ] + ], + device=device, + dtype=dtype, + ).expand(2, 2, 7, 7) + det = kornia.feature.BlobDoGSingle(1.0, 1.6).to(device) + scores = det(inp) + self.assert_close(scores, expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 2, 9, 11 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.feature.dog_response_single, (img), nondet_tol=1e-4) diff --git a/tests/feature/test_scale_space_detector.py b/tests/feature/test_scale_space_detector.py new file mode 100644 index 0000000..62a0232 --- /dev/null +++ b/tests/feature/test_scale_space_detector.py @@ -0,0 +1,193 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch + +import kornia +from kornia.feature.scale_space_detector import MultiResolutionDetector, ScaleSpaceDetector, get_default_detector_config +from kornia.geometry.subpix import ConvQuadInterp3d + +from testing.base import BaseTester + + +class TestScaleSpaceDetector(BaseTester): + def test_shape(self, device, dtype): + inp = torch.rand(1, 1, 32, 32, device=device, dtype=dtype) + n_feats = 10 + det = ScaleSpaceDetector(n_feats).to(device, dtype) + lafs, resps = det(inp) + assert lafs.shape == torch.Size([1, n_feats, 2, 3]) + assert resps.shape == torch.Size([1, n_feats]) + + def test_shape_batch(self, device, dtype): + inp = torch.rand(7, 1, 32, 32, device=device, dtype=dtype) + n_feats = 10 + det = ScaleSpaceDetector(n_feats).to(device, dtype) + lafs, resps = det(inp) + assert lafs.shape == torch.Size([7, n_feats, 2, 3]) + assert resps.shape == torch.Size([7, n_feats]) + + def test_toy(self, device, dtype): + inp = torch.zeros(1, 1, 33, 33, device=device, dtype=dtype) + inp[:, :, 13:-13, 13:-13] = 1.0 + n_feats = 1 + det = ScaleSpaceDetector(n_feats, resp_module=kornia.feature.BlobHessian(), mr_size=3.0).to(device, dtype) + lafs, resps = det(inp) + expected_laf = torch.tensor([[[[8.4260, 0.0000, 16.0], [0.0, 8.4260, 16.0]]]], device=device, dtype=dtype) + expected_resp = torch.tensor([[0.1159]], device=device, dtype=dtype) + self.assert_close(lafs, expected_laf, rtol=0.001, atol=1e-03) + self.assert_close(resps, expected_resp, rtol=0.001, atol=1e-03) + + def test_toy_mask(self, device, dtype): + inp = torch.zeros(1, 1, 33, 33, device=device, dtype=dtype) + inp[:, :, 13:-13, 13:-13] = 1.0 + + mask = torch.zeros(1, 1, 33, 33, device=device, dtype=dtype) + mask[:, :, 1:-1, 3:-3] = 1.0 + + n_feats = 1 + det = ScaleSpaceDetector(n_feats, resp_module=kornia.feature.BlobHessian(), mr_size=3.0).to(device, dtype) + lafs, resps = det(inp, mask) + expected_laf = torch.tensor([[[[8.4260, 0.0000, 16.0], [0.0, 8.4260, 16.0]]]], device=device, dtype=dtype) + expected_resp = torch.tensor([[0.1159]], device=device, dtype=dtype) + self.assert_close(lafs, expected_laf, rtol=0.001, atol=1e-03) + self.assert_close(resps, expected_resp, rtol=0.001, atol=1e-03) + + def test_minima_are_also_good(self, device, dtype): + # Image with a bright blob (local max) and dark blob (local min). + # With minima_are_also_good=True both should contribute to detections. + inp = torch.ones(1, 1, 33, 33, device=device, dtype=dtype) * 0.5 + inp[:, :, 10:14, 10:14] = 1.0 # bright blob → local maximum + inp[:, :, 10:14, 20:24] = 0.0 # dark blob → local minimum + n_feats = 2 + det_max_only = ScaleSpaceDetector(n_feats, resp_module=kornia.feature.BlobHessian(), mr_size=3.0).to( + device, dtype + ) + det_minmax = ScaleSpaceDetector( + n_feats, resp_module=kornia.feature.BlobHessian(), mr_size=3.0, minima_are_also_good=True + ).to(device, dtype) + lafs_max, resps_max = det_max_only(inp) + lafs_minmax, resps_minmax = det_minmax(inp) + assert lafs_max.shape == torch.Size([1, n_feats, 2, 3]) + assert lafs_minmax.shape == torch.Size([1, n_feats, 2, 3]) + # minmax detector should find a higher total response magnitude (it sees both blobs). + assert resps_minmax.abs().sum() >= resps_max.abs().sum() + + def test_scale_space_response_mode(self, device, dtype): + # Smoke test: scale_space_response=True uses a different internal code path. + # BlobDoG operates on the 5D scale-space tensor directly. + inp = torch.rand(1, 1, 32, 32, device=device, dtype=dtype) + n_feats = 5 + det = ScaleSpaceDetector(n_feats, resp_module=kornia.feature.BlobDoG(), scale_space_response=True).to( + device, dtype + ) + lafs, resps = det(inp) + assert lafs.shape == torch.Size([1, n_feats, 2, 3]) + assert resps.shape == torch.Size([1, n_feats]) + + def test_few_detections_padding(self, device, dtype): + # Constant image → very few (possibly zero) NMS candidates; output must still + # have the requested shape because the detect() method pads with zeros. + inp = torch.ones(1, 1, 32, 32, device=device, dtype=dtype) + n_feats = 20 + det = ScaleSpaceDetector(n_feats, subpix_module=ConvQuadInterp3d(10)).to(device, dtype) + lafs, resps = det(inp) + assert lafs.shape == torch.Size([1, n_feats, 2, 3]) + assert resps.shape == torch.Size([1, n_feats]) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 1, 7, 7 + patches = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + # Use ConvQuadInterp3d for gradcheck — IterativeQuadInterp3d uses non-differentiable + # indexed in-place assignments that are incompatible with torch.autograd.gradcheck. + det = ScaleSpaceDetector(2, subpix_module=ConvQuadInterp3d(10)).to(device) + self.gradcheck(det, patches, nondet_tol=1e-4) + + +class TestMultiResolutionDetector(BaseTester): + def _make_detector(self, num_features: int = 50, **config_overrides): + cfg = get_default_detector_config() + cfg.update(config_overrides) + return MultiResolutionDetector(kornia.feature.BlobHessian(), num_features=num_features, config=cfg) + + def test_shape(self, device, dtype): + inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype) + det = self._make_detector().to(device, dtype) + lafs, resps = det(inp) + assert lafs.shape == torch.Size([1, 50, 2, 3]) + assert resps.shape == torch.Size([1, 50]) + + def test_shape_non_square(self, device, dtype): + inp = torch.rand(1, 1, 48, 96, device=device, dtype=dtype) + det = self._make_detector().to(device, dtype) + lafs, _ = det(inp) + assert lafs.shape == torch.Size([1, 50, 2, 3]) + + def test_lafs_inside_image(self, device, dtype): + # All detected LAF centers should lie within the image boundaries. + inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype) + det = self._make_detector(num_features=20).to(device, dtype) + lafs, _ = det(inp) + cx = lafs[0, :, 0, 2] + cy = lafs[0, :, 1, 2] + assert (cx >= 0).all() and (cx <= 64).all() + assert (cy >= 0).all() and (cy <= 64).all() + + def test_no_upscale_levels(self, device, dtype): + # up_levels=0 disables the upsampling branch; should still produce valid output. + inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype) + cfg = get_default_detector_config() + cfg["up_levels"] = 0 + cfg["pyramid_levels"] = 2 + det = MultiResolutionDetector(kornia.feature.BlobHessian(), num_features=20, config=cfg).to(device, dtype) + lafs, _ = det(inp) + assert lafs.shape == torch.Size([1, 20, 2, 3]) + + def test_with_upscale_levels(self, device, dtype): + # up_levels > 0 exercises the upsampling code path. + inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype) + cfg = get_default_detector_config() + cfg["up_levels"] = 2 + cfg["pyramid_levels"] = 1 + det = MultiResolutionDetector(kornia.feature.BlobHessian(), num_features=20, config=cfg).to(device, dtype) + lafs, _ = det(inp) + assert lafs.shape == torch.Size([1, 20, 2, 3]) + + def test_score_threshold_reduces_detections(self, device, dtype): + # A very high score_threshold should leave no real detections: all returned responses + # will be the sentinel fill value (very negative), while shape remains fixed. + inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype) + det_no_thresh = self._make_detector(num_features=50).to(device, dtype) + det_high_thresh = MultiResolutionDetector( + kornia.feature.BlobHessian(), num_features=50, score_threshold=1e6 + ).to(device, dtype) + lafs_no_thresh, resps_no_thresh = det_no_thresh(inp) + lafs_high_thresh, resps_high_thresh = det_high_thresh(inp) + assert lafs_high_thresh.shape == lafs_no_thresh.shape + # With an impossibly high threshold all slots contain the fill sentinel (< 0), + # while real detections always have positive responses. + assert resps_no_thresh.max().item() > 0 + assert (resps_high_thresh <= 0).all() + + def test_smoke_with_blob_image(self, device, dtype): + # Synthetic image with a bright blob — detector should find it. + inp = torch.zeros(1, 1, 64, 64, device=device, dtype=dtype) + inp[:, :, 28:36, 28:36] = 1.0 + det = self._make_detector(num_features=5).to(device, dtype) + lafs, resps = det(inp) + assert lafs.shape == torch.Size([1, 5, 2, 3]) + assert resps.abs().max().item() > 0 diff --git a/tests/feature/test_siftdesc.py b/tests/feature/test_siftdesc.py new file mode 100644 index 0000000..e63a15a --- /dev/null +++ b/tests/feature/test_siftdesc.py @@ -0,0 +1,115 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.feature.siftdesc import ( + DenseSIFTDescriptor, + SIFTDescriptor, + get_sift_bin_ksize_stride_pad, + get_sift_pooling_kernel, +) + +from testing.base import BaseTester + + +@pytest.mark.parametrize("ksize", [5, 13, 25]) +def test_get_sift_pooling_kernel(ksize): + kernel = get_sift_pooling_kernel(ksize) + assert kernel.shape == (ksize, ksize) + + +@pytest.mark.parametrize("ps,n_bins,ksize,stride,pad", [(41, 3, 20, 13, 5), (32, 4, 12, 8, 3)]) +def test_get_sift_bin_ksize_stride_pad(ps, n_bins, ksize, stride, pad): + out = get_sift_bin_ksize_stride_pad(ps, n_bins) + assert out == (ksize, stride, pad) + + +class TestSIFTDescriptor(BaseTester): + def test_shape(self, device, dtype): + inp = torch.ones(1, 1, 32, 32, device=device, dtype=dtype) + sift = SIFTDescriptor(32).to(device, dtype) + out = sift(inp) + assert out.shape == (1, 128) + + def test_batch_shape(self, device, dtype): + inp = torch.ones(2, 1, 15, 15, device=device, dtype=dtype) + sift = SIFTDescriptor(15).to(device, dtype) + out = sift(inp) + assert out.shape == (2, 128) + + def test_batch_shape_non_std(self, device, dtype): + inp = torch.ones(3, 1, 19, 19, device=device, dtype=dtype) + sift = SIFTDescriptor(19, 5, 3).to(device, dtype) + out = sift(inp) + assert out.shape == (3, (3**2) * 5) + + def test_toy(self, device, dtype): + patch = torch.ones(1, 1, 6, 6, device=device, dtype=dtype) + patch[0, 0, :, 3:] = 0 + sift = SIFTDescriptor(6, num_ang_bins=4, num_spatial_bins=1, clipval=0.2, rootsift=False).to(device, dtype) + out = sift(patch) + expected = torch.tensor([[0, 0, 1.0, 0]], device=device, dtype=dtype) + self.assert_close(out, expected, atol=1e-3, rtol=1e-3) + + def test_gradcheck(self, device): + dtype = torch.float64 + batch_size, channels, height, width = 1, 1, 15, 15 + patches = torch.rand(batch_size, channels, height, width, device=device, dtype=dtype) + sift = SIFTDescriptor(15).to(device, dtype) + self.gradcheck(sift, (patches,), nondet_tol=1e-4) + + @pytest.mark.skip("Compiled functions can't take variable number") + def test_jit(self, device, dtype): + B, C, H, W = 1, 1, 32, 32 + patches = torch.ones(B, C, H, W, device=device, dtype=dtype) + model = SIFTDescriptor(41).to(patches.device, patches.dtype).eval() + model_jit = torch.jit.script(SIFTDescriptor(41).to(patches.device, patches.dtype).eval()) + self.assert_close(model(patches), model_jit(patches)) + + +class TestDenseSIFTDescriptor(BaseTester): + def test_shape_default(self, device, dtype): + bs, h, w = 1, 20, 15 + inp = torch.rand(1, 1, h, w, device=device, dtype=dtype) + sift = DenseSIFTDescriptor().to(device, dtype) + out = sift(inp) + assert out.shape == torch.Size([bs, 128, h, w]) + + def test_batch_shape(self, device, dtype): + bs, h, w = 2, 32, 15 + inp = torch.rand(bs, 1, h, w, device=device, dtype=dtype) + sift = DenseSIFTDescriptor().to(device, dtype) + out = sift(inp) + assert out.shape == torch.Size([bs, 128, h, w]) + + def test_batch_shape_custom(self, device, dtype): + bs, h, w = 2, 40, 30 + inp = torch.rand(bs, 1, h, w, device=device, dtype=dtype) + sift = DenseSIFTDescriptor(5, 3, 3, padding=1, stride=2).to(device, dtype) + out = sift(inp) + assert out.shape == torch.Size([bs, 45, h // 2, w // 2]) + + def test_print(self, device): + sift = DenseSIFTDescriptor() + sift.__repr__() + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 1, 16, 16 + patches = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(DenseSIFTDescriptor(4, 2, 2), (patches), nondet_tol=1e-4) diff --git a/tests/feature/test_sold2.py b/tests/feature/test_sold2.py new file mode 100644 index 0000000..d6c927c --- /dev/null +++ b/tests/feature/test_sold2.py @@ -0,0 +1,80 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.feature.sold2 import SOLD2, SOLD2_detector + +from testing.base import BaseTester + + +class TestSOLD2_detector(BaseTester): + @pytest.mark.slow + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_shape(self, device, batch_size, dtype): + inp = torch.ones(batch_size, 1, 64, 64, device=device, dtype=dtype) + sold2 = SOLD2_detector(pretrained=False).to(device, dtype) + out = sold2(inp) + assert out["junction_heatmap"].shape == (batch_size, 64, 64) + assert out["line_heatmap"].shape == (batch_size, 64, 64) + + @pytest.mark.skip("Takes ages to run") + def test_gradcheck(self, device): + img = torch.rand(2, 1, 128, 128, device=device, dtype=torch.float64) + sold2 = SOLD2_detector(pretrained=False).to(img.device, img.dtype) + + def proxy_forward(x): + return sold2.forward(x)["junction_heatmap"] + + self.gradcheck(proxy_forward, (img,), eps=1e-4, atol=1e-4) + + @pytest.mark.skip("Does not like recursive definition of Hourglass in backbones.py l.134.") + def test_jit(self, device, dtype): + B, C, H, W = 2, 1, 128, 128 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + model = SOLD2_detector().to(img.device, img.dtype).eval() + model_jit = torch.jit.script(model) + self.assert_close(model(img), model_jit(img)) + + +class TestSOLD2(BaseTester): + @pytest.mark.slow + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_shape(self, device, batch_size, dtype): + inp = torch.ones(batch_size, 1, 64, 64, device=device, dtype=dtype) + sold2 = SOLD2(pretrained=False).to(device, dtype) + out = sold2(inp) + assert out["dense_desc"].shape == (batch_size, 128, 16, 16) + + @pytest.mark.skip("Takes ages to run") + def test_gradcheck(self, device): + img = torch.rand(2, 1, 256, 256, device=device, dtype=torch.float64) + sold2 = SOLD2(pretrained=False).to(img.device, img.dtype) + + def proxy_forward(x): + return sold2.forward(x)["dense_desc"] + + self.gradcheck(proxy_forward, (img,), eps=1e-4, atol=1e-4) + + @pytest.mark.skip("Does not like recursive definition of Hourglass in backbones.py l.134.") + def test_jit(self, device, dtype): + B, C, H, W = 2, 1, 256, 256 + img = torch.ones(B, C, H, W, device=device, dtype=dtype) + model = SOLD2().to(img.device, img.dtype).eval() + model_jit = torch.jit.script(model) + self.assert_close(model(img), model_jit(img)) diff --git a/tests/feature/test_sosnet.py b/tests/feature/test_sosnet.py new file mode 100644 index 0000000..7e37c46 --- /dev/null +++ b/tests/feature/test_sosnet.py @@ -0,0 +1,52 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.feature import SOSNet + +from testing.base import BaseTester + + +class TestSOSNet(BaseTester): + def test_shape(self, device): + inp = torch.ones(1, 1, 32, 32, device=device) + sosnet = SOSNet(pretrained=False).to(device) + sosnet.eval() # batchnorm with size 1 is not allowed in train mode + out = sosnet(inp) + assert out.shape == (1, 128) + + def test_shape_batch(self, device): + inp = torch.ones(16, 1, 32, 32, device=device) + sosnet = SOSNet(pretrained=False).to(device) + out = sosnet(inp) + assert out.shape == (16, 128) + + @pytest.mark.skip("jacobian not well computed") + def test_gradcheck(self, device): + patches = torch.rand(2, 1, 32, 32, device=device, dtype=torch.float64) + sosnet = SOSNet(pretrained=False).to(patches.device, patches.dtype) + self.gradcheck(sosnet, (patches,), eps=1e-4, atol=1e-4) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 1, 32, 32 + patches = torch.ones(B, C, H, W, device=device, dtype=dtype) + model = SOSNet().to(patches.device, patches.dtype).eval() + model_jit = torch.jit.script(SOSNet().to(patches.device, patches.dtype).eval()) + self.assert_close(model(patches), model_jit(patches)) diff --git a/tests/feature/test_steerer.py b/tests/feature/test_steerer.py new file mode 100644 index 0000000..ee14318 --- /dev/null +++ b/tests/feature/test_steerer.py @@ -0,0 +1,98 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.feature.steerers import DiscreteSteerer + +from testing.base import BaseTester + + +class TestDiscreteSteerer(BaseTester): + @pytest.mark.parametrize("num_desc, desc_dim, steerer_power", [(1, 4, 1), (2, 128, 7), (32, 128, 11)]) + def test_shape(self, num_desc, desc_dim, steerer_power, device, dtype): + desc = torch.rand(num_desc, desc_dim, device=device, dtype=dtype) + generator = torch.rand(desc_dim, desc_dim, device=device, dtype=dtype) + steerer = DiscreteSteerer(generator) + desc = steerer.steer_descriptions(desc, steerer_power=steerer_power) + assert desc.shape == (num_desc, desc_dim) + + @pytest.mark.parametrize("normalize", [True, False]) + def test_steering(self, device, normalize): + generator = torch.tensor([[0.0, 1], [-1, 0]], device=device) + desc = torch.rand(16, 2, device=device) + steerer = DiscreteSteerer(generator) + desc_out = steerer.steer_descriptions(desc, steerer_power=3, normalize=normalize) + if normalize: + desc = torch.nn.functional.normalize(desc, dim=-1) + desc = desc[:, [1, 0]] + desc[:, 0] = -desc[:, 0] + assert torch.allclose(desc, desc_out, atol=1e-6) + + @pytest.mark.parametrize("generator_type", ["C4", "SO2"]) + @pytest.mark.parametrize("steerer_order", [2, 14]) + def test_default(self, device, generator_type, steerer_order): + steerer = DiscreteSteerer.create_dedode_default( + generator_type=generator_type, + steerer_order=steerer_order, + ).to(device) + assert isinstance(steerer, DiscreteSteerer) + shape = (96, 256) + desc = torch.randn(*shape, device=device) + desc = steerer(desc) + assert desc.shape == shape + + @pytest.mark.parametrize("steerer_power", [0, 1, 5, -1]) + def test_steerer_power_extremes(self, device, steerer_power): + generator = torch.tensor([[0.0, 1], [-1, 0]], device=device) + desc = torch.rand(8, 2, device=device) + steerer = DiscreteSteerer(generator) + out = steerer.steer_descriptions(desc, steerer_power=steerer_power) + assert out.shape == desc.shape + + def test_normalization_preserves_norm(self, device): + generator = torch.tensor([[0.0, 1], [-1, 0]], device=device) + desc = torch.rand(8, 2, device=device) + steerer = DiscreteSteerer(generator) + out = steerer.steer_descriptions(desc, steerer_power=1, normalize=True) + orig_norm = torch.norm(out, dim=-1) + assert torch.allclose(orig_norm, torch.ones_like(orig_norm), atol=1e-6) + + def test_invalid_generator_type_raises(self): + with pytest.raises(ValueError): + DiscreteSteerer.create_dedode_default(generator_type="INVALID") + + def test_gradient_flow(self, device): + generator = torch.tensor([[0.0, 1], [-1, 0]], device=device) + desc = torch.rand(4, 2, device=device, requires_grad=True) + steerer = DiscreteSteerer(generator) + out = steerer.steer_descriptions(desc, steerer_power=2) + loss = out.sum() + loss.backward() + assert desc.grad is not None + assert torch.all(desc.grad != 0) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + def test_cpu_vs_cuda_consistency(self): + generator = torch.tensor([[0.0, 1], [-1, 0]]) + desc = torch.rand(4, 2) + cpu_steerer = DiscreteSteerer(generator) + cuda_steerer = DiscreteSteerer(generator.to("cuda")) + out_cpu = cpu_steerer.steer_descriptions(desc, steerer_power=2) + out_cuda = cuda_steerer.steer_descriptions(desc.to("cuda"), steerer_power=2).cpu() + assert torch.allclose(out_cpu, out_cuda, atol=1e-6) diff --git a/tests/feature/test_tfeat.py b/tests/feature/test_tfeat.py new file mode 100644 index 0000000..f656fb7 --- /dev/null +++ b/tests/feature/test_tfeat.py @@ -0,0 +1,61 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.feature import TFeat + +from testing.base import BaseTester + + +class TestTFeat(BaseTester): + def test_shape(self, device): + inp = torch.ones(1, 1, 32, 32, device=device) + tfeat = TFeat().to(device) + tfeat.eval() # batchnorm with size 1 is not allowed in train mode + out = tfeat(inp) + assert out.shape == (1, 128) + + @pytest.mark.slow + def test_pretrained(self, device): + inp = torch.ones(1, 1, 32, 32, device=device) + tfeat = TFeat(True).to(device) + tfeat.eval() # batchnorm with size 1 is not allowed in train mode + out = tfeat(inp) + assert out.shape == (1, 128) + + def test_shape_batch(self, device): + inp = torch.ones(16, 1, 32, 32, device=device) + tfeat = TFeat().to(device) + out = tfeat(inp) + assert out.shape == (16, 128) + + @pytest.mark.skip("jacobian not well computed") + def test_gradcheck(self, device): + patches = torch.rand(2, 1, 32, 32, device=device, dtype=torch.float64) + tfeat = TFeat().to(patches.device, patches.dtype) + self.gradcheck(tfeat, (patches,), eps=1e-2, atol=1e-2) + + @pytest.mark.slow + @pytest.mark.jit() + def test_jit(self, device, dtype): + B, C, H, W = 2, 1, 32, 32 + patches = torch.ones(B, C, H, W, device=device, dtype=dtype) + tfeat = TFeat(True).to(patches.device, patches.dtype).eval() + tfeat_jit = torch.jit.script(TFeat(True).to(patches.device, patches.dtype).eval()) + self.assert_close(tfeat_jit(patches), tfeat(patches)) diff --git a/tests/feature/test_xfeat.py b/tests/feature/test_xfeat.py new file mode 100644 index 0000000..8ff27c9 --- /dev/null +++ b/tests/feature/test_xfeat.py @@ -0,0 +1,423 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.feature.lightglue import LightGlue +from kornia.feature.xfeat import InterpolateSparse2d, XFeat, XFeatModel + +from testing.base import BaseTester + +# --------------------------------------------------------------------------- +# XFeatModel backbone tests +# --------------------------------------------------------------------------- + + +class TestXFeatModel(BaseTester): + def test_smoke(self, device, dtype): + model = XFeatModel().to(device, dtype) + x = torch.rand(1, 3, 64, 64, device=device, dtype=dtype) + feats, kpts, heatmap = model(x) + assert feats.shape == (1, 64, 8, 8) + assert kpts.shape == (1, 65, 8, 8) + assert heatmap.shape == (1, 1, 8, 8) + + def test_cardinality(self, device, dtype): + B = 2 + model = XFeatModel().to(device, dtype) + x = torch.rand(B, 1, 96, 128, device=device, dtype=dtype) + feats, kpts, heatmap = model(x) + assert feats.shape == (B, 64, 12, 16) + assert kpts.shape == (B, 65, 12, 16) + assert heatmap.shape == (B, 1, 12, 16) + + def test_exception(self, device, dtype): + pass # XFeatModel has no checked exceptions on forward + + def test_gradcheck(self, device): + pass # InstanceNorm2d inside torch.no_grad() breaks end-to-end gradcheck + + def test_dynamo(self, device, dtype, torch_optimizer): + model = XFeatModel().to(device, dtype) + x = torch.rand(1, 3, 64, 64, device=device, dtype=dtype) + op = torch_optimizer(model) + feats_c, kpts_c, hm_c = op(x) + feats, kpts, hm = model(x) + if dtype in (torch.float32, torch.float64): + self.assert_close(feats_c, feats, rtol=1e-3, atol=1e-4) + self.assert_close(kpts_c, kpts, rtol=1e-3, atol=1e-4) + self.assert_close(hm_c, hm, rtol=1e-3, atol=1e-4) + else: + self.assert_close(feats_c, feats) + self.assert_close(kpts_c, kpts) + self.assert_close(hm_c, hm) + + def test_heatmap_in_zero_one(self, device, dtype): + model = XFeatModel().to(device, dtype) + x = torch.rand(1, 3, 64, 64, device=device, dtype=dtype) + _feats, _kpts, heatmap = model(x) + assert heatmap.min() >= 0.0 + assert heatmap.max() <= 1.0 + + def test_grayscale_and_rgb_same_shape(self, device, dtype): + model = XFeatModel().to(device, dtype) + gray = torch.rand(1, 1, 64, 64, device=device, dtype=dtype) + rgb = torch.rand(1, 3, 64, 64, device=device, dtype=dtype) + f_gray, _, _ = model(gray) + f_rgb, _, _ = model(rgb) + assert f_gray.shape == f_rgb.shape + + +# --------------------------------------------------------------------------- +# InterpolateSparse2d tests +# --------------------------------------------------------------------------- + + +class TestInterpolateSparse2d(BaseTester): + def test_smoke(self, device, dtype): + interp = InterpolateSparse2d("bilinear").to(device) + x = torch.rand(1, 32, 8, 8, device=device, dtype=dtype) + pos = torch.zeros(1, 5, 2, device=device, dtype=dtype) + out = interp(x, pos, 8, 8) + assert out.shape == (1, 5, 32) + + def test_cardinality(self, device, dtype): + interp = InterpolateSparse2d("bicubic").to(device) + B, C, H, W, N = 2, 16, 32, 48, 20 + x = torch.rand(B, C, H, W, device=device, dtype=dtype) + pos = torch.rand(B, N, 2, device=device, dtype=dtype) * torch.tensor([W - 1, H - 1], device=device, dtype=dtype) + out = interp(x, pos, H, W) + assert out.shape == (B, N, C) + + def test_exception(self, device, dtype): + pass + + def test_gradcheck(self, device): + interp = InterpolateSparse2d("bilinear") + x = torch.rand(1, 4, 8, 8, device=device, dtype=torch.float64, requires_grad=True) + pos = torch.rand(1, 3, 2, device=device, dtype=torch.float64) * 7 + self.gradcheck(interp, (x, pos, 8, 8), requires_grad=[True, False, False, False]) + + def test_dynamo(self, device, dtype, torch_optimizer): + interp = InterpolateSparse2d("bilinear") + x = torch.rand(1, 8, 16, 16, device=device, dtype=dtype) + pos = torch.rand(1, 5, 2, device=device, dtype=dtype) * 15 + op = torch_optimizer(interp) + self.assert_close(op(x, pos, 16, 16), interp(x, pos, 16, 16)) + + def test_normgrid_center(self, device, dtype): + interp = InterpolateSparse2d() + coords = torch.tensor([[[32.0, 24.0]]], device=device, dtype=dtype) + grid = interp.normgrid(coords, 49, 65) + assert grid.abs().max() < 0.1 # close to center -> close to 0 + + +# --------------------------------------------------------------------------- +# XFeat end-to-end tests +# --------------------------------------------------------------------------- + + +class TestXFeat(BaseTester): + def test_smoke(self, device, dtype): + model = XFeat().to(device) + x = torch.rand(1, 3, 64, 64, device=device) + out = model.detectAndCompute(x) + assert isinstance(out, list) + assert len(out) == 1 + assert "keypoints" in out[0] + assert "scores" in out[0] + assert "descriptors" in out[0] + + def test_cardinality(self, device, dtype): + model = XFeat().to(device) + B = 2 + x = torch.rand(B, 3, 64, 64, device=device) + out = model.detectAndCompute(x) + assert len(out) == B + for item in out: + N = item["keypoints"].shape[0] + assert item["scores"].shape == (N,) + assert item["descriptors"].shape == (N, 64) + + def test_exception(self, device, dtype): + model = XFeat().to(device) + with pytest.raises(Exception): + model.detectAndCompute(torch.rand(3, 64, 64, device=device)) # missing batch dim + + def test_gradcheck(self, device): + pass # inference_mode on detectAndCompute; gradcheck not applicable + + def test_dynamo(self, device, dtype, torch_optimizer): + pass # inference_mode and dynamic NMS control flow; not torch.compile compatible + + def test_descriptors_normalized(self, device, dtype): + model = XFeat().to(device) + x = torch.rand(1, 3, 64, 64, device=device) + out = model.detectAndCompute(x) + if out[0]["descriptors"].numel() == 0: + pytest.skip("No keypoints detected on random input") + norms = out[0]["descriptors"].norm(dim=-1) + self.assert_close(norms, torch.ones_like(norms), atol=1e-5, rtol=1e-5) + + def test_top_k_respected(self, device, dtype): + top_k = 16 + model = XFeat(top_k=top_k).to(device) + x = torch.rand(1, 3, 128, 128, device=device) + out = model.detectAndCompute(x, top_k=top_k) + assert out[0]["keypoints"].shape[0] <= top_k + + def test_keypoint_coordinates_in_image(self, device, dtype): + H, W = 64, 96 + model = XFeat().to(device) + x = torch.rand(1, 3, H, W, device=device) + out = model.detectAndCompute(x) + if out[0]["keypoints"].numel() == 0: + pytest.skip("No keypoints detected on random input") + kpts = out[0]["keypoints"].float() + assert (kpts[:, 0] >= 0).all() and (kpts[:, 0] < W).all() + assert (kpts[:, 1] >= 0).all() and (kpts[:, 1] < H).all() + + def test_dense_output_shapes(self, device, dtype): + top_k = 32 + model = XFeat(top_k=top_k).to(device) + x = torch.rand(1, 3, 64, 64, device=device) + out = model.detectAndComputeDense(x, top_k=top_k, multiscale=False) + assert "keypoints" in out + assert "descriptors" in out + assert "scales" in out + assert out["keypoints"].shape[-1] == 2 + assert out["descriptors"].shape[-1] == 64 + assert out["keypoints"].shape[1] == out["descriptors"].shape[1] + + def test_match_xfeat_returns_paired_keypoints(self, device, dtype): + model = XFeat().to(device) + img1 = torch.rand(1, 3, 64, 64, device=device) + img2 = torch.rand(1, 3, 64, 64, device=device) + mkpts0, mkpts1 = model.match_xfeat(img1, img2) + assert mkpts0.shape == mkpts1.shape + assert mkpts0.shape[-1] == 2 + + @pytest.mark.slow + def test_pretrained_smoke(self, device): + model = XFeat.from_pretrained().to(device) + x = torch.rand(1, 3, 256, 256, device=device) + out = model.detectAndCompute(x) + assert len(out) == 1 + assert out[0]["keypoints"].shape[-1] == 2 + + +# --------------------------------------------------------------------------- +# LighterGlue (xfeat config in LightGlue) tests +# --------------------------------------------------------------------------- + + +def _make_lighterglue(device: torch.device, dtype: torch.dtype) -> LightGlue: + """LightGlue with xfeat architecture and random weights (no download).""" + return ( + LightGlue( + features=None, + input_dim=64, + descriptor_dim=96, + n_layers=6, + num_heads=1, + depth_confidence=-1, + width_confidence=-1, + flash=False, + ) + .to(device, dtype) + .eval() + ) + + +def _make_lighterglue_data(device: torch.device, dtype: torch.dtype, B: int = 1, M: int = 20, N: int = 15) -> dict: + return { + "image0": { + "keypoints": torch.rand(B, M, 2, device=device, dtype=dtype) * 64, + "descriptors": torch.rand(B, M, 64, device=device, dtype=dtype), + "image_size": torch.tensor([[64, 64]], device=device, dtype=dtype).expand(B, -1), + }, + "image1": { + "keypoints": torch.rand(B, N, 2, device=device, dtype=dtype) * 64, + "descriptors": torch.rand(B, N, 64, device=device, dtype=dtype), + "image_size": torch.tensor([[64, 64]], device=device, dtype=dtype).expand(B, -1), + }, + } + + +def test_lighterglue_smoke(): + device = torch.device("cpu") + dtype = torch.float32 + lg = _make_lighterglue(device, dtype) + data = _make_lighterglue_data(device, dtype) + with torch.no_grad(): + out = lg(data) + assert "matches0" in out + assert "matching_scores0" in out + + +def test_lighterglue_cardinality(): + B, M, N = 1, 20, 15 + device = torch.device("cpu") + dtype = torch.float32 + lg = _make_lighterglue(device, dtype) + data = _make_lighterglue_data(device, dtype, B=B, M=M, N=N) + with torch.no_grad(): + out = lg(data) + assert out["matches0"].shape == (B, M) + assert out["matches1"].shape == (B, N) + assert out["matching_scores0"].shape == (B, M) + + +def test_lighterglue_architecture(): + """The xfeat LighterGlue config creates a 6-layer, 1-head, 96-dim model.""" + lg = _make_lighterglue(torch.device("cpu"), torch.float32) + assert lg.conf.n_layers == 6 + assert lg.conf.num_heads == 1 + assert lg.conf.descriptor_dim == 96 + assert lg.conf.input_dim == 64 + assert len(lg.transformers) == 6 + + +def test_lighterglue_scores_in_range(): + device = torch.device("cpu") + dtype = torch.float32 + lg = _make_lighterglue(device, dtype) + data = _make_lighterglue_data(device, dtype) + with torch.no_grad(): + out = lg(data) + scores = out["matching_scores0"] + assert (scores >= 0).all() + assert (scores <= 1).all() + + +def test_lighterglue_matches_are_valid_indices(): + B, M, N = 1, 30, 25 + device = torch.device("cpu") + dtype = torch.float32 + lg = _make_lighterglue(device, dtype) + data = _make_lighterglue_data(device, dtype, B=B, M=M, N=N) + with torch.no_grad(): + out = lg(data) + m0 = out["matches0"] + assert (m0 >= -1).all() + assert (m0 < N).all() + + +def test_lighterglue_wrong_features(): + """LightGlue rejects unknown feature types.""" + with pytest.raises(Exception): + LightGlue(features="nonexistent_feature") + + +@pytest.mark.slow +def test_lighterglue_pretrained_smoke(): + """Download xfeat-lighterglue weights and run a forward pass.""" + lg = LightGlue(features="xfeat", depth_confidence=-1, width_confidence=-1).to("cpu").eval() + B, M, N = 1, 50, 50 + data = _make_lighterglue_data(torch.device("cpu"), torch.float32, B=B, M=M, N=N) + with torch.no_grad(): + out = lg(data) + assert "matches0" in out + + +# --------------------------------------------------------------------------- +# Reference-data tests (kornia/data_test xfeat_reference.pt) +# --------------------------------------------------------------------------- + + +def _nn_match_fraction(kpts_a: torch.Tensor, kpts_b: torch.Tensor, max_dist: float) -> float: + """Fraction of points in ``kpts_a`` whose nearest neighbour in ``kpts_b`` is within ``max_dist`` px.""" + if kpts_a.numel() == 0 or kpts_b.numel() == 0: + return 0.0 + dists = torch.cdist(kpts_a.float(), kpts_b.float()) # (Na, Nb) + return (dists.min(dim=1).values < max_dist).float().mean().item() + + +@pytest.mark.slow +@pytest.mark.parametrize("data", ["xfeat_outdoor"], indirect=True) +def test_xfeat_reference_keypoints(device, data): + """XFeat keypoints reproduce the reference set under NN matching (order-independent).""" + xfeat = XFeat.from_pretrained(top_k=1024).to(device) + + out1 = xfeat.detectAndCompute(data["img1"].to(device))[0] + out2 = xfeat.detectAndCompute(data["img2"].to(device))[0] + + ref_kpts0 = data["xfeat_kpts0"].to(device) + ref_kpts1 = data["xfeat_kpts1"].to(device) + + # At least 99 % of computed keypoints must lie within 3 px of a reference keypoint + assert _nn_match_fraction(out1["keypoints"], ref_kpts0, max_dist=3.0) > 0.99 + assert _nn_match_fraction(out2["keypoints"], ref_kpts1, max_dist=3.0) > 0.99 + # Symmetric: reference keypoints are also covered by computed set + assert _nn_match_fraction(ref_kpts0, out1["keypoints"], max_dist=3.0) > 0.99 + assert _nn_match_fraction(ref_kpts1, out2["keypoints"], max_dist=3.0) > 0.99 + + +@pytest.mark.slow +@pytest.mark.parametrize("data", ["xfeat_outdoor"], indirect=True) +def test_lighterglue_reference_matches(device, data): + """LighterGlue matched pixel-coordinate pairs overlap ≥95 % with reference (order-independent).""" + xfeat = XFeat.from_pretrained(top_k=1024).to(device) + img1 = data["img1"].to(device) + img2 = data["img2"].to(device) + + out1 = xfeat.detectAndCompute(img1)[0] + out2 = xfeat.detectAndCompute(img2)[0] + + H, W = img1.shape[-2:] + lg = LightGlue(features="xfeat", depth_confidence=-1, width_confidence=-1).to(device).eval() + lg_data = { + "image0": { + "keypoints": out1["keypoints"].unsqueeze(0), + "descriptors": out1["descriptors"].unsqueeze(0), + "image_size": torch.tensor([[W, H]], dtype=torch.float32, device=device), + }, + "image1": { + "keypoints": out2["keypoints"].unsqueeze(0), + "descriptors": out2["descriptors"].unsqueeze(0), + "image_size": torch.tensor([[W, H]], dtype=torch.float32, device=device), + }, + } + with torch.no_grad(): + lg_out = lg(lg_data) + + # Build (x0, y0, x1, y1) match-coordinate tensors for computed and reference results + m0_comp = lg_out["matches0"].squeeze(0) # (N,) indices into kpts1, -1 = unmatched + valid_comp = m0_comp > -1 + if valid_comp.any(): + comp_pairs = torch.cat( + [out1["keypoints"][valid_comp], out2["keypoints"][m0_comp[valid_comp]]], dim=-1 + ) # (K_comp, 4) + else: + comp_pairs = torch.zeros(0, 4, device=device) + + ref_m0 = data["lighterglue_matches0"].to(device) + ref_kpts0 = data["xfeat_kpts0"].to(device) + ref_kpts1 = data["xfeat_kpts1"].to(device) + valid_ref = ref_m0 > -1 + ref_pairs = torch.cat([ref_kpts0[valid_ref], ref_kpts1[ref_m0[valid_ref]]], dim=-1) # (K_ref, 4) + + # Number of matches should be in the same ballpark (within 2x) + n_comp = valid_comp.sum().item() + n_ref = valid_ref.sum().item() + assert n_comp > n_ref / 2, f"Too few matches: {n_comp} vs reference {n_ref}" + assert n_comp < n_ref * 2, f"Too many matches: {n_comp} vs reference {n_ref}" + + # NN matching in joint (x0,y0,x1,y1) space with max L2 distance 2.0 + frac = _nn_match_fraction(ref_pairs, comp_pairs, max_dist=2.0) + assert frac > 0.95, f"Only {frac:.1%} of reference matches reproduced within joint distance 2.0 px" diff --git a/tests/filters/test_bilateral.py b/tests/filters/test_bilateral.py new file mode 100644 index 0000000..d7e7430 --- /dev/null +++ b/tests/filters/test_bilateral.py @@ -0,0 +1,357 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.filters import BilateralBlur, JointBilateralBlur, bilateral_blur, joint_bilateral_blur + +from testing.base import BaseTester + + +class TestBilateralBlur(BaseTester): + @pytest.mark.parametrize("shape", [(1, 1, 8, 15), (2, 3, 11, 7)]) + @pytest.mark.parametrize("kernel_size", [5, (3, 5)]) + @pytest.mark.parametrize("color_distance_type", ["l1", "l2"]) + def test_smoke(self, shape, kernel_size, color_distance_type, device, dtype): + inp = torch.zeros(shape, device=device, dtype=dtype) + + # tensor sigmas -> with batch dim + sigma_color = torch.rand(shape[0], device=device, dtype=dtype) + sigma_space = torch.rand(shape[0], 2, device=device, dtype=dtype) + actual_A = bilateral_blur(inp, kernel_size, sigma_color, sigma_space, "reflect", color_distance_type) + assert isinstance(actual_A, torch.Tensor) + assert actual_A.shape == shape + + # float and tuple sigmas -> same sigmas across batch + sigma_color_ = sigma_color[0].item() + sigma_space_ = tuple(sigma_space[0].cpu().numpy()) + actual_B = bilateral_blur(inp, kernel_size, sigma_color_, sigma_space_, "reflect", color_distance_type) + assert isinstance(actual_B, torch.Tensor) + assert actual_B.shape == shape + + self.assert_close(actual_A[0], actual_B[0]) + + @pytest.mark.parametrize("shape", [(1, 1, 8, 15), (2, 3, 11, 7)]) + @pytest.mark.parametrize("kernel_size", [5, (3, 5)]) + def test_cardinality(self, shape, kernel_size, device, dtype): + inp = torch.zeros(shape, device=device, dtype=dtype) + actual = bilateral_blur(inp, kernel_size, 0.1, (1, 1)) + assert actual.shape == shape + + def test_exception(self): + from kornia.core.exceptions import TypeCheckError + + with pytest.raises(TypeCheckError) as errinfo: + bilateral_blur(torch.rand(1, 1, 5, 5), 3, 1, 1) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(ValueError) as errinfo: + bilateral_blur(torch.rand(1, 1, 5, 5), 3, 0.1, (1, 1), color_distance_type="l3") + assert "color_distance_type only accepts l1 or l2" in str(errinfo) + + def test_noncontiguous(self, device, dtype): + batch_size = 3 + inp = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + + actual = bilateral_blur(inp, 3, 1, (1, 1)) + assert actual.is_contiguous() + + def test_gradcheck(self, device): + img = torch.rand(1, 2, 5, 4, device=device, dtype=torch.float64) + sigma_color = torch.rand(1, device=device, dtype=torch.float64) + sigma_space = torch.rand(1, 2, device=device, dtype=torch.float64) + + self.gradcheck(bilateral_blur, (img, 3, 1, (1, 1)), nondet_tol=1e-4) + self.gradcheck(bilateral_blur, (img, 3, sigma_color, (1, 1)), nondet_tol=1e-4) + self.gradcheck(bilateral_blur, (img, 3, 1, sigma_space), nondet_tol=1e-4) + self.gradcheck(bilateral_blur, (img, 3, sigma_color, sigma_space), nondet_tol=1e-4) + + @pytest.mark.parametrize("shape", [(1, 1, 8, 15), (2, 3, 11, 7)]) + @pytest.mark.parametrize("kernel_size", [5, (3, 5)]) + @pytest.mark.parametrize("sigma_color", [1, 0.1]) + @pytest.mark.parametrize("sigma_space", [(1, 1), (1.5, 1)]) + @pytest.mark.parametrize("color_distance_type", ["l1", "l2"]) + def test_module(self, shape, kernel_size, sigma_color, sigma_space, color_distance_type, device, dtype): + img = torch.rand(shape, device=device, dtype=dtype) + params = (kernel_size, sigma_color, sigma_space, "reflect", color_distance_type) + + op = bilateral_blur + op_module = BilateralBlur(*params) + self.assert_close(op_module(img), op(img, *params)) + + @pytest.mark.parametrize("kernel_size", [5, (5, 7)]) + @pytest.mark.parametrize("color_distance_type", ["l1", "l2"]) + def test_dynamo(self, kernel_size, color_distance_type, device, dtype, torch_optimizer): + data = torch.ones(2, 3, 8, 8, device=device, dtype=dtype) + op = BilateralBlur(kernel_size, 1, (1, 1), color_distance_type=color_distance_type) + op_optimized = torch_optimizer(op) + + self.assert_close(op(data), op_optimized(data)) + + sigma_color = torch.rand(1, device=device, dtype=dtype) + sigma_space = torch.rand(1, 2, device=device, dtype=dtype) + op = BilateralBlur(kernel_size, sigma_color, sigma_space, color_distance_type=color_distance_type) + op_optimized = torch_optimizer(op) + + self.assert_close(op(data), op_optimized(data)) + + def test_opencv_grayscale(self, device, dtype): + img = [[95, 130, 108, 228], [98, 142, 187, 166], [114, 166, 190, 141], [150, 83, 174, 216]] + img = torch.tensor(img, device=device, dtype=dtype).view(1, 1, 4, 4) / 255 + + kernel_size = 5 + sigma_color = 0.1 + sigma_distance = (0.5, 0.5) + + # Expected output generated with OpenCV: + # import cv2 + # expected = cv2.bilateralFilter(img[0, 0].numpy(), 5, 0.1, 0.5) + expected = [ + [0.38708255, 0.5060622, 0.43372786, 0.8876763], + [0.39813757, 0.55695623, 0.72320986, 0.6593296], + [0.4527661, 0.6484203, 0.7295754, 0.5705908], + [0.5774919, 0.32919288, 0.6949335, 0.83184093], + ] + expected = torch.tensor(expected, device=device, dtype=dtype).view(1, 1, 4, 4) + + out = bilateral_blur(img, kernel_size, sigma_color, sigma_distance) + self.assert_close(out, expected, rtol=1e-2, atol=1e-2) + + def test_opencv_rgb(self, device, dtype): + img = [ + [[170, 189, 182, 255], [169, 209, 216, 215], [196, 213, 228, 191], [207, 126, 224, 249]], + [[61, 104, 74, 225], [65, 112, 176, 148], [78, 147, 176, 120], [124, 61, 155, 211]], + [[73, 111, 90, 175], [77, 117, 163, 130], [83, 139, 163, 120], [132, 84, 137, 155]], + ] + img = torch.tensor(img, device=device, dtype=dtype).view(1, 3, 4, 4) / 255 + + kernel_size = 5 + sigma_color = 0.1 + sigma_distance = (0.5, 0.5) + + # Expected output generated with OpenCV: + # import cv2 + # expected = cv2.bilateralFilter(img[0].permute(1, 2, 0).numpy(), 5, 0.1, 0.5) + expected = [ + [ + [0.6658919, 0.7486991, 0.7140039, 0.9999949], + [0.6656203, 0.815614, 0.852062, 0.84256846], + [0.7658699, 0.83580506, 0.88873357, 0.7496973], + [0.8123873, 0.49414372, 0.87789816, 0.97619873], + ], + [ + [0.24242306, 0.40987095, 0.29138556, 0.8823465], + [0.2543548, 0.43856043, 0.68934506, 0.58119816], + [0.3045888, 0.5758538, 0.6885629, 0.47137713], + [0.48865014, 0.23922202, 0.6099074, 0.82698977], + ], + [ + [0.28948042, 0.43686634, 0.35377124, 0.686273], + [0.30078027, 0.4582056, 0.63826954, 0.5113827], + [0.32491508, 0.5446742, 0.63721484, 0.47087318], + [0.51836246, 0.32941142, 0.5409612, 0.60793126], + ], + ] + expected = torch.tensor(expected, device=device, dtype=dtype).view(1, 3, 4, 4) + + out = bilateral_blur(img, kernel_size, sigma_color, sigma_distance) + self.assert_close(out, expected, rtol=1e-2, atol=1e-2) + + +class TestJointBilateralBlur(BaseTester): + @pytest.mark.parametrize("input_depth", [1, 3]) + @pytest.mark.parametrize("guidance_depth", [1, 3]) + def test_smoke(self, input_depth, guidance_depth, device, dtype): + b, h, w = 2, 8, 15 + kernel_size = 5 + sigma_color = 0.1 + sigma_space = (2, 2) + inp = torch.rand(b, input_depth, h, w, device=device, dtype=dtype) + guide = torch.rand(b, guidance_depth, h, w, device=device, dtype=dtype) + + out = joint_bilateral_blur(inp, guide, kernel_size, sigma_color, sigma_space) + assert isinstance(out, torch.Tensor) + assert out.shape == (b, input_depth, h, w) + + def test_same_input(self, device, dtype): + shape = (2, 3, 8, 15) + kernel_size = 5 + sigma_color = 0.1 + sigma_space = (2, 2) + inp = torch.rand(shape, device=device, dtype=dtype) + + out1 = joint_bilateral_blur(inp, inp, kernel_size, sigma_color, sigma_space) + out2 = bilateral_blur(inp, kernel_size, sigma_color, sigma_space) + self.assert_close(out1, out2) + + @pytest.mark.parametrize("shape", [(1, 1, 8, 15), (2, 3, 11, 7)]) + @pytest.mark.parametrize("kernel_size", [5, (3, 5)]) + def test_cardinality(self, shape, kernel_size, device, dtype): + inp = torch.zeros(shape, device=device, dtype=dtype) + guide = torch.zeros(shape, device=device, dtype=dtype) + actual = joint_bilateral_blur(inp, guide, kernel_size, 0.1, (1, 1)) + assert actual.shape == shape + + def test_exception(self): + inp = torch.rand(1, 1, 5, 5) + guide = torch.rand(1, 1, 5, 5) + + from kornia.core.exceptions import TypeCheckError + + with pytest.raises(TypeCheckError) as errinfo: + joint_bilateral_blur(inp, guide, 3, 1, 1) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(Exception) as errinfo: + joint_bilateral_blur(inp, torch.randn(1, 1, 2, 4), 3, 1, (1, 1)) + assert "guidance and input should have the same" in str(errinfo) + + with pytest.raises(Exception) as errinfo: + joint_bilateral_blur(inp, torch.randn(2, 1, 5, 5), 3, 1, (1, 1)) + assert "guidance and input should have the same" in str(errinfo) + + with pytest.raises(ValueError) as errinfo: + joint_bilateral_blur(inp, guide, 3, 0.1, (1, 1), color_distance_type="l3") + assert "color_distance_type only accepts l1 or l2" in str(errinfo) + + def test_noncontiguous(self, device, dtype): + batch_size = 3 + inp = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + guide = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + + actual = joint_bilateral_blur(inp, guide, 3, 1, (1, 1)) + assert actual.is_contiguous() + + def test_gradcheck(self, device): + img = torch.rand(1, 2, 5, 4, device=device, dtype=torch.float64) + guide = torch.rand(1, 2, 5, 4, device=device, dtype=torch.float64) + self.gradcheck(joint_bilateral_blur, (img, guide, 3, 1, (1, 1)), nondet_tol=1e-4) + + def test_module(self, device, dtype): + shape = (2, 3, 11, 7) + kernel_size = 5 + sigma_color = 0.1 + sigma_space = (2, 2) + img = torch.rand(shape, device=device, dtype=dtype) + guide = torch.rand(shape, device=device, dtype=dtype) + params = (kernel_size, sigma_color, sigma_space) + + op = joint_bilateral_blur + op_module = JointBilateralBlur(*params) + self.assert_close(op_module(img, guide), op(img, guide, *params)) + + @pytest.mark.parametrize("kernel_size", [5, (5, 7)]) + @pytest.mark.parametrize("color_distance_type", ["l1", "l2"]) + def test_dynamo(self, kernel_size, color_distance_type, device, dtype, torch_optimizer): + data = torch.rand(2, 3, 8, 8, device=device, dtype=dtype) + guide = torch.rand(2, 3, 8, 8, device=device, dtype=dtype) + op = JointBilateralBlur(kernel_size, 1, (1, 1), color_distance_type=color_distance_type) + op_optimized = torch_optimizer(op) + + self.assert_close(op(data, guide), op_optimized(data, guide)) + + sigma_color = torch.rand(1, device=device, dtype=dtype) + sigma_space = torch.rand(1, 2, device=device, dtype=dtype) + op = JointBilateralBlur(kernel_size, sigma_color, sigma_space, color_distance_type=color_distance_type) + op_optimized = torch_optimizer(op) + + self.assert_close(op(data, guide), op_optimized(data, guide)) + + def test_opencv_grayscale(self, device, dtype): + img = [[95, 130, 108, 228], [98, 142, 187, 166], [114, 166, 190, 141], [150, 83, 174, 216]] + img = torch.tensor(img, device=device, dtype=dtype).view(1, 1, 4, 4) / 255 + + guide = [[161, 87, 93, 6], [91, 182, 97, 154], [70, 123, 109, 70], [119, 28, 60, 109]] + guide = torch.tensor(guide, device=device, dtype=dtype).view(1, 1, 4, 4) / 255 + + kernel_size = 5 + sigma_color = 0.1 + sigma_distance = (0.5, 0.5) + + # Expected output generated with OpenCV: + # import cv2 + # expected = cv2.ximgproc.jointBilateralFilter( + # guide.squeeze().numpy(), + # img.squeeze().numpy(), + # kernel_size, + # sigma_color, + # sigma_distance[0], + # ) + expected = [ + [0.38221005, 0.5027215, 0.49131155, 0.8937083], + [0.3976327, 0.55548316, 0.69680846, 0.65291953], + [0.44903287, 0.65470666, 0.7295845, 0.5840189], + [0.5867507, 0.3472942, 0.66494286, 0.81431836], + ] + expected = torch.tensor(expected, device=device, dtype=dtype).view(1, 1, 4, 4) + + out = joint_bilateral_blur(img, guide, kernel_size, sigma_color, sigma_distance) + self.assert_close(out, expected) + + def test_opencv_rgb(self, device, dtype): + img = [ + [[170, 189, 182, 255], [169, 209, 216, 215], [196, 213, 228, 191], [207, 126, 224, 249]], + [[61, 104, 74, 225], [65, 112, 176, 148], [78, 147, 176, 120], [124, 61, 155, 211]], + [[73, 111, 90, 175], [77, 117, 163, 130], [83, 139, 163, 120], [132, 84, 137, 155]], + ] + img = torch.tensor(img, device=device, dtype=dtype).view(1, 3, 4, 4) / 255 + + guide = [ + [[136, 196, 198, 21], [149, 185, 196, 141], [115, 110, 87, 155], [126, 82, 109, 207]], + [[188, 42, 48, 0], [73, 200, 58, 173], [53, 140, 130, 34], [129, 3, 41, 73]], + [[85, 36, 51, 0], [33, 82, 37, 87], [35, 70, 57, 36], [50, 10, 30, 43]], + ] + guide = torch.tensor(guide, device=device, dtype=dtype).view(1, 3, 4, 4) / 255 + + kernel_size = 5 + sigma_color = 0.1 + sigma_distance = (0.5, 0.5) + + # Expected output generated with OpenCV: + # import cv2 + # expected = cv2.ximgproc.jointBilateralFilter( + # guide.squeeze().permute(1, 2, 0).numpy(), + # img.squeeze().permute(1, 2, 0).numpy(), + # kernel_size, + # sigma_color, + # sigma_distance[0], + # ).transpose(2, 0, 1) + expected = [ + [ + [0.6671455, 0.74172455, 0.7328562, 1.0], + [0.66403687, 0.81948805, 0.8357967, 0.8431371], + [0.7673652, 0.836736, 0.8925936, 0.7494889], + [0.8120746, 0.49431974, 0.8778994, 0.9764218], + ], + [ + [0.23984201, 0.40574068, 0.35012922, 0.88235295], + [0.2555589, 0.43905944, 0.65692204, 0.58039117], + [0.30529743, 0.5791127, 0.68724936, 0.47124338], + [0.48745763, 0.23940866, 0.6072882, 0.82737976], + ], + [ + [0.2868149, 0.43398118, 0.39570162, 0.6862745], + [0.30228183, 0.45868847, 0.6153678, 0.50980365], + [0.3252297, 0.5474381, 0.6367775, 0.47098273], + [0.51799667, 0.32952034, 0.53696644, 0.6078228], + ], + ] + expected = torch.tensor(expected, device=device, dtype=dtype).view(1, 3, 4, 4) + + out = joint_bilateral_blur(img, guide, kernel_size, sigma_color, sigma_distance) + self.assert_close(out, expected) diff --git a/tests/filters/test_blur.py b/tests/filters/test_blur.py new file mode 100644 index 0000000..6f4fd87 --- /dev/null +++ b/tests/filters/test_blur.py @@ -0,0 +1,164 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.filters import BoxBlur, box_blur + +from testing.base import BaseTester + + +class TestBoxBlur(BaseTester): + @pytest.mark.parametrize("kernel_size", [5, (3, 5)]) + def test_smoke(self, kernel_size, device, dtype): + data = torch.rand(1, 1, 10, 10, device=device, dtype=dtype) + + bb = BoxBlur(kernel_size, "reflect") + actual = bb(data) + assert actual.shape == (1, 1, 10, 10) + + @pytest.mark.parametrize("kernel_size", [5, (3, 5)]) + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_separable(self, batch_size, kernel_size, device, dtype): + data = torch.randn(batch_size, 3, 10, 10, device=device, dtype=dtype) + out1 = box_blur(data, kernel_size, separable=False) + out2 = box_blur(data, kernel_size, separable=True) + self.assert_close(out1, out2) + + @pytest.mark.parametrize("kernel_size", [5, (3, 5)]) + def test_separable_module(self, kernel_size, device, dtype): + # BoxBlur with separable=True exercises the register_buffer(kernel_y/kernel_x) branch + data = torch.rand(1, 3, 8, 8, device=device, dtype=dtype) + blur_sep = BoxBlur(kernel_size, "reflect", separable=True) + blur_ref = BoxBlur(kernel_size, "reflect", separable=False) + self.assert_close(blur_sep(data), blur_ref(data)) + + def test_exception(self): + data = torch.rand(1, 1, 3, 3) + + with pytest.raises(Exception) as errinfo: + box_blur(data, (1,)) + assert "2D Kernel size should have a length of 2." in str(errinfo) + + @pytest.mark.parametrize("kernel_size", [(3, 3), 5, (5, 7)]) + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_cardinality(self, batch_size, kernel_size, device, dtype): + inp = torch.zeros(batch_size, 3, 4, 4, device=device, dtype=dtype) + blur = BoxBlur(kernel_size) + actual = blur(inp) + expected = (batch_size, 3, 4, 4) + assert actual.shape == expected + + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_kernel_3x3(self, batch_size, device, dtype): + inp = torch.tensor( + [ + [ + [ + [1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0], + [2.0, 2.0, 2.0, 2.0, 2.0], + [2.0, 2.0, 2.0, 2.0, 2.0], + ] + ] + ], + device=device, + dtype=dtype, + ).repeat(batch_size, 1, 1, 1) + + kernel_size = (3, 3) + actual = box_blur(inp, kernel_size) + expected = torch.tensor(35.0 * batch_size, device=device, dtype=dtype) + + self.assert_close(actual.sum(), expected) + + @pytest.mark.parametrize("batch_size", [None, 1, 3]) + def test_kernel_5x5(self, batch_size, device, dtype): + inp = torch.tensor( + [ + [ + [ + [1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0], + [2.0, 2.0, 2.0, 2.0, 2.0], + [2.0, 2.0, 2.0, 2.0, 2.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + if batch_size: + inp = inp.repeat(batch_size, 1, 1, 1) + + kernel_size = (5, 5) + + actual = box_blur(inp, kernel_size) + expected = inp.sum((1, 2, 3)) / torch.mul(*kernel_size) + + self.assert_close(actual[:, 0, 2, 2], expected) + + def test_kernel_3x1(self, device, dtype): + inp = torch.arange(16, device=device, dtype=dtype).view(1, 1, 4, 4) + + ky, kx = 3, 1 + actual = box_blur(inp, (ky, kx)) + + self.assert_close(actual[0, 0, 0, 0], torch.tensor((4 + 0 + 4) / 3, device=device, dtype=dtype)) + self.assert_close(actual[0, 0, 1, 0], torch.tensor((0 + 4 + 8) / 3, device=device, dtype=dtype)) + + @pytest.mark.parametrize("separable", [False, True]) + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_noncontiguous(self, batch_size, separable, device, dtype): + inp = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + + actual = box_blur(inp, 3, separable=separable) + + assert actual.is_contiguous() + + @pytest.mark.parametrize("kernel_size", [(3, 3), 5, (5, 7)]) + def test_gradcheck(self, kernel_size, device): + batch_size, channels, height, width = 1, 2, 5, 4 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + fast_mode = "cpu" in str(device) # Disable fast mode for GPU + self.gradcheck(box_blur, (img, kernel_size), fast_mode=fast_mode) + + @pytest.mark.parametrize("kernel_size", [(3, 3), 5, (5, 7)]) + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_module(self, kernel_size, batch_size, device, dtype): + op = box_blur + op_module = BoxBlur + + img = torch.rand(batch_size, 3, 4, 5, device=device, dtype=dtype) + actual = op_module(kernel_size)(img) + expected = op(img, kernel_size) + + self.assert_close(actual, expected) + + @pytest.mark.parametrize("separable", [False, True]) + @pytest.mark.parametrize("kernel_size", [5, (5, 7)]) + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_dynamo(self, batch_size, kernel_size, separable, device, dtype, torch_optimizer): + data = torch.ones(batch_size, 3, 10, 10, device=device, dtype=dtype) + op = BoxBlur(kernel_size, separable=separable) + op_optimized = torch_optimizer(op) + + self.assert_close(op(data), op_optimized(data)) diff --git a/tests/filters/test_blur_pool.py b/tests/filters/test_blur_pool.py new file mode 100644 index 0000000..11faccf --- /dev/null +++ b/tests/filters/test_blur_pool.py @@ -0,0 +1,215 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.filters import ( + BlurPool2D, + EdgeAwareBlurPool2D, + MaxBlurPool2D, + blur_pool2d, + edge_aware_blur_pool2d, + max_blur_pool2d, +) + +from testing.base import BaseTester + + +class TestMaxBlurPool(BaseTester): + @pytest.mark.parametrize("kernel_size", [3, (5, 5)]) + @pytest.mark.parametrize("ceil_mode", [True, False]) + def test_smoke(self, kernel_size, ceil_mode, device, dtype): + data = torch.rand(1, 1, 10, 10, device=device, dtype=dtype) + actual = MaxBlurPool2D(kernel_size, ceil_mode=ceil_mode)(data) + + assert actual.shape == (1, 1, 5, 5) + + @pytest.mark.parametrize("ceil_mode", [True, False]) + @pytest.mark.parametrize("kernel_size", [3, (5, 5)]) + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_cardinality(self, batch_size, kernel_size, ceil_mode, device, dtype): + data = torch.zeros(batch_size, 4, 4, 8, device=device, dtype=dtype) + blur = MaxBlurPool2D(kernel_size, ceil_mode=ceil_mode) + assert blur(data).shape == (batch_size, 4, 2, 4) + + def test_exception(self): + data = torch.rand(1, 1, 3, 3) + with pytest.raises(Exception) as errinfo: + MaxBlurPool2D((3, 5))(data) + assert "Invalid kernel shape. Expect CxC_outxNxN" in str(errinfo) + + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_noncontiguous(self, batch_size, device, dtype): + inp = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + + actual = max_blur_pool2d(inp, 3) + + assert actual.is_contiguous() + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 2, 5, 4 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(max_blur_pool2d, (img, 3)) + + @pytest.mark.parametrize("kernel_size", [(3, 3), 5]) + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_module(self, kernel_size, batch_size, device, dtype): + op = max_blur_pool2d + op_module = MaxBlurPool2D + + img = torch.rand(batch_size, 3, 4, 5, device=device, dtype=dtype) + actual = op_module(kernel_size)(img) + expected = op(img, kernel_size) + self.assert_close(actual, expected) + + @pytest.mark.parametrize("kernel_size", [3, (5, 5)]) + @pytest.mark.parametrize("batch_size", [1, 2]) + @pytest.mark.parametrize("ceil_mode", [True, False]) + def test_dynamo(self, batch_size, kernel_size, ceil_mode, device, dtype, torch_optimizer): + data = torch.ones(batch_size, 3, 10, 10, device=device, dtype=dtype) + op = MaxBlurPool2D(kernel_size, ceil_mode=ceil_mode) + op_optimized = torch_optimizer(op) + + self.assert_close(op(data), op_optimized(data)) + + +class TestBlurPool(BaseTester): + @pytest.mark.parametrize("kernel_size", [3, (5, 5)]) + @pytest.mark.parametrize("stride", [1, 2]) + def test_smoke(self, kernel_size, stride, device, dtype): + data = torch.rand(1, 1, 10, 10, device=device, dtype=dtype) + actual = BlurPool2D(kernel_size, stride=stride)(data) + expected = (1, 1, int(10 / stride), int(10 / stride)) + assert actual.shape == expected + + @pytest.mark.parametrize("kernel_size", [3, (5, 5)]) + @pytest.mark.parametrize("batch_size", [1, 2]) + @pytest.mark.parametrize("stride", [1, 2]) + def test_cardinality(self, batch_size, kernel_size, stride, device, dtype): + data = torch.zeros(batch_size, 4, 4, 8, device=device, dtype=dtype) + actual = BlurPool2D(kernel_size, stride=stride)(data) + expected = (batch_size, 4, int(4 / stride), int(8 / stride)) + assert actual.shape == expected + + def test_exception(self): + data = torch.rand(1, 1, 3, 3) + with pytest.raises(Exception) as errinfo: + BlurPool2D((3, 5))(data) + assert "Invalid kernel shape. Expect CxC_(out, None)xNxN" in str(errinfo) + + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_noncontiguous(self, batch_size, device, dtype): + inp = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + + actual = blur_pool2d(inp, 3) + assert actual.is_contiguous() + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 2, 5, 4 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(blur_pool2d, (img, 3)) + + @pytest.mark.parametrize("kernel_size", [3, (5, 5)]) + @pytest.mark.parametrize("batch_size", [1, 2]) + @pytest.mark.parametrize("stride", [1, 2]) + def test_module(self, batch_size, kernel_size, stride, device, dtype): + op = blur_pool2d + op_module = BlurPool2D + + img = torch.rand(batch_size, 3, 4, 5, device=device, dtype=dtype) + actual = op_module(kernel_size)(img) + expected = op(img, kernel_size) + self.assert_close(actual, expected) + + @pytest.mark.parametrize("kernel_size", [3, (5, 5)]) + @pytest.mark.parametrize("batch_size", [1, 2]) + @pytest.mark.parametrize("stride", [1, 2]) + def test_dynamo(self, batch_size, kernel_size, stride, device, dtype, torch_optimizer): + data = torch.ones(batch_size, 3, 10, 10, device=device, dtype=dtype) + op = BlurPool2D(kernel_size, stride=stride) + op_optimized = torch_optimizer(op) + + self.assert_close(op(data), op_optimized(data)) + + +class TestEdgeAwareBlurPool(BaseTester): + @pytest.mark.parametrize("kernel_size", [3, (5, 5)]) + @pytest.mark.parametrize("batch_size", [1, 2]) + @pytest.mark.parametrize("edge_threshold", [1.25, 2.5]) + @pytest.mark.parametrize("edge_dilation_kernel_size", [3, 5]) + def test_smoke(self, kernel_size, batch_size, edge_threshold, edge_dilation_kernel_size, device, dtype): + data = torch.zeros(batch_size, 3, 8, 8, device=device, dtype=dtype) + actual = edge_aware_blur_pool2d(data, kernel_size, edge_threshold, edge_dilation_kernel_size) + assert actual.shape == data.shape + + @pytest.mark.parametrize("kernel_size", [3, (5, 5)]) + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_cardinality(self, kernel_size, batch_size, device, dtype): + inp = torch.zeros(batch_size, 3, 8, 8, device=device, dtype=dtype) + blur = edge_aware_blur_pool2d(inp, kernel_size=kernel_size) + assert blur.shape == inp.shape + + def test_exception(self): + from kornia.core.exceptions import BaseError, ShapeError + + with pytest.raises(ShapeError) as errinfo: + data = torch.rand(1, 3, 3) + edge_aware_blur_pool2d(data, 3) + assert "Shape dimension mismatch" in str(errinfo.value) or "Expected shape" in str(errinfo.value) + with pytest.raises(BaseError) as errinfo: + data = torch.rand(1, 1, 3, 3) + edge_aware_blur_pool2d(data, 3, edge_threshold=-1) + assert "edge threshold should be positive, but got" in str(errinfo.value) + + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_noncontiguous(self, batch_size, device, dtype): + inp = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + + actual = edge_aware_blur_pool2d(inp, 3) + assert actual.is_contiguous() + + @pytest.mark.parametrize("kernel_size", [3, (5, 5)]) + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_module(self, kernel_size, batch_size, device, dtype): + op = edge_aware_blur_pool2d + op_module = EdgeAwareBlurPool2D + + img = torch.rand(batch_size, 3, 4, 5, device=device, dtype=dtype) + actual = op_module(kernel_size)(img) + expected = op(img, kernel_size) + self.assert_close(actual, expected) + + def test_gradcheck(self, device): + img = torch.rand((1, 2, 5, 4), device=device, dtype=torch.float64) + self.gradcheck(edge_aware_blur_pool2d, (img, 3)) + + def test_smooth(self, device, dtype): + img = torch.ones(1, 1, 5, 5, device=device, dtype=dtype) + img[0, 0, :, :2] = 0 + blur = edge_aware_blur_pool2d(img, kernel_size=3, edge_threshold=32.0) + self.assert_close(img, blur) + + @pytest.mark.parametrize("kernel_size", [3, (5, 5)]) + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_dynamo(self, batch_size, kernel_size, device, dtype, torch_optimizer): + op = edge_aware_blur_pool2d + data = torch.rand(batch_size, 3, 4, 5, device=device, dtype=dtype) + op = EdgeAwareBlurPool2D(kernel_size) + op_optimized = torch_optimizer(op) + + self.assert_close(op(data), op_optimized(data)) diff --git a/tests/filters/test_canny.py b/tests/filters/test_canny.py new file mode 100644 index 0000000..66dc267 --- /dev/null +++ b/tests/filters/test_canny.py @@ -0,0 +1,335 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.core._compat import torch_version +from kornia.filters import Canny, canny + +from testing.base import BaseTester + + +class TestCanny(BaseTester): + @pytest.mark.parametrize("batch_size", [1, 2]) + @pytest.mark.parametrize("kernel_size", [3, (5, 7)]) + @pytest.mark.parametrize("sigma", [(1.5, 1.0), (2.5, 0.5)]) + @pytest.mark.parametrize("hysteresis", [False, True]) + @pytest.mark.parametrize("low_threshold,high_threshold", [(0.1, 0.2), (0.3, 0.5)]) + def test_smoke(self, batch_size, kernel_size, sigma, hysteresis, low_threshold, high_threshold, device, dtype): + inp = torch.zeros(batch_size, 3, 4, 4, device=device, dtype=dtype) + + op = Canny(low_threshold, high_threshold, kernel_size, sigma, hysteresis) + actual = op(inp) + assert len(actual) == 2 + assert actual[0].shape == (batch_size, 1, 4, 4) + assert actual[1].shape == (batch_size, 1, 4, 4) + + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_cardinality(self, batch_size, device, dtype): + inp = torch.zeros(batch_size, 3, 4, 4, device=device, dtype=dtype) + + op = Canny() + magnitude, edges = op(inp) + + assert magnitude.shape == (batch_size, 1, 4, 4) + assert edges.shape == (batch_size, 1, 4, 4) + + def test_exception(self, device, dtype): + from kornia.core.exceptions import BaseError, ShapeError, TypeCheckError + + with pytest.raises(BaseError) as errinfo: + Canny(0.3, 0.2) + assert "low_threshold should be smaller than the high_threshold" in str(errinfo.value) + + with pytest.raises(BaseError) as errinfo: + Canny(-2, 0.3) + assert "Invalid low threshold." in str(errinfo.value) + + with pytest.raises(BaseError) as errinfo: + Canny(0.1, 3) + assert "Invalid high threshold." in str(errinfo.value) + + with pytest.raises(TypeCheckError) as errinfo: + canny(1) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + inp = torch.zeros(3, 4, 4, device=device, dtype=dtype) + with pytest.raises(ShapeError) as errinfo: + canny(inp) + assert "Shape dimension mismatch" in str(errinfo.value) or "Expected shape" in str(errinfo.value) + + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_noncontiguous(self, batch_size, device, dtype): + inp = torch.rand(batch_size, 3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + + magnitude, edges = canny(inp) + + assert magnitude.is_contiguous() + assert edges.is_contiguous() + + def test_magnitude(self, device, dtype): + inp = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected_magnitude = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.2458, 0.9672, 1.2458, 0.0], + [0.0, 0.9672, 0.0, 0.9672, 0.0], + [0.0, 1.2458, 0.9672, 1.2458, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected_edges = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 1.0, 0.0, 1.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + magnitude, edges = canny(inp) + + self.assert_close(magnitude, expected_magnitude, atol=1e-4, rtol=1e-4) + self.assert_close(edges, expected_edges, atol=1e-4, rtol=1e-4) + + def test_magnitude_hyst(self, device, dtype): + inp = torch.tensor( + [ + [ + [ + [0.5, 0.4, 0.5, 0.45, 0.1], + [0.3, 0.2, 0.3, 0.0, 0.3], + [0.5, 1.0, 1.0, 0.6, 0.75], + [0.2, 0.4, 0.6, 0.0, 0.5], + [0.1, 0.35, 0.35, 0.26, 0.1], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected_magnitude = torch.tensor( + [ + [ + [ + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.4858, 0.5594, 0.6878, 0.6977, 0.5602], + [0.1129, 0.0000, 0.0000, 0.4531, 0.0000], + [0.6115, 0.5859, 0.6110, 0.6766, 0.5160], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected_edges = torch.tensor( + [ + [ + [ + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [1.0000, 1.0000, 1.0000, 1.0000, 1.0000], + [1.0000, 0.0000, 0.0000, 1.0000, 0.0000], + [1.0000, 1.0000, 1.0000, 1.0000, 1.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + ] + ] + ], + device=device, + dtype=dtype, + ) + + magnitude, edges = canny(inp, hysteresis=True) + + self.assert_close(magnitude, expected_magnitude, atol=1e-4, rtol=1e-4) + self.assert_close(edges, expected_edges, atol=1e-4, rtol=1e-4) + + def test_magnitude_hyst_false(self, device, dtype): + inp = torch.tensor( + [ + [ + [ + [0.5, 0.4, 0.5, 0.45, 0.1], + [0.3, 0.2, 0.3, 0.0, 0.3], + [0.5, 1.0, 1.0, 0.6, 0.75], + [0.2, 0.4, 0.6, 0.0, 0.5], + [0.1, 0.35, 0.35, 0.26, 0.1], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected_magnitude = torch.tensor( + [ + [ + [ + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.4858, 0.5594, 0.6878, 0.6977, 0.5602], + [0.1129, 0.0000, 0.0000, 0.4531, 0.0000], + [0.6115, 0.5859, 0.6110, 0.6766, 0.5160], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected_edges = torch.tensor( + [ + [ + [ + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [1.0000, 1.0000, 1.0000, 1.0000, 1.0000], + [0.5000, 0.0000, 0.0000, 1.0000, 0.0000], + [1.0000, 1.0000, 1.0000, 1.0000, 1.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + ] + ] + ], + device=device, + dtype=dtype, + ) + + magnitude, edges = canny(inp, hysteresis=False) + + self.assert_close(magnitude, expected_magnitude, atol=1e-4, rtol=1e-4) + self.assert_close(edges, expected_edges, atol=1e-4, rtol=1e-4) + + def test_magnitude_threshold(self, device, dtype): + inp = torch.tensor( + [ + [ + [ + [0.5, 0.4, 0.5, 0.45, 0.1], + [0.3, 0.2, 0.3, 0.0, 0.3], + [0.5, 1.0, 1.0, 0.6, 0.75], + [0.2, 0.4, 0.6, 0.0, 0.5], + [0.1, 0.35, 0.35, 0.26, 0.1], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected_magnitude = torch.tensor( + [ + [ + [ + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.4858, 0.5594, 0.6878, 0.6977, 0.5602], + [0.1129, 0.0000, 0.0000, 0.4531, 0.0000], + [0.6115, 0.5859, 0.6110, 0.6766, 0.5160], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected_edges = torch.tensor( + [ + [ + [ + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + ] + ] + ], + device=device, + dtype=dtype, + ) + + magnitude, edges = canny(inp, low_threshold=0.3, high_threshold=0.9) + + self.assert_close(magnitude, expected_magnitude, atol=1e-4, rtol=1e-4) + self.assert_close(edges, expected_edges, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + if "cuda" in str(device): + pytest.skip("RuntimeError: Backward is not reentrant, i.e., running backward,") + batch_size, channels, height, width = 1, 1, 3, 4 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(canny, img) + + def test_module(self, device, dtype): + img = torch.rand(2, 3, 4, 5, device=device, dtype=dtype) + op = canny + op_module = Canny() + expected_magnitude, expected_edges = op(img) + actual_magnitude, actual_edges = op_module(img) + self.assert_close(actual_magnitude, expected_magnitude) + self.assert_close(actual_edges, expected_edges) + + @pytest.mark.parametrize("kernel_size", [5, (5, 7)]) + @pytest.mark.parametrize("batch_size", [1, 2]) + @pytest.mark.skipif(torch_version() in {"2.0.0", "2.0.1"}, reason="Not working on 2.0") + def test_dynamo(self, batch_size, kernel_size, device, dtype, torch_optimizer): + if ( + torch_version() in {"2.1.1", "2.1.2", "2.2.2", "2.3.1"} + and dtype == torch.float64 + and (isinstance(kernel_size, int) or kernel_size[0] == kernel_size[1]) + ): + pytest.skip("Canny compiled failing into fp64 for kernel sizes where kx and ky are equals") + data = torch.ones(batch_size, 3, 10, 10, device=device, dtype=dtype) + op = Canny(kernel_size=kernel_size) + op_optimized = torch_optimizer(op) + + expected_magnitude, expected_edges = op(data) + actual_magnitude, actual_edges = op_optimized(data) + + self.assert_close(actual_magnitude, expected_magnitude) + self.assert_close(actual_edges, expected_edges) diff --git a/tests/filters/test_dissolving.py b/tests/filters/test_dissolving.py new file mode 100644 index 0000000..fa16881 --- /dev/null +++ b/tests/filters/test_dissolving.py @@ -0,0 +1,64 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.core._compat import torch_version_le +from kornia.filters.dissolving import StableDiffusionDissolving + +WEIGHTS_CACHE_DIR = "weights/" + + +@pytest.mark.slow +@pytest.mark.skipif( + torch_version_le(2, 0, 1), + reason="Skipped for torch versions <= 2.0.1: transformers clip model needs distributed tensor.", +) +class TestStableDiffusionDissolving: + @pytest.fixture(scope="class") + def sdm_2_1(self): + return StableDiffusionDissolving(version="1.5", cache_dir=WEIGHTS_CACHE_DIR) + + @pytest.fixture(scope="class") + def dummy_image(self): + # Create a dummy image tensor with shape [B, C, H, W], where B is the batch size. + return torch.rand(1, 3, 64, 64) + + def test_init(self, sdm_2_1): + assert isinstance(sdm_2_1, StableDiffusionDissolving), "Initialization failed" + + def test_encode_tensor_to_latent(self, sdm_2_1, dummy_image): + latents = sdm_2_1.model.encode_tensor_to_latent(dummy_image) + assert isinstance(latents, torch.Tensor), "Latent encoding failed" + assert latents.shape == (1, 4, 8, 8), "Latent shape mismatch" + + def test_decode_tensor_to_latent(self, sdm_2_1, dummy_image): + latents = sdm_2_1.model.encode_tensor_to_latent(dummy_image) + reconstructed_image = sdm_2_1.model.decode_tensor_to_latent(latents) + assert isinstance(reconstructed_image, torch.Tensor), "Latent decoding failed" + assert reconstructed_image.shape == dummy_image.shape, "Reconstructed image shape mismatch" + + def test_dissolve(self, sdm_2_1, dummy_image): + step_number = 500 # Test with a middle step + dissolved_image = sdm_2_1(dummy_image, step_number) + assert isinstance(dissolved_image, torch.Tensor), "Dissolve failed" + assert dissolved_image.shape == dummy_image.shape, "Dissolved image shape mismatch" + + def test_invalid_version(self): + with pytest.raises(NotImplementedError): + StableDiffusionDissolving(version="invalid_version") diff --git a/tests/filters/test_filters.py b/tests/filters/test_filters.py new file mode 100644 index 0000000..e8cd238 --- /dev/null +++ b/tests/filters/test_filters.py @@ -0,0 +1,1165 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import pytest +import torch + +from kornia.core._compat import torch_version_le +from kornia.filters import ( + convolve2d, + convolve3d, + correlate2d, + correlate3d, + fft_conv, + filter2d, + filter2d_separable, + filter3d, +) + +from testing.base import BaseTester + + +class TestFilter2D(BaseTester): + @pytest.mark.parametrize("border_type", ["constant", "reflect", "replicate", "circular"]) + @pytest.mark.parametrize("normalized", [True, False]) + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_smoke(self, border_type, normalized, padding, device, dtype): + kernel = torch.rand(1, 3, 3, device=device, dtype=dtype) + _, height, width = kernel.shape + sample = torch.ones(1, 1, 7, 8, device=device, dtype=dtype) + b, c, h, w = sample.shape + + actual = filter2d(sample, kernel, border_type, normalized, padding) + assert isinstance(actual, torch.Tensor) + assert actual.shape in {(b, c, h, w), (b, c, h - height + 1, w - width + 1)} + + @pytest.mark.parametrize("batch_size", [2, 3, 6, 8]) + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_cardinality(self, batch_size, padding, device, dtype): + B: int = batch_size + kernel = torch.rand(1, 3, 3, device=device, dtype=dtype) + _, height, width = kernel.shape + sample = torch.ones(B, 3, 7, 8, device=device, dtype=dtype) + b, c, h, w = sample.shape + out = filter2d(sample, kernel, padding=padding) + if padding == "same": + assert out.shape == (b, c, h, w) + else: + assert out.shape == (b, c, h - height + 1, w - width + 1) + + def test_conv(self, device, dtype): + inp = torch.zeros(1, 1, 5, 5, device=device, dtype=dtype) + inp[..., 2, 2] = 1.0 + kernel = torch.arange(1, 10).reshape(3, 3).to(device, dtype)[None] + corr_expected = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 9.0, 8.0, 7.0, 0.0], + [0.0, 6.0, 5.0, 4.0, 0.0], + [0.0, 3.0, 2.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + conv_expected = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 2.0, 3.0, 0.0], + [0.0, 4.0, 5.0, 6.0, 0.0], + [0.0, 7.0, 8.0, 9.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + out_corr = filter2d(inp, kernel, behaviour="corr") + self.assert_close(out_corr, corr_expected) + out_conv = filter2d(inp, kernel, behaviour="conv") + self.assert_close(out_conv, conv_expected) + + def test_exception(self): + from kornia.core.exceptions import ShapeError, TypeCheckError + + k = torch.ones(1, 1, 1) + data = torch.ones(1, 1, 1, 1) + with pytest.raises(TypeCheckError) as errinfo: + filter2d(1, k) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(TypeCheckError) as errinfo: + filter2d(data, 1) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(ShapeError) as errinfo: + filter2d(torch.ones(1), k) + assert "Shape dimension mismatch" in str(errinfo.value) + assert "['B', 'C', 'H', 'W']" in str(errinfo.value) + + with pytest.raises(ShapeError) as errinfo: + filter2d(data, torch.ones(1)) + assert "Shape dimension mismatch" in str(errinfo.value) + assert "['B', 'H', 'W']" in str(errinfo.value) + + with pytest.raises(Exception) as errinfo: + filter2d(data, k, border_type="a") + assert "Invalid border, a. Ex" in str(errinfo) + + with pytest.raises(Exception) as errinfo: + filter2d(data, k, padding="a") + assert "Invalid padding mode, a. Ex" in str(errinfo) + + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_mean_filter(self, padding, device, dtype): + kernel = torch.ones(1, 3, 3, device=device, dtype=dtype) + sample = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 5.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + actual = filter2d(sample, kernel, padding=padding) + + if padding == "same": + expected_same = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + self.assert_close(actual, expected_same) + else: + expected_valid = torch.tensor( + [[[[5.0, 5.0, 5.0], [5.0, 5.0, 5.0], [5.0, 5.0, 5.0]]]], device=device, dtype=dtype + ) + + self.assert_close(actual, expected_valid) + + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_mean_filter_2batch_2ch(self, padding, device, dtype): + kernel = torch.ones(1, 3, 3, device=device, dtype=dtype) + sample = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 5.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ).expand(2, 2, -1, -1) + + actual = filter2d(sample, kernel, padding=padding) + + if padding == "same": + expected_same = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ).expand(2, 2, -1, -1) + + self.assert_close(actual, expected_same) + else: + expected_valid = torch.tensor( + [[[[5.0, 5.0, 5.0], [5.0, 5.0, 5.0], [5.0, 5.0, 5.0]]]], device=device, dtype=dtype + ).expand(2, 2, -1, -1) + self.assert_close(actual, expected_valid) + + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_normalized_mean_filter(self, padding, device, dtype): + kernel = torch.ones(1, 3, 3, device=device, dtype=dtype) + sample = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 5.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ).expand(2, 2, -1, -1) + + nv: float = 5.0 / 9 # normalization value + actual = filter2d(sample, kernel, normalized=True, padding=padding) + + if padding == "same": + expected_same = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, nv, nv, nv, 0.0], + [0.0, nv, nv, nv, 0.0], + [0.0, nv, nv, nv, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ).expand(2, 2, -1, -1) + + self.assert_close(actual, expected_same) + else: + expected_valid = torch.tensor( + [[[[nv, nv, nv], [nv, nv, nv], [nv, nv, nv]]]], device=device, dtype=dtype + ).expand(2, 2, -1, -1) + + self.assert_close(actual, expected_valid) + + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_even_sized_filter(self, padding, device, dtype): + kernel = torch.ones(1, 2, 2, device=device, dtype=dtype) + sample = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 5.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + actual = filter2d(sample, kernel, padding=padding) + + if padding == "same": + expected_same = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 5.0, 5.0, 0.0, 0.0], + [0.0, 5.0, 5.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + self.assert_close(actual, expected_same) + else: + expected_valid = torch.tensor( + [[[[0.0, 0.0, 0.0, 0.0], [0.0, 5.0, 5.0, 0.0], [0.0, 5.0, 5.0, 0.0], [0.0, 0.0, 0.0, 0.0]]]], + device=device, + dtype=dtype, + ) + + self.assert_close(actual, expected_valid) + + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_mix_sized_filter_padding_same(self, padding, device, dtype): + kernel = torch.ones(1, 5, 6, device=device, dtype=dtype) + sample_ = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected_same = torch.tensor( + [ + [ + [ + [2.0, 2.0, 2.0, 2.0, 2.0, 0.0], + [3.0, 3.0, 3.0, 3.0, 3.0, 0.0], + [3.0, 3.0, 3.0, 3.0, 3.0, 0.0], + [3.0, 3.0, 3.0, 3.0, 3.0, 0.0], + [2.0, 2.0, 2.0, 2.0, 2.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + actual = filter2d(sample_, kernel, padding="same", border_type="constant") + self.assert_close(actual, expected_same) + + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_noncontiguous(self, padding, device, dtype): + batch_size = 3 + inp = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + kernel = torch.ones(1, 2, 2, device=device, dtype=dtype) + + actual = filter2d(inp, kernel, padding=padding) + assert actual.is_contiguous() + + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_separable(self, padding, device, dtype): + batch_size = 3 + inp = torch.rand(3, 9, 9, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + kernel_x = torch.ones(1, 3, device=device, dtype=dtype) + kernel_y = torch.ones(1, 3, device=device, dtype=dtype) + kernel = kernel_y.t() @ kernel_x + out = filter2d(inp, kernel[None], padding=padding) + out_sep = filter2d_separable(inp, kernel_x, kernel_y, padding=padding) + self.assert_close(out, out_sep) + + def test_gradcheck(self, device): + kernel = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + sample = torch.ones(1, 1, 7, 8, device=device, dtype=torch.float64) + + # evaluate function gradient + self.gradcheck(filter2d, (sample, kernel), nondet_tol=1e-8) + + @pytest.mark.skip(reason="filter2d do not have a module") + def test_module(self): ... + + @pytest.mark.parametrize("normalized", [True, False]) + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_dynamo(self, normalized, padding, device, dtype, torch_optimizer): + kernel = torch.rand(1, 3, 3, device=device, dtype=dtype) + data = torch.ones(2, 3, 10, 10, device=device, dtype=dtype) + op = filter2d + op_optimized = torch_optimizer(op) + + expected = op(data, kernel, padding=padding, normalized=normalized) + actual = op_optimized(data, kernel, padding=padding, normalized=normalized) + + self.assert_close(actual, expected) + + +class TestFilter3D(BaseTester): + @pytest.mark.parametrize("border_type", ["constant", "reflect", "replicate", "circular"]) + @pytest.mark.parametrize("normalized", [True, False]) + def test_smoke(self, border_type, normalized, device, dtype): + if torch_version_le(1, 9, 1) and border_type == "reflect": + pytest.skip(reason="Reflect border is not implemented for 3D on torch < 1.9.1") + + kernel = torch.rand(1, 3, 3, 3, device=device, dtype=dtype) + data = torch.ones(1, 1, 6, 7, 8, device=device, dtype=dtype) + actual = filter3d(data, kernel, border_type, normalized) + + assert isinstance(actual, torch.Tensor) + assert actual.shape == data.shape + + @pytest.mark.parametrize("batch_size", [2, 3, 6, 8]) + def test_cardinality(self, batch_size, device, dtype): + kernel = torch.rand(1, 3, 3, 3, device=device, dtype=dtype) + data = torch.ones(batch_size, 3, 6, 7, 8, device=device, dtype=dtype) + assert filter3d(data, kernel).shape == data.shape + + def test_exception(self): + from kornia.core.exceptions import ShapeError, TypeCheckError + + k = torch.ones(1, 1, 1, 1) + data = torch.ones(1, 1, 1, 1, 1) + with pytest.raises(TypeCheckError) as errinfo: + filter3d(1, k) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(TypeCheckError) as errinfo: + filter3d(data, 1) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(ShapeError) as errinfo: + filter3d(torch.ones(1), k) + assert "Shape dimension mismatch" in str(errinfo.value) or "Expected shape" in str(errinfo.value) + + with pytest.raises(ShapeError) as errinfo: + filter3d(data, torch.ones(1)) + assert "Shape dimension mismatch" in str(errinfo.value) or "Expected shape" in str(errinfo.value) + + with pytest.raises(Exception) as errinfo: + filter3d(data, k, border_type="a") + assert "Invalid border, gotcha a. Ex" in str(errinfo) + + def test_mean_filter(self, device, dtype): + kernel = torch.ones(1, 3, 3, 3, device=device, dtype=dtype) + sample = torch.tensor( + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 5.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + ] + ] + ], + device=device, + dtype=dtype, + ) + + actual = filter3d(sample, kernel) + self.assert_close(actual, expected) + + def test_mean_filter_2batch_2ch(self, device, dtype): + kernel = torch.ones(1, 3, 3, 3, device=device, dtype=dtype) + sample = torch.tensor( + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 5.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + ] + ] + ], + device=device, + dtype=dtype, + ) + sample = sample.expand(2, 2, -1, -1, -1) + + expected = torch.tensor( + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + ] + ] + ], + device=device, + dtype=dtype, + ) + expected = expected.expand(2, 2, -1, -1, -1) + + actual = filter3d(sample, kernel) + self.assert_close(actual, expected) + + def test_normalized_mean_filter(self, device, dtype): + kernel = torch.ones(1, 3, 3, 3, device=device, dtype=dtype) + sample = torch.tensor( + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 5.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + ] + ] + ], + device=device, + dtype=dtype, + ) + sample = sample.expand(2, 2, -1, -1, -1) + + nv = 5.0 / 27 # normalization value + expected = torch.tensor( + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, nv, nv, nv, 0.0], + [0.0, nv, nv, nv, 0.0], + [0.0, nv, nv, nv, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, nv, nv, nv, 0.0], + [0.0, nv, nv, nv, 0.0], + [0.0, nv, nv, nv, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, nv, nv, nv, 0.0], + [0.0, nv, nv, nv, 0.0], + [0.0, nv, nv, nv, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + ] + ] + ], + device=device, + dtype=dtype, + ) + expected = expected.expand(2, 2, -1, -1, -1) + + actual = filter3d(sample, kernel, normalized=True) + + self.assert_close(actual, expected) + + def test_even_sized_filter(self, device, dtype): + kernel = torch.ones(1, 2, 2, 2, device=device, dtype=dtype) + sample = torch.tensor( + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 5.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 5.0, 5.0, 0.0, 0.0], + [0.0, 5.0, 5.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 5.0, 5.0, 0.0, 0.0], + [0.0, 5.0, 5.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + ] + ] + ], + device=device, + dtype=dtype, + ) + + actual = filter3d(sample, kernel) + self.assert_close(actual, expected) + + def test_noncontiguous(self, device, dtype): + batch_size = 3 + inp = torch.rand(3, 5, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1, -1) + kernel = torch.ones(1, 2, 2, 2, device=device, dtype=dtype) + + actual = filter3d(inp, kernel) + assert actual.is_contiguous() + + def test_gradcheck(self, device): + kernel = torch.rand(1, 3, 3, 3, device=device, dtype=torch.float64) + sample = torch.ones(1, 1, 6, 7, 8, device=device, dtype=torch.float64) + + # evaluate function gradient + self.gradcheck(filter3d, (sample, kernel), nondet_tol=1e-8) + + @pytest.mark.skip(reason="filter3d do not have a module") + def test_module(self): ... + + @pytest.mark.parametrize("normalized", [True, False]) + def test_dynamo(self, normalized, device, dtype, torch_optimizer): + kernel = torch.rand(1, 3, 3, 3, device=device, dtype=dtype) + data = torch.ones(2, 3, 4, 10, 10, device=device, dtype=dtype) + op = filter3d + op_optimized = torch_optimizer(op) + + expected = op(data, kernel, normalized=normalized) + actual = op_optimized(data, kernel, normalized=normalized) + + self.assert_close(actual, expected) + + +class TestFilter2D_fftconv(BaseTester): + @pytest.mark.parametrize("border_type", ["constant", "reflect", "replicate", "circular"]) + @pytest.mark.parametrize("normalized", [True, False]) + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_smoke(self, border_type, normalized, padding, device, dtype): + kernel = torch.rand(1, 3, 3, device=device, dtype=dtype) + _, height, width = kernel.shape + sample = torch.ones(1, 1, 7, 8, device=device, dtype=dtype) + b, c, h, w = sample.shape + + actual = fft_conv(sample, kernel, border_type, normalized, padding) + assert isinstance(actual, torch.Tensor) + assert actual.shape in {(b, c, h, w), (b, c, h - height + 1, w - width + 1)} + + @pytest.mark.parametrize("batch_size", [2, 3, 6, 8]) + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_cardinality(self, batch_size, padding, device, dtype): + B: int = batch_size + kernel = torch.rand(1, 3, 3, device=device, dtype=dtype) + _, height, width = kernel.shape + sample = torch.ones(B, 3, 7, 8, device=device, dtype=dtype) + b, c, h, w = sample.shape + out = fft_conv(sample, kernel, padding=padding) + if padding == "same": + assert out.shape == (b, c, h, w) + else: + assert out.shape == (b, c, h - height + 1, w - width + 1) + + def test_conv(self, device, dtype): + inp = torch.zeros(1, 1, 5, 5, device=device, dtype=dtype) + inp[..., 2, 2] = 1.0 + kernel = torch.arange(1, 10).reshape(3, 3).to(device, dtype)[None] + corr_expected = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 9.0, 8.0, 7.0, 0.0], + [0.0, 6.0, 5.0, 4.0, 0.0], + [0.0, 3.0, 2.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + conv_expected = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 2.0, 3.0, 0.0], + [0.0, 4.0, 5.0, 6.0, 0.0], + [0.0, 7.0, 8.0, 9.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + out_corr = fft_conv(inp, kernel, behaviour="corr") + self.assert_close(out_corr, corr_expected, atol=1e-6, rtol=1e-6) + out_conv = fft_conv(inp, kernel, behaviour="conv") + self.assert_close(out_conv, conv_expected, atol=1e-6, rtol=1e-6) + + def test_exception(self): + from kornia.core.exceptions import ShapeError, TypeCheckError + + k = torch.ones(1, 1, 1) + data = torch.ones(1, 1, 1, 1) + with pytest.raises(TypeCheckError) as errinfo: + fft_conv(1, k) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(TypeCheckError) as errinfo: + fft_conv(data, 1) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(ShapeError) as errinfo: + fft_conv(torch.ones(1), k) + assert "Shape dimension mismatch" in str(errinfo.value) + assert "['B', 'C', 'H', 'W']" in str(errinfo.value) + + with pytest.raises(ShapeError) as errinfo: + fft_conv(data, torch.ones(1)) + assert "Shape dimension mismatch" in str(errinfo.value) + assert "['B', 'H', 'W']" in str(errinfo.value) + + with pytest.raises(Exception) as errinfo: + fft_conv(data, k, border_type="a") + assert "Invalid border, a. Ex" in str(errinfo) + + with pytest.raises(Exception) as errinfo: + fft_conv(data, k, padding="a") + assert "Invalid padding mode, a. Ex" in str(errinfo) + + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_mean_filter(self, padding, device, dtype): + kernel = torch.ones(1, 3, 3, device=device, dtype=dtype) + sample = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 5.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + actual = fft_conv(sample, kernel, padding=padding) + + if padding == "same": + expected_same = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + self.assert_close(actual, expected_same) + else: + expected_valid = torch.tensor( + [[[[5.0, 5.0, 5.0], [5.0, 5.0, 5.0], [5.0, 5.0, 5.0]]]], device=device, dtype=dtype + ) + + self.assert_close(actual, expected_valid) + + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_mean_filter_2batch_2ch(self, padding, device, dtype): + kernel = torch.ones(1, 3, 3, device=device, dtype=dtype) + sample = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 5.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ).expand(2, 2, -1, -1) + + actual = fft_conv(sample, kernel, padding=padding) + + if padding == "same": + expected_same = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 5.0, 5.0, 5.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ).expand(2, 2, -1, -1) + + self.assert_close(actual, expected_same) + else: + expected_valid = torch.tensor( + [[[[5.0, 5.0, 5.0], [5.0, 5.0, 5.0], [5.0, 5.0, 5.0]]]], device=device, dtype=dtype + ).expand(2, 2, -1, -1) + self.assert_close(actual, expected_valid) + + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_normalized_mean_filter(self, padding, device, dtype): + kernel = torch.ones(1, 3, 3, device=device, dtype=dtype) + sample = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 5.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ).expand(2, 2, -1, -1) + + nv: float = 5.0 / 9 # normalization value + actual = fft_conv(sample, kernel, normalized=True, padding=padding) + + if padding == "same": + expected_same = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, nv, nv, nv, 0.0], + [0.0, nv, nv, nv, 0.0], + [0.0, nv, nv, nv, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ).expand(2, 2, -1, -1) + + self.assert_close(actual, expected_same) + else: + expected_valid = torch.tensor( + [[[[nv, nv, nv], [nv, nv, nv], [nv, nv, nv]]]], device=device, dtype=dtype + ).expand(2, 2, -1, -1) + + self.assert_close(actual, expected_valid) + + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_even_sized_filter(self, padding, device, dtype): + kernel = torch.ones(1, 2, 2, device=device, dtype=dtype) + sample = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 5.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + actual = fft_conv(sample, kernel, padding=padding) + + if padding == "same": + expected_same = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 5.0, 5.0, 0.0, 0.0], + [0.0, 5.0, 5.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + self.assert_close(actual, expected_same) + else: + expected_valid = torch.tensor( + [[[[0.0, 0.0, 0.0, 0.0], [0.0, 5.0, 5.0, 0.0], [0.0, 5.0, 5.0, 0.0], [0.0, 0.0, 0.0, 0.0]]]], + device=device, + dtype=dtype, + ) + + self.assert_close(actual, expected_valid) + + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_mix_sized_filter_padding_same(self, padding, device, dtype): + kernel = torch.ones(1, 5, 6, device=device, dtype=dtype) + sample_ = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected_same = torch.tensor( + [ + [ + [ + [2.0, 2.0, 2.0, 2.0, 2.0, 0.0], + [3.0, 3.0, 3.0, 3.0, 3.0, 0.0], + [3.0, 3.0, 3.0, 3.0, 3.0, 0.0], + [3.0, 3.0, 3.0, 3.0, 3.0, 0.0], + [2.0, 2.0, 2.0, 2.0, 2.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + actual = fft_conv(sample_, kernel, padding="same", border_type="constant") + self.assert_close(actual, expected_same) + + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_noncontiguous(self, padding, device, dtype): + batch_size = 3 + inp = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + kernel = torch.ones(1, 2, 2, device=device, dtype=dtype) + + actual = fft_conv(inp, kernel, padding=padding) + assert actual.is_contiguous() + + def test_gradcheck(self, device): + kernel = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + sample = torch.ones(1, 1, 7, 8, device=device, dtype=torch.float64) + + # evaluate function gradient + self.gradcheck(fft_conv, (sample, kernel), nondet_tol=1e-8) + + @pytest.mark.skip(reason="filter2d do not have a module") + def test_module(self): ... + + @pytest.mark.parametrize("normalized", [True, False]) + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_dynamo(self, normalized, padding, device, dtype, torch_optimizer): + kernel = torch.rand(1, 3, 3, device=device, dtype=dtype) + data = torch.ones(2, 3, 10, 10, device=device, dtype=dtype) + op = fft_conv + op_optimized = torch_optimizer(op) + + expected = op(data, kernel, padding=padding, normalized=normalized) + actual = op_optimized(data, kernel, padding=padding, normalized=normalized) + + self.assert_close(actual, expected) + + +class TestCorrelate2d: + def test_equivalent_to_filter2d_corr(self, device, dtype): + inp = torch.rand(1, 1, 7, 8, device=device, dtype=dtype) + kernel = torch.rand(1, 3, 3, device=device, dtype=dtype) + expected = filter2d(inp, kernel, behaviour="corr") + result = correlate2d(inp, kernel) + assert torch.allclose(result, expected) + + @pytest.mark.parametrize("border_type", ["constant", "reflect", "replicate", "circular"]) + @pytest.mark.parametrize("padding", ["same", "valid"]) + def test_smoke(self, border_type, padding, device, dtype): + inp = torch.ones(1, 1, 7, 8, device=device, dtype=dtype) + kernel = torch.rand(1, 3, 3, device=device, dtype=dtype) + out = correlate2d(inp, kernel, border_type=border_type, padding=padding) + assert isinstance(out, torch.Tensor) + + +class TestConvolve2d: + def test_equivalent_to_filter2d_conv(self, device, dtype): + inp = torch.rand(1, 1, 7, 8, device=device, dtype=dtype) + kernel = torch.rand(1, 3, 3, device=device, dtype=dtype) + expected = filter2d(inp, kernel, behaviour="conv") + result = convolve2d(inp, kernel) + assert torch.allclose(result, expected) + + def test_differs_from_correlate_asymmetric_kernel(self, device, dtype): + inp = torch.rand(1, 1, 7, 8, device=device, dtype=dtype) + # Asymmetric kernel so corr != conv + kernel = torch.tensor([[[1.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]], device=device, dtype=dtype) + corr = correlate2d(inp, kernel) + conv = convolve2d(inp, kernel) + assert not torch.allclose(corr, conv) + + +class TestCorrelate3d: + def test_equivalent_to_filter3d_corr(self, device, dtype): + inp = torch.rand(1, 1, 5, 7, 8, device=device, dtype=dtype) + kernel = torch.rand(1, 3, 3, 3, device=device, dtype=dtype) + expected = filter3d(inp, kernel, behaviour="corr") + result = correlate3d(inp, kernel) + assert torch.allclose(result, expected) + + +class TestConvolve3d: + def test_equivalent_to_filter3d_conv(self, device, dtype): + inp = torch.rand(1, 1, 5, 7, 8, device=device, dtype=dtype) + kernel = torch.rand(1, 3, 3, 3, device=device, dtype=dtype) + expected = filter3d(inp, kernel, behaviour="conv") + result = convolve3d(inp, kernel) + assert torch.allclose(result, expected) diff --git a/tests/filters/test_gaussian.py b/tests/filters/test_gaussian.py new file mode 100644 index 0000000..998ef35 --- /dev/null +++ b/tests/filters/test_gaussian.py @@ -0,0 +1,473 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import warnings + +import pytest +import torch + +from kornia.filters import ( + GaussianBlur2d, + gaussian, + gaussian_blur2d, + get_gaussian_discrete_kernel1d, + get_gaussian_erf_kernel1d, + get_gaussian_kernel1d, + get_gaussian_kernel2d, + get_gaussian_kernel3d, +) + +from testing.base import BaseTester, assert_close + + +@pytest.mark.parametrize( + "window_size, sigma, mean, expected", + [ + (5, 1.0, None, torch.tensor([[0.0545, 0.2442, 0.4026, 0.2442, 0.0545]])), + ( + 11, + 5.0, + None, + torch.tensor([[0.0663, 0.0794, 0.0914, 0.1010, 0.1072, 0.1094, 0.1072, 0.1010, 0.0914, 0.0794, 0.0663]]), + ), + ( + 11, + 5.0, + 8.0, + torch.tensor([[0.0343, 0.0463, 0.0600, 0.0747, 0.0895, 0.1029, 0.1138, 0.1208, 0.1232, 0.1208, 0.1138]]), + ), + ( + 11, + 11.0, + 3.0, + torch.tensor([[0.0926, 0.0946, 0.0957, 0.0961, 0.0957, 0.0946, 0.0926, 0.0900, 0.0867, 0.0828, 0.0785]]), + ), + ], +) +def test_gaussian(window_size, sigma, mean, expected, device, dtype): + expected = expected.to(device=device, dtype=dtype) + result = gaussian(window_size, sigma, mean=mean, device=device, dtype=dtype) + assert_close(result, expected, atol=1e-4, rtol=1e-4) + + +@pytest.mark.parametrize("window_size", [5, 11]) +@pytest.mark.parametrize("sigma", [1.5, 5.0]) +def test_get_gaussian_kernel1d_float(window_size, sigma, device, dtype): + actual = get_gaussian_kernel1d(window_size, sigma, device=device, dtype=dtype) + expected = torch.ones(1, device=device, dtype=dtype) + + assert actual.shape == (1, window_size) + assert_close(actual.sum(), expected.sum()) + + +@pytest.mark.parametrize("window_size", [5, 11]) +@pytest.mark.parametrize("sigma", [[[1.5]], [[1.5], [5.0]], [[1.5], [5.0]]]) +def test_get_gaussian_kernel1d_tensor(window_size, sigma, device, dtype): + sigma = torch.tensor(sigma, device=device, dtype=dtype) + bs = sigma.shape[0] + + actual = get_gaussian_kernel1d(window_size, sigma) + expected = torch.ones(bs, device=device, dtype=dtype) + + assert actual.shape == (bs, window_size) + assert_close(actual.sum(), expected.sum()) + + +@pytest.mark.parametrize("ksize_x", [5, 11]) +@pytest.mark.parametrize("ksize_y", [3, 7]) +@pytest.mark.parametrize("sigma", [(1.5, 1.5), (2.1, 2.1)]) +def test_get_gaussian_kernel2d_float(ksize_x, ksize_y, sigma, device, dtype): + actual = get_gaussian_kernel2d((ksize_y, ksize_x), sigma, device=device, dtype=dtype) + expected = torch.ones(1, device=device, dtype=dtype) + + assert actual.shape == (1, ksize_y, ksize_x) + assert_close(actual.sum(), expected.sum()) + + +@pytest.mark.parametrize("ksize_x", [5, 11]) +@pytest.mark.parametrize("ksize_y", [3, 7]) +@pytest.mark.parametrize("sigma", ([[1.5, 2.1], [1.5, 2.1], [5.0, 2.7]], [[1.5, 2.1], [3.5, 2.1]])) +def test_get_gaussian_kernel2d_tensor(ksize_x, ksize_y, sigma, device, dtype): + sigma = torch.tensor(sigma, device=device, dtype=dtype) + bs = sigma.shape[0] + + actual = get_gaussian_kernel2d((ksize_y, ksize_x), sigma) + expected = torch.ones(bs, device=device, dtype=dtype) + + assert actual.shape == (bs, ksize_y, ksize_x) + assert_close(actual.sum(), expected.sum()) + + +@pytest.mark.parametrize("ksize_x", [5, 11]) +@pytest.mark.parametrize("ksize_y", [3, 7]) +@pytest.mark.parametrize("ksize_z", [9, 3]) +@pytest.mark.parametrize("sigma", [(1.5, 1.5, 3.5), (2.1, 1.5, 2.1)]) +def test_get_gaussian_kernel3d_float(ksize_x, ksize_y, ksize_z, sigma, device, dtype): + actual = get_gaussian_kernel3d((ksize_z, ksize_y, ksize_x), sigma, device=device, dtype=dtype) + expected = torch.ones(1, device=device, dtype=dtype) + + assert actual.shape == (1, ksize_z, ksize_y, ksize_x) + assert_close(actual.sum(), expected.sum()) + + +@pytest.mark.parametrize("ksize_x", [5, 11]) +@pytest.mark.parametrize("ksize_y", [3, 7]) +@pytest.mark.parametrize("ksize_z", [9, 3]) +@pytest.mark.parametrize( + "sigma", ([[1.5, 2.1, 3.5], [1.5, 2.1, 1.5], [5.0, 2.7, 2.1]], [[1.5, 3.5, 2.1], [1.2, 3.5, 2.1]]) +) +def test_get_gaussian_kernel3d_tensor(ksize_x, ksize_y, ksize_z, sigma, device, dtype): + sigma = torch.tensor(sigma, device=device, dtype=dtype) + bs = sigma.shape[0] + + actual = get_gaussian_kernel3d((ksize_z, ksize_y, ksize_x), sigma) + expected = torch.ones(bs, device=device, dtype=dtype) + + assert actual.shape == (bs, ksize_z, ksize_y, ksize_x) + assert_close(actual.sum(), expected.sum()) + + +@pytest.mark.parametrize("window_size", [5, 11]) +@pytest.mark.parametrize("sigma", [1.5, 5.0]) +def test_get_discrete_gaussian_erf_kernel1d_float(window_size, sigma, device, dtype): + actual = get_gaussian_erf_kernel1d(window_size, sigma, device=device, dtype=dtype) + expected = torch.ones(1, device=device, dtype=dtype) + + assert actual.shape == (1, window_size) + assert_close(actual.sum(), expected.sum()) + + +@pytest.mark.parametrize("window_size", [5, 11]) +@pytest.mark.parametrize("sigma", [[[1.5]], [[1.5], [5.0]], [[1.5], [5.0]]]) +def test_get_discrete_gaussian_erf_kernel1d_tensor(window_size, sigma, device, dtype): + sigma = torch.tensor(sigma, device=device, dtype=dtype) + bs = sigma.shape[0] + + actual = get_gaussian_erf_kernel1d(window_size, sigma) + expected = torch.ones(bs, device=device, dtype=dtype) + + assert actual.shape == (bs, window_size) + assert_close(actual.sum(), expected.sum()) + + +@pytest.mark.parametrize("window_size", [5, 11]) +@pytest.mark.parametrize("sigma", [1.5, 5.0]) +def test_get_gaussian_discrete_kernel1d_float(window_size, sigma, device, dtype): + actual = get_gaussian_discrete_kernel1d(window_size, sigma, device=device, dtype=dtype) + expected = torch.ones(1, device=device, dtype=dtype) + + assert actual.shape == (1, window_size) + assert_close(actual.sum(), expected.sum()) + + +@pytest.mark.parametrize("window_size", [5, 11]) +@pytest.mark.parametrize("sigma", [[[1.5]], [[1.5], [5.0]], [[1.5], [5.0]]]) +def test_get_gaussian_discrete_kernel1d_tensor(window_size, sigma, device, dtype): + sigma = torch.tensor(sigma, device=device, dtype=dtype) + bs = sigma.shape[0] + + actual = get_gaussian_discrete_kernel1d(window_size, sigma) + expected = torch.ones(bs, device=device, dtype=dtype) + + assert actual.shape == (bs, window_size) + assert_close(actual.sum(), expected.sum()) + + +@pytest.mark.parametrize("ksize_x", [5, 11]) +@pytest.mark.parametrize("ksize_y", [3, 7]) +@pytest.mark.parametrize("sigma", [(1.5, 1.5), (2.1, 2.1)]) +def test_gaussian_blur2d_float(ksize_x, ksize_y, sigma, device, dtype): + sample = torch.rand(1, 3, 16, 16, device=device, dtype=dtype) + + actual = gaussian_blur2d(sample, (ksize_y, ksize_x), sigma, "replicate", separable=False) + actual_sep = gaussian_blur2d(sample, (ksize_y, ksize_x), sigma, "replicate", separable=True) + + assert_close(actual, actual_sep) + + +@pytest.mark.parametrize("ksize_x", [5, 11]) +@pytest.mark.parametrize("ksize_y", [3, 7]) +@pytest.mark.parametrize("sigma", ([[1.5, 2.1], [1.5, 2.1], [5.0, 2.7]], [[1.5, 2.1], [3.5, 2.1]])) +def test_gaussian_blur2d_tensor(ksize_x, ksize_y, sigma, device, dtype): + sigma = torch.tensor(sigma, device=device, dtype=dtype) + bs = sigma.shape[0] + sample = torch.rand(bs, 3, 16, 16, device=device, dtype=dtype) + actual = gaussian_blur2d(sample, (ksize_y, ksize_x), sigma, "replicate", separable=False) + actual_sep = gaussian_blur2d(sample, (ksize_y, ksize_x), sigma, "replicate", separable=True) + + assert_close(actual, actual_sep) + + +class TestGaussianBlur2d(BaseTester): + @pytest.mark.parametrize("shape", [(1, 4, 8, 15), (2, 3, 11, 7)]) + @pytest.mark.parametrize("kernel_size", [3, (5, 5), (5, 7)]) + @pytest.mark.parametrize("separable", [False, True]) + def test_smoke(self, shape, kernel_size, separable, device, dtype): + B, C, H, W = shape + data = torch.rand(B, C, H, W, device=device, dtype=dtype) + sigma_tensor = torch.rand(B, 2, device=device, dtype=dtype) + + actual_A = gaussian_blur2d(data, kernel_size, sigma_tensor, "reflect", separable) + assert isinstance(actual_A, torch.Tensor) + assert actual_A.shape == shape + + sigma = tuple(sigma_tensor[0, ...].cpu().numpy().tolist()) + actual_B = gaussian_blur2d(data, kernel_size, sigma, "reflect", separable) + assert isinstance(actual_B, torch.Tensor) + assert actual_B.shape == shape + + # Just the first item of the batch use the same sigma + self.assert_close(actual_A[0, ...], actual_B[0, ...]) + + @pytest.mark.parametrize("shape", [(1, 4, 8, 15), (2, 3, 11, 7)]) + @pytest.mark.parametrize("kernel_size", [3, (5, 5), (5, 7)]) + def test_cardinality(self, shape, kernel_size, device, dtype): + sigma = (1.5, 2.1) + sample = torch.rand(shape, device=device, dtype=dtype) + actual = gaussian_blur2d(sample, kernel_size, sigma, "replicate") + assert actual.shape == shape + + def test_exception(self): + from kornia.core.exceptions import TypeCheckError + + # input should be a tensor + with pytest.raises(TypeCheckError) as errinfo: + gaussian_blur2d(1, 3, (1.0, 1.0)) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + # Sigma should be a tuple or a tensor + with pytest.raises(TypeCheckError) as errinfo: + gaussian_blur2d(torch.rand(1, 1, 1, 1), 3, 1.0) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + def test_noncontiguous(self, device, dtype): + batch_size = 3 + sample = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + + kernel_size = (3, 3) + sigma = (1.5, 2.1) + actual = gaussian_blur2d(sample, kernel_size, sigma, "replicate") + assert actual.is_contiguous() + + def test_gradcheck(self, device): + # test parameters + batch_shape = (1, 3, 5, 5) + kernel_size = (3, 3) + sigma = (1.5, 2.1) + + # evaluate function gradient + sample = torch.rand(batch_shape, device=device, dtype=torch.float64) + self.gradcheck(gaussian_blur2d, (sample, kernel_size, sigma, "replicate")) + + @pytest.mark.parametrize("kernel_size", [3, (5, 5), (5, 7)]) + @pytest.mark.parametrize("sigma", [(1.5, 2.1), (0.5, 0.5)]) + def test_module(self, kernel_size, sigma, device, dtype): + params = [kernel_size, sigma] + op = gaussian_blur2d + op_module = GaussianBlur2d(*params) + + img = torch.ones(1, 3, 5, 5, device=device, dtype=dtype) + + self.assert_close(op(img, *params), op_module(img)) + sigma_tensor = torch.tensor([sigma], device=device, dtype=dtype) + params = [kernel_size, sigma_tensor] + op_module = GaussianBlur2d(*params) + + self.assert_close(op(img, *params), op_module(img)) + + @pytest.mark.parametrize("kernel_size", [3, (5, 5), (5, 7)]) + @pytest.mark.parametrize("sigma", [(1.5, 2.1), (0.5, 0.5)]) + def test_dynamo(self, kernel_size, sigma, device, dtype, torch_optimizer): + data = torch.ones(1, 3, 5, 5, device=device, dtype=dtype) + + op = GaussianBlur2d(kernel_size, sigma) + op_optimized = torch_optimizer(op) + + self.assert_close(op(data), op_optimized(data)) + + sigma_tensor = torch.tensor([sigma], device=device, dtype=dtype) + op = GaussianBlur2d(kernel_size, sigma_tensor) + op_optimized = torch_optimizer(op) + + self.assert_close(op(data), op_optimized(data)) + + @pytest.mark.parametrize("kernel_size", [3, (5, 5)]) + @pytest.mark.parametrize("sigma", [(1.5, 2.1), (0.5, 0.5)]) + def test_dynamo_functional(self, kernel_size, sigma, device, dtype, torch_optimizer): + """Test that functional gaussian_blur2d works with torch.compile / torch._dynamo.""" + data = torch.ones(1, 3, 5, 5, device=device, dtype=dtype) + + # Test functional form + # Test functional form + def op(x): + return gaussian_blur2d(x, kernel_size, sigma, "reflect", separable=True) + + op_optimized = torch_optimizer(op) + + self.assert_close(op(data), op_optimized(data)) + + def test_onnx_export(self, device, dtype): + """Test that GaussianBlur2d can be exported via torch.onnx.export.""" + kernel_size = (3, 3) + sigma = (1.5, 1.5) + + # Create model and sample input + model = GaussianBlur2d(kernel_size, sigma) + sample_input = torch.ones(1, 3, 8, 8, device=device, dtype=dtype) + + # Test ONNX export - just ensure it doesn't error + # TODO: think of an absctraction + try: + import os + import tempfile + + # Suppress onnxscript deprecation warnings (Python 3.15 compatibility) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning, module="onnxscript.converter") + with tempfile.TemporaryDirectory() as tmpdir: + onnx_path = os.path.join(tmpdir, "gaussian_blur2d.onnx") + torch.onnx.export( + model, + sample_input, + onnx_path, + input_names=["input"], + output_names=["output"], + opset_version=17, + ) + # Verify the file was created + assert os.path.exists(onnx_path) + except Exception as e: + pytest.skip(f"ONNX export not supported: {e}") + + def test_sigma_negative_raises_exception(self, device, dtype): + """Test that negative sigma raises an exception.""" + sample = torch.rand(1, 3, 5, 5, device=device, dtype=dtype) + with pytest.raises(Exception) as errinfo: + gaussian_blur2d(sample, (3, 3), (-1.5, 1.5)) + assert "sigma must be positive" in str(errinfo.value) + + def test_sigma_zero_raises_exception(self, device, dtype): + """Test that zero sigma raises an exception.""" + sample = torch.rand(1, 3, 5, 5, device=device, dtype=dtype) + with pytest.raises(Exception) as errinfo: + gaussian_blur2d(sample, (3, 3), (0.0, 1.5)) + assert "sigma must be positive" in str(errinfo.value) + + def test_sigma_tensor_negative_raises_exception(self, device, dtype): + """Test that negative sigma tensor raises an exception.""" + sample = torch.rand(2, 3, 5, 5, device=device, dtype=dtype) + sigma = torch.tensor([[-1.5, 1.5], [1.5, 1.5]], device=device, dtype=dtype) + with pytest.raises(Exception) as errinfo: + gaussian_blur2d(sample, (3, 3), sigma) + assert "sigma must be positive" in str(errinfo.value) + + def test_kernel_size_even_raises_exception(self, device, dtype): + """Test that even kernel size raises an exception.""" + sample = torch.rand(1, 3, 5, 5, device=device, dtype=dtype) + with pytest.raises(Exception) as errinfo: + gaussian_blur2d(sample, 4, (1.5, 1.5)) + assert "Kernel size must be" in str(errinfo.value) + + def test_kernel_size_zero_raises_exception(self, device, dtype): + """Test that zero kernel size raises an exception.""" + sample = torch.rand(1, 3, 5, 5, device=device, dtype=dtype) + with pytest.raises(Exception) as errinfo: + gaussian_blur2d(sample, 0, (1.5, 1.5)) + assert "Kernel size must be" in str(errinfo.value) + + def test_very_small_sigma(self, device, dtype): + """Test with very small positive sigma values (should not raise).""" + sample = torch.rand(1, 3, 5, 5, device=device, dtype=dtype) + # Should not raise - any positive value is valid + output = gaussian_blur2d(sample, (3, 3), (0.01, 0.01)) + assert output.shape == sample.shape + + def test_very_large_sigma(self, device, dtype): + """Test with very large sigma values (should not raise).""" + sample = torch.rand(1, 3, 5, 5, device=device, dtype=dtype) + # Should not raise - large sigma just blurs more + output = gaussian_blur2d(sample, (3, 3), (100.0, 100.0)) + assert output.shape == sample.shape + + @pytest.mark.parametrize("sigma_value", [0.1, 1.0, 5.0, 10.0]) + def test_sigma_range(self, sigma_value, device, dtype): + """Test various sigma values across a reasonable range.""" + sample = torch.rand(1, 3, 8, 8, device=device, dtype=dtype) + output = gaussian_blur2d(sample, (5, 5), (sigma_value, sigma_value)) + assert output.shape == sample.shape + + def test_single_pixel_image(self, device, dtype): + """Test with minimal spatial dimensions (3x3 - smallest for 3x3 kernel).""" + sample = torch.rand(1, 3, 3, 3, device=device, dtype=dtype) + output = gaussian_blur2d(sample, 3, (1.5, 1.5)) + assert output.shape == sample.shape + + def test_large_batch_size(self, device, dtype): + """Test with large batch size.""" + sample = torch.rand(32, 3, 8, 8, device=device, dtype=dtype) + output = gaussian_blur2d(sample, (3, 3), (1.5, 1.5)) + assert output.shape == sample.shape + + def test_many_channels(self, device, dtype): + """Test with many channels (e.g., video or multi-spectral).""" + sample = torch.rand(1, 64, 8, 8, device=device, dtype=dtype) + output = gaussian_blur2d(sample, (3, 3), (1.5, 1.5)) + assert output.shape == sample.shape + + def test_batched_sigma_mismatched_batch_size(self, device, dtype): + """Test that batched sigma uses first batch element when shapes don't match.""" + # Note: The function broadcasts sigma, so mismatched batch size is allowed + # but only the first sigma in the batch is used for all input samples + sample = torch.rand(4, 3, 8, 8, device=device, dtype=dtype) + sigma = torch.tensor([[1.5, 1.5], [2.0, 2.0]], device=device, dtype=dtype) + # Should not raise - will use broadcasting behavior + output = gaussian_blur2d(sample, (3, 3), sigma) + assert output.shape == sample.shape + + def test_all_border_types(self, device, dtype): + """Test that all supported border types work.""" + sample = torch.rand(1, 3, 8, 8, device=device, dtype=dtype) + for border_type in ["constant", "reflect", "replicate", "circular"]: + output = gaussian_blur2d(sample, (3, 3), (1.5, 1.5), border_type=border_type) + assert output.shape == sample.shape + + def test_separable_vs_non_separable_small_kernel(self, device, dtype): + """Test that separable and non-separable modes produce similar results.""" + sample = torch.rand(1, 3, 16, 16, device=device, dtype=dtype) + sigma = (1.5, 1.5) + output_sep = gaussian_blur2d(sample, (3, 3), sigma, separable=True) + output_non_sep = gaussian_blur2d(sample, (3, 3), sigma, separable=False) + assert_close(output_sep, output_non_sep, atol=1e-4, rtol=1e-4) + + def test_different_kernel_sizes(self, device, dtype): + """Test with various kernel sizes.""" + sample = torch.rand(1, 3, 16, 16, device=device, dtype=dtype) + for ksize in [3, 5, 7, 11, (3, 5), (5, 7)]: + output = gaussian_blur2d(sample, ksize, (1.5, 1.5)) + assert output.shape == sample.shape + + def test_output_dtype_preserved(self, dtype, device): + """Test that output dtype matches input dtype.""" + sample = torch.rand(1, 3, 8, 8, device=device, dtype=dtype) + output = gaussian_blur2d(sample, (3, 3), (1.5, 1.5)) + assert output.dtype == dtype + + def test_output_device_preserved(self, device, dtype): + """Test that output device matches input device.""" + sample = torch.rand(1, 3, 8, 8, device=device, dtype=dtype) + output = gaussian_blur2d(sample, (3, 3), (1.5, 1.5)) + assert output.device.type == sample.device.type diff --git a/tests/filters/test_guided.py b/tests/filters/test_guided.py new file mode 100644 index 0000000..f70d6f1 --- /dev/null +++ b/tests/filters/test_guided.py @@ -0,0 +1,213 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.core._compat import torch_version +from kornia.filters import GuidedBlur, guided_blur + +from testing.base import BaseTester + + +class TestGuidedBlur(BaseTester): + @pytest.mark.parametrize("batch_size", [1, 2]) + @pytest.mark.parametrize("guide_dim", [1, 3]) + @pytest.mark.parametrize("input_dim", [1, 3]) + @pytest.mark.parametrize("kernel_size", [5, (3, 5)]) + @pytest.mark.parametrize("eps", [0.1, 0.01]) + def test_smoke(self, batch_size, guide_dim, input_dim, kernel_size, eps, device, dtype): + H, W = 8, 16 + guide = torch.randn(batch_size, guide_dim, H, W, device=device, dtype=dtype) + inp = torch.randn(batch_size, input_dim, H, W, device=device, dtype=dtype) + + # tensor eps -> with batch dim + eps = torch.rand(batch_size, device=device, dtype=dtype) + actual_A = guided_blur(guide, inp, kernel_size, eps) + assert isinstance(actual_A, torch.Tensor) + assert actual_A.shape == (batch_size, input_dim, H, W) + + # float and tuple sigmas -> same sigmas across batch + eps_ = eps[0].item() + actual_B = guided_blur(guide, inp, kernel_size, eps_) + assert isinstance(actual_B, torch.Tensor) + assert actual_B.shape == (batch_size, input_dim, H, W) + + self.assert_close(actual_A[0], actual_B[0]) + + # fast guided filter + actual_C = guided_blur(guide, inp, kernel_size, eps_, subsample=4) + assert isinstance(actual_C, torch.Tensor) + assert actual_C.shape == (batch_size, input_dim, H, W) + + # self-guidance + actual_D = guided_blur(inp, inp, kernel_size, eps_) + assert isinstance(actual_D, torch.Tensor) + assert actual_D.shape == (batch_size, input_dim, H, W) + + @pytest.mark.parametrize("shape", [(1, 1, 8, 15), (2, 3, 11, 7)]) + @pytest.mark.parametrize("kernel_size", [5, (3, 5)]) + def test_cardinality(self, shape, kernel_size, device, dtype): + guide = torch.zeros(shape, device=device, dtype=dtype) + inp = torch.zeros(shape, device=device, dtype=dtype) + actual = guided_blur(guide, inp, kernel_size, 0.1) + assert actual.shape == shape + + def test_exception(self): + from kornia.core.exceptions import BaseError, TypeCheckError + + with pytest.raises(TypeCheckError) as errinfo: + guided_blur(torch.rand(1, 1, 5, 5), 3, 3, 0.1) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(BaseError) as errinfo: + guided_blur(torch.rand(1, 1, 5, 5), torch.rand(2, 1, 5, 5), 3, 0.1) + assert "same batch size and spatial dimensions" in str(errinfo.value) + + def test_noncontiguous(self, device, dtype): + batch_size = 3 + guide = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + inp = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + + actual = guided_blur(guide, inp, 3, 0.1) + assert actual.is_contiguous() + + def test_gradcheck(self, device): + guide = torch.rand(1, 2, 5, 4, device=device, dtype=torch.float64) + img = torch.rand(1, 2, 5, 4, device=device, dtype=torch.float64) + self.gradcheck(guided_blur, (guide, img, 3, 0.1), nondet_tol=1e-4) + + eps = torch.rand(1, device=device, dtype=torch.float64) + self.gradcheck(guided_blur, (guide, img, 3, eps), nondet_tol=1e-4) + + @pytest.mark.parametrize("shape", [(1, 1, 8, 16), (2, 3, 12, 8)]) + @pytest.mark.parametrize("kernel_size", [5, (3, 5)]) + @pytest.mark.parametrize("eps", [0.1, 0.01]) + @pytest.mark.parametrize("subsample", [1, 2]) + def test_module(self, shape, kernel_size, eps, subsample, device, dtype): + guide = torch.rand(shape, device=device, dtype=dtype) + img = torch.rand(shape, device=device, dtype=dtype) + + op = guided_blur + op_module = GuidedBlur(kernel_size, eps, subsample=subsample) + self.assert_close(op_module(guide, img), op(guide, img, kernel_size, eps, subsample=subsample)) + + @pytest.mark.skipif( + torch_version() in {"1.9.1", "2.1.0", "2.1.1", "2.1.2"}, + reason=( + "https://github.com/pytorch/pytorch/issues/110696 " + "- Failing with: Argument of Integer should be of numeric type, got s3 + 3." + ), + ) + @pytest.mark.parametrize("kernel_size", [5, (5, 7)]) + @pytest.mark.parametrize("subsample", [1, 2]) + def test_dynamo(self, kernel_size, subsample, device, dtype, torch_optimizer): + guide = torch.ones(2, 3, 8, 8, device=device, dtype=dtype) + data = torch.ones(2, 3, 8, 8, device=device, dtype=dtype) + op = GuidedBlur(kernel_size, 0.1, subsample=subsample) + op_optimized = torch_optimizer(op) + + self.assert_close(op(guide, data), op_optimized(guide, data)) + + op = GuidedBlur(kernel_size, torch.tensor(0.1, device=device, dtype=dtype), subsample=subsample) + op_optimized = torch_optimizer(op) + + self.assert_close(op(guide, data), op_optimized(guide, data)) + + def test_opencv_grayscale(self, device, dtype): + guide = [[100, 130, 58, 36], [215, 142, 173, 166], [114, 150, 190, 60], [23, 83, 84, 216]] + guide = torch.tensor(guide, device=device, dtype=dtype).view(1, 1, 4, 4) / 255 + + img = [[95, 130, 108, 228], [98, 142, 187, 166], [114, 166, 190, 141], [150, 83, 174, 216]] + img = torch.tensor(img, device=device, dtype=dtype).view(1, 1, 4, 4) / 255 + + kernel_size = 3 + eps = 0.01 + + # Expected output generated with OpenCV: + # import cv2 + # expected = cv2.ximgproc.guidedFilter( + # guide.squeeze().numpy(), + # img.squeeze().numpy(), + # (kernel_size - 1) // 2, + # eps, + # ) + expected = [ + [0.4487294, 0.5163902, 0.5981981, 0.70094436], + [0.4850059, 0.53724647, 0.62616897, 0.6686147], + [0.5010369, 0.5631456, 0.6808387, 0.5960593], + [0.5304646, 0.53203756, 0.57674146, 0.80308396], + ] + expected = torch.tensor(expected, device=device, dtype=dtype).view(1, 1, 4, 4) + + # OpenCV uses hard-coded BORDER_REFLECT mode, which also reflects the outermost pixels + # https://github.com/opencv/opencv_contrib/blob/853144ef93c4ffa55661619b861539090943c5b6/modules/ximgproc/src/guided_filter.cpp#L162 + # PyTorch's `reflect` border type corresponds to OpenCV's BORDER_REFLECT_101 + # To match the border's behavior, we use kernel_size = 3 and border_type="replicate" for testing + out = guided_blur(guide, img, kernel_size, eps, border_type="replicate") + self.assert_close(out, expected) + + def test_opencv_rgb(self, device, dtype): + guide = [ + [[170, 89, 182, 255], [199, 209, 216, 205], [196, 213, 218, 191], [207, 126, 224, 249]], + [[61, 104, 274, 225], [65, 112, 14, 148], [78, 247, 176, 120], [124, 69, 155, 211]], + [[73, 111, 94, 175], [77, 117, 123, 130], [83, 139, 163, 120], [132, 84, 137, 155]], + ] + guide = torch.tensor(guide, device=device, dtype=dtype).view(1, 3, 4, 4) / 255 + + img = [ + [[170, 189, 182, 255], [169, 239, 206, 215], [196, 213, 28, 191], [207, 16, 234, 240]], + [[61, 144, 74, 225], [20, 112, 176, 148], [34, 147, 116, 120], [124, 61, 155, 211]], + [[73, 111, 90, 175], [177, 117, 163, 130], [89, 139, 163, 120], [132, 84, 137, 135]], + ] + img = torch.tensor(img, device=device, dtype=dtype).view(1, 3, 4, 4) / 255 + + kernel_size = 3 + eps = 0.01 + + # Expected output generated with OpenCV: + # import cv2 + # expected = cv2.ximgproc.guidedFilter( + # guide.squeeze().permute(1, 2, 0).numpy(), + # img.squeeze().permute(1, 2, 0).numpy(), + # (kernel_size - 1) // 2, + # eps, + # ).transpose(2, 0, 1) + expected = [ + [ + [0.7039907, 0.7277061, 0.7474556, 0.904094], + [0.7095674, 0.76176095, 0.77444744, 0.7774203], + [0.67807436, 0.7721572, 0.70001286, 0.7042719], + [0.73099065, 0.28477466, 0.7464762, 0.8454268], + ], + [ + [0.25627214, 0.4922768, 0.3593133, 0.76788116], + [0.21797341, 0.42890117, 0.56577384, 0.58102953], + [0.25184435, 0.5643642, 0.59704626, 0.5153022], + [0.42154774, 0.24721909, 0.56817913, 0.7258603], + ], + [ + [0.431774, 0.40672457, 0.39094293, 0.63833976], + [0.47457936, 0.51558167, 0.58189815, 0.5340911], + [0.45442006, 0.5345709, 0.5615816, 0.5071402], + [0.49547666, 0.37159446, 0.5301453, 0.55153173], + ], + ] + expected = torch.tensor(expected, device=device, dtype=dtype).view(1, 3, 4, 4) + + out = guided_blur(guide, img, kernel_size, eps, border_type="replicate") + self.assert_close(out, expected) diff --git a/tests/filters/test_hanning.py b/tests/filters/test_hanning.py new file mode 100644 index 0000000..51f7033 --- /dev/null +++ b/tests/filters/test_hanning.py @@ -0,0 +1,54 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.filters import get_hanning_kernel1d, get_hanning_kernel2d + +from testing.base import assert_close + + +@pytest.mark.parametrize("window_size", [5, 11]) +def test_get_hanning_kernel(window_size, device, dtype): + kernel = get_hanning_kernel1d(window_size, dtype=dtype, device=device) + assert kernel.shape == (window_size,) + assert kernel.max().item() == pytest.approx(1.0) + + +@pytest.mark.parametrize("ksize_x", [5, 11]) +@pytest.mark.parametrize("ksize_y", [3, 7]) +def test_get_hanning_kernel2d(ksize_x, ksize_y, device, dtype): + kernel = get_hanning_kernel2d((ksize_x, ksize_y), dtype=dtype, device=device) + assert kernel.shape == (ksize_x, ksize_y) + assert kernel.max().item() == pytest.approx(1.0) + + +def test_get_hanning_kernel1d_5(device, dtype): + kernel = get_hanning_kernel1d(5, dtype=dtype, device=device) + expected = torch.tensor([0, 0.5, 1.0, 0.5, 0], dtype=dtype, device=device) + assert kernel.shape == (5,) + assert_close(kernel, expected) + + +def test_get_hanning_kernel2d_3x4(device, dtype): + kernel = get_hanning_kernel2d((3, 4), dtype=dtype, device=device) + expected = torch.tensor( + [[0.0, 0.00, 0.00, 0.0], [0.0, 0.75, 0.75, 0.0], [0.0, 0.00, 0.00, 0.0]], dtype=dtype, device=device + ) + assert kernel.shape == (3, 4) + assert_close(kernel, expected) diff --git a/tests/filters/test_in_range.py b/tests/filters/test_in_range.py new file mode 100644 index 0000000..eeeb7c0 --- /dev/null +++ b/tests/filters/test_in_range.py @@ -0,0 +1,153 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re + +import pytest +import torch + +from kornia.core._compat import torch_version +from kornia.filters import InRange, in_range + +from testing.base import BaseTester, assert_close + + +def test_in_range(device, dtype): + torch.manual_seed(1) + # Generate on CPU first so the expected mask is device-independent, then move to target device. + input_tensor = torch.rand(1, 3, 3, 3).to(dtype=dtype).to(device=device) + expected = torch.tensor([[[[1.0, 1.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]]]], device=device, dtype=dtype) + lower = (0.2, 0.3, 0.4) + upper = (0.8, 0.9, 1.0) + result = in_range(input_tensor, lower, upper, return_mask=True) + + assert_close(result, expected, atol=1e-4, rtol=1e-4) + + +class TestInRange(BaseTester): + def _get_expected(self, device, dtype): + return torch.tensor( + [[[[1.0, 1.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]]]], + device=device, + dtype=dtype, + ) + + def test_smoke(self, device, dtype): + torch.manual_seed(1) + # Generate on CPU first so the expected mask is device-independent, then move to target device. + input_tensor = torch.rand(1, 3, 3, 3).to(dtype=dtype).to(device=device) + expected = self._get_expected(device=device, dtype=dtype) + res = InRange(lower=(0.2, 0.3, 0.4), upper=(0.8, 0.9, 1.0), return_mask=True)(input_tensor) + assert expected.shape == res.shape + self.assert_close(res, expected, rtol=1e-4, atol=1e-4) + + @pytest.mark.parametrize( + "input_shape, lower, upper", + [ + ((1, 3, 3, 3), (0.2, 0.2, 0.2), (0.6, 0.6, 0.6)), + ((2, 3, 3, 3), (0.2, 0.2, 0.2), (0.6, 0.6, 0.6)), + ((5, 5, 3, 3), (0.2, 0.2, 0.2, 0.2, 0.2), (0.6, 0.6, 0.6, 0.6, 0.6)), + ((3, 3), (0.2,), (0.6,)), + ((2, 3, 3), (0.2, 0.2), (0.6, 0.6)), + ], + ) + def test_cardinality(self, input_shape, lower, upper, device, dtype): + input_tensor = torch.rand(input_shape, device=device, dtype=dtype) + res = InRange(lower=lower, upper=upper, return_mask=True)(input_tensor) + + if len(input_tensor.shape) == 2: + assert res.shape == (res.shape[-2], res.shape[-1]) + elif len(input_tensor.shape) == 3: + assert res.shape == (1, res.shape[-2], res.shape[-1]) + else: + assert res.shape == (res.shape[0], 1, res.shape[-2], res.shape[-1]) + + def test_exception(self, device, dtype): + input_tensor = torch.rand(1, 3, 3, 3, device=device, dtype=dtype) + with pytest.raises(Exception, match=r"Invalid `lower` and `upper` format. Should be tuple or torch\.Tensor\."): + InRange(lower=3, upper=3)(input_tensor) + + with pytest.raises(Exception, match=r"Invalid `lower` and `upper` format. Should be tuple or torch\.Tensor\."): + InRange(lower=[0.2, 0.2], upper=[0.2, 0.2])(input_tensor) + + with pytest.raises(Exception, match=r"Invalid `lower` and `upper` format. Should be tuple or torch\.Tensor\."): + InRange(lower=(0.2), upper=(0.2))(input_tensor) + + with pytest.raises( + ValueError, match=r"Shape of `lower`, `upper` and `input` image channels must have same shape." + ): + InRange(lower=(0.2,), upper=(0.2,))(input_tensor) + + with pytest.raises( + ValueError, + match=re.escape( + "`lower` and `upper` bounds as Tensors must have compatible shapes with the input (B, C, 1, 1)." + ), + ): + lower = torch.tensor([0.2, 0.2, 0.2]) + upper = torch.tensor([0.6, 0.6, 0.6]) + InRange(lower=lower, upper=upper)(input_tensor) + + with pytest.raises(Exception, match=r"Invalid `return_mask` format. Should be boolean."): + lower = torch.tensor([0.2, 0.2, 0.2]) + upper = torch.tensor([0.6, 0.6, 0.6]) + InRange(lower=lower, upper=upper, return_mask=2)(input_tensor) + + def test_tensor_bounds_return_masked_input(self, device, dtype): + # Exercises the Tensor-bounds branch (lines 132-139) with return_mask=False (line 148) + inp = torch.ones(1, 3, 4, 4, device=device, dtype=dtype) * 0.5 + lower = torch.tensor([0.2, 0.2, 0.2], device=device, dtype=dtype).reshape(1, 3, 1, 1) + upper = torch.tensor([0.8, 0.8, 0.8], device=device, dtype=dtype).reshape(1, 3, 1, 1) + out = in_range(inp, lower, upper, return_mask=False) + # All pixels are in range, so output == input + self.assert_close(out, inp) + + def test_noncontiguous(self, device, dtype): + batch_size = 3 + inp = torch.rand(1, 3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + actual = InRange((0.2, 0.2, 0.2), (0.6, 0.6, 0.6), return_mask=True)(inp) + assert actual.is_contiguous() + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 3, 5, 5 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(in_range, (img, (0.2, 0.2, 0.2), (0.6, 0.6, 0.6), True)) + + @pytest.mark.parametrize( + "input_shape, lower, upper", + [ + ((1, 3, 3, 3), (0.2, 0.2, 0.2), (0.6, 0.6, 0.6)), + ((2, 3, 3, 3), (0.2, 0.2, 0.2), (0.6, 0.6, 0.6)), + ((3, 3), (0.2,), (0.6,)), + ], + ) + def test_module(self, input_shape, lower, upper, device, dtype): + img = torch.rand(input_shape, device=device, dtype=dtype) + op = in_range + op_module = InRange(lower=lower, upper=upper, return_mask=True) + actual = op_module(img) + expected = op(img, lower, upper, True) + self.assert_close(actual, expected) + + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_dynamo(self, batch_size, device, dtype, torch_optimizer): + if device == torch.device("cpu") and torch_version() in {"2.3.0", "2.3.1"}: + pytest.skip("Failing to compile on CPU see pytorch/pytorch#126619") + data = torch.rand(batch_size, 3, 5, 5, device=device, dtype=dtype) + op = InRange(lower=(0.2, 0.2, 0.2), upper=(0.6, 0.6, 0.6), return_mask=True) + op_optimized = torch_optimizer(op) + self.assert_close(op(data), op_optimized(data)) diff --git a/tests/filters/test_laplacian.py b/tests/filters/test_laplacian.py new file mode 100644 index 0000000..7590396 --- /dev/null +++ b/tests/filters/test_laplacian.py @@ -0,0 +1,121 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.filters import Laplacian, get_laplacian_kernel1d, get_laplacian_kernel2d, laplacian + +from testing.base import BaseTester, assert_close + + +@pytest.mark.parametrize("window_size", [5, 11]) +def test_get_laplacian_kernel1d(window_size, device, dtype): + actual = get_laplacian_kernel1d(window_size, device=device, dtype=dtype) + expected = torch.zeros(1, device=device, dtype=dtype) + + assert actual.shape == (window_size,) + assert_close(actual.sum(), expected.sum()) + + +@pytest.mark.parametrize("window_size", [5, 11, (3, 3)]) +def test_get_laplacian_kernel2d(window_size, device, dtype): + actual = get_laplacian_kernel2d(window_size, device=device, dtype=dtype) + expected = torch.zeros(1, device=device, dtype=dtype) + expected_shape = window_size if isinstance(window_size, tuple) else (window_size, window_size) + + assert actual.shape == expected_shape + assert_close(actual.sum(), expected.sum()) + + +def test_get_laplacian_kernel1d_exact(device, dtype): + actual = get_laplacian_kernel1d(5, device=device, dtype=dtype) + expected = torch.tensor([1.0, 1.0, -4.0, 1.0, 1.0], device=device, dtype=dtype) + assert_close(expected, actual) + + +def test_get_laplacian_kernel2d_exact(device, dtype): + actual = get_laplacian_kernel2d(7, device=device, dtype=dtype) + expected = torch.tensor( + [ + [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, -48.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + ], + device=device, + dtype=dtype, + ) + assert_close(expected, actual) + + +class TestLaplacian(BaseTester): + @pytest.mark.parametrize("shape", [(1, 4, 8, 15), (2, 3, 11, 7)]) + @pytest.mark.parametrize("kernel_size", [5, (11, 7), (3, 3)]) + @pytest.mark.parametrize("normalized", [True, False]) + def test_smoke(self, shape, kernel_size, normalized, device, dtype): + data = torch.rand(shape, device=device, dtype=dtype) + actual = laplacian(data, kernel_size, "reflect", normalized) + assert isinstance(actual, torch.Tensor) + assert actual.shape == shape + + @pytest.mark.parametrize("shape", [(1, 4, 8, 15), (2, 3, 11, 7)]) + @pytest.mark.parametrize("kernel_size", [5, (11, 7), 3]) + def test_cardinality(self, shape, kernel_size, device, dtype): + sample = torch.rand(shape, device=device, dtype=dtype) + actual = laplacian(sample, kernel_size) + assert actual.shape == shape + + @pytest.mark.skip(reason="Nothing to test.") + def test_exception(self): ... + + def test_noncontiguous(self, device, dtype): + batch_size = 3 + sample = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + + kernel_size = 3 + actual = laplacian(sample, kernel_size) + assert actual.is_contiguous() + + def test_gradcheck(self, device): + # test parameters + batch_shape = (1, 2, 5, 7) + kernel_size = 3 + + # evaluate function gradient + sample = torch.rand(batch_shape, device=device, dtype=torch.float64) + self.gradcheck(laplacian, (sample, kernel_size)) + + def test_module(self, device, dtype): + params = [3] + op = laplacian + op_module = Laplacian(*params) + + img = torch.ones(1, 3, 5, 5, device=device, dtype=dtype) + self.assert_close(op(img, *params), op_module(img)) + + @pytest.mark.parametrize("kernel_size", [5, (5, 7)]) + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_dynamo(self, batch_size, kernel_size, device, dtype, torch_optimizer): + data = torch.ones(batch_size, 3, 10, 10, device=device, dtype=dtype) + op = Laplacian(kernel_size) + op_optimized = torch_optimizer(op) + + self.assert_close(op(data), op_optimized(data)) diff --git a/tests/filters/test_median.py b/tests/filters/test_median.py new file mode 100644 index 0000000..d02c0bc --- /dev/null +++ b/tests/filters/test_median.py @@ -0,0 +1,125 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.filters import MedianBlur, median_blur + +from testing.base import BaseTester + + +class TestMedianBlur(BaseTester): + def test_smoke(self, device, dtype): + inp = torch.zeros(1, 3, 4, 4, device=device, dtype=dtype) + actual = median_blur(inp, 3) + assert isinstance(actual, torch.Tensor) + + @pytest.mark.parametrize("batch_size", [0, 1, 2]) + @pytest.mark.parametrize("kernel_size", [3, (5, 7)]) + def test_cardinality(self, batch_size, kernel_size, device, dtype): + inp = torch.zeros(batch_size, 3, 4, 4, device=device, dtype=dtype) + actual = median_blur(inp, kernel_size) + assert actual.shape == (batch_size, 3, 4, 4) + + def test_exception(self, device, dtype): + from kornia.core.exceptions import ShapeError, TypeCheckError + + with pytest.raises(TypeCheckError) as errinfo: + median_blur(1, 1) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(ShapeError) as errinfo: + median_blur(torch.ones(1, 1, device=device, dtype=dtype), 1) + assert "Shape dimension mismatch" in str(errinfo.value) or "Expected shape" in str(errinfo.value) + + def test_kernel_3x3(self, device, dtype): + inp = torch.tensor( + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 3.0, 7.0, 5.0, 0.0], + [0.0, 3.0, 1.0, 1.0, 0.0], + [0.0, 6.0, 9.0, 2.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [36.0, 7.0, 25.0, 0.0, 0.0], + [3.0, 14.0, 1.0, 0.0, 0.0], + [65.0, 59.0, 2.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + ], + device=device, + dtype=dtype, + ).repeat(2, 1, 1, 1) + + kernel_size = (3, 3) + actual = median_blur(inp, kernel_size) + self.assert_close(actual[0, 0, 2, 2], torch.tensor(3.0, device=device, dtype=dtype)) + self.assert_close(actual[0, 1, 1, 1], torch.tensor(14.0, device=device, dtype=dtype)) + + def test_kernel_3x1(self, device, dtype): + inp = torch.tensor( + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 3.0, 7.0, 5.0, 0.0], + [0.0, 3.0, 1.0, 1.0, 0.0], + [0.0, 6.0, 9.0, 2.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + device=device, + dtype=dtype, + ).view(1, 1, 5, 5) + + ky, kx = 3, 1 + actual = median_blur(inp, (ky, kx)) + + self.assert_close(actual[0, 0, 2, 2], torch.tensor(7.0, device=device, dtype=dtype)) + self.assert_close(actual[0, 0, 1, 1], torch.tensor(3.0, device=device, dtype=dtype)) + + def test_noncontiguous(self, device, dtype): + batch_size = 3 + inp = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + + kernel_size = (3, 3) + actual = median_blur(inp, kernel_size) + assert actual.is_contiguous() + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 2, 5, 4 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(median_blur, (img, (5, 3))) + + def test_module(self, device, dtype): + kernel_size = (3, 5) + img = torch.rand(2, 3, 4, 5, device=device, dtype=dtype) + op = median_blur + op_module = MedianBlur((3, 5)) + actual = op_module(img) + expected = op(img, kernel_size) + self.assert_close(actual, expected) + + @pytest.mark.parametrize("kernel_size", [5, (5, 7)]) + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_dynamo(self, batch_size, kernel_size, device, dtype, torch_optimizer): + data = torch.ones(batch_size, 3, 10, 10, device=device, dtype=dtype) + op = MedianBlur(kernel_size) + op_optimized = torch_optimizer(op) + + self.assert_close(op(data), op_optimized(data)) diff --git a/tests/filters/test_motion.py b/tests/filters/test_motion.py new file mode 100644 index 0000000..b040bab --- /dev/null +++ b/tests/filters/test_motion.py @@ -0,0 +1,208 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.filters import ( + MotionBlur, + MotionBlur3D, + get_motion_kernel2d, + get_motion_kernel3d, + motion_blur, + motion_blur3d, +) + +from testing.base import BaseTester + + +class TestMotionBlur(BaseTester): + @pytest.mark.parametrize("shape", [(1, 4, 8, 15), (2, 3, 11, 7)]) + @pytest.mark.parametrize("kernel_size", [3, 5]) + @pytest.mark.parametrize("angle", [36.0, 200.0]) + @pytest.mark.parametrize("direction", [-0.9, 0.0, 0.9]) + @pytest.mark.parametrize("mode", ["bilinear", "nearest"]) + @pytest.mark.parametrize("params_as_tensor", [True, False]) + def test_smoke(self, shape, kernel_size, angle, direction, mode, params_as_tensor, device, dtype): + B, _C, _H, _W = shape + data = torch.rand(shape, device=device, dtype=dtype) + + if params_as_tensor is True: + angle = torch.tensor([angle], device=device, dtype=dtype).repeat(B) + direction = torch.tensor([direction], device=device, dtype=dtype).repeat(B) + actual = motion_blur(data, kernel_size, angle, direction, "constant", mode) + + assert isinstance(actual, torch.Tensor) + assert actual.shape == shape + + @pytest.mark.parametrize("shape", [(1, 4, 8, 15), (2, 3, 11, 7)]) + def test_cardinality(self, shape, device, dtype): + ksize = 5 + angle = 200.0 + direction = 0.3 + + sample = torch.rand(shape, device=device, dtype=dtype) + motion = MotionBlur(ksize, angle, direction) + assert motion(sample).shape == shape + + @pytest.mark.skip(reason="nothing to test") + def test_exception(self): ... + + @pytest.mark.parametrize("batch_size", [1, 3]) + @pytest.mark.parametrize("ksize", [3, 11]) + @pytest.mark.parametrize("angle", [0.0, 360.0]) + @pytest.mark.parametrize("direction", [-1.0, 1.0]) + @pytest.mark.parametrize("params_as_tensor", [True, False]) + def test_get_motion_kernel2d(self, batch_size, ksize, angle, direction, params_as_tensor, device, dtype): + if params_as_tensor is True: + angle = torch.tensor([angle], device=device, dtype=dtype).repeat(batch_size) + direction = torch.tensor([direction], device=device, dtype=dtype).repeat(batch_size) + else: + batch_size = 1 + device = None + dtype = None + + actual = get_motion_kernel2d(ksize, angle, direction) + expected = torch.ones(1, device=device, dtype=dtype) * batch_size + assert actual.shape == (batch_size, ksize, ksize) + self.assert_close(actual.sum(), expected.sum()) + + def test_noncontiguous(self, device, dtype): + batch_size = 3 + inp = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + + kernel_size = 3 + angle = 200.0 + direction = 0.3 + actual = motion_blur(inp, kernel_size, angle, direction) + assert actual.is_contiguous() + + def test_gradcheck(self, device): + batch_shape = (1, 3, 4, 5) + ksize = 9 + angle = 34.0 + direction = -0.2 + + sample = torch.rand(batch_shape, device=device, dtype=torch.float64) + self.gradcheck(motion_blur, (sample, ksize, angle, direction, "replicate"), nondet_tol=1e-8) + + def test_module(self, device, dtype): + params = [3, 20.0, 0.5] + op = motion_blur + op_module = MotionBlur(*params) + img = torch.ones(1, 3, 5, 5, device=device, dtype=dtype) + + self.assert_close(op(img, *params), op_module(img)) + + @pytest.mark.skip(reason="After the op be optimized the results are not the same") + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_dynamo(self, batch_size, device, dtype, torch_optimizer): + # TODO: FIX op + data = torch.ones(batch_size, 3, 10, 10, device=device, dtype=dtype) + op = MotionBlur(3, 36.0, 0.5) + op_optimized = torch_optimizer(op) + + self.assert_close(op(data), op_optimized(data)) + + +class TestMotionBlur3D(BaseTester): + @pytest.mark.parametrize("shape", [(1, 4, 3, 8, 15), (2, 2, 3, 11, 7)]) + @pytest.mark.parametrize("kernel_size", [3, 5]) + @pytest.mark.parametrize("angle", [(36.0, 15.0, 200.0), (200.0, 10.0, 150.0)]) + @pytest.mark.parametrize("direction", [-0.9, 0.0, 0.9]) + @pytest.mark.parametrize("mode", ["bilinear", "nearest"]) + @pytest.mark.parametrize("params_as_tensor", [True, False]) + def test_smoke(self, shape, kernel_size, angle, direction, mode, params_as_tensor, device, dtype): + B, _C, _D, _H, _W = shape + data = torch.rand(shape, device=device, dtype=dtype) + + if params_as_tensor is True: + angle = torch.tensor([angle], device=device, dtype=dtype).expand(B, 3) + direction = torch.tensor([direction], device=device, dtype=dtype).repeat(B) + actual = motion_blur3d(data, kernel_size, angle, direction, "constant", mode) + + assert isinstance(actual, torch.Tensor) + assert actual.shape == shape + + @pytest.mark.parametrize("shape", [(1, 4, 1, 8, 15), (2, 3, 1, 11, 7)]) + def test_cardinality(self, shape, device, dtype): + ksize = 5 + angle = (200.0, 15.0, 120.0) + direction = 0.3 + + sample = torch.rand(shape, device=device, dtype=dtype) + motion = MotionBlur3D(ksize, angle, direction) + assert motion(sample).shape == shape + + @pytest.mark.skip(reason="nothing to test") + def test_exception(self): ... + + @pytest.mark.parametrize("batch_size", [1, 3]) + @pytest.mark.parametrize("ksize", [3, 11]) + @pytest.mark.parametrize("angle", [(0.0, 360.0, 150.0)]) + @pytest.mark.parametrize("direction", [-1.0, 1.0]) + @pytest.mark.parametrize("params_as_tensor", [True, False]) + def test_get_motion_kernel3d(self, batch_size, ksize, angle, direction, params_as_tensor, device, dtype): + if params_as_tensor is True: + angle = torch.tensor([angle], device=device, dtype=dtype).repeat(batch_size, 1) + direction = torch.tensor([direction], device=device, dtype=dtype).repeat(batch_size) + else: + batch_size = 1 + device = None + dtype = None + + actual = get_motion_kernel3d(ksize, angle, direction) + expected = torch.ones(1, device=device, dtype=dtype) * batch_size + assert actual.shape == (batch_size, ksize, ksize, ksize) + self.assert_close(actual.sum(), expected.sum()) + + def test_noncontiguous(self, device, dtype): + batch_size = 3 + inp = torch.rand(3, 1, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1, -1) + + kernel_size = 3 + angle = (0.0, 360.0, 150.0) + direction = 0.3 + actual = motion_blur3d(inp, kernel_size, angle, direction) + assert actual.is_contiguous() + + def test_gradcheck(self, device): + batch_shape = (1, 3, 1, 4, 5) + ksize = 9 + angle = (0.0, 360.0, 150.0) + direction = -0.2 + + sample = torch.rand(batch_shape, device=device, dtype=torch.float64) + self.gradcheck(motion_blur3d, (sample, ksize, angle, direction, "replicate"), nondet_tol=1e-8) + + def test_module(self, device, dtype): + params = [3, (0.0, 360.0, 150.0), 0.5] + op = motion_blur3d + op_module = MotionBlur3D(*params) + img = torch.ones(1, 3, 1, 5, 5, device=device, dtype=dtype) + + self.assert_close(op(img, *params), op_module(img)) + + @pytest.mark.skip(reason="After the op be optimized the results are not the same") + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_dynamo(self, batch_size, device, dtype, torch_optimizer): + # TODO: Fix the operation to works after dynamo optimize + data = torch.ones(batch_size, 3, 1, 10, 10, device=device, dtype=dtype) + op = MotionBlur3D(3, (0.0, 360.0, 150.0), 0.5) + op_optimized = torch_optimizer(op) + + self.assert_close(op(data), op_optimized(data)) diff --git a/tests/filters/test_otsu_thresholding.py b/tests/filters/test_otsu_thresholding.py new file mode 100644 index 0000000..9d847c8 --- /dev/null +++ b/tests/filters/test_otsu_thresholding.py @@ -0,0 +1,117 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.filters.otsu_thresholding import OtsuThreshold, otsu_threshold + +from testing.base import BaseTester, assert_close + + +class TestOtsuThreshold(BaseTester): + def test_smoke(self, device, dtype): + img = torch.rand(1, 3, 5, 5, device=device, dtype=dtype) + op = OtsuThreshold() + thresh_result, _thresh_value = op(img, nbins=4) + assert thresh_result.shape == img.shape + + @pytest.mark.parametrize("input_shape", [(3, 3), (1, 3, 3), (1, 1, 3, 3), (2, 1, 1, 3, 3)]) + def test_transform_input_shapes(self, input_shape, device, dtype): + img = torch.rand(input_shape, device=device, dtype=dtype) + op = OtsuThreshold() + flat, orig_shape = op.transform_input(img) + assert orig_shape == img.shape + assert flat.ndim == 2 + + def test_otsu_threshold_consistency(self, device, dtype): + torch.manual_seed(0) + img = torch.rand(1, 4, 6, 1, device=device, dtype=dtype) + out_func_tensor, _out_func_value = otsu_threshold(img, nbins=3, return_mask=False) + out_class_tensor, _out_class_value = OtsuThreshold()(img, nbins=3) + assert_close(out_func_tensor, out_class_tensor) + + def test_invalid_dim(self, device, dtype): + img = torch.rand(1, 1, 1, 1, 3, 3, device=device, dtype=dtype) + op = OtsuThreshold() + with pytest.raises(ValueError, match="Unsupported tensor dimensionality"): + op.transform_input(img) + + def test_gradcheck(self, device, dtype): + img = torch.rand(1, 1, 5, 5, device=device, dtype=dtype, requires_grad=True) + self.gradcheck(otsu_threshold, (img, 3, True, False)) + + def test_differentiable_tensor_otsu(self, device, dtype): + differentiable_input = torch.rand(1, 1, 5, 5, device=device, dtype=dtype, requires_grad=True) + + input = differentiable_input.clone().detach().requires_grad_(False) + + op = OtsuThreshold() + diff_thresh_result, _diff_thresh_value = op(input, slow_and_differentiable=True) + thresh_result, _thresh_value = op(input) + self.assert_close(diff_thresh_result, thresh_result) + + def test_threshold_result(self, device, dtype): + input = torch.tensor( + [[10, 10, 10, 10], [10, 10, 10, 10], [200, 200, 200, 200], [200, 200, 200, 200]], device=device, dtype=dtype + ) + + expected = torch.tensor( + [[0, 0, 0, 0], [0, 0, 0, 0], [200, 200, 200, 200], [200, 200, 200, 200]], device=device, dtype=dtype + ) + + op = OtsuThreshold() + thresh_result, _thresh_value = op(input) + self.assert_close(thresh_result, expected) + + def test_gradual_threshold(self, device, dtype): + input = torch.tensor([[10, 20, 30], [40, 50, 60], [70, 80, 90]], device=device, dtype=dtype) + + expected = torch.tensor([[0, 0, 0], [0, 50, 60], [70, 80, 90]], device=device, dtype=dtype) + + op = OtsuThreshold() + thresh_result, _thresh_value = op(input) + self.assert_close(thresh_result, expected) + + def test_uniform_result(self, device, dtype): + input = torch.tensor( + [[10, 10, 10, 10], [10, 10, 10, 10], [10, 10, 10, 10], [10, 10, 10, 10]], device=device, dtype=dtype + ) + + expected = torch.tensor( + [[10, 10, 10, 10], [10, 10, 10, 10], [10, 10, 10, 10], [10, 10, 10, 10]], device=device, dtype=dtype + ) + + op = OtsuThreshold() + thresh_result, _thresh_value = op(input) + self.assert_close(thresh_result, expected) + + +def test_mask(device, dtype): + input = torch.tensor([[10, 20, 30], [40, 50, 60], [70, 80, 90]], device=device, dtype=dtype) + + expected = torch.tensor([[0, 0, 0], [0, 1, 1], [1, 1, 1]], device=device, dtype=torch.bool) + + thresh_result, _thresh_value = otsu_threshold(input, return_mask=True) + assert_close(thresh_result, expected) + + +@pytest.mark.parametrize("shape", [(1, 3, 5, 5), (2, 1, 10, 10)]) +def test_otsu_threshold_basic(shape, device, dtype): + img = torch.rand(shape, device=device, dtype=dtype) + thresh_result, _thresh_value = otsu_threshold(img) + assert thresh_result.shape == img.shape diff --git a/tests/filters/test_sobel.py b/tests/filters/test_sobel.py new file mode 100644 index 0000000..aea1a30 --- /dev/null +++ b/tests/filters/test_sobel.py @@ -0,0 +1,538 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.core._compat import torch_version +from kornia.filters import Sobel, SpatialGradient, SpatialGradient3d, sobel, spatial_gradient, spatial_gradient3d + +from testing.base import BaseTester + + +class TestSpatialGradient(BaseTester): + @pytest.mark.parametrize("batch_size", [1, 2]) + @pytest.mark.parametrize("mode", ["sobel", "diff"]) + @pytest.mark.parametrize("order", [1, 2]) + @pytest.mark.parametrize("normalized", [True, False]) + def test_smoke(self, batch_size, mode, order, normalized, device, dtype): + data = torch.zeros(batch_size, 3, 4, 4, device=device, dtype=dtype) + actual = SpatialGradient(mode, order, normalized)(data) + assert isinstance(actual, torch.Tensor) + + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_cardinality(self, batch_size, device, dtype): + inp = torch.zeros(batch_size, 3, 4, 4, device=device, dtype=dtype) + assert SpatialGradient()(inp).shape == (batch_size, 3, 2, 4, 4) + + def test_exception(self): + from kornia.core.exceptions import ShapeError, TypeCheckError + + with pytest.raises(TypeCheckError) as errinfo: + spatial_gradient(1) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(ShapeError) as errinfo: + spatial_gradient(torch.zeros(1)) + assert "Shape dimension mismatch" in str(errinfo.value) or "Expected shape" in str(errinfo.value) + + def test_edges(self, device, dtype): + inp = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [ + [ + [ + [ + [0.0, 1.0, 0.0, -1.0, 0.0], + [1.0, 3.0, 0.0, -3.0, -1.0], + [2.0, 4.0, 0.0, -4.0, -2.0], + [1.0, 3.0, 0.0, -3.0, -1.0], + [0.0, 1.0, 0.0, -1.0, 0.0], + ], + [ + [0.0, 1.0, 2.0, 1.0, 0.0], + [1.0, 3.0, 4.0, 3.0, 1.0], + [0.0, 0.0, 0.0, 0.0, 0], + [-1.0, -3.0, -4.0, -3.0, -1], + [0.0, -1.0, -2.0, -1.0, 0.0], + ], + ] + ] + ], + device=device, + dtype=dtype, + ) + + edges = spatial_gradient(inp, normalized=False) + self.assert_close(edges, expected) + + def test_edges_norm(self, device, dtype): + inp = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected = ( + torch.tensor( + [ + [ + [ + [ + [0.0, 1.0, 0.0, -1.0, 0.0], + [1.0, 3.0, 0.0, -3.0, -1.0], + [2.0, 4.0, 0.0, -4.0, -2.0], + [1.0, 3.0, 0.0, -3.0, -1.0], + [0.0, 1.0, 0.0, -1.0, 0.0], + ], + [ + [0.0, 1.0, 2.0, 1.0, 0.0], + [1.0, 3.0, 4.0, 3.0, 1.0], + [0.0, 0.0, 0.0, 0.0, 0], + [-1.0, -3.0, -4.0, -3.0, -1], + [0.0, -1.0, -2.0, -1.0, 0.0], + ], + ] + ] + ], + device=device, + dtype=dtype, + ) + / 8.0 + ) + + edges = spatial_gradient(inp, normalized=True) + self.assert_close(edges, expected) + + def test_edges_sep(self, device, dtype): + inp = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, -1.0, 0.0], + [1.0, 1.0, 0.0, -1.0, -1.0], + [0.0, 1.0, 0.0, -1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, -1.0, -1.0, -1.0, 0.0], + [0.0, 0.0, -1.0, 0.0, 0.0], + ], + ] + ] + ], + device=device, + dtype=dtype, + ) + + edges = spatial_gradient(inp, "diff", normalized=False) + self.assert_close(edges, expected) + + def test_edges_sep_norm(self, device, dtype): + inp = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected = ( + torch.tensor( + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, -1.0, 0.0], + [1.0, 1.0, 0.0, -1.0, -1.0], + [0.0, 1.0, 0.0, -1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, -1.0, -1.0, -1.0, 0.0], + [0.0, 0.0, -1.0, 0.0, 0.0], + ], + ] + ] + ], + device=device, + dtype=dtype, + ) + / 2.0 + ) + + edges = spatial_gradient(inp, "diff", normalized=True) + self.assert_close(edges, expected) + + def test_noncontiguous(self, device, dtype): + batch_size = 3 + inp = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + + actual = spatial_gradient(inp) + + assert inp.is_contiguous() is False + assert actual.is_contiguous() + assert actual.shape == (3, 3, 2, 5, 5) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 1, 3, 4 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(spatial_gradient, (img,)) + + def test_module(self, device, dtype): + img = torch.rand(2, 3, 4, 5, device=device, dtype=dtype) + op = spatial_gradient + op_module = SpatialGradient() + expected = op(img) + actual = op_module(img) + self.assert_close(actual, expected) + + @pytest.mark.parametrize("mode", ["sobel", "diff"]) + @pytest.mark.parametrize("order", [1, 2]) + @pytest.mark.parametrize("batch_size", [1, 2]) + @pytest.mark.xfail(torch_version() in {"2.0.1"}, reason="random failing") + def test_dynamo(self, batch_size, order, mode, device, dtype, torch_optimizer): + data = torch.ones(batch_size, 3, 10, 10, device=device, dtype=dtype) + if order == 1 and dtype == torch.float64: + # TODO: FIX order 1 spatial gradient with fp64 on dynamo + pytest.xfail(reason="Order 1 on spatial gradient may be wrong computed for float64 on dynamo") + op = SpatialGradient(mode, order) + op_optimized = torch_optimizer(op) + + self.assert_close(op(data), op_optimized(data)) + + +class TestSpatialGradient3d(BaseTester): + @pytest.mark.parametrize("batch_size", [1, 2]) + @pytest.mark.parametrize("mode", ["diff"]) # TODO: add support to 'sobel' + @pytest.mark.parametrize("order", [1, 2]) + def test_smoke(self, batch_size, mode, order, device, dtype): + data = torch.ones(batch_size, 3, 2, 7, 4, device=device, dtype=dtype) + actual = SpatialGradient3d(mode, order)(data) + assert isinstance(actual, torch.Tensor) + + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_cardinality(self, batch_size, device, dtype): + inp = torch.zeros(batch_size, 2, 4, 5, 6, device=device, dtype=dtype) + sobel = SpatialGradient3d() + assert sobel(inp).shape == (batch_size, 2, 3, 4, 5, 6) + + def test_exception(self): + from kornia.core.exceptions import ShapeError, TypeCheckError + + with pytest.raises(TypeCheckError) as errinfo: + spatial_gradient3d(1) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(ShapeError) as errinfo: + spatial_gradient3d(torch.zeros(1)) + assert "Shape dimension mismatch" in str(errinfo.value) or "Expected shape" in str(errinfo.value) + + def test_edges(self, device, dtype): + inp = torch.tensor( + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [ + [ + [ + [ + [ + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.5000, 0.0000, -0.5000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + ], + [ + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.5000, 0.0000, -0.5000, 0.0000], + [0.5000, 0.5000, 0.0000, -0.5000, -0.5000], + [0.0000, 0.5000, 0.0000, -0.5000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + ], + [ + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.5000, 0.0000, -0.5000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + ], + ], + [ + [ + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.5000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, -0.5000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + ], + [ + [0.0000, 0.0000, 0.5000, 0.0000, 0.0000], + [0.0000, 0.5000, 0.5000, 0.5000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, -0.5000, -0.5000, -0.5000, 0.0000], + [0.0000, 0.0000, -0.5000, 0.0000, 0.0000], + ], + [ + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.5000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, -0.5000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + ], + ], + [ + [ + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.5000, 0.0000, 0.0000], + [0.0000, 0.5000, 0.0000, 0.5000, 0.0000], + [0.0000, 0.0000, 0.5000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + ], + [ + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + ], + [ + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [0.0000, 0.0000, -0.5000, 0.0000, 0.0000], + [0.0000, -0.5000, 0.0000, -0.5000, 0.0000], + [0.0000, 0.0000, -0.5000, 0.0000, 0.0000], + [0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + ], + ], + ] + ] + ], + device=device, + dtype=dtype, + ) + + edges = spatial_gradient3d(inp) + self.assert_close(edges, expected) + + def test_gradcheck(self, device): + img = torch.rand(1, 1, 1, 3, 4, device=device, dtype=torch.float64) + fast_mode = "cpu" in str(device) # disable fast mode on gpu + self.gradcheck(spatial_gradient3d, (img,), fast_mode=fast_mode) + + def test_module(self, device, dtype): + img = torch.rand(2, 3, 1, 4, 5, device=device, dtype=dtype) + op = spatial_gradient3d + op_module = SpatialGradient3d() + expected = op(img) + actual = op_module(img) + self.assert_close(actual, expected) + + @pytest.mark.parametrize("mode", ["diff"]) + @pytest.mark.parametrize("order", [1, 2]) + def test_dynamo(self, mode, order, device, dtype, torch_optimizer): + data = torch.ones(1, 3, 1, 10, 10, device=device, dtype=dtype) + op = SpatialGradient3d(mode, order) + op_optimized = torch_optimizer(op) + + self.assert_close(op(data), op_optimized(data)) + + +class TestSobel(BaseTester): + @pytest.mark.parametrize("batch_size", [1, 2]) + @pytest.mark.parametrize("normalized", [True, False]) + def test_smoke(self, batch_size, normalized, device, dtype): + inp = torch.zeros(batch_size, 3, 4, 7, device=device, dtype=dtype) + actual = Sobel()(inp) + + assert isinstance(actual, torch.Tensor) + assert actual.shape == (batch_size, 3, 4, 7) + + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_cardinality(self, batch_size, device, dtype): + inp = torch.zeros(batch_size, 3, 4, 7, device=device, dtype=dtype) + assert Sobel()(inp).shape == (batch_size, 3, 4, 7) + + def test_exception(self): + from kornia.core.exceptions import ShapeError, TypeCheckError + + with pytest.raises(TypeCheckError) as errinfo: + sobel(1) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(ShapeError) as errinfo: + sobel(torch.zeros(1)) + assert "Shape dimension mismatch" in str(errinfo.value) or "Expected shape" in str(errinfo.value) + + def test_magnitude(self, device, dtype): + inp = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 1.0, 1.0, 1.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [ + [ + [ + [0.0, 1.4142, 2.0, 1.4142, 0.0], + [1.4142, 4.2426, 4.00, 4.2426, 1.4142], + [2.0, 4.0000, 0.00, 4.0000, 2.0], + [1.4142, 4.2426, 4.00, 4.2426, 1.4142], + [0.0, 1.4142, 2.0, 1.4142, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + + edges = sobel(inp, normalized=False, eps=0.0) + self.assert_close(edges, expected) + + def test_noncontiguous(self, device, dtype): + batch_size = 3 + inp = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + + op = Sobel() + actual = op(inp) + + assert inp.is_contiguous() is False + assert actual.is_contiguous() + assert actual.shape == (3, 3, 5, 5) + + @pytest.mark.parametrize("normalized", [True, False]) + def test_gradcheck(self, normalized, device): + batch_size, channels, height, width = 1, 1, 3, 4 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(sobel, (img, normalized)) + + def test_module(self, device, dtype): + img = torch.rand(2, 3, 4, 5, device=device, dtype=dtype) + op = sobel + op_module = Sobel() + expected = op(img) + actual = op_module(img) + self.assert_close(actual, expected) + + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_dynamo(self, batch_size, device, dtype, torch_optimizer): + if dtype == torch.float64: + # TODO: investigate sobel for float64 with dynamo + pytest.xfail(reason="The sobel results can be different after dynamo on fp64") + data = torch.ones(batch_size, 3, 10, 10, device=device, dtype=dtype) + op = Sobel() + op_optimized = torch_optimizer(op) + + self.assert_close(op(data), op_optimized(data)) diff --git a/tests/filters/test_unsharp_mask.py b/tests/filters/test_unsharp_mask.py new file mode 100644 index 0000000..c57db82 --- /dev/null +++ b/tests/filters/test_unsharp_mask.py @@ -0,0 +1,89 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.filters import UnsharpMask, unsharp_mask + +from testing.base import BaseTester + + +class Testunsharp(BaseTester): + @pytest.mark.parametrize("shape", [(1, 4, 8, 15), (2, 3, 11, 7)]) + @pytest.mark.parametrize("kernel_size", [3, (5, 3)]) + @pytest.mark.parametrize("sigma", [(3.0, 1.0), (0.5, 0.1)]) + @pytest.mark.parametrize("params_as_tensor", [True, False]) + def test_smoke(self, shape, kernel_size, sigma, params_as_tensor, device, dtype): + if params_as_tensor is True: + sigma = torch.tensor([sigma], device=device, dtype=dtype).repeat(shape[0], 1) + + data = torch.ones(shape, device=device, dtype=dtype) + actual = unsharp_mask(data, kernel_size, sigma, "replicate") + assert isinstance(actual, torch.Tensor) + assert actual.shape == shape + + @pytest.mark.parametrize("shape", [(1, 4, 8, 15), (2, 3, 11, 7)]) + def test_cardinality(self, shape, device, dtype): + kernel_size = (5, 7) + sigma = (1.5, 2.1) + + data = torch.ones(shape, device=device, dtype=dtype) + actual = unsharp_mask(data, kernel_size, sigma, "replicate") + assert actual.shape == shape + + @pytest.mark.skip(reason="nothing to test") + def test_exception(self): ... + + def test_noncontiguous(self, device, dtype): + batch_size = 3 + data = torch.rand(3, 5, 5, device=device, dtype=dtype).expand(batch_size, -1, -1, -1) + + kernel_size = (3, 3) + sigma = (1.5, 2.1) + actual = unsharp_mask(data, kernel_size, sigma, "replicate") + assert actual.is_contiguous() + + def test_gradcheck(self, device): + # test parameters + shape = (1, 3, 5, 5) + kernel_size = (3, 3) + sigma = (1.5, 2.1) + + # evaluate function gradient + data = torch.rand(shape, device=device, dtype=torch.float64) + self.gradcheck(unsharp_mask, (data, kernel_size, sigma, "replicate")) + + def test_module(self, device, dtype): + params = [(3, 3), (1.5, 1.5)] + op = unsharp_mask + op_module = UnsharpMask(*params) + + img = torch.ones(1, 3, 5, 5, device=device, dtype=dtype) + self.assert_close(op(img, *params), op_module(img)) + + @pytest.mark.parametrize("sigma", [(3.0, 1.0), (0.5, 0.1)]) + @pytest.mark.parametrize("params_as_tensor", [True, False]) + def test_dynamo(self, sigma, params_as_tensor, device, dtype, torch_optimizer): + if params_as_tensor is True: + sigma = torch.tensor([sigma], device=device, dtype=dtype) + + data = torch.ones(1, 3, 10, 10, device=device, dtype=dtype) + op = UnsharpMask(3, sigma) + op_optimized = torch_optimizer(op) + + self.assert_close(op(data), op_optimized(data)) diff --git a/tests/geometry/__init__.py b/tests/geometry/__init__.py new file mode 100644 index 0000000..4d273d5 --- /dev/null +++ b/tests/geometry/__init__.py @@ -0,0 +1,16 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/geometry/calibration/test_distort.py b/tests/geometry/calibration/test_distort.py new file mode 100644 index 0000000..2dfe3ae --- /dev/null +++ b/tests/geometry/calibration/test_distort.py @@ -0,0 +1,117 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.geometry.calibration.distort import distort_points + +from testing.base import BaseTester + + +class TestDistortPoints(BaseTester): + def test_smoke(self, device, dtype): + points = torch.rand(1, 2, device=device, dtype=dtype) + K = torch.rand(3, 3, device=device, dtype=dtype) + distCoeff = torch.rand(4, device=device, dtype=dtype) + pointsu = distort_points(points, K, distCoeff) + assert points.shape == pointsu.shape + + new_K = torch.rand(3, 3, device=device, dtype=dtype) + pointsu = distort_points(points, K, distCoeff, new_K) + assert points.shape == pointsu.shape + + def test_smoke_batch(self, device, dtype): + points = torch.rand(1, 1, 2, device=device, dtype=dtype) + K = torch.rand(1, 3, 3, device=device, dtype=dtype) + distCoeff = torch.rand(1, 4, device=device, dtype=dtype) + pointsu = distort_points(points, K, distCoeff) + assert points.shape == pointsu.shape + + new_K = torch.rand(1, 3, 3, device=device, dtype=dtype) + pointsu = distort_points(points, K, distCoeff, new_K) + assert points.shape == pointsu.shape + + @pytest.mark.parametrize( + "batch_size, num_points, num_distcoeff", [(1, 3, 4), (2, 4, 5), (3, 5, 8), (4, 6, 12), (5, 7, 14)] + ) + def test_shape(self, batch_size, num_points, num_distcoeff, device, dtype): + B, N, Ndist = batch_size, num_points, num_distcoeff + + points = torch.rand(B, N, 2, device=device, dtype=dtype) + K = torch.rand(B, 3, 3, device=device, dtype=dtype) + new_K = torch.rand(B, 3, 3, device=device, dtype=dtype) + distCoeff = torch.rand(B, Ndist, device=device, dtype=dtype) + + pointsu = distort_points(points, K, distCoeff, new_K) + assert pointsu.shape == (B, N, 2) + + def test_gradcheck(self, device): + # ── ORIGINAL (partial): only points had requires_grad ────────────── + # This left distCoeff, K, and new_K untested — meaning nobody had + # verified that gradients actually flow through distortion coefficients. + # That matters: without coefficient gradients you cannot optimise camera + # intrinsics end-to-end via gradient descent (NeRF, bundle adjustment). + # + # ── FIX: enable requires_grad on ALL differentiable inputs ────────── + points = torch.rand(1, 8, 2, device=device, dtype=torch.float64, requires_grad=True) + K = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True) + new_K = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True) + distCoeff = torch.rand(1, 4, device=device, dtype=torch.float64, requires_grad=True) + + assert self.gradcheck(distort_points, (points, K, distCoeff, new_K), raise_exception=True, fast_mode=True) + + def test_gradcheck_distcoeff_only(self, device): + """Gradient flows through distortion coefficients independently. + + This is the critical use-case test: optimising distCoeff via gradient + descent (e.g. camera calibration refinement during NeRF training) requires + that d(output)/d(distCoeff) is non-zero and correctly computed. + + We fix points and K, and only differentiate through distCoeff — isolating + the coefficient gradient path from the point gradient path. + """ + points = torch.rand(1, 8, 2, device=device, dtype=torch.float64) + K = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + new_K = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + distCoeff = torch.rand(1, 4, device=device, dtype=torch.float64, requires_grad=True) + + assert self.gradcheck(distort_points, (points, K, distCoeff, new_K), raise_exception=True, fast_mode=True) + + def test_gradcheck_K_only(self, device): + """Gradient flows through the camera intrinsic matrix K independently. + + Verifies that d(output)/d(K) is correctly computed, which is required + for joint optimisation of intrinsics and distortion parameters. + """ + points = torch.rand(1, 8, 2, device=device, dtype=torch.float64) + K = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True) + new_K = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + distCoeff = torch.rand(1, 4, device=device, dtype=torch.float64) + + assert self.gradcheck(distort_points, (points, K, distCoeff, new_K), raise_exception=True, fast_mode=True) + + def test_jit(self, device, dtype): + points = torch.rand(1, 1, 2, device=device, dtype=dtype) + K = torch.rand(1, 3, 3, device=device, dtype=dtype) + new_K = torch.rand(1, 3, 3, device=device, dtype=dtype) + distCoeff = torch.rand(1, 4, device=device, dtype=dtype) + inputs = (points, K, distCoeff, new_K) + + op = distort_points + op_jit = torch.jit.script(op) + self.assert_close(op(*inputs), op_jit(*inputs)) diff --git a/tests/geometry/calibration/test_pnp.py b/tests/geometry/calibration/test_pnp.py new file mode 100644 index 0000000..c431302 --- /dev/null +++ b/tests/geometry/calibration/test_pnp.py @@ -0,0 +1,183 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia +from kornia.geometry.calibration.pnp import _mean_isotropic_scale_normalize + +from testing.base import BaseTester + + +class TestSolvePnpDlt(BaseTester): + @staticmethod + def _get_samples(shape, low, high, device, dtype): + """Return a tensor having the given shape and whose values are in the range [low, high)""" + return ((high - low) * torch.rand(shape, device=device, dtype=dtype)) + low + + @staticmethod + def _project_to_image(world_points, world_to_cam_4x4, repeated_intrinsics): + r"""Projects points in the world coordinate system to the image coordinate system. + + Since cam_points will have shape (B, N, 3), repeated_intrinsics should have shape (B, N, 3, 3) so that + kornia.geometry.project_points can be used. + """ + cam_points = kornia.geometry.transform_points(world_to_cam_4x4, world_points) + img_points = kornia.geometry.project_points(cam_points, repeated_intrinsics) + + return img_points + + @staticmethod + def _get_world_points_and_img_points(cam_points, world_to_cam_4x4, repeated_intrinsics): + r"""Calculates world_points and img_points. + + Since cam_points will have shape (B, N, 3), repeated_intrinsics should have shape (B, N, 3, 3) so that + kornia.geometry.project_points can be used. + """ + cam_to_world_4x4 = kornia.geometry.inverse_transformation(world_to_cam_4x4) + world_points = kornia.geometry.transform_points(cam_to_world_4x4, cam_points) + img_points = kornia.geometry.project_points(cam_points, repeated_intrinsics) + + return world_points, img_points + + def _get_test_data(self, num_points, device, dtype): + """Creates some test data. + + Batch size is fixed to 2 for all tests. + """ + batch_size = 2 + torch.manual_seed(84) + + tau = 2 * 3.141592653589793 + axis_angle_1 = self._get_samples(shape=(1, 3), low=-tau, high=tau, dtype=dtype, device=device) + axis_angle_2 = self._get_samples(shape=(1, 3), low=-tau, high=tau, dtype=dtype, device=device) + rotation_1 = kornia.geometry.axis_angle_to_rotation_matrix(axis_angle_1) + rotation_2 = kornia.geometry.axis_angle_to_rotation_matrix(axis_angle_2) + + translation_1 = self._get_samples(shape=(3,), low=-100, high=100, dtype=dtype, device=device) + translation_2 = self._get_samples(shape=(3,), low=-100, high=100, dtype=dtype, device=device) + + temp = torch.eye(4, dtype=dtype, device=device) + world_to_cam_mats = temp.unsqueeze(0).repeat(batch_size, 1, 1) + world_to_cam_mats[0, :3, :3] = torch.squeeze(rotation_1) + world_to_cam_mats[0, :3, 3] = translation_1 + world_to_cam_mats[1, :3, :3] = torch.squeeze(rotation_2) + world_to_cam_mats[1, :3, 3] = translation_2 + + intrinsic_1 = torch.tensor( + [[500.0, 0.0, 250.0], [0.0, 500.0, 250.0], [0.0, 0.0, 1.0]], dtype=dtype, device=device + ) + + intrinsic_2 = torch.tensor( + [[1000.0, 0.0, 550.0], [0.0, 750.0, 200.0], [0.0, 0.0, 1.0]], dtype=dtype, device=device + ) + + intrinsics = torch.stack([intrinsic_1, intrinsic_2], dim=0) + + cam_points_xy = self._get_samples( + shape=(batch_size, num_points, 2), low=-100, high=100, dtype=dtype, device=device + ) + cam_points_z = self._get_samples( + shape=(batch_size, num_points, 1), low=0.5, high=100, dtype=dtype, device=device + ) + cam_points = torch.cat([cam_points_xy, cam_points_z], dim=-1) + + repeated_intrinsics = intrinsics.unsqueeze(1).repeat(1, num_points, 1, 1) + world_points, img_points = self._get_world_points_and_img_points( + cam_points, world_to_cam_mats, repeated_intrinsics + ) + world_to_cam_3x4 = world_to_cam_mats[:, :3, :] + + return intrinsics, world_to_cam_3x4, world_points, img_points + + @pytest.mark.parametrize("num_points", (6, 20)) + def test_smoke(self, num_points, device, dtype): + intrinsics, _, world_points, img_points = self._get_test_data(num_points, device, dtype) + batch_size = world_points.shape[0] + + pred_world_to_cam = kornia.geometry.solve_pnp_dlt(world_points, img_points, intrinsics) + assert pred_world_to_cam.shape == (batch_size, 3, 4) + + @pytest.mark.parametrize("num_points", (6,)) + def test_gradcheck(self, num_points, device): + intrinsics, _, world_points, img_points = self._get_test_data(num_points, device, torch.float64) + self.gradcheck(kornia.geometry.solve_pnp_dlt, (world_points, img_points, intrinsics)) + + @pytest.mark.parametrize("num_points", (8,)) + def test_gradcheck_weights(self, num_points, device): + intrinsics, _, world_points, img_points = self._get_test_data(num_points, device, torch.float64) + weights = torch.rand(*world_points.shape[:2], device=device, dtype=torch.float64).abs() + self.gradcheck(kornia.geometry.solve_pnp_dlt, (world_points, img_points, intrinsics, weights)) + + @pytest.mark.parametrize("num_points", (6, 20)) + def test_pred_world_to_cam(self, num_points, device, dtype): + intrinsics, gt_world_to_cam, world_points, img_points = self._get_test_data(num_points, device, dtype) + pred_world_to_cam = kornia.geometry.solve_pnp_dlt(world_points, img_points, intrinsics) + self.assert_close(pred_world_to_cam, gt_world_to_cam, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("num_points", (16, 20)) + def test_pred_world_to_cam_weighted(self, num_points, device, dtype): + intrinsics, gt_world_to_cam, world_points, img_points = self._get_test_data(num_points, device, dtype) + weights = torch.ones(*world_points.shape[:2], device=device, dtype=dtype) + pred_world_to_cam = kornia.geometry.solve_pnp_dlt(world_points, img_points, intrinsics, weights) + self.assert_close(pred_world_to_cam, gt_world_to_cam, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("num_points", (25,)) + def test_pred_world_to_cam_weighted_rand(self, num_points, device, dtype): + torch.manual_seed(0) + intrinsics, gt_world_to_cam, world_points, img_points = self._get_test_data(num_points, device, dtype) + weights = torch.ones(*world_points.shape[:2], device=device, dtype=dtype) + weights[0, 0:2] = 1e-9 + world_points[0, 0:1] *= 0.1 + world_points[0, 1:2] += 100 + pred_world_to_cam = kornia.geometry.solve_pnp_dlt(world_points, img_points, intrinsics, weights) + self.assert_close(pred_world_to_cam, gt_world_to_cam, atol=1e-4, rtol=1e-3) + + @pytest.mark.parametrize("num_points", (6, 20)) + def test_project(self, num_points, device, dtype): + intrinsics, _, world_points, img_points = self._get_test_data(num_points, device, dtype) + + pred_world_to_cam = kornia.geometry.solve_pnp_dlt(world_points, img_points, intrinsics) + + pred_world_to_cam_4x4 = kornia.core.ops.eye_like(4, pred_world_to_cam) + pred_world_to_cam_4x4[:, :3, :] = pred_world_to_cam + + repeated_intrinsics = intrinsics.unsqueeze(1).repeat(1, num_points, 1, 1) + pred_img_points = self._project_to_image(world_points, pred_world_to_cam_4x4, repeated_intrinsics) + + self.assert_close(pred_img_points, img_points, atol=1e-3, rtol=1e-3) + + +class TestNormalization(BaseTester): + @pytest.mark.parametrize("dimension", (2, 3, 5)) + def test_smoke(self, dimension, device, dtype): + batch_size = 10 + num_points = 100 + points = torch.rand((batch_size, num_points, dimension), device=device, dtype=dtype) + points_norm, transform = _mean_isotropic_scale_normalize(points) + + assert points_norm.shape == (batch_size, num_points, dimension) + assert transform.shape == (batch_size, dimension + 1, dimension + 1) + + @pytest.mark.parametrize("dimension", (2, 3, 5)) + def test_gradcheck(self, dimension, device): + batch_size = 3 + num_points = 5 + points = torch.rand((batch_size, num_points, dimension), device=device, dtype=torch.float64) + + self.gradcheck(_mean_isotropic_scale_normalize, (points,)) diff --git a/tests/geometry/calibration/test_undistort.py b/tests/geometry/calibration/test_undistort.py new file mode 100644 index 0000000..b77786a --- /dev/null +++ b/tests/geometry/calibration/test_undistort.py @@ -0,0 +1,373 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.geometry.calibration.undistort import undistort_image, undistort_points + +from testing.base import BaseTester + + +class TestUndistortPoints(BaseTester): + def test_smoke(self, device, dtype): + points = torch.rand(1, 2, device=device, dtype=dtype) + K = torch.rand(3, 3, device=device, dtype=dtype) + distCoeff = torch.rand(4, device=device, dtype=dtype) + pointsu = undistort_points(points, K, distCoeff) + assert points.shape == pointsu.shape + + new_K = torch.rand(3, 3, device=device, dtype=dtype) + pointsu = undistort_points(points, K, distCoeff, new_K) + assert points.shape == pointsu.shape + + def test_smoke_batch(self, device, dtype): + points = torch.rand(1, 1, 2, device=device, dtype=dtype) + K = torch.rand(1, 3, 3, device=device, dtype=dtype) + distCoeff = torch.rand(1, 4, device=device, dtype=dtype) + pointsu = undistort_points(points, K, distCoeff) + assert points.shape == pointsu.shape + + new_K = torch.rand(1, 3, 3, device=device, dtype=dtype) + pointsu = undistort_points(points, K, distCoeff, new_K) + assert points.shape == pointsu.shape + + @pytest.mark.parametrize( + "batch_size, num_points, num_distcoeff", [(1, 3, 4), (2, 4, 5), (3, 5, 8), (4, 6, 12), (5, 7, 14)] + ) + def test_shape(self, batch_size, num_points, num_distcoeff, device, dtype): + B, N, Ndist = batch_size, num_points, num_distcoeff + + points = torch.rand(B, N, 2, device=device, dtype=dtype) + K = torch.rand(B, 3, 3, device=device, dtype=dtype) + distCoeff = torch.rand(B, Ndist, device=device, dtype=dtype) + + pointsu = undistort_points(points, K, distCoeff) + assert pointsu.shape == (B, N, 2) + + new_K = torch.rand(B, 3, 3, device=device, dtype=dtype) + pointsu = undistort_points(points, K, distCoeff, new_K) + assert pointsu.shape == (B, N, 2) + + def test_opencv_five_coeff(self, device, dtype): + # Test using 5 distortion coefficients + pts = torch.tensor( + [[1028.0374, 788.7520], [1025.1218, 716.8726], [1022.1792, 645.1857]], device=device, dtype=dtype + ) + + K = torch.tensor( + [[1.7315e03, 0.0000e00, 6.2289e02], [0.0000e00, 1.7320e03, 5.3537e02], [0.0000e00, 0.0000e00, 1.0000e00]], + device=device, + dtype=dtype, + ) + + dist = torch.tensor([-0.1007, 0.2650, -0.0018, 0.0007, -0.2597], device=device, dtype=dtype) + + # Expected output generated with OpenCV: + # import cv2 + # ptsu_expected = cv2.undistortPoints(pts.numpy().reshape(-1,1,2), K.numpy(), + # dist1.numpy(), None, None, K.numpy()).reshape(-1,2) + ptsu_expected = torch.tensor( + [[1030.5992, 790.65533], [1027.3059, 718.10020], [1024.0700, 645.90600]], device=device, dtype=dtype + ) + ptsu = undistort_points(pts, K, dist) + self.assert_close(ptsu, ptsu_expected, rtol=1e-4, atol=1e-4) + + new_K = K * 2 + new_K[2, 2] = 1 + # Expected output generated with OpenCV: + # import cv2 + # ptsu_expected = cv2.undistortPoints(pts.numpy().reshape(-1,1,2), K.numpy(), + # dist.numpy(), None, None, new_K.numpy()).reshape(-1,2) + print(ptsu_expected) + ptsu_expected = 2 * torch.tensor( + [[1030.5992, 790.65533], [1027.3059, 718.10020], [1024.0700, 645.90600]], device=device, dtype=dtype + ) + print(ptsu_expected) + ptsu = undistort_points(pts, K, dist, new_K) + self.assert_close(ptsu, ptsu_expected, rtol=1e-4, atol=1e-4) + + def test_opencv_all_coeff(self, device, dtype): + # Test using 14 distortion coefficients + pts = torch.tensor( + [[1028.0374, 788.7520], [1025.1218, 716.8726], [1022.1792, 645.1857]], device=device, dtype=dtype + ) + + K = torch.tensor( + [[1.7315e03, 0.0000e00, 6.2289e02], [0.0000e00, 1.7320e03, 5.3537e02], [0.0000e00, 0.0000e00, 1.0000e00]], + device=device, + dtype=dtype, + ) + + dist = torch.tensor( + [ + -5.6388e-02, + 2.3881e-01, + 8.3374e-02, + 2.0710e-03, + 7.1349e00, + 5.6335e-02, + -3.1738e-01, + 4.9981e00, + -4.0287e-03, + -2.8246e-02, + -8.6064e-02, + 1.5543e-02, + -1.7322e-01, + 2.3154e-03, + ], + device=device, + dtype=dtype, + ) + + # Expected output generated with OpenCV: + # import cv2 + # ptsu_expected = cv2.undistortPoints(pts.numpy().reshape(-1,1,2), K.numpy(), + # dist2.numpy(), None, None, K.numpy()).reshape(-1,2) + ptsu_expected = torch.tensor( + [[1030.8245, 786.3807], [1027.5505, 715.0732], [1024.2753, 644.0319]], device=device, dtype=dtype + ) + ptsu = undistort_points(pts, K, dist) + self.assert_close(ptsu, ptsu_expected, rtol=1e-4, atol=1e-4) + + new_K = K * 2 + new_K[2, 2] = 1 + # Expected output generated with OpenCV: + # import cv2 + # ptsu_expected = cv2.undistortPoints(pts.numpy().reshape(-1,1,2), K.numpy(), + # dist.numpy(), None, None, new_K.numpy()).reshape(-1,2) + print(ptsu_expected) + ptsu_expected = 2 * torch.tensor( + [[1030.8245, 786.3807], [1027.5505, 715.0732], [1024.2753, 644.0319]], device=device, dtype=dtype + ) + print(ptsu_expected) + ptsu = undistort_points(pts, K, dist, new_K) + self.assert_close(ptsu, ptsu_expected, rtol=1e-4, atol=1e-4) + + def test_opencv_stereo(self, device, dtype): + # Udistort stereo points with data given in two batches using 14 distortion coefficients + pts = torch.tensor( + [ + [[1028.0374, 788.7520], [1025.1218, 716.8726], [1022.1792, 645.1857]], + [[345.9135, 847.9113], [344.0880, 773.9890], [342.2381, 700.3029]], + ], + device=device, + dtype=dtype, + ) + + K = torch.tensor( + [ + [ + [3.3197e03, 0.0000e00, 6.1813e02], + [0.0000e00, 3.3309e03, 5.2281e02], + [0.0000e00, 0.0000e00, 1.0000e00], + ], + [ + [1.9206e03, 0.0000e00, 6.1395e02], + [0.0000e00, 1.9265e03, 7.7164e02], + [0.0000e00, 0.0000e00, 1.0000e00], + ], + ], + device=device, + dtype=dtype, + ) + + dist = torch.tensor( + [ + [ + -5.6388e-02, + 2.3881e-01, + 8.3374e-02, + 2.0710e-03, + 7.1349e00, + 5.6335e-02, + -3.1738e-01, + 4.9981e00, + -4.0287e-03, + -2.8246e-02, + -8.6064e-02, + 1.5543e-02, + -1.7322e-01, + 2.3154e-03, + ], + [ + 1.4050e-03, + -3.0691e00, + -1.0209e-01, + -2.3687e-02, + -1.7082e02, + 4.3593e-03, + -3.1904e00, + -1.7050e02, + 1.7854e-02, + 1.8999e-02, + 9.9122e-02, + 3.6675e-02, + 3.0816e-03, + -5.7133e-02, + ], + ], + device=device, + dtype=dtype, + ) + + # Expected output generated with OpenCV: + # import cv2 + # ptsu_expected1 = cv2.undistortPoints(pts[0].numpy().reshape(-1,1,2), K[0].numpy(), + # dist[0].numpy(), None, None, K[0].numpy()).reshape(-1,2) + # ptsu_expected2 = cv2.undistortPoints(pts[1].numpy().reshape(-1,1,2), K[1].numpy(), + # dist[1].numpy(), None, None, K[1].numpy()).reshape(-1,2) + ptsu_expected1 = torch.tensor( + [[1029.3234, 785.4813], [1026.1599, 714.3689], [1023.02045, 643.5359]], device=device, dtype=dtype + ) + + ptsu_expected2 = torch.tensor( + [[344.04456, 848.7696], [344.27606, 774.1254], [344.47018, 700.8522]], device=device, dtype=dtype + ) + + ptsu = undistort_points(pts, K, dist) + self.assert_close(ptsu[0], ptsu_expected1, rtol=1e-4, atol=1e-4) + self.assert_close(ptsu[1], ptsu_expected2, rtol=1e-4, atol=1e-4) + + def test_gradcheck(self, device): + points = torch.rand(1, 8, 2, device=device, dtype=torch.float64, requires_grad=True) + K = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + new_K = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + distCoeff = torch.rand(1, 4, device=device, dtype=torch.float64) + + self.gradcheck(undistort_points, (points, K, distCoeff, new_K), requires_grad=(True, False, False, False)) + + def test_dynamo(self, device, dtype, torch_optimizer): + points = torch.rand(1, 1, 2, device=device, dtype=dtype) + K = torch.rand(1, 3, 3, device=device, dtype=dtype) + new_K = torch.rand(1, 3, 3, device=device, dtype=dtype) + distCoeff = torch.rand(1, 4, device=device, dtype=dtype) + inputs = (points, K, distCoeff, new_K) + + op = undistort_points + op_optimized = torch_optimizer(op) + self.assert_close(op(*inputs), op_optimized(*inputs)) + + +class TestUndistortImage(BaseTester): + def test_shape(self, device, dtype): + im = torch.rand(1, 3, 5, 5, device=device, dtype=dtype) + K = torch.rand(3, 3, device=device, dtype=dtype) + distCoeff = torch.rand(4, device=device, dtype=dtype) + + imu = undistort_image(im, K, distCoeff) + assert imu.shape == (1, 3, 5, 5) + + def test_shape_minimum_dims(self, device, dtype): + im = torch.rand(3, 5, 5, device=device, dtype=dtype) + K = torch.rand(3, 3, device=device, dtype=dtype) + distCoeff = torch.rand(4, device=device, dtype=dtype) + + imu = undistort_image(im, K, distCoeff) + assert imu.shape == (3, 5, 5) + + def test_shape_extra_dims(self, device, dtype): + im = torch.rand(1, 1, 3, 5, 5, device=device, dtype=dtype).tile(3, 2, 1, 1, 1) + K = torch.rand(1, 1, 3, 3, device=device, dtype=dtype).tile(3, 2, 1, 1) + distCoeff = torch.rand(1, 1, 4, device=device, dtype=dtype).tile(3, 2, 1) + + imu = undistort_image(im, K, distCoeff) + assert imu.shape == (3, 2, 3, 5, 5) + self.assert_close(imu[0], imu[1]) + + def test_exception(self, device, dtype): + with pytest.raises(ValueError): + im = torch.rand(5, 5, device=device, dtype=dtype) + K = torch.rand(3, 3, device=device, dtype=dtype) + distCoeff = torch.rand(4, device=device, dtype=dtype) + undistort_image(im, K, distCoeff) + + with pytest.raises(ValueError): + im = torch.rand(3, 5, 5, device=device, dtype=dtype) + K = torch.rand(4, 4, device=device, dtype=dtype) + distCoeff = torch.rand(4, device=device, dtype=dtype) + undistort_image(im, K, distCoeff) + + with pytest.raises(ValueError): + im = torch.rand(3, 5, 5, device=device, dtype=dtype) + K = torch.rand(3, 3, device=device, dtype=dtype) + distCoeff = torch.rand(6, device=device, dtype=dtype) + undistort_image(im, K, distCoeff) + + with pytest.raises(ValueError): + im = torch.randint(0, 256, (3, 5, 5), device=device, dtype=torch.uint8) + K = torch.rand(3, 3, device=device, dtype=dtype) + distCoeff = torch.rand(4, device=device, dtype=dtype) + undistort_image(im, K, distCoeff) + + with pytest.raises(ValueError): + im = torch.rand(1, 1, 3, 5, 5, device=device, dtype=dtype) + K = torch.rand(1, 3, 3, device=device, dtype=dtype) + distCoeff = torch.rand(1, 4, device=device, dtype=dtype) + undistort_image(im, K, distCoeff) + + def test_opencv(self, device, dtype): + im = torch.tensor( + [ + [ + [ + [116, 75, 230, 5, 32], + [9, 182, 97, 213, 3], + [91, 10, 33, 141, 230], + [229, 63, 221, 244, 61], + [19, 137, 23, 59, 227], + ] + ] + ], + device=device, + dtype=dtype, + ) + + K = torch.tensor([[2, 0, 2], [0, 2, 2], [0, 0, 1]], device=device, dtype=dtype) + + dist = torch.tensor([0.2290, 0.9565, 0.0083, 0.0475], device=device, dtype=dtype) + + # Expected output generated with OpenCV: + # import cv2 + # imu_expected = cv2.undistort(np.uint8(im[0,0].numpy()), K.numpy(), dist.numpy()) + imu_expected = torch.tensor( + [[[[0, 0, 0, 0, 0], [0, 124, 112, 82, 0], [0, 13, 33, 158, 0], [0, 108, 197, 150, 0], [0, 0, 0, 0, 0]]]], + device=device, + dtype=dtype, + ) + + imu = undistort_image(im / 255.0, K, dist) + self.assert_close(imu, imu_expected / 255.0, rtol=1e-2, atol=1e-2) + + def test_gradcheck(self, device): + im = torch.rand(1, 1, 15, 15, device=device, dtype=torch.float64, requires_grad=True) + K = torch.rand(3, 3, device=device, dtype=torch.float64) + distCoeff = torch.rand(4, device=device, dtype=torch.float64) + + self.gradcheck(undistort_image, (im, K, distCoeff), requires_grad=(True, False, False)) + + @pytest.mark.xfail(reason="Some times this seems to random fail") + def test_dynamo(self, device, dtype, torch_optimizer): + # TODO: check if `undistort_image` fully support dynamo + im = torch.rand(1, 3, 5, 5, device=device, dtype=dtype) + K = torch.rand(3, 3, device=device, dtype=dtype) + distCoeff = torch.rand(4, device=device, dtype=dtype) + inputs = (im, K, distCoeff) + + op = undistort_image + op_optimized = torch_optimizer(op) + self.assert_close(op(*inputs), op_optimized(*inputs)) diff --git a/tests/geometry/camera/test_distortion.py b/tests/geometry/camera/test_distortion.py new file mode 100644 index 0000000..4c8aa24 --- /dev/null +++ b/tests/geometry/camera/test_distortion.py @@ -0,0 +1,257 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.geometry.camera.distortion_affine import ( + distort_points_affine, + dx_distort_points_affine, + undistort_points_affine, +) +from kornia.geometry.camera.distortion_kannala_brandt import ( + distort_points_kannala_brandt, + dx_distort_points_kannala_brandt, + undistort_points_kannala_brandt, +) + +from testing.base import BaseTester + + +class TestDistortionAffine(BaseTester): + def test_smoke(self, device, dtype): + points = torch.tensor([1.0, 2.0], device=device, dtype=dtype) + params = torch.tensor([600.0, 600.0, 319.5, 239.5], device=device, dtype=dtype) + assert distort_points_affine(points, params) is not None + + def _test_cardinality_distort_batch(self, device, dtype, batch_size): + batch_tuple = (batch_size,) if batch_size is not None else () + points = torch.rand(batch_tuple + (2,), device=device, dtype=dtype) + params = torch.rand(batch_tuple + (4,), device=device, dtype=dtype) + assert distort_points_affine(points, params).shape == batch_tuple + (2,) + + def _test_cardinality_undistort_batch(self, device, dtype, batch_size): + batch_tuple = (batch_size,) if batch_size is not None else () + points = torch.rand(batch_tuple + (2,), device=device, dtype=dtype) + params = torch.rand(batch_tuple + (4,), device=device, dtype=dtype) + assert undistort_points_affine(points, params).shape == batch_tuple + (2,) + + @pytest.mark.parametrize("batch_size", [None, 1, 2, 3]) + def test_cardinality(self, device, dtype, batch_size): + self._test_cardinality_distort_batch(device, dtype, batch_size) + self._test_cardinality_undistort_batch(device, dtype, batch_size) + + # NOTE: data generated with sophus-rs + def test_distort_points_roundtrip(self, device, dtype): + points = torch.tensor( + [ + [0.0, 0.0], + [1.0, 400.0], + [320.0, 240.0], + [319.5, 239.5], + [100.0, 40.0], + [639.0, 479.0], + ], + device=device, + dtype=dtype, + ) + params = torch.tensor([[600.0, 600.0, 319.5, 239.5]], device=device, dtype=dtype) + expected = torch.tensor( + [ + [319.5, 239.5], + [919.5, 240239.5], + [192319.5, 144239.5], + [192019.5, 143939.5], + [60319.5, 24239.5], + [383719.5, 287639.5], + ], + device=device, + dtype=dtype, + ) + points_distorted = distort_points_affine(points, params) + self.assert_close(points_distorted, expected) + self.assert_close(points, undistort_points_affine(points_distorted, params)) + + def test_dx_distort_points(self, device, dtype): + points = torch.tensor([1.0, 2.0], device=device, dtype=dtype) + params = torch.tensor([600.0, 600.0, 319.5, 239.5], device=device, dtype=dtype) + expected = torch.tensor([[600.0, 0.0], [0.0, 600.0]], device=device, dtype=dtype) + self.assert_close(dx_distort_points_affine(points, params), expected) + + def test_exception(self, device, dtype) -> None: + from kornia.core.exceptions import ShapeError + + points = torch.tensor([1.0, 2.0], device=device, dtype=dtype) + params = torch.tensor([600.0, 600.0, 319.5], device=device, dtype=dtype) + with pytest.raises(ShapeError): + distort_points_affine(points, params) + + def _test_gradcheck_distort(self, device): + points = torch.tensor([1.0, 2.0], device=device, dtype=torch.float64) + params = torch.tensor([600.0, 600.0, 319.5, 239.5], device=device, dtype=torch.float64) + self.gradcheck(distort_points_affine, (points, params)) + + def _test_gradcheck_undistort(self, device): + points = torch.tensor([601.0, 602.0], device=device, dtype=torch.float64) + params = torch.tensor([600.0, 600.0, 319.5, 239.5], device=device, dtype=torch.float64) + self.gradcheck(undistort_points_affine, (points, params)) + + def test_gradcheck(self, device) -> None: + self._test_gradcheck_distort(device) + self._test_gradcheck_undistort(device) + + def _test_jit_distort(self, device, dtype) -> None: + points = torch.tensor([1.0, 2.0], device=device, dtype=dtype) + params = torch.tensor([600.0, 600.0, 319.5, 239.5], device=device, dtype=dtype) + op_script = torch.jit.script(distort_points_affine) + actual = op_script(points, params) + expected = distort_points_affine(points, params) + self.assert_close(actual, expected) + + def _test_jit_undistort(self, device, dtype) -> None: + points = torch.tensor([601.0, 602.0], device=device, dtype=dtype) + params = torch.tensor([600.0, 600.0, 319.5, 239.5], device=device, dtype=dtype) + op_script = torch.jit.script(undistort_points_affine) + actual = op_script(points, params) + expected = undistort_points_affine(points, params) + self.assert_close(actual, expected) + + def test_jit(self, device, dtype) -> None: + self._test_jit_distort(device, dtype) + self._test_jit_undistort(device, dtype) + + +class TestDistortionKannalaBrandt(BaseTester): + def test_smoke(self, device, dtype) -> None: + points = torch.tensor([1.0, 2.0], device=device, dtype=dtype) + params = torch.tensor([600.0, 600.0, 319.5, 239.5, 0.1, 0.2, 0.3, 0.4], device=device, dtype=dtype) + assert distort_points_kannala_brandt(points, params) is not None + + def _test_cardinality_distort_batch(self, device, dtype, batch_size): + batch_tuple = (batch_size,) if batch_size is not None else () + points = torch.rand(batch_tuple + (2,), device=device, dtype=dtype) + params = torch.rand(batch_tuple + (8,), device=device, dtype=dtype) + assert distort_points_kannala_brandt(points, params).shape == batch_tuple + (2,) + + def _test_cardinality_undistort_batch(self, device, dtype, batch_size): + batch_tuple = (batch_size,) if batch_size is not None else () + points = torch.rand(batch_tuple + (2,), device=device, dtype=dtype) + params = torch.rand(batch_tuple + (8,), device=device, dtype=dtype) + assert undistort_points_kannala_brandt(points, params).shape == batch_tuple + (2,) + + @pytest.mark.parametrize("batch_size", [None, 1, 2, 3]) + def test_cardinality(self, device, dtype, batch_size): + self._test_cardinality_distort_batch(device, dtype, batch_size) + self._test_cardinality_undistort_batch(device, dtype, batch_size) + + # NOTE: data generated with sophus-rs + def test_distort_points_roundtrip(self, device, dtype) -> None: + points = torch.tensor( + [ + [0.0, 0.0], + [1.0, 400.0], + [320.0, 240.0], + [319.5, 239.5], + [100.0, 40.0], + [639.0, 479.0], + ], + device=device, + dtype=dtype, + ) + params = torch.tensor( + [[1000.0, 1000.0, 320.0, 280.0, 0.1, 0.01, 0.001, 0.0001]], + device=device, + dtype=dtype, + ) + expected = torch.tensor( + [ + [320.0, 280.0], + [325.1949172763466, 2357.966910538644], + [1982.378709731326, 1526.7840322984944], + [1982.6832644475849, 1526.3619462760455], + [2235.6822069661744, 1046.2728827864696], + [1984.8663275417607, 1527.9983895031353], + ], + device=device, + dtype=dtype, + ) + points_distorted = distort_points_kannala_brandt(points, params) + self.assert_close(points_distorted, expected) + self.assert_close(points, undistort_points_kannala_brandt(points_distorted, params)) + + def test_dx_distort_points_kannala_brandt(self, device, dtype) -> None: + points = torch.tensor([1.0, 2.0], device=device, dtype=dtype) + params = torch.tensor([600.0, 600.0, 319.5, 239.5, 0.1, 0.2, 0.3, 0.4], device=device, dtype=dtype) + expected = torch.tensor( + [ + [1191.5316162109375, 282.3212890625], + [282.3212890625, 1615.0135498046875], + ], + device=device, + dtype=dtype, + ) + self.assert_close(dx_distort_points_kannala_brandt(points, params), expected) + + def test_exception(self, device, dtype) -> None: + from kornia.core.exceptions import ShapeError + + points = torch.tensor([1.0, 2.0], device=device, dtype=dtype) + params = torch.tensor([600.0, 600.0, 319.5], device=device, dtype=dtype) + with pytest.raises(ShapeError): + distort_points_kannala_brandt(points, params) + + def _test_gradcheck_distort(self, device): + points = torch.tensor([1.0, 2.0], device=device, dtype=torch.float64) + params = torch.tensor( + [600.0, 600.0, 319.5, 239.5, 0.1, 0.2, 0.3, 0.4], + device=device, + dtype=torch.float64, + ) + self.gradcheck(distort_points_kannala_brandt, (points, params)) + + def _test_gradcheck_undistort(self, device): + points = torch.tensor([919.5000, 1439.5000], device=device, dtype=torch.float64) + params = torch.tensor( + [600.0, 600.0, 319.5, 239.5, 0.1, 0.2, 0.3, 0.4], + device=device, + dtype=torch.float64, + ) + self.gradcheck(undistort_points_kannala_brandt, (points, params)) + + def test_gradcheck(self, device) -> None: + self._test_gradcheck_distort(device) + self._test_gradcheck_undistort(device) + + def _test_jit_distort(self, device, dtype) -> None: + points = torch.tensor([1.0, 2.0], device=device, dtype=dtype) + params = torch.tensor([600.0, 600.0, 319.5, 239.5, 0.1, 0.2, 0.3, 0.4], device=device, dtype=dtype) + op_script = torch.jit.script(distort_points_kannala_brandt) + actual = op_script(points, params) + expected = distort_points_kannala_brandt(points, params) + self.assert_close(actual, expected) + + def _test_jit_undistort(self, device, dtype) -> None: + points = torch.tensor([919.5000, 1439.5000], device=device, dtype=dtype) + params = torch.tensor([600.0, 600.0, 319.5, 239.5, 0.1, 0.2, 0.3, 0.4], device=device, dtype=dtype) + op_script = torch.jit.script(undistort_points_kannala_brandt) + actual = op_script(points, params) + expected = undistort_points_kannala_brandt(points, params) + self.assert_close(actual, expected) + + def test_jit(self, device, dtype) -> None: + self._test_jit_distort(device, dtype) + self._test_jit_undistort(device, dtype) diff --git a/tests/geometry/camera/test_perspective.py b/tests/geometry/camera/test_perspective.py new file mode 100644 index 0000000..2c2bd6d --- /dev/null +++ b/tests/geometry/camera/test_perspective.py @@ -0,0 +1,131 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch + +import kornia + +from testing.base import BaseTester + + +class TestProjectPoints(BaseTester): + def test_smoke(self, device, dtype): + point_3d = torch.zeros(1, 3, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(1, -1, -1) + point_2d = kornia.geometry.camera.project_points(point_3d, camera_matrix) + assert point_2d.shape == (1, 2) + + def test_smoke_batch(self, device, dtype): + point_3d = torch.zeros(2, 3, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(2, -1, -1) + point_2d = kornia.geometry.camera.project_points(point_3d, camera_matrix) + assert point_2d.shape == (2, 2) + + def test_smoke_batch_multi(self, device, dtype): + point_3d = torch.zeros(2, 4, 3, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(2, 4, -1, -1) + point_2d = kornia.geometry.camera.project_points(point_3d, camera_matrix) + assert point_2d.shape == (2, 4, 2) + + def test_project_and_unproject(self, device, dtype): + point_3d = torch.tensor([[10.0, 2.0, 30.0]], device=device, dtype=dtype) + depth = point_3d[..., -1:] + camera_matrix = torch.tensor( + [[[2746.0, 0.0, 991.0], [0.0, 2748.0, 619.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype + ) + point_2d = kornia.geometry.camera.project_points(point_3d, camera_matrix) + point_3d_hat = kornia.geometry.camera.unproject_points(point_2d, depth, camera_matrix) + self.assert_close(point_3d, point_3d_hat, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + # TODO: point [0, 0, 0] crashes + points_3d = torch.ones(1, 3, device=device) + camera_matrix = torch.eye(3, device=device).expand(1, -1, -1) + + # evaluate function gradient + self.gradcheck(kornia.geometry.camera.project_points, (points_3d, camera_matrix)) + + def test_jit(self, device, dtype): + points_3d = torch.zeros(1, 3, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(1, -1, -1) + op = kornia.geometry.camera.project_points + op_jit = torch.jit.script(op) + self.assert_close(op(points_3d, camera_matrix), op_jit(points_3d, camera_matrix)) + + +class TestUnprojectPoints(BaseTester): + def test_smoke(self, device, dtype): + points_2d = torch.zeros(1, 2, device=device, dtype=dtype) + depth = torch.ones(1, 1, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(1, -1, -1) + point_3d = kornia.geometry.camera.unproject_points(points_2d, depth, camera_matrix) + assert point_3d.shape == (1, 3) + + def test_smoke_batch(self, device, dtype): + points_2d = torch.zeros(2, 2, device=device, dtype=dtype) + depth = torch.ones(2, 1, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(2, -1, -1) + point_3d = kornia.geometry.camera.unproject_points(points_2d, depth, camera_matrix) + assert point_3d.shape == (2, 3) + + def test_smoke_multi_batch(self, device, dtype): + points_2d = torch.zeros(2, 3, 2, device=device, dtype=dtype) + depth = torch.ones(2, 3, 1, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(2, 3, -1, -1) + point_3d = kornia.geometry.camera.unproject_points(points_2d, depth, camera_matrix) + assert point_3d.shape == (2, 3, 3) + + def test_unproject_center(self, device, dtype): + point_2d = torch.tensor([[0.0, 0.0]], device=device, dtype=dtype) + depth = torch.tensor([[2.0]], device=device, dtype=dtype) + camera_matrix = torch.tensor([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], device=device, dtype=dtype) + expected = torch.tensor([[0.0, 0.0, 2.0]], device=device, dtype=dtype) + actual = kornia.geometry.camera.unproject_points(point_2d, depth, camera_matrix) + self.assert_close(actual, expected, atol=1e-4, rtol=1e-4) + + def test_unproject_center_normalize(self, device, dtype): + point_2d = torch.tensor([[0.0, 0.0]], device=device, dtype=dtype) + depth = torch.tensor([[2.0]], device=device, dtype=dtype) + camera_matrix = torch.tensor([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], device=device, dtype=dtype) + expected = torch.tensor([[0.0, 0.0, 2.0]], device=device, dtype=dtype) + actual = kornia.geometry.camera.unproject_points(point_2d, depth, camera_matrix, True) + self.assert_close(actual, expected, atol=1e-4, rtol=1e-4) + + def test_unproject_and_project(self, device, dtype): + point_2d = torch.tensor([[0.0, 0.0]], device=device, dtype=dtype) + depth = torch.tensor([[2.0]], device=device, dtype=dtype) + camera_matrix = torch.tensor([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], device=device, dtype=dtype) + point_3d = kornia.geometry.camera.unproject_points(point_2d, depth, camera_matrix) + point_2d_hat = kornia.geometry.camera.project_points(point_3d, camera_matrix) + self.assert_close(point_2d, point_2d_hat, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + points_2d = torch.zeros(1, 2, device=device, dtype=torch.float64) + depth = torch.ones(1, 1, device=device, dtype=torch.float64) + camera_matrix = torch.eye(3, device=device, dtype=torch.float64).expand(1, -1, -1) + + # evaluate function gradient + self.gradcheck(kornia.geometry.camera.unproject_points, (points_2d, depth, camera_matrix)) + + def test_jit(self, device, dtype): + points_2d = torch.zeros(1, 2, device=device, dtype=dtype) + depth = torch.ones(1, 1, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(1, -1, -1) + args = (points_2d, depth, camera_matrix) + op = kornia.geometry.camera.unproject_points + op_jit = torch.jit.script(op) + self.assert_close(op(*args), op_jit(*args)) diff --git a/tests/geometry/camera/test_pinhole.py b/tests/geometry/camera/test_pinhole.py new file mode 100644 index 0000000..4ff2e5e --- /dev/null +++ b/tests/geometry/camera/test_pinhole.py @@ -0,0 +1,488 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestCam2Pixel(BaseTester): + def _create_intrinsics(self, batch_size, fx, fy, cx, cy, device, dtype): + temp = torch.eye(4, device=device, dtype=dtype) + temp[0, 0], temp[0, 2] = fx, cx + temp[1, 1], temp[1, 2] = fy, cy + intrinsics = temp.expand(batch_size, -1, -1) + return intrinsics + + def _create_intrinsics_inv(self, batch_size, fx, fy, cx, cy, device, dtype): + temp = torch.eye(4, device=device, dtype=dtype) + temp[0, 0], temp[0, 2] = 1 / fx, -cx / fx + temp[1, 1], temp[1, 2] = 1 / fy, -cy / fy + intrinsics_inv = temp.expand(batch_size, -1, -1) + return intrinsics_inv + + def _get_samples(self, shape, low, high, device, dtype): + """Return a tensor having the given shape and whose values are in the range [low, high)""" + return ((high - low) * torch.rand(shape, device=device, dtype=dtype)) + low + + @pytest.mark.parametrize("batch_size", (1,)) + def test_smoke(self, batch_size, device, dtype): + H, W = 250, 500 + fx, fy = W, H + cx, cy = W / 2, H / 2 + eps = 1e-12 + seed = 77 + low, high = -500, 500 + + intrinsics = self._create_intrinsics(batch_size, fx, fy, cx, cy, device=device, dtype=dtype) + + # Setting the projection matrix to the intrinsic matrix for + # simplicity (i.e. assuming that the RT matrix is an identity matrix) + proj_mat = intrinsics + + torch.manual_seed(seed) + cam_coords_src = self._get_samples((batch_size, H, W, 3), low, high, device, dtype) + + pixel_coords_dst = kornia.geometry.camera.cam2pixel( + cam_coords_src=cam_coords_src, dst_proj_src=proj_mat, eps=eps + ) + assert pixel_coords_dst.shape == (batch_size, H, W, 2) + + @pytest.mark.parametrize("batch_size", (1, 2, 5)) + def test_consistency(self, batch_size, device, dtype): + H, W = 250, 500 + fx, fy = W, H + cx, cy = W / 2, H / 2 + eps = 1e-12 + seed = 77 + # Use normalized image-plane coords so that projected pixel values stay in [0,W) x [0,H). + # cam_x/z in [-0.5, 0.5] gives pixel_x in [cx - fx/2, cx + fx/2] = [0, W). + low_norm, high_norm = -0.45, 0.45 + low_z, high_z = 1.0, 500.0 + + intrinsics = self._create_intrinsics(batch_size, fx, fy, cx, cy, device=device, dtype=dtype) + intrinsics_inv = self._create_intrinsics_inv(batch_size, fx, fy, cx, cy, device=device, dtype=dtype) + + # Setting the projection matrix to the intrinsic matrix for + # simplicity (i.e. assuming that the RT matrix is an identity matrix) + proj_mat = intrinsics + + torch.manual_seed(seed) + # Generate z first, then x,y as z * normalized_coord so pixel coords stay in image bounds + cam_coords_z = self._get_samples((batch_size, H, W, 1), low_z, high_z, device, dtype) + cam_coords_xy = self._get_samples((batch_size, H, W, 2), low_norm, high_norm, device, dtype) * cam_coords_z + cam_coords_input = torch.cat([cam_coords_xy, cam_coords_z], dim=-1) + + pixel_coords_output = kornia.geometry.camera.cam2pixel( + cam_coords_src=cam_coords_input, dst_proj_src=proj_mat, eps=eps + ) + + last_ch = torch.ones((batch_size, H, W, 1), device=device, dtype=dtype) + pixel_coords_concat = torch.cat([pixel_coords_output, last_ch], axis=-1) + + depth = cam_coords_input[..., 2:3].permute(0, 3, 1, 2).contiguous() + cam_coords_output = kornia.geometry.camera.pixel2cam( + depth=depth, intrinsics_inv=intrinsics_inv, pixel_coords=pixel_coords_concat + ) + + self.assert_close(cam_coords_output, cam_coords_input, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_size", (1,)) + def test_gradcheck(self, batch_size, device): + dtype = torch.float64 + H, W = 10, 20 + fx, fy = W, H + cx, cy = W / 2, H / 2 + eps = 1e-12 + seed = 77 + low, high = -500, 500 + atol, rtol = 1e-5, 1e-3 + + # Different tolerances for the below case. + if (device.type == "cuda") and (dtype == torch.float64): + atol, rtol = 1e-4, 1e-2 + + # If contiguous() is not called, gradcheck fails + intrinsics = self._create_intrinsics(batch_size, fx, fy, cx, cy, device=device, dtype=dtype).contiguous() + + # Setting the projection matrix to the intrinsic matrix for + # simplicity (i.e. assuming that the RT matrix is an identity matrix) + proj_mat = intrinsics + + torch.manual_seed(seed) + cam_coords_src = self._get_samples((batch_size, H, W, 3), low, high, device, dtype) + + self.gradcheck(kornia.geometry.camera.cam2pixel, (cam_coords_src, proj_mat, eps), atol=atol, rtol=rtol) + + +class TestPixel2Cam(BaseTester): + def _create_intrinsics(self, batch_size, fx, fy, cx, cy, device, dtype): + temp = torch.eye(4, device=device, dtype=dtype) + temp[0, 0], temp[0, 2] = fx, cx + temp[1, 1], temp[1, 2] = fy, cy + intrinsics = temp.expand(batch_size, -1, -1) + return intrinsics + + def _create_intrinsics_inv(self, batch_size, fx, fy, cx, cy, device, dtype): + temp = torch.eye(4, device=device, dtype=dtype) + temp[0, 0], temp[0, 2] = 1 / fx, -cx / fx + temp[1, 1], temp[1, 2] = 1 / fy, -cy / fy + intrinsics_inv = temp.expand(batch_size, -1, -1) + return intrinsics_inv + + def _get_samples(self, shape, low, high, device, dtype): + """Return a tensor having the given shape and whose values are in the range [low, high)""" + return ((high - low) * torch.rand(shape, device=device, dtype=dtype)) + low + + @pytest.mark.parametrize("batch_size", (1, 2, 5)) + def test_smoke(self, batch_size, device, dtype): + H, W = 250, 500 + fx, fy = W, H + cx, cy = W / 2, H / 2 + seed = 77 + low_1, high_1 = -500, 500 + low_2, high_2 = -(max(W, H) * 3), (max(W, H) * 3) + + torch.manual_seed(seed) + depth = self._get_samples((batch_size, 1, H, W), low_1, high_1, device, dtype) + pixel_coords = self._get_samples((batch_size, H, W, 2), low_2, high_2, device, dtype) + + last_ch = torch.ones((batch_size, H, W, 1), device=device, dtype=dtype) + pixel_coords_input = torch.cat([pixel_coords, last_ch], axis=-1) + + intrinsics_inv = self._create_intrinsics_inv(batch_size, fx, fy, cx, cy, device=device, dtype=dtype) + + output = kornia.geometry.camera.pixel2cam( + depth=depth, intrinsics_inv=intrinsics_inv, pixel_coords=pixel_coords_input + ) + + assert output.shape == (batch_size, H, W, 3) + + @pytest.mark.parametrize("batch_size", (1, 2, 5)) + def test_consistency(self, batch_size, device, dtype): + H, W = 250, 500 + fx, fy = W, H + cx, cy = W / 2, H / 2 + eps = 1e-12 + seed = 77 + # Depth must be positive and bounded away from zero to avoid 1/z blow-up. + # Pixel coords restricted to image bounds [0,W) x [0,H) to avoid TF32 precision issues + # from large coordinate values in matrix multiplication. + low_1, high_1 = 1.0, 500.0 + low_2x, high_2x = 0.0, float(W) + low_2y, high_2y = 0.0, float(H) + + torch.manual_seed(seed) + depth = self._get_samples((batch_size, 1, H, W), low_1, high_1, device, dtype) + pixel_coords_x = self._get_samples((batch_size, H, W, 1), low_2x, high_2x, device, dtype) + pixel_coords_y = self._get_samples((batch_size, H, W, 1), low_2y, high_2y, device, dtype) + pixel_coords = torch.cat([pixel_coords_x, pixel_coords_y], dim=-1) + + last_ch = torch.ones((batch_size, H, W, 1), device=device, dtype=dtype) + pixel_coords_input = torch.cat([pixel_coords, last_ch], axis=-1) + + intrinsics = self._create_intrinsics(batch_size, fx, fy, cx, cy, device=device, dtype=dtype) + intrinsics_inv = self._create_intrinsics_inv(batch_size, fx, fy, cx, cy, device=device, dtype=dtype) + + cam_coords = kornia.geometry.camera.pixel2cam( + depth=depth, intrinsics_inv=intrinsics_inv, pixel_coords=pixel_coords_input + ) + + # Setting the projection matrix to the intrinsic matrix for + # simplicity (i.e. assuming that the RT matrix is an identity matrix) + proj_mat = intrinsics + pixel_coords_output = kornia.geometry.camera.cam2pixel( + cam_coords_src=cam_coords, dst_proj_src=proj_mat, eps=eps + ) + pixel_coords_concat = torch.cat([pixel_coords_output, last_ch], axis=-1) + + self.assert_close(pixel_coords_concat, pixel_coords_input, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_size", (1,)) + @pytest.mark.slow + def test_gradcheck(self, batch_size, device): + dtype = torch.float64 + H, W = 10, 20 + fx, fy = W, H + cx, cy = W / 2, H / 2 + seed = 77 + low_1, high_1 = -500, 500 + low_2, high_2 = -(max(W, H) * 3), (max(W, H) * 3) + + torch.manual_seed(seed) + depth = self._get_samples((batch_size, 1, H, W), low_1, high_1, device, dtype) + pixel_coords = self._get_samples((batch_size, H, W, 2), low_2, high_2, device, dtype) + + last_ch = torch.ones((batch_size, H, W, 1), device=device, dtype=dtype) + pixel_coords_input = torch.cat([pixel_coords, last_ch], axis=-1) + + # If contiguous() is not called, gradcheck fails + intrinsics_inv = self._create_intrinsics_inv( + batch_size, fx, fy, cx, cy, device=device, dtype=dtype + ).contiguous() + + self.gradcheck(kornia.geometry.camera.pixel2cam, (depth, intrinsics_inv, pixel_coords_input), fast_mode=False) + + +class TestPinholeCamera(BaseTester): + def _create_intrinsics(self, batch_size, fx, fy, cx, cy, device, dtype): + intrinsics = torch.eye(4, device=device, dtype=dtype) + intrinsics[..., 0, 0] = fx + intrinsics[..., 1, 1] = fy + intrinsics[..., 0, 2] = cx + intrinsics[..., 1, 2] = cy + return intrinsics.expand(batch_size, -1, -1) + + def _create_extrinsics(self, batch_size, tx, ty, tz, device, dtype): + extrinsics = torch.eye(4, device=device, dtype=dtype) + extrinsics[..., 0, -1] = tx + extrinsics[..., 1, -1] = ty + extrinsics[..., 2, -1] = tz + return extrinsics.expand(batch_size, -1, -1) + + def _create_extrinsics_with_rotation(self, batch_size, alpha, beta, gamma, tx, ty, tz, device, dtype): + Rx = torch.eye(3, device=device, dtype=dtype) + Rx[1, 1] = math.cos(alpha) + Rx[1, 2] = math.sin(alpha) + Rx[2, 1] = -Rx[1, 2] + Rx[2, 2] = Rx[1, 1] + + Ry = torch.eye(3, device=device, dtype=dtype) + Ry[0, 0] = math.cos(beta) + Ry[0, 2] = -math.sin(beta) + Ry[2, 0] = -Ry[0, 2] + Ry[2, 2] = Ry[0, 0] + + Rz = torch.eye(3, device=device, dtype=dtype) + Rz[0, 0] = math.cos(gamma) + Rz[0, 1] = math.sin(gamma) + Rz[1, 0] = -Rz[0, 1] + Rz[1, 1] = Rz[0, 0] + + Ryz = torch.matmul(Ry, Rz) + R = torch.matmul(Rx, Ryz) + + extrinsics = torch.eye(4, device=device, dtype=dtype) + extrinsics[..., 0, -1] = tx + extrinsics[..., 1, -1] = ty + extrinsics[..., 2, -1] = tz + extrinsics[:3, :3] = R + + return extrinsics.expand(batch_size, -1, -1) + + def test_smoke(self, device, dtype): + intrinsics = torch.eye(4, device=device, dtype=dtype)[None] + extrinsics = torch.eye(4, device=device, dtype=dtype)[None] + height = torch.ones(1, device=device, dtype=dtype) + width = torch.ones(1, device=device, dtype=dtype) + pinhole = kornia.geometry.camera.PinholeCamera(intrinsics, extrinsics, height, width) + assert isinstance(pinhole, kornia.geometry.camera.PinholeCamera) + + def test_pinhole_camera_attributes(self, device, dtype): + batch_size = 1 + height, width = 4, 6 + fx, fy, cx, cy = 1, 2, width / 2, height / 2 + tx, ty, tz = 1, 2, 3 + + intrinsics = self._create_intrinsics(batch_size, fx, fy, cx, cy, device=device, dtype=dtype) + extrinsics = self._create_extrinsics(batch_size, tx, ty, tz, device=device, dtype=dtype) + height = torch.ones(batch_size, device=device, dtype=dtype) * height + width = torch.ones(batch_size, device=device, dtype=dtype) * width + + pinhole = kornia.geometry.camera.PinholeCamera(intrinsics, extrinsics, height, width) + + assert pinhole.batch_size == batch_size + assert pinhole.fx.item() == fx + assert pinhole.fy.item() == fy + assert pinhole.cx.item() == cx + assert pinhole.cy.item() == cy + assert pinhole.tx.item() == tx + assert pinhole.ty.item() == ty + assert pinhole.tz.item() == tz + assert pinhole.height.item() == height + assert pinhole.width.item() == width + assert pinhole.rt_matrix.shape == (batch_size, 3, 4) + assert pinhole.camera_matrix.shape == (batch_size, 3, 3) + assert pinhole.rotation_matrix.shape == (batch_size, 3, 3) + assert pinhole.translation_vector.shape == (batch_size, 3, 1) + + def test_pinhole_camera_translation_setters(self, device, dtype): + batch_size = 1 + height, width = 4, 6 + fx, fy, cx, cy = 1, 2, width / 2, height / 2 + tx, ty, tz = 1, 2, 3 + + intrinsics = self._create_intrinsics(batch_size, fx, fy, cx, cy, device=device, dtype=dtype) + extrinsics = self._create_extrinsics(batch_size, tx, ty, tz, device=device, dtype=dtype) + height = torch.ones(batch_size, device=device, dtype=dtype) * height + width = torch.ones(batch_size, device=device, dtype=dtype) * width + + pinhole = kornia.geometry.camera.PinholeCamera(intrinsics, extrinsics, height, width) + + assert pinhole.tx.item() == tx + assert pinhole.ty.item() == ty + assert pinhole.tz.item() == tz + + # add offset + pinhole.tx += 3.0 + pinhole.ty += 2.0 + pinhole.tz += 1.0 + + assert pinhole.tx.item() == tx + 3.0 + assert pinhole.ty.item() == ty + 2.0 + assert pinhole.tz.item() == tz + 1.0 + + # set to zero + pinhole.tx = 0.0 + pinhole.ty = 0.0 + pinhole.tz = 0.0 + + assert pinhole.tx.item() == 0.0 + assert pinhole.ty.item() == 0.0 + assert pinhole.tz.item() == 0.0 + + def test_pinhole_camera_attributes_batch2(self, device, dtype): + batch_size = 2 + height, width = 4, 6 + fx, fy, cx, cy = 1, 2, width / 2, height / 2 + tx, ty, tz = 1, 2, 3 + + intrinsics = self._create_intrinsics(batch_size, fx, fy, cx, cy, device=device, dtype=dtype) + extrinsics = self._create_extrinsics(batch_size, tx, ty, tz, device=device, dtype=dtype) + height = torch.ones(batch_size, device=device, dtype=dtype) * height + width = torch.ones(batch_size, device=device, dtype=dtype) * width + + pinhole = kornia.geometry.camera.PinholeCamera(intrinsics, extrinsics, height, width) + + assert pinhole.batch_size == batch_size + assert pinhole.fx.shape[0] == batch_size + assert pinhole.fy.shape[0] == batch_size + assert pinhole.cx.shape[0] == batch_size + assert pinhole.cy.shape[0] == batch_size + assert pinhole.tx.shape[0] == batch_size + assert pinhole.ty.shape[0] == batch_size + assert pinhole.tz.shape[0] == batch_size + assert pinhole.height.shape[0] == batch_size + assert pinhole.width.shape[0] == batch_size + assert pinhole.rt_matrix.shape == (batch_size, 3, 4) + assert pinhole.camera_matrix.shape == (batch_size, 3, 3) + assert pinhole.rotation_matrix.shape == (batch_size, 3, 3) + assert pinhole.translation_vector.shape == (batch_size, 3, 1) + + def test_pinhole_camera_scale(self, device, dtype): + batch_size = 2 + height, width = 4, 6 + fx, fy, cx, cy = 1, 2, width / 2, height / 2 + tx, ty, tz = 1, 2, 3 + scale_val = 2.0 + + intrinsics = self._create_intrinsics(batch_size, fx, fy, cx, cy, device=device, dtype=dtype) + extrinsics = self._create_extrinsics(batch_size, tx, ty, tz, device=device, dtype=dtype) + height = torch.ones(batch_size, device=device, dtype=dtype) * height + width = torch.ones(batch_size, device=device, dtype=dtype) * width + scale_factor = torch.ones(batch_size, device=device, dtype=dtype) * scale_val + + pinhole = kornia.geometry.camera.PinholeCamera(intrinsics, extrinsics, height, width) + pinhole_scale = pinhole.scale(scale_factor) + + self.assert_close( + pinhole_scale.intrinsics[..., 0, 0], pinhole.intrinsics[..., 0, 0] * scale_val, atol=1e-4, rtol=1e-4 + ) # fx + self.assert_close( + pinhole_scale.intrinsics[..., 1, 1], pinhole.intrinsics[..., 1, 1] * scale_val, atol=1e-4, rtol=1e-4 + ) # fy + self.assert_close( + pinhole_scale.intrinsics[..., 0, 2], pinhole.intrinsics[..., 0, 2] * scale_val, atol=1e-4, rtol=1e-4 + ) # cx + self.assert_close( + pinhole_scale.intrinsics[..., 1, 2], pinhole.intrinsics[..., 1, 2] * scale_val, atol=1e-4, rtol=1e-4 + ) # cy + self.assert_close(pinhole_scale.height, pinhole.height * scale_val, atol=1e-4, rtol=1e-4) + self.assert_close(pinhole_scale.width, pinhole.width * scale_val, atol=1e-4, rtol=1e-4) + + def test_pinhole_camera_scale_inplace(self, device, dtype): + batch_size = 2 + height, width = 4, 6 + fx, fy, cx, cy = 1, 2, width / 2, height / 2 + tx, ty, tz = 1, 2, 3 + scale_val = 2.0 + + intrinsics = self._create_intrinsics(batch_size, fx, fy, cx, cy, device=device, dtype=dtype) + extrinsics = self._create_extrinsics(batch_size, tx, ty, tz, device=device, dtype=dtype) + height = torch.ones(batch_size, device=device, dtype=dtype) * height + width = torch.ones(batch_size, device=device, dtype=dtype) * width + scale_factor = torch.ones(batch_size, device=device, dtype=dtype) * scale_val + + pinhole = kornia.geometry.camera.PinholeCamera(intrinsics, extrinsics, height, width) + pinhole_scale = pinhole.clone() + pinhole_scale.scale_(scale_factor) + + self.assert_close( + pinhole_scale.intrinsics[..., 0, 0], pinhole.intrinsics[..., 0, 0] * scale_val, atol=1e-4, rtol=1e-4 + ) # fx + self.assert_close( + pinhole_scale.intrinsics[..., 1, 1], pinhole.intrinsics[..., 1, 1] * scale_val, atol=1e-4, rtol=1e-4 + ) # fy + self.assert_close( + pinhole_scale.intrinsics[..., 0, 2], pinhole.intrinsics[..., 0, 2] * scale_val, atol=1e-4, rtol=1e-4 + ) # cx + self.assert_close( + pinhole_scale.intrinsics[..., 1, 2], pinhole.intrinsics[..., 1, 2] * scale_val, atol=1e-4, rtol=1e-4 + ) # cy + self.assert_close(pinhole_scale.height, pinhole.height * scale_val, atol=1e-4, rtol=1e-4) + self.assert_close(pinhole_scale.width, pinhole.width * scale_val, atol=1e-4, rtol=1e-4) + + def test_pinhole_camera_project_and_unproject(self, device, dtype): + batch_size = 5 + n = 2 # Point per batch + height, width = 4, 6 + fx, fy, cx, cy = 1, 2, width / 2, height / 2 + alpha, beta, gamma = 0.0, 0.0, 0.4 + tx, ty, tz = 0, 0, 3 + + intrinsics = self._create_intrinsics(batch_size, fx, fy, cx, cy, device=device, dtype=dtype) + extrinsics = self._create_extrinsics_with_rotation( + batch_size, alpha, beta, gamma, tx, ty, tz, device=device, dtype=dtype + ) + + height = torch.ones(batch_size, device=device, dtype=dtype) * height + width = torch.ones(batch_size, device=device, dtype=dtype) * width + + pinhole = kornia.geometry.camera.PinholeCamera(intrinsics, extrinsics, height, width) + + point_3d = torch.rand((batch_size, n, 3), device=device, dtype=dtype) + + depth = point_3d[..., -1:] + tz + + point_2d = pinhole.project(point_3d) + point_3d_hat = pinhole.unproject(point_2d, depth) + self.assert_close(point_3d, point_3d_hat, atol=1e-4, rtol=1e-4) + + def test_pinhole_camera_device(self, device, dtype): + batch_size = 5 + intrinsics = torch.rand((batch_size, 4, 4), device=device, dtype=dtype) + extrinsics = torch.rand((batch_size, 4, 4), device=device, dtype=dtype) + height = torch.randint(low=5, high=9, size=(batch_size,), device=device) + width = torch.randint(low=5, high=9, size=(batch_size,), device=device) + + pinhole = kornia.geometry.camera.PinholeCamera(intrinsics, extrinsics, height, width) + assert pinhole.device() == intrinsics.device diff --git a/tests/geometry/camera/test_projections.py b/tests/geometry/camera/test_projections.py new file mode 100644 index 0000000..c47f251 --- /dev/null +++ b/tests/geometry/camera/test_projections.py @@ -0,0 +1,247 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.geometry.camera.projection_orthographic import ( + dx_project_points_orthographic, + project_points_orthographic, + unproject_points_orthographic, +) +from kornia.geometry.camera.projection_z1 import dx_project_points_z1, project_points_z1, unproject_points_z1 + +from testing.base import BaseTester + + +class TestProjectionZ1(BaseTester): + def test_smoke(self, device, dtype): + points = torch.tensor([1.0, 2.0, 3.0], device=device, dtype=dtype) + assert project_points_z1(points) is not None + + def _test_cardinality_unproject_batch(self, device, dtype, batch_size): + batch_tuple = (batch_size,) if batch_size is not None else () + points = torch.rand(batch_tuple + (3,), device=device, dtype=dtype) + assert project_points_z1(points).shape == batch_tuple + (2,) + + def _test_cardinality_project_batch(self, device, dtype, batch_size): + batch_tuple = (batch_size,) if batch_size is not None else () + points = torch.rand(batch_tuple + (2,), device=device, dtype=dtype) + assert unproject_points_z1(points).shape == batch_tuple + (3,) + + @pytest.mark.parametrize("batch_size", [None, 1, 2, 3]) + def test_cardinality(self, device, dtype, batch_size): + self._test_cardinality_project_batch(device, dtype, batch_size) + self._test_cardinality_unproject_batch(device, dtype, batch_size) + + def test_project_points_z1(self, device, dtype): + points = torch.tensor([1.0, 2.0, 3.0], device=device, dtype=dtype) + expected = torch.tensor([0.3333333432674408, 0.6666666865348816], device=device, dtype=dtype) + self.assert_close(project_points_z1(points), expected) + + def test_project_points_z1_batch(self, device, dtype): + points = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], device=device, dtype=dtype) + expected = torch.tensor( + [ + [0.3333333432674408, 0.6666666865348816], + [0.6666666865348816, 0.8333333730697632], + ], + device=device, + dtype=dtype, + ) + self.assert_close(project_points_z1(points), expected) + + def test_project_points_z1_invalid(self, device, dtype): + # NOTE: this is a corner case where the depth is 0.0 and the point is at infinity + # the projection is not defined and the function returns inf. The second point + # is behind the camera which is not a valid point and the user should handle it. + points = torch.tensor([[1.0, 2.0, 0.0], [4.0, 5.0, -1.0]], device=device, dtype=dtype) + expected = torch.tensor([[float("inf"), float("inf")], [-4.0, -5.0]], device=device, dtype=dtype) + self.assert_close(project_points_z1(points), expected) + + def test_unproject_points_z1(self, device, dtype): + points = torch.tensor([1.0, 2.0], device=device, dtype=dtype) + expected = torch.tensor([1.0, 2.0, 1.0], device=device, dtype=dtype) + self.assert_close(unproject_points_z1(points), expected) + + def test_unproject_points_z1_batch(self, device, dtype): + points = torch.tensor([[1.0, 2.0], [3.0, 4.0]], device=device, dtype=dtype) + expected = torch.tensor([[1.0, 2.0, 1.0], [3.0, 4.0, 1.0]], device=device, dtype=dtype) + self.assert_close(unproject_points_z1(points), expected) + + def test_project_unproject(self, device, dtype): + points = torch.tensor([1.0, 2.0, 2.0], device=device, dtype=dtype) + extension = torch.tensor([2.0], device=device, dtype=dtype) + self.assert_close(unproject_points_z1(project_points_z1(points), extension), points) + + def test_unproject_points_z1_extension(self, device, dtype): + points = torch.tensor([1.0, 2.0], device=device, dtype=dtype) + extension = torch.tensor([2.0], device=device, dtype=dtype) + expected = torch.tensor([2.0, 4.0, 2.0], device=device, dtype=dtype) + self.assert_close(unproject_points_z1(points, extension), expected) + + def test_unproject_points_z1_batch_extension(self, device, dtype): + points = torch.tensor([[1.0, 2.0], [3.0, 4.0]], device=device, dtype=dtype) + extension = torch.tensor([2.0, 3.0], device=device, dtype=dtype) + expected = torch.tensor([[2.0, 4.0, 2.0], [9.0, 12.0, 3.0]], device=device, dtype=dtype) + self.assert_close(unproject_points_z1(points, extension), expected) + + def test_dx_proj_x(self, device, dtype): + points = torch.tensor([1.0, 2.0, 3.0], device=device, dtype=dtype) + expected = torch.tensor( + [ + [0.3333333432674408, 0.0, -0.1111111119389534], + [0.0, 0.3333333432674408, -0.2222222238779068], + ], + device=device, + dtype=dtype, + ) + self.assert_close(dx_project_points_z1(points), expected) + + def test_exception(self, device, dtype) -> None: + from kornia.core.exceptions import ShapeError + + points = torch.tensor([1.0, 2.0, 3.0], device=device, dtype=dtype) + extension = torch.tensor([2.0], device=device, dtype=dtype) + with pytest.raises(ShapeError): + unproject_points_z1(points, extension) + + def _test_gradcheck_unproject(self, device): + points = torch.tensor([1.0, 2.0], device=device, dtype=torch.float64) + extension = torch.tensor([2.0], device=device, dtype=torch.float64) + self.gradcheck(unproject_points_z1, (points, extension)) + + def _test_gradcheck_project(self, device): + points = torch.tensor([1.0, 2.0, 3.0], device=device, dtype=torch.float64) + self.gradcheck(project_points_z1, (points,)) + + def test_gradcheck(self, device) -> None: + self._test_gradcheck_project(device) + self._test_gradcheck_unproject(device) + + def _test_jit_unproject(self, device, dtype) -> None: + points = torch.tensor([1.0, 2.0], device=device, dtype=dtype) + extension = torch.tensor([2.0], device=device, dtype=dtype) + op_script = torch.jit.script(unproject_points_z1) + actual = op_script(points, extension) + expected = unproject_points_z1(points, extension) + self.assert_close(actual, expected) + + def _test_jit_project(self, device, dtype) -> None: + points = torch.tensor([1.0, 2.0, 3.0], device=device, dtype=dtype) + op_script = torch.jit.script(project_points_z1) + actual = op_script(points) + expected = project_points_z1(points) + self.assert_close(actual, expected) + + def test_jit(self, device, dtype) -> None: + self._test_jit_project(device, dtype) + self._test_jit_unproject(device, dtype) + + +class TestProjectionOrthographic(BaseTester): + def test_smoke(self, device, dtype): + points = torch.tensor([1.0, 2.0, 3.0], device=device, dtype=dtype) + assert project_points_orthographic(points) is not None + + def _test_cardinality_unproject_batch(self, device, dtype, batch_size): + batch_tuple = (batch_size,) if batch_size is not None else () + points = torch.rand(batch_tuple + (3,), device=device, dtype=dtype) + assert project_points_orthographic(points).shape == batch_tuple + (2,) + + def _test_cardinality_project_batch(self, device, dtype, batch_size): + batch_tuple = (batch_size,) if batch_size is not None else () + points = torch.rand(batch_tuple + (2,), device=device, dtype=dtype) + extension = torch.rand(batch_tuple, device=device, dtype=dtype) + assert unproject_points_orthographic(points, extension).shape == batch_tuple + (3,) + + @pytest.mark.parametrize("batch_size", [None, 1, 2, 3]) + def test_cardinality(self, device, dtype, batch_size): + self._test_cardinality_project_batch(device, dtype, batch_size) + self._test_cardinality_unproject_batch(device, dtype, batch_size) + + def test_project_points_orthographic(self, device, dtype): + points = torch.tensor([1.0, 2.0, 3.0], device=device, dtype=dtype) + expected = torch.tensor([1.0, 2.0], device=device, dtype=dtype) + self.assert_close(project_points_orthographic(points), expected) + + def test_project_points_orthographic_batch(self, device, dtype): + points = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], device=device, dtype=dtype) + expected = torch.tensor([[1.0, 2.0], [4.0, 5.0]], device=device, dtype=dtype) + self.assert_close(project_points_orthographic(points), expected) + + def test_unproject_points_orthographic_extension(self, device, dtype): + points = torch.tensor([1.0, 2.0], device=device, dtype=dtype) + extension = torch.tensor([2.0], device=device, dtype=dtype) + expected = torch.tensor([1.0, 2.0, 2.0], device=device, dtype=dtype) + self.assert_close(unproject_points_orthographic(points, extension), expected) + + def test_unproject_points_orthographic_batch_extension(self, device, dtype): + points = torch.tensor([[1.0, 2.0], [3.0, 4.0]], device=device, dtype=dtype) + extension = torch.tensor([2.0, 3.0], device=device, dtype=dtype) + expected = torch.tensor([[1.0, 2.0, 2.0], [3.0, 4.0, 3.0]], device=device, dtype=dtype) + self.assert_close(unproject_points_orthographic(points, extension), expected) + + def test_project_unproject(self, device, dtype): + points = torch.tensor([1.0, 2.0, 2.0], device=device, dtype=dtype) + extension = torch.tensor([2.0], device=device, dtype=dtype) + self.assert_close(unproject_points_orthographic(project_points_orthographic(points), extension), points) + + def test_dx_proj_x(self, device, dtype): + points = torch.tensor([1.0, 2.0, 3.0], device=device, dtype=dtype) + expected = torch.tensor([1.0], device=device, dtype=dtype) + self.assert_close(dx_project_points_orthographic(points), expected) + + def test_exception(self, device, dtype) -> None: + from kornia.core.exceptions import ShapeError + + points = torch.tensor([1.0, 2.0, 3.0], device=device, dtype=dtype) + extension = torch.tensor([2.0], device=device, dtype=dtype) + with pytest.raises(ShapeError): + unproject_points_orthographic(points, extension) + + def _test_gradcheck_project(self, device): + points = torch.tensor([1.0, 2.0, 3.0], device=device, dtype=torch.float64) + self.gradcheck(project_points_orthographic, (points,)) + + def _test_gradcheck_unproject(self, device): + points = torch.tensor([1.0, 2.0], device=device, dtype=torch.float64) + extension = torch.tensor([2.0], device=device, dtype=torch.float64) + self.gradcheck(unproject_points_orthographic, (points, extension)) + + def test_gradcheck(self, device) -> None: + self._test_gradcheck_project(device) + self._test_gradcheck_unproject(device) + + def _test_jit_project(self, device, dtype) -> None: + points = torch.tensor([1.0, 2.0, 3.0], device=device, dtype=dtype) + op_script = torch.jit.script(project_points_orthographic) + actual = op_script(points) + expected = project_points_orthographic(points) + self.assert_close(actual, expected) + + def _test_jit_unproject(self, device, dtype) -> None: + points = torch.tensor([1.0, 2.0], device=device, dtype=dtype) + extension = torch.tensor([2.0], device=device, dtype=dtype) + op_script = torch.jit.script(unproject_points_orthographic) + actual = op_script(points, extension) + expected = unproject_points_orthographic(points, extension) + self.assert_close(actual, expected) + + def test_jit(self, device, dtype) -> None: + self._test_jit_project(device, dtype) + self._test_jit_unproject(device, dtype) diff --git a/tests/geometry/camera/test_stereo.py b/tests/geometry/camera/test_stereo.py new file mode 100644 index 0000000..33c9d6b --- /dev/null +++ b/tests/geometry/camera/test_stereo.py @@ -0,0 +1,270 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.geometry.camera import StereoCamera + +from testing.base import BaseTester + + +@pytest.fixture(params=[1, 2, 4]) +def batch_size(request): + return request.param + + +class _TestParams: + """Collection of test parameters for smoke test.""" + + height = 4 + width = 6 + fx = 1 + fy = 2 + cx = width / 2 + cy = height / 2 + + +class _RealTestData(BaseTester): + """Collection of data from a real stereo setup.""" + + @property + def height(self): + return 375 + + @property + def width(self): + return 1242 + + @staticmethod + def _get_real_left_camera(batch_size, device, dtype): + cam = torch.tensor( + [ + 9.9640068207290187e02, + 0.0, + 3.7502582168579102e02, + 0.0, + 0.0, + 9.9640068207290187e02, + 2.4026374816894531e02, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + ], + device=device, + dtype=dtype, + ).reshape(3, 4) + return cam.expand(batch_size, -1, -1) + + @staticmethod + def _get_real_right_camera(batch_size, device, dtype): + cam = torch.tensor( + [ + 9.9640068207290187e02, + 0.0, + 3.7502582168579102e02, + -5.4301732344712009e03, + 0.0, + 9.9640068207290187e02, + 2.4026374816894531e02, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + ], + device=device, + dtype=dtype, + ).reshape(3, 4) + return cam.expand(batch_size, -1, -1) + + @staticmethod + def _get_real_stereo_camera(batch_size, device, dtype): + return ( + _RealTestData._get_real_left_camera(batch_size, device, dtype), + _RealTestData._get_real_right_camera(batch_size, device, dtype), + ) + + @staticmethod + def _get_real_disparity(batch_size, device, dtype): + # First 10 cols of 1 row in a real disparity map. + disp = torch.tensor( + [ + [ + [ + [67.5039], + [67.5078], + [67.5117], + [67.5156], + [67.5195], + [67.5234], + [67.5273], + [67.5312], + [67.5352], + [67.5391], + ] + ] + ], + device=device, + dtype=dtype, + ).permute(0, 2, 3, 1) + return disp.expand(batch_size, -1, -1, -1) + + @staticmethod + def _get_real_point_cloud(batch_size, device, dtype): + # First 10 cols of 1 row in the ground truth point cloud computed from above disparity map. + pc = torch.tensor( + [ + [ + [[-30.2769, -19.3972, 80.4424]], + [[-30.1945, -19.3961, 80.4377]], + [[-30.1120, -19.3950, 80.4330]], + [[-30.0295, -19.3938, 80.4284]], + [[-29.9471, -19.3927, 80.4237]], + [[-29.8646, -19.3916, 80.4191]], + [[-29.7822, -19.3905, 80.4144]], + [[-29.6998, -19.3893, 80.4098]], + [[-29.6174, -19.3882, 80.4051]], + [[-29.5350, -19.3871, 80.4005]], + ] + ], + device=device, + dtype=dtype, + ) + + return pc.expand(batch_size, -1, -1, -1) + + +class _SmokeTestData: + """Collection of smoke test data.""" + + @staticmethod + def _create_rectified_camera(params, batch_size, device, dtype, tx_fx=None): + intrinsics = torch.zeros((3, 4), device=device, dtype=dtype) + intrinsics[..., 0, 0] = params.fx + intrinsics[..., 1, 1] = params.fy + intrinsics[..., 0, 2] = params.cx + intrinsics[..., 1, 2] = params.cy + + if tx_fx: + intrinsics[..., 0, 3] = tx_fx + + return intrinsics.expand(batch_size, -1, -1) + + @staticmethod + def _create_left_camera(batch_size, device, dtype): + return _SmokeTestData._create_rectified_camera(_TestParams, batch_size, device, dtype) + + @staticmethod + def _create_right_camera(batch_size, device, dtype, tx_fx): + return _SmokeTestData._create_rectified_camera(_TestParams, batch_size, device, dtype, tx_fx=tx_fx) + + @staticmethod + def _create_stereo_camera(batch_size, device, dtype, tx_fx): + left_rectified_camera = _SmokeTestData._create_left_camera(batch_size, device, dtype) + right_rectified_camera = _SmokeTestData._create_right_camera(batch_size, device, dtype, tx_fx) + return left_rectified_camera, right_rectified_camera + + +class TestStereoCamera(BaseTester): + """Test class for :class:`~kornia.geometry.camera.stereo.StereoCamera`""" + + @staticmethod + def _create_disparity_tensor(batch_size, height, width, max_disparity, device, dtype): + size = (batch_size, height, width, 1) + return torch.randint(size=size, low=0, high=max_disparity, device=device, dtype=dtype) + + @staticmethod + def test_stereo_camera_attributes_smoke(batch_size, device, dtype): + """Test proper setup of the class for smoke data.""" + tx_fx = -10 + left_rectified_camera, right_rectified_camera = _SmokeTestData._create_stereo_camera( + batch_size, device, dtype, tx_fx + ) + + stereo_camera = StereoCamera(left_rectified_camera, right_rectified_camera) + + def _assert_all(x, y): + assert torch.all(torch.eq(x, y)) + + _assert_all(stereo_camera.fx, _TestParams.fx) + _assert_all(stereo_camera.fy, _TestParams.fy) + _assert_all(stereo_camera.cx_left, _TestParams.cx) + _assert_all(stereo_camera.cy, _TestParams.cy) + _assert_all(stereo_camera.tx, -tx_fx / _TestParams.fx) + + assert stereo_camera.Q.shape == (batch_size, 4, 4) + assert stereo_camera.Q.dtype in (torch.float16, torch.float32, torch.float64) + + def test_stereo_camera_attributes_real(self, batch_size, device, dtype): + """Test proper setup of the class for real data.""" + left_rectified_camera, right_rectified_camera = _RealTestData._get_real_stereo_camera(batch_size, device, dtype) + + stereo_camera = StereoCamera(left_rectified_camera, right_rectified_camera) + self.assert_close(stereo_camera.fx, left_rectified_camera[..., 0, 0]) + self.assert_close(stereo_camera.fy, left_rectified_camera[..., 1, 1]) + self.assert_close(stereo_camera.cx_left, left_rectified_camera[..., 0, 2]) + self.assert_close(stereo_camera.cy, left_rectified_camera[..., 1, 2]) + self.assert_close(stereo_camera.tx, -right_rectified_camera[..., 0, 3] / right_rectified_camera[..., 0, 0]) + assert stereo_camera.Q.shape == (batch_size, 4, 4) + assert stereo_camera.Q.dtype in (torch.float16, torch.float32, torch.float64) + + def test_reproject_disparity_to_3D_smoke(self, batch_size, device, dtype): + """Test reprojecting of disparity to 3D for smoke data.""" + tx_fx = -10 + left_rectified_camera, right_rectified_camera = _SmokeTestData._create_stereo_camera( + batch_size, device, dtype, tx_fx + ) + disparity_tensor = self._create_disparity_tensor( + batch_size, _TestParams.height, _TestParams.width, max_disparity=2, device=device, dtype=dtype + ) + stereo_camera = StereoCamera(left_rectified_camera, right_rectified_camera) + xyz = stereo_camera.reproject_disparity_to_3D(disparity_tensor) + + assert xyz.shape == (batch_size, _TestParams.height, _TestParams.width, 3) + assert xyz.dtype in (torch.float16, torch.float32, torch.float64) + assert xyz.device == device + + def test_reproject_disparity_to_3D_real(self, batch_size, device, dtype): + """Test reprojecting of disparity to 3D for known outcome.""" + disparity_tensor = _RealTestData._get_real_disparity(batch_size, device, dtype) + xyz_gt = _RealTestData._get_real_point_cloud(batch_size, device, dtype) + + left_rectified_camera, right_rectified_camera = _RealTestData._get_real_stereo_camera(batch_size, device, dtype) + stereo_camera = StereoCamera(left_rectified_camera, right_rectified_camera) + + xyz = stereo_camera.reproject_disparity_to_3D(disparity_tensor) + + self.assert_close(xyz, xyz_gt) + + def test_reproject_disparity_to_3D_simple(self, batch_size, device, dtype): + """Test reprojecting of disparity to 3D for real data.""" + height, width = _RealTestData().height, _RealTestData().width + max_disparity = 80 + disparity_tensor = self._create_disparity_tensor( + batch_size, height, width, max_disparity=max_disparity, device=device, dtype=dtype + ) + left_rectified_camera, right_rectified_camera = _RealTestData._get_real_stereo_camera(batch_size, device, dtype) + stereo_camera = StereoCamera(left_rectified_camera, right_rectified_camera) + + xyz = stereo_camera.reproject_disparity_to_3D(disparity_tensor) + + assert xyz.shape == (batch_size, height, width, 3) + assert xyz.dtype in (torch.float16, torch.float32, torch.float64) + assert xyz.dtype == dtype diff --git a/tests/geometry/epipolar/test_epipolar_metrics.py b/tests/geometry/epipolar/test_epipolar_metrics.py new file mode 100644 index 0000000..ffeb3d9 --- /dev/null +++ b/tests/geometry/epipolar/test_epipolar_metrics.py @@ -0,0 +1,215 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch + +import kornia.geometry.epipolar as epi + +from testing.base import BaseTester +from testing.geometry.create import create_random_fundamental_matrix + + +class TestSymmetricalEpipolarDistance(BaseTester): + def test_smoke(self, device, dtype): + pts1 = torch.rand(1, 4, 3, device=device, dtype=dtype) + pts2 = torch.rand(1, 4, 3, device=device, dtype=dtype) + Fm = create_random_fundamental_matrix(1, dtype=dtype, device=device) + assert epi.symmetrical_epipolar_distance(pts1, pts2, Fm).shape == (1, 4) + + def test_batch(self, device, dtype): + batch_size = 5 + pts1 = torch.rand(batch_size, 4, 3, device=device, dtype=dtype) + pts2 = torch.rand(batch_size, 4, 3, device=device, dtype=dtype) + Fm = create_random_fundamental_matrix(1, dtype=dtype, device=device) + assert epi.symmetrical_epipolar_distance(pts1, pts2, Fm).shape == (5, 4) + + def test_frames(self, device, dtype): + batch_size, num_frames, num_points = 5, 3, 4 + pts1 = torch.rand(batch_size, num_frames, num_points, 3, device=device, dtype=dtype) + pts2 = torch.rand(batch_size, num_frames, num_points, 3, device=device, dtype=dtype) + Fm = torch.stack( + [create_random_fundamental_matrix(1, dtype=dtype, device=device) for _ in range(num_frames)], dim=1 + ) + dist_frame_by_frame = torch.stack( + [ + epi.symmetrical_epipolar_distance(pts1[:, t, ...], pts2[:, t, ...], Fm[:, t, ...]) + for t in range(num_frames) + ], + dim=1, + ) + dist_all_frames = epi.symmetrical_epipolar_distance(pts1, pts2, Fm) + self.assert_close(dist_frame_by_frame, dist_all_frames, atol=1e-6, rtol=1e-6) + + def test_gradcheck(self, device): + # generate input data + batch_size, num_points, num_dims = 2, 3, 2 + points1 = torch.rand(batch_size, num_points, num_dims, device=device, dtype=torch.float64, requires_grad=True) + points2 = torch.rand(batch_size, num_points, num_dims, device=device, dtype=torch.float64) + Fm = create_random_fundamental_matrix(batch_size, dtype=torch.float64, device=device) + + self.gradcheck(epi.symmetrical_epipolar_distance, (points1, points2, Fm), requires_grad=(True, False, False)) + + def test_shift(self, device, dtype): + pts1 = torch.zeros(3, 2, device=device, dtype=dtype)[None] + pts2 = torch.tensor([[2, 0.0], [2, 1], [2, 2.0]], device=device, dtype=dtype)[None] + Fm = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, -1.0], [0.0, 1.0, 0.0]], dtype=dtype, device=device)[None] + expected = torch.tensor([0.0, 2.0, 8.0], device=device, dtype=dtype)[None] + self.assert_close(epi.symmetrical_epipolar_distance(pts1, pts2, Fm), expected, atol=1e-4, rtol=1e-4) + + +class TestSampsonEpipolarDistance(BaseTester): + def test_smoke(self, device, dtype): + pts1 = torch.rand(1, 4, 3, device=device, dtype=dtype) + pts2 = torch.rand(1, 4, 3, device=device, dtype=dtype) + Fm = create_random_fundamental_matrix(1, dtype=dtype, device=device) + + assert epi.sampson_epipolar_distance(pts1, pts2, Fm).shape == (1, 4) + + def test_batch(self, device, dtype): + batch_size = 5 + pts1 = torch.rand(batch_size, 4, 3, device=device, dtype=dtype) + pts2 = torch.rand(batch_size, 4, 3, device=device, dtype=dtype) + Fm = create_random_fundamental_matrix(1, dtype=dtype, device=device) + assert epi.sampson_epipolar_distance(pts1, pts2, Fm).shape == (5, 4) + + def test_frames(self, device, dtype): + batch_size, num_frames, num_points = 5, 3, 4 + pts1 = torch.rand(batch_size, num_frames, num_points, 3, device=device, dtype=dtype) + pts2 = torch.rand(batch_size, num_frames, num_points, 3, device=device, dtype=dtype) + Fm = torch.stack( + [create_random_fundamental_matrix(1, dtype=dtype, device=device) for _ in range(num_frames)], dim=1 + ) + dist_frame_by_frame = torch.stack( + [epi.sampson_epipolar_distance(pts1[:, t, ...], pts2[:, t, ...], Fm[:, t, ...]) for t in range(num_frames)], + dim=1, + ) + dist_all_frames = epi.sampson_epipolar_distance(pts1, pts2, Fm) + self.assert_close(dist_frame_by_frame, dist_all_frames, atol=1e-6, rtol=1e-6) + + def test_shift(self, device, dtype): + pts1 = torch.zeros(3, 2, device=device, dtype=dtype)[None] + pts2 = torch.tensor([[2, 0.0], [2, 1], [2, 2.0]], device=device, dtype=dtype)[None] + Fm = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, -1.0], [0.0, 1.0, 0.0]], dtype=dtype, device=device)[None] + expected = torch.tensor([0.0, 0.5, 2.0], device=device, dtype=dtype)[None] + self.assert_close(epi.sampson_epipolar_distance(pts1, pts2, Fm), expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + # generate input data + batch_size, num_points, num_dims = 2, 3, 2 + points1 = torch.rand(batch_size, num_points, num_dims, device=device, dtype=torch.float64, requires_grad=True) + points2 = torch.rand(batch_size, num_points, num_dims, device=device, dtype=torch.float64) + Fm = create_random_fundamental_matrix(batch_size, dtype=torch.float64, device=device) + + self.gradcheck(epi.sampson_epipolar_distance, (points1, points2, Fm), requires_grad=(True, False, False)) + + +class TestLeftToRightEpipolarDistance(BaseTester): + def test_smoke(self, device, dtype): + pts1 = torch.rand(1, 4, 3, device=device, dtype=dtype) + pts2 = torch.rand(1, 4, 3, device=device, dtype=dtype) + Fm = create_random_fundamental_matrix(1, dtype=dtype, device=device) + + assert epi.left_to_right_epipolar_distance(pts1, pts2, Fm).shape == (1, 4) + + def test_batch(self, device, dtype): + batch_size = 5 + pts1 = torch.rand(batch_size, 4, 3, device=device, dtype=dtype) + pts2 = torch.rand(batch_size, 4, 3, device=device, dtype=dtype) + Fm = create_random_fundamental_matrix(1, dtype=dtype, device=device) + assert epi.left_to_right_epipolar_distance(pts1, pts2, Fm).shape == (5, 4) + + def test_frames(self, device, dtype): + batch_size, num_frames, num_points = 5, 3, 4 + pts1 = torch.rand(batch_size, num_frames, num_points, 3, device=device, dtype=dtype) + pts2 = torch.rand(batch_size, num_frames, num_points, 3, device=device, dtype=dtype) + Fm = torch.stack( + [create_random_fundamental_matrix(1, dtype=dtype, device=device) for _ in range(num_frames)], dim=1 + ) + dist_frame_by_frame = torch.stack( + [ + epi.left_to_right_epipolar_distance(pts1[:, t, ...], pts2[:, t, ...], Fm[:, t, ...]) + for t in range(num_frames) + ], + dim=1, + ) + dist_all_frames = epi.left_to_right_epipolar_distance(pts1, pts2, Fm) + self.assert_close(dist_frame_by_frame, dist_all_frames, atol=1e-6, rtol=1e-6) + + def test_shift(self, device, dtype): + pts1 = torch.zeros(3, 2, device=device, dtype=dtype)[None] + pts2 = torch.tensor([[2, 0.0], [2, 1], [2, 2.0]], device=device, dtype=dtype)[None] + Fm = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, -1.0], [0.0, 1.0, 0.0]], dtype=dtype, device=device)[None] + expected = torch.tensor([0.0, 1.0, 2.0], device=device, dtype=dtype)[None] + self.assert_close(epi.left_to_right_epipolar_distance(pts1, pts2, Fm), expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + # generate input data + batch_size, num_points, num_dims = 2, 3, 2 + points1 = torch.rand(batch_size, num_points, num_dims, device=device, dtype=torch.float64, requires_grad=True) + points2 = torch.rand(batch_size, num_points, num_dims, device=device, dtype=torch.float64) + Fm = create_random_fundamental_matrix(batch_size, dtype=torch.float64, device=device) + + self.gradcheck(epi.left_to_right_epipolar_distance, (points1, points2, Fm), requires_grad=(True, False, False)) + + +class TestRightToLeftEpipolarDistance(BaseTester): + def test_smoke(self, device, dtype): + pts1 = torch.rand(1, 4, 3, device=device, dtype=dtype) + pts2 = torch.rand(1, 4, 3, device=device, dtype=dtype) + Fm = create_random_fundamental_matrix(1, dtype=dtype, device=device) + + assert epi.right_to_left_epipolar_distance(pts1, pts2, Fm).shape == (1, 4) + + def test_batch(self, device, dtype): + batch_size = 5 + pts1 = torch.rand(batch_size, 4, 3, device=device, dtype=dtype) + pts2 = torch.rand(batch_size, 4, 3, device=device, dtype=dtype) + Fm = create_random_fundamental_matrix(1, dtype=dtype, device=device) + assert epi.right_to_left_epipolar_distance(pts1, pts2, Fm).shape == (5, 4) + + def test_frames(self, device, dtype): + batch_size, num_frames, num_points = 5, 3, 4 + pts1 = torch.rand(batch_size, num_frames, num_points, 3, device=device, dtype=dtype) + pts2 = torch.rand(batch_size, num_frames, num_points, 3, device=device, dtype=dtype) + Fm = torch.stack( + [create_random_fundamental_matrix(1, dtype=dtype, device=device) for _ in range(num_frames)], dim=1 + ) + dist_frame_by_frame = torch.stack( + [ + epi.right_to_left_epipolar_distance(pts1[:, t, ...], pts2[:, t, ...], Fm[:, t, ...]) + for t in range(num_frames) + ], + dim=1, + ) + dist_all_frames = epi.right_to_left_epipolar_distance(pts1, pts2, Fm) + self.assert_close(dist_frame_by_frame, dist_all_frames, atol=1e-6, rtol=1e-6) + + def test_shift(self, device, dtype): + pts1 = torch.zeros(3, 2, device=device, dtype=dtype)[None] + pts2 = torch.tensor([[2, 0.0], [2, 1], [2, 2.0]], device=device, dtype=dtype)[None] + Fm = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, -1.0], [0.0, 1.0, 0.0]], dtype=dtype, device=device)[None] + expected = torch.tensor([0.0, 1.0, 2.0], device=device, dtype=dtype)[None] + self.assert_close(epi.right_to_left_epipolar_distance(pts1, pts2, Fm), expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + # generate input data + batch_size, num_points, num_dims = 2, 3, 2 + points1 = torch.rand(batch_size, num_points, num_dims, device=device, dtype=torch.float64, requires_grad=True) + points2 = torch.rand(batch_size, num_points, num_dims, device=device, dtype=torch.float64) + Fm = create_random_fundamental_matrix(batch_size, dtype=torch.float64, device=device) + + self.gradcheck(epi.right_to_left_epipolar_distance, (points1, points2, Fm), requires_grad=(True, False, False)) diff --git a/tests/geometry/epipolar/test_essential.py b/tests/geometry/epipolar/test_essential.py new file mode 100644 index 0000000..de448c3 --- /dev/null +++ b/tests/geometry/epipolar/test_essential.py @@ -0,0 +1,519 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia +import kornia.geometry.epipolar as epi + +from testing.base import BaseTester +from testing.geometry.create import generate_two_view_random_scene + + +class TestFindEssential(BaseTester): + def test_smoke(self, device, dtype): + points1 = torch.rand(1, 5, 2, device=device, dtype=dtype) + points2 = torch.rand(1, 5, 2, device=device, dtype=dtype) + weights = torch.ones(1, 5, device=device, dtype=dtype) + E_mat = epi.essential.find_essential(points1, points2, weights) + assert E_mat.shape == (1, 10, 3, 3) + + @pytest.mark.parametrize("batch_size, num_points", [(1, 5), (2, 6), (3, 7), (1000, 5)]) + def test_shape(self, batch_size, num_points, device, dtype): + B, N = batch_size, num_points + points1 = torch.rand(B, N, 2, device=device, dtype=dtype) + points2 = torch.rand(B, N, 2, device=device, dtype=dtype) + weights = torch.ones(B, N, device=device, dtype=dtype) + E_mat = epi.essential.find_essential(points1, points2, weights) + assert E_mat.shape == (B, 10, 3, 3) + + @pytest.mark.parametrize("batch_size, num_points", [(1, 5), (2, 6), (3, 7)]) + def test_shape_noweights(self, batch_size, num_points, device, dtype): + B, N = batch_size, num_points + points1 = torch.rand(B, N, 2, device=device, dtype=dtype) + points2 = torch.rand(B, N, 2, device=device, dtype=dtype) + weights = None + E_mat = epi.essential.find_essential(points1, points2, weights) + assert E_mat.shape == (B, 10, 3, 3) + + def test_epipolar_constraint(self, device, dtype): + calibrated_x1 = torch.tensor( + [[[0.0640, 0.7799], [-0.2011, 0.2836], [-0.1355, 0.2907], [0.0520, 1.0086], [-0.0361, 0.6533]]], + device=device, + dtype=dtype, + ) + calibrated_x2 = torch.tensor( + [[[0.3470, -0.4274], [-0.1818, -0.1281], [-0.1766, -0.1617], [0.4066, -0.0706], [0.1137, 0.0363]]], + device=device, + dtype=dtype, + ) + + E = epi.essential.find_essential(calibrated_x1, calibrated_x2) + if torch.all(E != 0): + distance = epi.symmetrical_epipolar_distance(calibrated_x1, calibrated_x2, E) + distance = torch.nan_to_num(distance, nan=1e8) + # Note : here we check only the best model, although all solutions are returned + mean_error = distance.mean(-1).min() + self.assert_close(mean_error, torch.tensor(0.0, device=device, dtype=dtype), atol=1e-4, rtol=1e-4) + + def test_synthetic_sampson(self, device, dtype): + calibrated_x1 = torch.tensor( + [[[0.0640, 0.7799], [-0.2011, 0.2836], [-0.1355, 0.2907], [0.0520, 1.0086], [-0.0361, 0.6533]]], + device=device, + dtype=dtype, + ) + calibrated_x2 = torch.tensor( + [[[0.3470, -0.4274], [-0.1818, -0.1281], [-0.1766, -0.1617], [0.4066, -0.0706], [0.1137, 0.0363]]], + device=device, + dtype=dtype, + ) + + weights = torch.ones_like(calibrated_x2)[..., 0] + E_est = epi.essential.find_essential(calibrated_x1, calibrated_x2, weights) + error = epi.sampson_epipolar_distance(calibrated_x1, calibrated_x2, E_est) + error = torch.nan_to_num(error, nan=1e8) + self.assert_close( + error[:, torch.argmin(error.mean(-1))], + torch.zeros((calibrated_x1.shape[:2]), device=device, dtype=dtype), + atol=1e-4, + rtol=1e-4, + ) + + @pytest.mark.parametrize("batch_size, num_points", [(5, 5), (10, 5)]) + def test_degenerate_case(self, batch_size, num_points, device, dtype): + B, N = batch_size, num_points + points1_deg = torch.rand(B, N, 2, device=device, dtype=dtype) + weights = torch.ones_like(points1_deg)[..., 0] + E_mat_deg = epi.essential.find_essential(points1_deg, points1_deg, weights) + assert E_mat_deg.shape == (B, 10, 3, 3) + + +class TestEssentialFromFundamental(BaseTester): + def test_smoke(self, device, dtype): + F_mat = torch.rand(1, 3, 3, device=device, dtype=dtype) + K1 = torch.rand(1, 3, 3, device=device, dtype=dtype) + K2 = torch.rand(1, 3, 3, device=device, dtype=dtype) + E_mat = epi.essential_from_fundamental(F_mat, K1, K2) + assert E_mat.shape == (1, 3, 3) + + @pytest.mark.parametrize("batch_size", [1, 2, 4, 7]) + def test_shape(self, batch_size, device, dtype): + B: int = batch_size + F_mat = torch.rand(B, 3, 3, device=device, dtype=dtype) + K1 = torch.rand(B, 3, 3, device=device, dtype=dtype) + K2 = torch.rand(1, 3, 3, device=device, dtype=dtype) # check broadcasting + E_mat = epi.essential_from_fundamental(F_mat, K1, K2) + assert E_mat.shape == (B, 3, 3) + + @pytest.mark.xfail(reason="TODO: fix #685") + def test_from_to_fundamental(self, device, dtype): + F_mat = torch.rand(1, 3, 3, device=device, dtype=dtype) + K1 = torch.rand(1, 3, 3, device=device, dtype=dtype) + K2 = torch.rand(1, 3, 3, device=device, dtype=dtype) + E_mat = epi.essential_from_fundamental(F_mat, K1, K2) + F_hat = epi.fundamental_from_essential(E_mat, K1, K2) + self.assert_close(F_mat, F_hat, atol=1e-4, rtol=1e-4) + + def test_shape_large(self, device, dtype): + F_mat = torch.rand(1, 2, 3, 3, device=device, dtype=dtype) + K1 = torch.rand(1, 2, 3, 3, device=device, dtype=dtype) + K2 = torch.rand(1, 1, 3, 3, device=device, dtype=dtype) # check broadcasting + E_mat = epi.essential_from_fundamental(F_mat, K1, K2) + assert E_mat.shape == (1, 2, 3, 3) + + def test_from_fundamental(self, device, dtype): + scene = generate_two_view_random_scene(device, dtype) + + F_mat = scene["F"] + + K1 = scene["K1"] + K2 = scene["K2"] + + E_mat = epi.essential_from_fundamental(F_mat, K1, K2) + F_hat = epi.fundamental_from_essential(E_mat, K1, K2) + + F_mat_norm = epi.normalize_transformation(F_mat) + F_hat_norm = epi.normalize_transformation(F_hat) + self.assert_close(F_mat_norm, F_hat_norm) + + def test_gradcheck(self, device): + F_mat = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True) + K1 = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + K2 = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + self.gradcheck(epi.essential_from_fundamental, (F_mat, K1, K2), requires_grad=(True, False, False)) + + +class TestRelativeCameraMotion(BaseTester): + def test_smoke(self, device, dtype): + R1 = torch.rand(1, 3, 3, device=device, dtype=dtype) + t1 = torch.rand(1, 3, 1, device=device, dtype=dtype) + R2 = torch.rand(1, 3, 3, device=device, dtype=dtype) + t2 = torch.rand(1, 3, 1, device=device, dtype=dtype) + R, t = epi.relative_camera_motion(R1, t1, R2, t2) + assert R.shape == (1, 3, 3) + assert t.shape == (1, 3, 1) + + @pytest.mark.parametrize("batch_size", [1, 3, 5, 8]) + def test_shape(self, batch_size, device, dtype): + B: int = batch_size + R1 = torch.rand(B, 3, 3, device=device, dtype=dtype) + t1 = torch.rand(B, 3, 1, device=device, dtype=dtype) + R2 = torch.rand(1, 3, 3, device=device, dtype=dtype) # check broadcasting + t2 = torch.rand(B, 3, 1, device=device, dtype=dtype) + R, t = epi.relative_camera_motion(R1, t1, R2, t2) + assert R.shape == (B, 3, 3) + assert t.shape == (B, 3, 1) + + def test_translation(self, device, dtype): + R1 = torch.tensor([[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + + t1 = torch.tensor([[[10.0], [0.0], [0.0]]]).type_as(R1) + + R2 = kornia.core.ops.eye_like(3, R1) + t2 = kornia.core.ops.vec_like(3, t1) + + R_expected = R1.clone() + t_expected = -t1 + + R, t = epi.relative_camera_motion(R1, t1, R2, t2) + self.assert_close(R_expected, R) + self.assert_close(t_expected, t) + + def test_rotate_z(self, device, dtype): + R1 = torch.tensor([[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + + R2 = torch.tensor([[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + + t1 = kornia.core.ops.vec_like(3, R1) + t2 = kornia.core.ops.vec_like(3, R2) + + R_expected = R2.clone() + t_expected = t1 + + R, t = epi.relative_camera_motion(R1, t1, R2, t2) + self.assert_close(R_expected, R) + self.assert_close(t_expected, t) + + def test_gradcheck(self, device): + R1 = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True) + R2 = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + t1 = torch.rand(1, 3, 1, device=device, dtype=torch.float64) + t2 = torch.rand(1, 3, 1, device=device, dtype=torch.float64) + self.gradcheck(epi.relative_camera_motion, (R1, t1, R2, t2), requires_grad=(True, False, False, False)) + + +class TestEssentalFromRt(BaseTester): + def test_smoke(self, device, dtype): + R1 = torch.rand(1, 3, 3, device=device, dtype=dtype) + t1 = torch.rand(1, 3, 1, device=device, dtype=dtype) + R2 = torch.rand(1, 3, 3, device=device, dtype=dtype) + t2 = torch.rand(1, 3, 1, device=device, dtype=dtype) + E_mat = epi.essential_from_Rt(R1, t1, R2, t2) + assert E_mat.shape == (1, 3, 3) + + @pytest.mark.parametrize("batch_size", [1, 3, 5, 8]) + def test_shape(self, batch_size, device, dtype): + B: int = batch_size + R1 = torch.rand(B, 3, 3, device=device, dtype=dtype) + t1 = torch.rand(B, 3, 1, device=device, dtype=dtype) + R2 = torch.rand(1, 3, 3, device=device, dtype=dtype) # check broadcasting + t2 = torch.rand(B, 3, 1, device=device, dtype=dtype) + E_mat = epi.essential_from_Rt(R1, t1, R2, t2) + assert E_mat.shape == (B, 3, 3) + + @pytest.mark.xfail(reason="TODO: fix #685") + def test_from_fundamental_Rt(self, device, dtype): + scene = generate_two_view_random_scene(device, dtype) + + E_from_Rt = epi.essential_from_Rt(scene["R1"], scene["t1"], scene["R2"], scene["t2"]) + + E_from_F = epi.essential_from_fundamental(scene["F"], scene["K1"], scene["K2"]) + + E_from_Rt_norm = epi.normalize_transformation(E_from_Rt) + E_from_F_norm = epi.normalize_transformation(E_from_F) + # TODO: occasionally failed with error > 0.04 + self.assert_close(E_from_Rt_norm, E_from_F_norm, rtol=1e-3, atol=1e-3) + + def test_gradcheck(self, device): + R1 = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True) + R2 = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + t1 = torch.rand(1, 3, 1, device=device, dtype=torch.float64) + t2 = torch.rand(1, 3, 1, device=device, dtype=torch.float64) + self.gradcheck(epi.essential_from_Rt, (R1, t1, R2, t2), requires_grad=(True, False, False, False)) + + +class TestDecomposeEssentialMatrix(BaseTester): + def test_smoke(self, device, dtype): + E_mat = torch.rand(1, 3, 3, device=device, dtype=dtype) + R1, R2, t = epi.decompose_essential_matrix(E_mat) + assert R1.shape == (1, 3, 3) + assert R2.shape == (1, 3, 3) + assert t.shape == (1, 3, 1) + + @pytest.mark.parametrize("batch_shape", [(1, 3, 3), (2, 3, 3), (2, 1, 3, 3), (3, 2, 1, 3, 3)]) + def test_shape(self, batch_shape, device, dtype): + E_mat = torch.rand(batch_shape, device=device, dtype=dtype) + R1, R2, t = epi.decompose_essential_matrix(E_mat) + assert R1.shape == batch_shape + assert R2.shape == batch_shape + assert t.shape == batch_shape[:-1] + (1,) + + def test_gradcheck(self, device): + E_mat = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True) + + def eval_rot1(input): + return epi.decompose_essential_matrix(input)[0] + + def eval_rot2(input): + return epi.decompose_essential_matrix(input)[1] + + def eval_vec(input): + return epi.decompose_essential_matrix(input)[2] + + self.gradcheck(eval_rot1, (E_mat,)) + self.gradcheck(eval_rot2, (E_mat,)) + self.gradcheck(eval_vec, (E_mat,)) + + +class TestDecomposeEssentialMatrixNoSVD(BaseTester): + def test_smoke(self, device, dtype): + E_mat = torch.rand(1, 3, 3, device=device, dtype=dtype) + R1, R2, t = epi.decompose_essential_matrix_no_svd(E_mat) + assert R1.shape == (1, 3, 3) + assert R2.shape == (1, 3, 3) + assert t.shape == (1, 3, 1) + + @pytest.mark.parametrize("batch_shape", [(3, 3), (1, 3, 3), (2, 3, 3), (2, 1, 3, 3), (3, 2, 1, 3, 3)]) + def test_shape(self, batch_shape, device, dtype): + E_mat = torch.rand(batch_shape, device=device, dtype=dtype) + R1, R2, t = epi.decompose_essential_matrix_no_svd(E_mat) + if len(batch_shape) >= 2: + batch_shape = E_mat.view(-1, 3, 3).shape + assert R1.shape == batch_shape + assert R2.shape == batch_shape + assert t.shape == batch_shape[:-1] + (1,) + + def test_gradcheck(self, device): + E_mat = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True) + + def eval_rot1(input): + return epi.decompose_essential_matrix_no_svd(input)[0] + + def eval_rot2(input): + return epi.decompose_essential_matrix_no_svd(input)[1] + + def eval_vec(input): + return epi.decompose_essential_matrix_no_svd(input)[2] + + self.gradcheck(eval_rot1, (E_mat,)) + self.gradcheck(eval_rot2, (E_mat,)) + self.gradcheck(eval_vec, (E_mat,)) + + def test_correct_decompose(self): + E_mat = torch.tensor([[[0.2057, -3.8266, 3.1615], [4.5417, -1.0707, -2.2023], [-1.0975, 1.6386, -0.6590]]]) + R1, R2, t = epi.decompose_essential_matrix(E_mat) + R1_1, R2_1, t_1 = epi.decompose_essential_matrix_no_svd(E_mat) + # As the orders of two R solutions and t solutions might be different from epi.decompose_essential_matrix(), + # we have to check on the correct ones + rtol: float = 1e-4 + if (R1 - R1_1).abs().sum() < rtol: + self.assert_close(R1, R1_1) + self.assert_close(R2, R2_1) + else: + self.assert_close(R1, R2_1) + self.assert_close(R2, R1_1) + R1_1, R2_1 = R2_1, R1_1 + + if (t - t_1).abs().sum() < rtol: + self.assert_close(t, t_1) + else: + self.assert_close(t, -t_1) + t_1 = -t_1.clone() + self.assert_close( + epi.essential_from_Rt(R1_1, t_1, R2_1, -t_1), epi.essential_from_Rt(R1, t, R2, -t), rtol=1e-3, atol=1e-3 + ) + + @pytest.mark.xfail(reason="skip the tests where there are no solutions.") + def test_consistency(self, device, dtype): + scene = generate_two_view_random_scene(device, dtype) + + R1, t1 = scene["R1"], scene["t1"] + R2, t2 = scene["R2"], scene["t2"] + + E_mat = epi.essential_from_Rt(R1, t1, R2, t2) + + # compare the decomposed R and t to the method with svd + R1, R2, t = epi.decompose_essential_matrix(E_mat) + R1_1, R2_1, t_1 = epi.decompose_essential_matrix_no_svd(E_mat) + # As the orders of two R solutions and t solutions might be different from epi.decompose_essential_matrix(), + # we have to check on the correct ones + rtol: float = 1e-4 + if (R1 - R1_1).abs().sum() < rtol: + self.assert_close(R1, R1_1) + self.assert_close(R2, R2_1) + else: + self.assert_close(R1, R2_1) + self.assert_close(R2, R1_1) + R1_1, R2_1 = R2_1, R1_1 + + if (t - t_1).abs().sum() < rtol: + self.assert_close(t, t_1) + else: + self.assert_close(t, -t_1) + t_1 = -t_1.clone() + + self.assert_close(epi.essential_from_Rt(R1_1, t_1, R2_1, -t_1), epi.essential_from_Rt(R1, t, R2, -t)) + + +class TestMotionFromEssential(BaseTester): + def test_smoke(self, device, dtype): + E_mat = torch.rand(1, 3, 3, device=device, dtype=dtype) + Rs, Ts = epi.motion_from_essential(E_mat) + assert Rs.shape == (1, 4, 3, 3) + assert Ts.shape == (1, 4, 3, 1) + + @pytest.mark.parametrize("batch_shape", [(1, 3, 3), (2, 3, 3), (2, 1, 3, 3), (3, 2, 1, 3, 3)]) + def test_shape(self, batch_shape, device, dtype): + E_mat = torch.rand(batch_shape, device=device, dtype=dtype) + Rs, Ts = epi.motion_from_essential(E_mat) + assert Rs.shape == batch_shape[:-2] + (4, 3, 3) + assert Ts.shape == batch_shape[:-2] + (4, 3, 1) + + def test_two_view(self, device, dtype): + scene = generate_two_view_random_scene(device, dtype) + + R1, t1 = scene["R1"], scene["t1"] + R2, t2 = scene["R2"], scene["t2"] + + E_mat = epi.essential_from_Rt(R1, t1, R2, t2) + + R, t = epi.relative_camera_motion(R1, t1, R2, t2) + t = torch.nn.functional.normalize(t, dim=1) + + Rs, ts = epi.motion_from_essential(E_mat) + + rot_error = (Rs - R).abs().sum((-2, -1)) + vec_error = (ts - t).abs().sum(-1) + + rtol: float = 1e-4 + assert (rot_error < rtol).any() & (vec_error < rtol).any() + + def test_gradcheck(self, device): + E_mat = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True) + + def eval_rot(input): + return epi.motion_from_essential(input)[0] + + def eval_vec(input): + return epi.motion_from_essential(input)[1] + + self.gradcheck(eval_rot, (E_mat,)) + self.gradcheck(eval_vec, (E_mat,)) + + +class TestMotionFromEssentialChooseSolution(BaseTester): + def test_smoke(self, device, dtype): + E_mat = torch.rand(1, 3, 3, device=device, dtype=dtype) + K1 = torch.rand(1, 3, 3, device=device, dtype=dtype) + K2 = torch.rand(1, 3, 3, device=device, dtype=dtype) + x1 = torch.rand(1, 1, 2, device=device, dtype=dtype) + x2 = torch.rand(1, 1, 2, device=device, dtype=dtype) + R, t, X = epi.motion_from_essential_choose_solution(E_mat, K1, K2, x1, x2) + assert R.shape == (1, 3, 3) + assert t.shape == (1, 3, 1) + assert X.shape == (1, 1, 3) + + @pytest.mark.parametrize("batch_size, num_points", [(1, 3), (2, 3), (2, 8), (3, 2)]) + def test_shape(self, batch_size, num_points, device, dtype): + B, N = batch_size, num_points + E_mat = torch.rand(B, 3, 3, device=device, dtype=dtype) + K1 = torch.rand(B, 3, 3, device=device, dtype=dtype) + K2 = torch.rand(1, 3, 3, device=device, dtype=dtype) # check for broadcasting + x1 = torch.rand(B, N, 2, device=device, dtype=dtype) + x2 = torch.rand(B, 1, 2, device=device, dtype=dtype) # check for broadcasting + R, t, X = epi.motion_from_essential_choose_solution(E_mat, K1, K2, x1, x2) + assert R.shape == (B, 3, 3) + assert t.shape == (B, 3, 1) + assert X.shape == (B, N, 3) + + def test_masking(self, device, dtype): + E_mat = torch.rand(2, 3, 3, device=device, dtype=dtype) + K1 = torch.rand(2, 3, 3, device=device, dtype=dtype) + K2 = torch.rand(2, 3, 3, device=device, dtype=dtype) + x1 = torch.rand(2, 10, 2, device=device, dtype=dtype) + x2 = torch.rand(2, 10, 2, device=device, dtype=dtype) + + R, t, X = epi.motion_from_essential_choose_solution(E_mat, K1, K2, x1[:, 1:-1, :], x2[:, 1:-1, :]) + + mask = torch.zeros(2, 10, dtype=torch.bool, device=device) + mask[:, 1:-1] = True + Rm, tm, Xm = epi.motion_from_essential_choose_solution(E_mat, K1, K2, x1, x2, mask=mask) + + self.assert_close(R, Rm) + self.assert_close(t, tm) + self.assert_close(X, Xm[:, 1:-1, :]) + + @pytest.mark.parametrize("num_points", [10, 15, 20]) + def test_unbatched(self, num_points, device, dtype): + N = num_points + E_mat = torch.rand(3, 3, device=device, dtype=dtype) + K1 = torch.rand(3, 3, device=device, dtype=dtype) + K2 = torch.rand(3, 3, device=device, dtype=dtype) + x1 = torch.rand(N, 2, device=device, dtype=dtype) + x2 = torch.rand(N, 2, device=device, dtype=dtype) + + R, t, X = epi.motion_from_essential_choose_solution(E_mat, K1, K2, x1[1:-1, :], x2[1:-1, :]) + assert R.shape == (3, 3) + assert t.shape == (3, 1) + assert X.shape == (N - 2, 3) + + mask = torch.zeros(N, dtype=torch.bool, device=device) + mask[1:-1] = True + Rm, tm, Xm = epi.motion_from_essential_choose_solution(E_mat, K1, K2, x1, x2, mask=mask) + + self.assert_close(R, Rm) + self.assert_close(t, tm) + self.assert_close(X, Xm[1:-1, :]) + + def test_two_view(self, device, dtype): + scene = generate_two_view_random_scene(device, dtype) + + E_mat = epi.essential_from_Rt(scene["R1"], scene["t1"], scene["R2"], scene["t2"]) + + R, t = epi.relative_camera_motion(scene["R1"], scene["t1"], scene["R2"], scene["t2"]) + t = torch.nn.functional.normalize(t, dim=1) + + R_hat, t_hat, _ = epi.motion_from_essential_choose_solution( + E_mat, scene["K1"], scene["K2"], scene["x1"], scene["x2"] + ) + + self.assert_close(t, t_hat) + self.assert_close(R, R_hat, rtol=1e-4, atol=1e-4) + + def test_gradcheck(self, device): + E_mat = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True) + K1 = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + K2 = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + x1 = torch.rand(1, 2, 2, device=device, dtype=torch.float64) + x2 = torch.rand(1, 2, 2, device=device, dtype=torch.float64) + + self.gradcheck( + epi.motion_from_essential_choose_solution, + (E_mat, K1, K2, x1, x2), + requires_grad=(True, False, False, False, False), + ) diff --git a/tests/geometry/epipolar/test_fundamental.py b/tests/geometry/epipolar/test_fundamental.py new file mode 100644 index 0000000..b870d8b --- /dev/null +++ b/tests/geometry/epipolar/test_fundamental.py @@ -0,0 +1,557 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Dict + +import pytest +import torch + +import kornia.geometry.epipolar as epi + +from testing.base import BaseTester +from testing.geometry.create import create_random_fundamental_matrix, generate_two_view_random_scene + + +class TestNormalizePoints(BaseTester): + def test_smoke(self, device, dtype): + points = torch.rand(1, 1, 2, device=device, dtype=dtype) + output = epi.normalize_points(points) + assert len(output) == 2 + assert output[0].shape == (1, 1, 2) + assert output[1].shape == (1, 3, 3) + + @pytest.mark.parametrize("batch_size, num_points", [(1, 2), (2, 3), (3, 2)]) + def test_shape(self, batch_size, num_points, device, dtype): + B, N = batch_size, num_points + points = torch.rand(B, N, 2, device=device, dtype=dtype) + output = epi.normalize_points(points) + assert output[0].shape == (B, N, 2) + assert output[1].shape == (B, 3, 3) + + def test_mean_std(self, device, dtype): + points = torch.tensor([[[0.0, 0.0], [0.0, 2.0], [1.0, 1.0], [1.0, 3.0]]], device=device, dtype=dtype) + + points_norm, _ = epi.normalize_points(points) + points_std, points_mean = torch.std_mean(points_norm, dim=1) + + self.assert_close(points_mean, torch.zeros_like(points_mean)) + assert (points_std < 2.0).all() + + def test_gradcheck(self, device): + points = torch.rand(2, 3, 2, device=device, requires_grad=True, dtype=torch.float64) + self.gradcheck(epi.normalize_points, (points,)) + + +class TestNormalizeTransformation(BaseTester): + def test_smoke(self, device, dtype): + trans = torch.rand(2, 2, device=device, dtype=dtype) + trans_norm = epi.normalize_transformation(trans) + assert trans_norm.shape == (2, 2) + + @pytest.mark.parametrize("batch_size, rows, cols", [(1, 2, 2), (2, 3, 3), (3, 4, 4), (2, 1, 2)]) + def test_shape(self, batch_size, rows, cols, device, dtype): + B, N, M = batch_size, rows, cols + trans = torch.rand(B, N, M, device=device, dtype=dtype) + trans_norm = epi.normalize_transformation(trans) + assert trans_norm.shape == (B, N, M) + + def test_check_last_val(self, device, dtype): + trans = torch.tensor([[[0.0, 0.0, 1.0], [0.0, 2.0, 0.0], [0.5, 0.0, 0.5]]], device=device, dtype=dtype) + + trans_expected = torch.tensor([[[0.0, 0.0, 2.0], [0.0, 4.0, 0.0], [1.0, 0.0, 1.0]]], device=device, dtype=dtype) + + trans_norm = epi.normalize_transformation(trans) + self.assert_close(trans_norm, trans_expected, atol=1e-4, rtol=1e-4) + + def test_check_corner_case(self, device, dtype): + trans = torch.tensor([[[0.0, 0.0, 1.0], [0.0, 2.0, 0.0], [0.5, 0.0, 0.0]]], device=device, dtype=dtype) + + trans_expected = trans.clone() + + trans_norm = epi.normalize_transformation(trans) + self.assert_close(trans_norm, trans_expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + trans = torch.rand(2, 3, 3, device=device, requires_grad=True, dtype=torch.float64) + self.gradcheck(epi.normalize_transformation, (trans,)) + + +class TestFindFundamental(BaseTester): + def test_smoke(self, device, dtype): + points1 = torch.rand(1, 8, 2, device=device, dtype=dtype) + points2 = torch.rand(1, 8, 2, device=device, dtype=dtype) + weights = torch.ones(1, 8, device=device, dtype=dtype) + F_mat = epi.find_fundamental(points1, points2, weights) + assert F_mat.shape == (1, 3, 3) + + @pytest.mark.parametrize("batch_size, num_points", [(1, 8), (2, 9), (3, 10)]) + def test_shape(self, batch_size, num_points, device, dtype): + B, N = batch_size, num_points + points1 = torch.rand(B, N, 2, device=device, dtype=dtype) + points2 = torch.rand(B, N, 2, device=device, dtype=dtype) + weights = torch.ones(B, N, device=device, dtype=dtype) + F_mat = epi.find_fundamental(points1, points2, weights) + assert F_mat.shape == (B, 3, 3) + + @pytest.mark.parametrize("batch_size, num_points", [(1, 8), (2, 8), (3, 10)]) + def test_shape_noweights(self, batch_size, num_points, device, dtype): + B, N = batch_size, num_points + points1 = torch.rand(B, N, 2, device=device, dtype=dtype) + points2 = torch.rand(B, N, 2, device=device, dtype=dtype) + weights = None + F_mat = epi.find_fundamental(points1, points2, weights) + assert F_mat.shape == (B, 3, 3) + + @pytest.mark.parametrize("batch_size", [1, 2, 3]) + def test_shape_7point(self, batch_size, device, dtype): + B = batch_size + points1 = torch.rand(B, 7, 2, device=device, dtype=dtype) + points2 = torch.rand(B, 7, 2, device=device, dtype=dtype) + torch.ones(B, 7, device=device, dtype=dtype) + F_mat = epi.find_fundamental(points1, points2, method="7POINT") + assert F_mat.shape == (B, 3, 3, 3) + + def test_opencv_svd(self, device, dtype): + points1 = torch.tensor( + [ + [ + [0.8569, 0.5982], + [0.0059, 0.9649], + [0.1968, 0.8846], + [0.6084, 0.3467], + [0.9633, 0.5274], + [0.8941, 0.8939], + [0.0863, 0.5133], + [0.2645, 0.8882], + [0.2411, 0.3045], + [0.8199, 0.4107], + ] + ], + device=device, + dtype=dtype, + ) + + points2 = torch.tensor( + [ + [ + [0.0928, 0.3013], + [0.0989, 0.9649], + [0.0341, 0.4827], + [0.8294, 0.4469], + [0.2230, 0.2998], + [0.1722, 0.8182], + [0.5264, 0.8869], + [0.8908, 0.1233], + [0.2338, 0.7663], + [0.4466, 0.5696], + ] + ], + device=device, + dtype=dtype, + ) + + weights = torch.ones(1, 10, device=device, dtype=dtype) + + # generated with OpenCV using above points + # import cv2 + # Fm_expected, _ = cv2.findFundamentalMat( + # points1.detach().numpy().reshape(-1, 1, 2), + # points2.detach().numpy().reshape(-1, 1, 2), cv2.FM_8POINT) + + Fm_expected = torch.tensor( + [ + [ + [-0.47408533, 0.22033807, -0.00346677], + [0.54935973, 1.31080955, -1.25028275], + [-0.36690215, -1.08143769, 1.0], + ] + ], + device=device, + dtype=dtype, + ) + + F_mat = epi.find_fundamental(points1, points2, weights) + self.assert_close(F_mat, Fm_expected, rtol=1e-4, atol=1e-4) + + def test_7point_opencv(self, device, dtype): + points1 = torch.tensor( + [ + [ + [0.8569, 0.5982], + [0.0059, 0.9649], + [0.1968, 0.8846], + [0.6084, 0.3467], + [0.9633, 0.5274], + [0.8941, 0.8939], + [0.0863, 0.5133], + ] + ], + device=device, + dtype=dtype, + ) + + points2 = torch.tensor( + [ + [ + [0.0928, 0.3013], + [0.0989, 0.9649], + [0.0341, 0.4827], + [0.8294, 0.4469], + [0.2230, 0.2998], + [0.1722, 0.8182], + [0.5264, 0.8869], + ] + ], + device=device, + dtype=dtype, + ) + + # generated with OpenCV using above points + # Fm_expected shape is 9x3 + # import cv2 + # Fm_expected, _ = cv2.findFundamentalMat( + # points1.detach().numpy().reshape(-1, 1, 2), + # points2.detach().numpy().reshape(-1, 1, 2), cv2.FM_7POINT) + + Fm_expected = torch.tensor( + [ + [ + [ + [-2.87490907, 5.41934672, 0.73871396], + [0.34010174, 3.70371623, -4.65517276], + [-0.1809933, -0.56577107, 1.0], + ], + [ + [0.14465888, 0.68711702, -0.65570944], + [0.53424758, 0.7988479, -0.75446946], + [-0.48201197, -1.05375511, 1.0], + ], + [ + [-0.0901827, 1.05515785, -0.54726062], + [0.51914823, 1.02476892, -1.05783979], + [-0.45860077, -1.01580301, 1.0], + ], + ] + ], + device=device, + dtype=dtype, + ) + # We need this voodoo, because the order of the solutions is not guaranteed by the algorithm. + F_mat = epi.find_fundamental(points1, points2, method="7POINT") + ordering = [] + for expected in Fm_expected[0]: + min_diff = float("inf") + for i, estimated in enumerate(F_mat[0]): + diff = (expected - estimated).abs().sum() + if diff < min_diff: + min_diff = diff + min_index = i + ordering.append(min_index) + F_mat[0] = F_mat[0][ordering] + self.assert_close(F_mat, Fm_expected, rtol=1e-3, atol=1e-3) + + def test_synthetic_sampson_7point(self, device, dtype): + scene: Dict[str, torch.Tensor] = generate_two_view_random_scene(device, dtype) + x1 = scene["x1"][:, :7, :] + x2 = scene["x2"][:, :7, :] + F_est = epi.find_fundamental(x1, x2, None, "7POINT") + for i in range(3): + F = F_est[0][i].unsqueeze(0) + if torch.all(F != 0): + error = epi.sampson_epipolar_distance(x1, x2, F) + self.assert_close(error, torch.zeros((F.shape[0], 7), device=device, dtype=dtype), atol=1e-4, rtol=1e-4) + + @pytest.mark.xfail() + def test_epipolar_constraint_7point(self, device, dtype): + scene: Dict[str, torch.Tensor] = generate_two_view_random_scene(device, dtype) + x1 = scene["x1"][:, :7, :] + x2 = scene["x2"][:, :7, :] + F_est = epi.find_fundamental(x1, x2, None, "7POINT") + for i in range(3): + F = F_est[0][i].unsqueeze(0) + if torch.all(F != 0): + distance = epi.symmetrical_epipolar_distance(x1, x2, F) + mean_error = distance.mean() + self.assert_close(mean_error, torch.tensor(0.0, device=device, dtype=dtype), atol=1e-4, rtol=1e-4) + + def test_synthetic_sampson(self, device, dtype): + scene: Dict[str, torch.Tensor] = generate_two_view_random_scene(device, dtype) + + x1 = scene["x1"] + x2 = scene["x2"] + + weights = torch.ones_like(x1)[..., 0] + F_est = epi.find_fundamental(x1, x2, weights) + + error = epi.sampson_epipolar_distance(x1, x2, F_est) + self.assert_close(error, torch.zeros((x1.shape[:2]), device=device, dtype=dtype), atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + points1 = torch.rand(1, 10, 2, device=device, dtype=torch.float64, requires_grad=True) + points2 = torch.rand(1, 10, 2, device=device, dtype=torch.float64) + weights = torch.ones(1, 10, device=device, dtype=torch.float64) + self.gradcheck(epi.find_fundamental, (points1, points2, weights)) + + +class TestComputeCorrespondEpilines(BaseTester): + def test_smoke(self, device, dtype): + point = torch.rand(1, 1, 2, device=device, dtype=dtype) + F_mat = torch.rand(1, 3, 3, device=device, dtype=dtype) + lines = epi.compute_correspond_epilines(point, F_mat) + assert lines.shape == (1, 1, 3) + + @pytest.mark.parametrize("batch_size, num_points", [(1, 2), (2, 3), (3, 2)]) + def test_shape(self, batch_size, num_points, device, dtype): + B, N = batch_size, num_points + point = torch.rand(B, N, 2, device=device, dtype=dtype) + F_mat = torch.rand(B, 3, 3, device=device, dtype=dtype) + lines = epi.compute_correspond_epilines(point, F_mat) + assert lines.shape == (B, N, 3) + + @pytest.mark.parametrize( + "batch_size, num_frames, num_points", + [(1, 1, 1), (1, 1, 2), (1, 2, 1), (1, 2, 2), (2, 1, 1), (2, 1, 2), (2, 2, 1), (2, 2, 2)], + ) + def test_volumetric(self, batch_size, num_frames, num_points, device, dtype): + B, T, N = batch_size, num_frames, num_points + point = torch.rand(B, T, N, 2, device=device, dtype=dtype) + F_mat = torch.rand(B, T, 3, 3, device=device, dtype=dtype) + + lines_T_hops = torch.zeros(B, T, N, 3, device=device, dtype=dtype) + for i in range(T): + lines_T_hops[:, i, ...] = epi.compute_correspond_epilines(point[:, i, ...], F_mat[:, i, ...]) + lines_one_hop = epi.compute_correspond_epilines(point, F_mat) + + self.assert_close(lines_T_hops, lines_one_hop, atol=2e-7, rtol=2e-7) + + def test_opencv(self, device, dtype): + point = torch.rand(1, 2, 2, device=device, dtype=dtype) + F_mat = torch.rand(1, 3, 3, device=device, dtype=dtype) + + point = torch.tensor([[[0.9794, 0.7994], [0.8163, 0.8500]]], device=device, dtype=dtype) + + F_mat = torch.tensor( + [[[0.1185, 0.4438, 0.9869], [0.5670, 0.9447, 0.4100], [0.1546, 0.2554, 0.4485]]], device=device, dtype=dtype + ) + + # generated with OpenCV using above points + # import cv2 + # lines_expected = cv2.computeCorrespondEpilines( + # point.detach().numpy().reshape(-1, 1, 2), 0, + # F_mat.detach().numpy()[0]).transpose(1, 0, 2) + + lines_expected = torch.tensor( + [[[0.64643687, 0.7629675, 0.35658622], [0.65710586, 0.7537983, 0.35616538]]], device=device, dtype=dtype + ) + + lines_est = epi.compute_correspond_epilines(point, F_mat) + self.assert_close(lines_est, lines_expected, rtol=1e-4, atol=1e-4) + + def test_gradcheck(self, device): + point = torch.rand(1, 4, 2, device=device, dtype=torch.float64, requires_grad=True) + F_mat = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + self.gradcheck(epi.compute_correspond_epilines, (point, F_mat), requires_grad=(True, False)) + + +class TestFundamentlFromEssential(BaseTester): + def test_smoke(self, device, dtype): + E_mat = torch.rand(1, 3, 3, device=device, dtype=dtype) + K1 = torch.rand(1, 3, 3, device=device, dtype=dtype) + K2 = torch.rand(1, 3, 3, device=device, dtype=dtype) + F_mat = epi.fundamental_from_essential(E_mat, K1, K2) + assert F_mat.shape == (1, 3, 3) + + @pytest.mark.parametrize("batch_size", [1, 2, 4, 7]) + def test_shape(self, batch_size, device, dtype): + B: int = batch_size + E_mat = torch.rand(B, 3, 3, device=device, dtype=dtype) + K1 = torch.rand(B, 3, 3, device=device, dtype=dtype) + K2 = torch.rand(1, 3, 3, device=device, dtype=dtype) # check broadcasting + F_mat = epi.fundamental_from_essential(E_mat, K1, K2) + assert F_mat.shape == (B, 3, 3) + + def test_shape_large(self, device, dtype): + E_mat = torch.rand(1, 2, 3, 3, device=device, dtype=dtype) + K1 = torch.rand(1, 2, 3, 3, device=device, dtype=dtype) + K2 = torch.rand(1, 1, 3, 3, device=device, dtype=dtype) # check broadcasting + F_mat = epi.fundamental_from_essential(E_mat, K1, K2) + assert F_mat.shape == (1, 2, 3, 3) + + def test_from_to_essential(self, device, dtype): + scene = generate_two_view_random_scene(device, dtype) + + F_mat = scene["F"] + E_mat = epi.essential_from_fundamental(F_mat, scene["K1"], scene["K2"]) + F_hat = epi.fundamental_from_essential(E_mat, scene["K1"], scene["K2"]) + + F_mat_norm = epi.normalize_transformation(F_mat) + F_hat_norm = epi.normalize_transformation(F_hat) + self.assert_close(F_mat_norm, F_hat_norm, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + E_mat = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True) + K1 = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + K2 = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + self.gradcheck(epi.fundamental_from_essential, (E_mat, K1, K2), requires_grad=(True, False, False)) + + +class TestFundamentalFromProjections(BaseTester): + def test_smoke(self, device, dtype): + P1 = torch.rand(1, 3, 4, device=device, dtype=dtype) + P2 = torch.rand(1, 3, 4, device=device, dtype=dtype) + F_mat = epi.fundamental_from_projections(P1, P2) + assert F_mat.shape == (1, 3, 3) + + @pytest.mark.parametrize("batch_size", [1, 2, 4, 7]) + def test_shape(self, batch_size, device, dtype): + B: int = batch_size + P1 = torch.rand(B, 3, 4, device=device, dtype=dtype) + P2 = torch.rand(B, 3, 4, device=device, dtype=dtype) + F_mat = epi.fundamental_from_projections(P1, P2) + assert F_mat.shape == (B, 3, 3) + + def test_shape_large(self, device, dtype): + P1 = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + P2 = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + F_mat = epi.fundamental_from_projections(P1, P2) + assert F_mat.shape == (1, 2, 3, 3) + + def test_from_to_projections(self, device, dtype): + P1 = torch.tensor( + [[[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [1.0, 0.0, 1.0, 0.0]]], device=device, dtype=dtype + ) + + P2 = torch.tensor( + [[[1.0, 1.0, 1.0, 3.0], [0.0, 2.0, 0.0, 3.0], [0.0, 1.0, 1.0, 0.0]]], device=device, dtype=dtype + ) + + F_mat = epi.fundamental_from_projections(P1, P2) + P_mat = epi.projections_from_fundamental(F_mat) + F_hat = epi.fundamental_from_projections(P_mat[..., 0], P_mat[..., 1]) + + F_mat_norm = epi.normalize_transformation(F_mat) + F_hat_norm = epi.normalize_transformation(F_hat) + self.assert_close(F_mat_norm, F_hat_norm, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + P1 = torch.rand(1, 3, 4, device=device, dtype=torch.float64, requires_grad=True) + P2 = torch.rand(1, 3, 4, device=device, dtype=torch.float64) + self.gradcheck(epi.fundamental_from_projections, (P1, P2), requires_grad=(True, False)) + + def test_batch_support_check(self, device, dtype): + P1_batch = torch.tensor( + [ + [ + [9.4692e02, -9.6658e02, 6.0862e02, -2.3076e05], + [-2.1829e02, 5.4163e02, 1.3445e03, -6.4387e05], + [-6.0675e-01, -6.9807e-01, 3.8021e-01, 3.8896e02], + ], + [ + [9.4692e02, -9.6658e02, 6.0862e02, -2.3076e05], + [-2.1829e02, 5.4163e02, 1.3445e03, -6.4387e05], + [-6.0675e-01, -6.9807e-01, 3.8021e-01, 3.8896e02], + ], + ], + device=device, + dtype=dtype, + ) + P1 = torch.tensor( + [ + [ + [9.4692e02, -9.6658e02, 6.0862e02, -2.3076e05], + [-2.1829e02, 5.4163e02, 1.3445e03, -6.4387e05], + [-6.0675e-01, -6.9807e-01, 3.8021e-01, 3.8896e02], + ] + ], + device=device, + dtype=dtype, + ) + P2_batch = torch.tensor( + [ + [ + [1.1518e03, -7.5822e02, 5.4764e02, -1.9764e05], + [-2.1548e02, 5.3102e02, 1.3492e03, -6.4731e05], + [-4.3727e-01, -7.8632e-01, 4.3646e-01, 3.4515e02], + ], + [ + [9.9595e02, -8.6464e02, 6.7959e02, -2.7517e05], + [-8.1716e01, 7.7826e02, 1.2395e03, -5.8137e05], + [-5.7090e-01, -6.0416e-01, 5.5594e-01, 2.8111e02], + ], + ], + device=device, + dtype=dtype, + ) + P2 = torch.tensor( + [ + [ + [1.1518e03, -7.5822e02, 5.4764e02, -1.9764e05], + [-2.1548e02, 5.3102e02, 1.3492e03, -6.4731e05], + [-4.3727e-01, -7.8632e-01, 4.3646e-01, 3.4515e02], + ] + ], + device=device, + dtype=dtype, + ) + + F_batch = epi.fundamental_from_projections(P1_batch, P2_batch) + F = epi.fundamental_from_projections(P1, P2) + self.assert_close(F_batch[0], F[0]) + + +class TestPerpendicular(BaseTester): + def test_shape(self, device, dtype): + lines = torch.rand(2, 4, 3, device=device, dtype=dtype) + points = torch.rand(2, 4, 2, device=device, dtype=dtype) + perp = epi.get_perpendicular(lines, points) + assert perp.shape == (2, 4, 3) + + def test_result(self, device, dtype): + points = torch.tensor([[[1.0, 0.0], [0.0, 1.0]]], device=device, dtype=dtype) + + lines = torch.tensor([[[1.0, -1.0, 0.0], [0.0, 1.0, 1.0]]], device=device, dtype=dtype) + perp = epi.get_perpendicular(lines, points) + expected = torch.tensor([[[1.0, 1.0, -1.0], [-1.0, 0.0, 0.0]]], device=device, dtype=dtype) + self.assert_close(perp, expected) + + def test_gradcheck(self, device): + pt = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True) + line = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + self.gradcheck(epi.get_perpendicular, (pt, line), requires_grad=(True, False)) + + +class TestGetClosestPointOnEpipolarLine(BaseTester): + def test_shape(self, device, dtype): + pts1 = torch.rand(2, 4, 2, device=device, dtype=dtype) + pts2 = torch.rand(2, 4, 2, device=device, dtype=dtype) + Fm = create_random_fundamental_matrix(1, device=device, dtype=dtype) + perp = epi.get_closest_point_on_epipolar_line(pts1, pts2, Fm) + assert perp.shape == (2, 4, 2) + + def test_shift(self, device, dtype): + pts1 = torch.zeros(3, 2, device=device, dtype=dtype)[None] + pts2 = torch.tensor([[2, 4.0], [2, 1], [2, 2.0]], device=device, dtype=dtype)[None] + Fm = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, -1.0], [0.0, 1.0, 0.0]], dtype=dtype, device=device)[None] + cp = epi.get_closest_point_on_epipolar_line(pts1, pts2, Fm) + expected = torch.tensor([[[2.0, 0.0], [2.0, 0.0], [2.0, 0.0]]], device=device, dtype=dtype) + self.assert_close(cp, expected) + + def test_gradcheck(self, device): + pts1 = torch.rand(2, 4, 2, device=device, dtype=torch.float64, requires_grad=True) + pts2 = torch.rand(2, 4, 2, device=device, dtype=torch.float64) + Fm = create_random_fundamental_matrix(1, dtype=torch.float64, device=device) + self.gradcheck(epi.get_closest_point_on_epipolar_line, (pts1, pts2, Fm), requires_grad=(True, False, False)) diff --git a/tests/geometry/epipolar/test_numeric.py b/tests/geometry/epipolar/test_numeric.py new file mode 100644 index 0000000..6df2140 --- /dev/null +++ b/tests/geometry/epipolar/test_numeric.py @@ -0,0 +1,109 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia +import kornia.geometry.epipolar as epi + +from testing.base import BaseTester + + +class TestSkewSymmetric(BaseTester): + def test_smoke(self, device, dtype): + vec = torch.rand(1, 3, device=device, dtype=dtype) + cross_product_matrix = epi.cross_product_matrix(vec) + assert cross_product_matrix.shape == (1, 3, 3) + + @pytest.mark.parametrize("batch_size", [1, 2, 4, 7]) + def test_shape(self, batch_size, device, dtype): + B = batch_size + vec = torch.rand(B, 3, device=device, dtype=dtype) + cross_product_matrix = epi.cross_product_matrix(vec) + assert cross_product_matrix.shape == (B, 3, 3) + + @pytest.mark.parametrize("shapes", [(1, 1), (1, 5), (2, 1), (2, 5), (4, 1), (4, 5)]) + def test_shapes(self, device, dtype, shapes): + input_shape = (*shapes, 3) + output_shape = (*shapes, 3, 3) + t = torch.rand(*input_shape, device=device, dtype=dtype) + cross_product_matrix = epi.cross_product_matrix(t) + assert cross_product_matrix.shape == output_shape + + @pytest.mark.parametrize("shapes", [(1, 1), (1, 5), (2, 1), (2, 5), (4, 1), (4, 5)]) + def test_funcional_shapes(self, device, dtype, shapes): + input_shape = (*shapes, 3) + t = torch.rand(*input_shape, device=device, dtype=dtype) + + # Feed batches + cross_product_matrices = [] + for i in range(t.shape[1]): + cross_product_matrices.append(epi.cross_product_matrix(t[:, i, ...])) + cross_product_matrix_parts = torch.stack(cross_product_matrices, dim=1) + + # Feed one-shot + cross_product_matrix_whole = epi.cross_product_matrix(t) + + self.assert_close(cross_product_matrix_parts, cross_product_matrix_whole) + + def test_mean_std(self, device, dtype): + vec = torch.tensor([[1.0, 2.0, 3.0]], device=device, dtype=dtype) + cross_product_matrix = epi.cross_product_matrix(vec) + self.assert_close(cross_product_matrix[..., 0, 1], -cross_product_matrix[..., 1, 0]) + self.assert_close(cross_product_matrix[..., 0, 2], -cross_product_matrix[..., 2, 0]) + self.assert_close(cross_product_matrix[..., 1, 2], -cross_product_matrix[..., 2, 1]) + + def test_gradcheck(self, device): + vec = torch.ones(2, 3, device=device, requires_grad=True, dtype=torch.float64) + assert self.gradcheck(epi.cross_product_matrix, (vec,), raise_exception=True, fast_mode=True) + + +class TestEyeLike: + def test_smoke(self, device, dtype): + image = torch.rand(1, 3, 4, 4, device=device, dtype=dtype) + identity = kornia.core.ops.eye_like(3, image) + assert identity.shape == (1, 3, 3) + assert identity.device == image.device + assert identity.dtype == image.dtype + + @pytest.mark.parametrize("batch_size, eye_size", [(1, 2), (2, 3), (3, 3), (2, 4)]) + def test_shape(self, batch_size, eye_size, device, dtype): + B, N = batch_size, eye_size + image = torch.rand(B, 3, 4, 4, device=device, dtype=dtype) + identity = kornia.core.ops.eye_like(N, image) + assert identity.shape == (B, N, N) + assert identity.device == image.device + assert identity.dtype == image.dtype + + +class TestVecLike: + def test_smoke(self, device, dtype): + image = torch.rand(1, 3, 4, 4, device=device, dtype=dtype) + vec = kornia.core.ops.vec_like(3, image) + assert vec.shape == (1, 3, 1) + assert vec.device == image.device + assert vec.dtype == image.dtype + + @pytest.mark.parametrize("batch_size, eye_size", [(1, 2), (2, 3), (3, 3), (2, 4)]) + def test_shape(self, batch_size, eye_size, device, dtype): + B, N = batch_size, eye_size + image = torch.rand(B, 3, 4, 4, device=device, dtype=dtype) + vec = kornia.core.ops.vec_like(N, image) + assert vec.shape == (B, N, 1) + assert vec.device == image.device + assert vec.dtype == image.dtype diff --git a/tests/geometry/epipolar/test_projection.py b/tests/geometry/epipolar/test_projection.py new file mode 100644 index 0000000..a5ac0a0 --- /dev/null +++ b/tests/geometry/epipolar/test_projection.py @@ -0,0 +1,227 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +from torch.autograd import gradcheck + +import kornia.geometry.epipolar as epi + +from testing.base import BaseTester + + +class TestIntrinsicsLike: + def test_smoke(self, device, dtype): + image = torch.rand(1, 3, 4, 4, device=device, dtype=dtype) + focal = torch.rand(1, device=device, dtype=dtype) + camera_matrix = epi.intrinsics_like(focal, image) + assert camera_matrix.shape == (1, 3, 3) + + @pytest.mark.parametrize("batch_size", [1, 2, 4, 9]) + def test_shape(self, batch_size, device, dtype): + B: int = batch_size + focal: float = 100.0 + image = torch.rand(B, 3, 4, 4, device=device, dtype=dtype) + camera_matrix = epi.intrinsics_like(focal, image) + assert camera_matrix.shape == (B, 3, 3) + assert camera_matrix.device == image.device + assert camera_matrix.dtype == image.dtype + + +class TestScaleIntrinsics(BaseTester): + def test_smoke_float(self, device, dtype): + scale_factor: float = 1.0 + camera_matrix = torch.rand(1, 3, 3, device=device, dtype=dtype) + camera_matrix_scale = epi.scale_intrinsics(camera_matrix, scale_factor) + assert camera_matrix_scale.shape == (1, 3, 3) + + def test_smoke_tensor(self, device, dtype): + scale_factor = torch.tensor(1.0) + camera_matrix = torch.rand(1, 3, 3, device=device, dtype=dtype) + camera_matrix_scale = epi.scale_intrinsics(camera_matrix, scale_factor) + assert camera_matrix_scale.shape == (1, 3, 3) + + @pytest.mark.parametrize("batch_size", [1, 2, 4, 9]) + def test_shape(self, batch_size, device, dtype): + B: int = batch_size + scale_factor = torch.rand(B, device=device, dtype=dtype) + camera_matrix = torch.rand(B, 3, 3, device=device, dtype=dtype) + camera_matrix_scale = epi.scale_intrinsics(camera_matrix, scale_factor) + assert camera_matrix_scale.shape == (B, 3, 3) + + def test_scale_double(self, device, dtype): + scale_factor = torch.tensor(0.5) + camera_matrix = torch.tensor( + [[[100.0, 0.0, 50.0], [0.0, 100.0, 50.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype + ) + + camera_matrix_expected = torch.tensor( + [[[50.0, 0.0, 25.0], [0.0, 50.0, 25.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype + ) + + camera_matrix_scale = epi.scale_intrinsics(camera_matrix, scale_factor) + self.assert_close(camera_matrix_scale, camera_matrix_expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + scale_factor = torch.ones(1, device=device, dtype=torch.float64, requires_grad=True) + camera_matrix = torch.ones(1, 3, 3, device=device, dtype=torch.float64) + assert self.gradcheck(epi.scale_intrinsics, (camera_matrix, scale_factor), raise_exception=True, fast_mode=True) + + +class TestProjectionFromKRt(BaseTester): + def test_smoke(self, device, dtype): + K = torch.rand(1, 3, 3, device=device, dtype=dtype) + R = torch.rand(1, 3, 3, device=device, dtype=dtype) + t = torch.rand(1, 3, 1, device=device, dtype=dtype) + P = epi.projection_from_KRt(K, R, t) + assert P.shape == (1, 3, 4) + + @pytest.mark.parametrize("batch_size", [1, 2, 4]) + def test_shape(self, batch_size, device, dtype): + B: int = batch_size + K = torch.rand(B, 3, 3, device=device, dtype=dtype) + R = torch.rand(B, 3, 3, device=device, dtype=dtype) + t = torch.rand(B, 3, 1, device=device, dtype=dtype) + P = epi.projection_from_KRt(K, R, t) + assert P.shape == (B, 3, 4) + + def test_simple(self, device, dtype): + K = torch.tensor([[[10.0, 0.0, 30.0], [0.0, 20.0, 40.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + + R = torch.tensor([[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + + t = torch.tensor([[[1.0], [2.0], [3.0]]], device=device, dtype=dtype) + + P_expected = torch.tensor( + [[[10.0, 0.0, 30.0, 100.0], [0.0, 20.0, 40.0, 160.0], [0.0, 0.0, 1.0, 3.0]]], device=device, dtype=dtype + ) + + P_estimated = epi.projection_from_KRt(K, R, t) + self.assert_close(P_estimated, P_expected, atol=1e-4, rtol=1e-4) + + def test_krt_from_projection(self, device, dtype): + P = torch.tensor( + [[[10.0, 0.0, 30.0, 100.0], [0.0, 20.0, 40.0, 160.0], [0.0, 0.0, 1.0, 3.0]]], device=device, dtype=dtype + ) + + K_expected = torch.tensor([[[10.0, 0.0, 30.0], [0.0, 20.0, 40.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + + R_expected = torch.tensor([[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + + t_expected = torch.tensor([[[1.0], [2.0], [3.0]]], device=device, dtype=dtype) + + K_estimated, R_estimated, t_estimated = epi.KRt_from_projection(P) + self.assert_close(K_estimated, K_expected, atol=1e-4, rtol=1e-4) + self.assert_close(R_estimated, R_expected, atol=1e-4, rtol=1e-4) + self.assert_close(t_estimated, t_expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + K = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True) + R = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + t = torch.rand(1, 3, 1, device=device, dtype=torch.float64) + assert gradcheck(epi.projection_from_KRt, (K, R, t), raise_exception=True, fast_mode=True) + + +class TestProjectionsFromFundamental(BaseTester): + def test_smoke(self, device, dtype): + F_mat = torch.rand(1, 3, 3, device=device, dtype=dtype) + P = epi.projections_from_fundamental(F_mat) + assert P.shape == (1, 3, 4, 2) + + @pytest.mark.parametrize("batch_size", [1, 2, 4]) + def test_shape(self, batch_size, device, dtype): + B: int = batch_size + F_mat = torch.rand(B, 3, 3, device=device, dtype=dtype) + P = epi.projections_from_fundamental(F_mat) + assert P.shape == (B, 3, 4, 2) + + def test_gradcheck(self, device): + F_mat = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True) + assert self.gradcheck(epi.projections_from_fundamental, (F_mat,), raise_exception=True, fast_mode=True) + + +class TestKRtFromProjection(BaseTester): + def test_smoke(self, device, dtype): + P = torch.randn(1, 3, 4, device=device, dtype=dtype) + K, R, t = epi.KRt_from_projection(P) + assert K.shape == (1, 3, 3) + assert R.shape == (1, 3, 3) + assert t.shape == (1, 3, 1) + + @pytest.mark.parametrize("batch_size", [1, 2, 4]) + def test_shape(self, batch_size, device, dtype): + B: int = batch_size + P = torch.rand(B, 3, 4, device=device, dtype=dtype) + K, R, t = epi.KRt_from_projection(P) + + assert K.shape == (B, 3, 3) + assert R.shape == (B, 3, 3) + assert t.shape == (B, 3, 1) + + def test_simple(self, device, dtype): + P = torch.tensor( + [[[308.0, 139.0, 231.0, 84.0], [481.0, 161.0, 358.0, 341.0], [384.0, 387.0, 459.0, 102.0]]], + device=device, + dtype=dtype, + ) + + K_expected = torch.tensor( + [[[17.006138, 122.441254, 390.211426], [0.0, 228.743622, 577.167480], [0.0, 0.0, 712.675232]]], + device=device, + dtype=dtype, + ) + + R_expected = torch.tensor( + [[[0.396559, 0.511023, -0.762625], [0.743249, -0.666318, -0.060006], [0.538815, 0.543024, 0.644052]]], + device=device, + dtype=dtype, + ) + + t_expected = torch.tensor([[[-6.477699], [1.129624], [0.143123]]], device=device, dtype=dtype) + + K_estimated, R_estimated, t_estimated = epi.KRt_from_projection(P) + self.assert_close(K_estimated, K_expected, atol=1e-4, rtol=1e-4) + self.assert_close(R_estimated, R_expected, atol=1e-4, rtol=1e-4) + self.assert_close(t_estimated, t_expected, atol=1e-4, rtol=1e-4) + + def test_projection_from_krt(self, device, dtype): + K = torch.tensor( + [[[17.006138, 122.441254, 390.211426], [0.0, 228.743622, 577.167480], [0.0, 0.0, 712.675232]]], + device=device, + dtype=dtype, + ) + + R = torch.tensor( + [[[0.396559, 0.511023, -0.762625], [0.743249, -0.666318, -0.060006], [0.538815, 0.543024, 0.644052]]], + device=device, + dtype=dtype, + ) + + t = torch.tensor([[[-6.477699], [1.129624], [0.143123]]], device=device, dtype=dtype) + + P_expected = torch.tensor( + [[[308.0, 139.0, 231.0, 84.0], [481.0, 161.0, 358.0, 341.0], [384.0, 387.0, 459.0, 102.0]]], + device=device, + dtype=dtype, + ) + + P_estimated = epi.projection_from_KRt(K, R, t) + self.assert_close(P_estimated, P_expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + P_mat = torch.rand(1, 3, 4, device=device, dtype=torch.float64, requires_grad=True) + assert self.gradcheck(epi.KRt_from_projection, (P_mat,), raise_exception=True, fast_mode=True) diff --git a/tests/geometry/epipolar/test_triangulation.py b/tests/geometry/epipolar/test_triangulation.py new file mode 100644 index 0000000..7441815 --- /dev/null +++ b/tests/geometry/epipolar/test_triangulation.py @@ -0,0 +1,327 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import functools +from typing import Dict + +import pytest +import torch + +import kornia +import kornia.geometry.epipolar as epi + +from testing.base import BaseTester + +SOLVERS = ["svd", "eigh", "cofactor"] + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_scene(device, dtype, num_views: int = 2, num_points: int = 10): + """Return a consistent synthetic two-view scene in float64 then cast.""" + scene: Dict[str, torch.Tensor] = epi.generate_scene(num_views, num_points) + return {k: v.to(device=device, dtype=dtype) for k, v in scene.items()} + + +def _project(P: torch.Tensor, X: torch.Tensor) -> torch.Tensor: + """Project (B, N, 3) points through a (B, 3, 4) camera matrix → (B, N, 2).""" + B, N = X.shape[:2] + Xh = torch.cat([X, torch.ones(B, N, 1, device=X.device, dtype=X.dtype)], dim=-1) + px = (P @ Xh.mT).mT + return px[..., :2] / px[..., 2:3] + + +class TestTriangulation(BaseTester): + # ------------------------------------------------------------------ + # Smoke — verify all three solvers produce the right output shape + # ------------------------------------------------------------------ + + @pytest.mark.parametrize("solver", SOLVERS) + def test_smoke(self, solver, device, dtype): + P1 = torch.rand(1, 3, 4, device=device, dtype=dtype) + P2 = torch.rand(1, 3, 4, device=device, dtype=dtype) + points1 = torch.rand(1, 1, 2, device=device, dtype=dtype) + points2 = torch.rand(1, 1, 2, device=device, dtype=dtype) + pts3d = epi.triangulate_points(P1, P2, points1, points2, solver=solver) + assert pts3d.shape == (1, 1, 3) + + # ------------------------------------------------------------------ + # Shape / cardinality + # ------------------------------------------------------------------ + + @pytest.mark.parametrize("batch_size, num_points", [(1, 3), (2, 4), (3, 5)]) + @pytest.mark.parametrize("solver", SOLVERS) + def test_shape(self, batch_size, num_points, solver, device, dtype): + B, N = batch_size, num_points + P1 = torch.rand(B, 3, 4, device=device, dtype=dtype) + P2 = torch.rand(1, 3, 4, device=device, dtype=dtype) + points1 = torch.rand(1, N, 2, device=device, dtype=dtype) + points2 = torch.rand(B, N, 2, device=device, dtype=dtype) + pts3d = epi.triangulate_points(P1, P2, points1, points2, solver=solver) + assert pts3d.shape == (B, N, 3) + + # ------------------------------------------------------------------ + # Exception: unknown solver + # ------------------------------------------------------------------ + + def test_exception_unknown_solver(self, device, dtype): + P1 = torch.rand(1, 3, 4, device=device, dtype=dtype) + P2 = torch.rand(1, 3, 4, device=device, dtype=dtype) + pts = torch.rand(1, 4, 2, device=device, dtype=dtype) + with pytest.raises(NotImplementedError, match="Unknown solver"): + epi.triangulate_points(P1, P2, pts, pts, solver="unknown") + + # ------------------------------------------------------------------ + # Two-view accuracy on a noise-free synthetic scene + # ------------------------------------------------------------------ + + def test_two_view(self, device, dtype): + torch.manual_seed(0) + num_views: int = 2 + num_points: int = 10 + scene = _make_scene(device, dtype, num_views, num_points) + + P1 = scene["P"][0:1] + P2 = scene["P"][1:2] + x1 = scene["points2d"][0:1] + x2 = scene["points2d"][1:2] + + X = epi.triangulate_points(P1, P2, x1, x2) + x_reprojected = kornia.geometry.transform_points(scene["P"], X.expand(num_views, -1, -1)) + + atol = {torch.float16: 1e-2, torch.bfloat16: 0.25, torch.float32: 1e-4}.get(dtype, 1e-4) + self.assert_close(scene["points3d"], X, rtol=atol, atol=atol) + self.assert_close(scene["points2d"], x_reprojected, rtol=atol, atol=atol) + + # ------------------------------------------------------------------ + # All solvers produce collinear results for noise-free data + # ------------------------------------------------------------------ + + def test_solver_consistency(self, device, dtype): + """All three solvers should agree (up to sign) on noise-free data.""" + torch.manual_seed(0) + B, N = 2, 20 + + # Build a pair of cameras and project known 3-D points. + R2 = torch.tensor( + [[0.9998, -0.0175, 0.0], [0.0175, 0.9998, 0.0], [0.0, 0.0, 1.0]], + device=device, + dtype=dtype, + ) + t2 = torch.tensor([[-0.5], [0.0], [0.0]], device=device, dtype=dtype) + + P1 = torch.eye(3, 4, device=device, dtype=dtype).unsqueeze(0).expand(B, -1, -1) + P2 = torch.cat([R2, t2], dim=-1).unsqueeze(0).expand(B, -1, -1) + + X_true = torch.rand(B, N, 3, device=device, dtype=dtype) + torch.tensor( + [0.0, 0.0, 3.0], device=device, dtype=dtype + ) + + pts1 = _project(P1, X_true) + pts2 = _project(P2, X_true) + + results = {s: epi.triangulate_points(P1, P2, pts1, pts2, solver=s) for s in SOLVERS} + + ref = results["svd"] + for _name, pts in results.items(): + # Check that the recovered direction matches (cosine similarity ≈ 1). + cos = (pts * ref).sum(-1) / (pts.norm(dim=-1).clamp(min=1e-8) * ref.norm(dim=-1).clamp(min=1e-8)) + atol = {torch.float16: 1e-2, torch.bfloat16: 1e-2, torch.float32: 1e-3}.get(dtype, 1e-6) + self.assert_close(cos.abs(), torch.ones_like(cos), atol=atol, rtol=0.0) + + # ------------------------------------------------------------------ + # Cofactor sign-alignment regression + # ------------------------------------------------------------------ + + def test_cofactor_sign_alignment(self, device, dtype): + """Cofactor solver must not produce NaN/Inf due to sign cancellation. + + Before the sign-alignment fix, the two 3x4 sub-systems could yield + opposite-signed null vectors whose sum cancelled to ~0, producing NaN + after dehomogenisation. This test constructs a noise-free two-view + scene with a modest baseline, triangulates with both the cofactor and + SVD solvers, and checks that the cofactor output is finite and + directionally consistent with the SVD result. + """ + if dtype in (torch.float16, torch.bfloat16): + pytest.skip("cofactor sign test only runs for float32/float64") + + torch.manual_seed(3) + B, N = 1, 8 + + P1 = torch.eye(3, 4, device=device, dtype=dtype).unsqueeze(0).expand(B, -1, -1) + R2 = torch.tensor( + [[0.9998, -0.0175, 0.0], [0.0175, 0.9998, 0.0], [0.0, 0.0, 1.0]], + device=device, + dtype=dtype, + ) + t2 = torch.tensor([[-1.0], [0.0], [0.0]], device=device, dtype=dtype) + P2 = torch.cat([R2, t2], dim=-1).unsqueeze(0).expand(B, -1, -1) + + X_true = torch.rand(B, N, 3, device=device, dtype=dtype) + torch.tensor( + [0.0, 0.0, 3.0], device=device, dtype=dtype + ) + + pts1 = _project(P1, X_true) + pts2 = _project(P2, X_true) + + X_cofactor = epi.triangulate_points(P1, P2, pts1, pts2, solver="cofactor") + X_svd = epi.triangulate_points(P1, P2, pts1, pts2, solver="svd") + + # Output must be finite — NaN would indicate the pre-fix cancellation bug. + assert torch.isfinite(X_cofactor).all(), "cofactor solver produced non-finite values" + + # Numerical closeness to SVD (noise-free → both solvers recover the same 3-D point). + # This also catches incorrect scale/depth, unlike a direction-only check. + atol = 1e-3 if dtype == torch.float32 else 1e-6 + self.assert_close(X_cofactor, X_svd, atol=atol, rtol=0.0) + + # ------------------------------------------------------------------ + # Gradcheck — default solver + # ------------------------------------------------------------------ + + def test_gradcheck(self, device): + points1 = torch.rand(1, 8, 2, device=device, dtype=torch.float64, requires_grad=True) + points2 = torch.rand(1, 8, 2, device=device, dtype=torch.float64) + P1 = kornia.core.ops.eye_like(3, points1) + P1 = torch.nn.functional.pad(P1, [0, 1]) + P2 = kornia.core.ops.eye_like(3, points2) + P2 = torch.nn.functional.pad(P2, [0, 1]) + assert self.gradcheck(epi.triangulate_points, (P1, P2, points1, points2), raise_exception=True, fast_mode=True) + + @pytest.mark.parametrize("solver", SOLVERS) + def test_gradcheck_all_solvers(self, solver, device): + points1 = torch.rand(1, 4, 2, device=device, dtype=torch.float64, requires_grad=True) + points2 = torch.rand(1, 4, 2, device=device, dtype=torch.float64) + P1 = kornia.core.ops.eye_like(3, points1) + P1 = torch.nn.functional.pad(P1, [0, 1]) + P2 = kornia.core.ops.eye_like(3, points2) + P2 = torch.nn.functional.pad(P2, [0, 1]) + fn = functools.partial(epi.triangulate_points, solver=solver) + assert self.gradcheck(fn, (P1, P2, points1, points2), raise_exception=True, fast_mode=True) + + # ------------------------------------------------------------------ + # Noisy correspondences — compare against OpenCV DLT reference + # ------------------------------------------------------------------ + + @pytest.mark.parametrize("solver", ["svd", "eigh"]) + def test_noisy_correspondences_dlt(self, solver, device, dtype): + """Triangulation from noisy correspondences matches the numpy DLT reference. + + Two-view setup: camera 1 at [I|0], camera 2 with ~5.7-degree rotation and + 1-unit rightward translation, 8 points at depth 3-4, Gaussian noise sigma=0.05. + + Expected output was pre-computed once with a point-by-point numpy SVD + implementation identical to ``cv2.triangulatePoints`` (seed 7). + + # Snippet used to generate X_expected (requires numpy only): + # import numpy as np, torch + # torch.manual_seed(7) + # ... (see test body for the full scene construction) + # for i in range(N): + # A = np.array([pts1[0,i]*P1[2]-P1[0], pts1[1,i]*P1[2]-P1[1], + # pts2[0,i]*P2[2]-P2[0], pts2[1,i]*P2[2]-P2[1]]) + # _, _, V = np.linalg.svd(A) + # X_expected[i] = V[-1, :3] / V[-1, 3] + """ + if dtype not in (torch.float32, torch.float64): + pytest.skip("noisy-correspondence test only runs for float32/float64") + + # Hardcoded inputs (torch.manual_seed(7), noise sigma=0.05) + P1 = torch.tensor( + [[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0]], + dtype=dtype, + device=device, + ) + P2 = torch.tensor( + [ + [0.9950041770935059, 0.0, 0.0998334214091301, -1.0], + [0.0, 1.0, 0.0, 0.0], + [-0.0998334214091301, 0.0, 0.9950041770935059, 0.0], + ], + dtype=dtype, + device=device, + ) + pts1 = torch.tensor( + [ + [-0.024943894271020096, 0.023484865267389687], + [0.0284050326736405, 0.22463705371976522], + [0.05363684307356541, 0.1995784905718564], + [0.1202179211117687, -0.002082513366088421], + [-0.05603666748438946, -0.07320451037059378], + [0.2234642952593915, 0.03488261645314251], + [0.04132917198453165, 0.19063330422731778], + [0.07586561872234336, 0.12222851867867218], + ], + dtype=dtype, + device=device, + ) + pts2 = torch.tensor( + [ + [-0.2005761556307751, 0.08063633664743011], + [-0.05218266808824606, 0.09602012894912612], + [-0.14514520805453554, 0.11014017584343802], + [-0.14273667377657795, 0.051925094545445555], + [-0.2219682806714787, -0.09313464409688944], + [-0.061311792967568134, 0.04181839051261021], + [-0.1400088642080817, 0.12810430456996066], + [-0.16590968636404868, 0.08471239170067128], + ], + dtype=dtype, + device=device, + ) + # Expected: numpy DLT (same algorithm as cv2.triangulatePoints) + X_expected = torch.tensor( + [ + [-0.08818743120343768, 0.188334626330844, 3.6220817756133172], + [0.16144758240818913, 0.8807090389866558, 5.4844829932547015], + [0.18216324334542242, 0.517464929751309, 3.3477856439676588], + [0.3350949547897918, 0.06786420490996858, 2.770735605435314], + [-0.20989477003241103, -0.3123833872441908, 3.754403690255068], + [0.5852768065399436, 0.0989402354921456, 2.6188660940360844], + [0.1480216292032729, 0.5668535581804692, 3.5667031501090993], + [0.22379169128122922, 0.3033298174456883, 2.9457656267879586], + ], + dtype=dtype, + device=device, + ) + + X_kornia = epi.triangulate_points( + P1.unsqueeze(0), + P2.unsqueeze(0), + pts1.unsqueeze(0), + pts2.unsqueeze(0), + solver=solver, + ).squeeze(0) + + atol = 1e-4 if dtype == torch.float64 else 1e-3 + self.assert_close(X_kornia, X_expected, atol=atol, rtol=0.0) + + # ------------------------------------------------------------------ + # Module-level import check + # ------------------------------------------------------------------ + + def test_module(self, device, dtype): + assert hasattr(epi, "triangulate_points") + P1 = torch.rand(1, 3, 4, device=device, dtype=dtype) + P2 = torch.rand(1, 3, 4, device=device, dtype=dtype) + pts = torch.rand(1, 3, 2, device=device, dtype=dtype) + out = epi.triangulate_points(P1, P2, pts, pts) + assert out.shape == (1, 3, 3) diff --git a/tests/geometry/liegroup/test_se2.py b/tests/geometry/liegroup/test_se2.py new file mode 100644 index 0000000..a15a382 --- /dev/null +++ b/tests/geometry/liegroup/test_se2.py @@ -0,0 +1,287 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.geometry.liegroup import Se2, So2 +from kornia.geometry.vector import Vector2 + +from testing.base import BaseTester + + +class TestSe2(BaseTester): + def _make_rand_data(self, device, dtype, input_shape): + batch_size = input_shape[0] + shape = input_shape[1:] if batch_size is None else input_shape + return torch.rand(shape, device=device, dtype=dtype) + + def test_smoke(self, device, dtype): + z = torch.rand((2,), dtype=torch.cfloat, device=device) + so2 = So2(z) + t = torch.rand((1, 2), device=device, dtype=dtype) + s = Se2(so2, t) + assert isinstance(s, Se2) + assert isinstance(s.r, So2) + self.assert_close(s.r.z.data, z) + self.assert_close(s.t, t) + + @pytest.mark.parametrize("input_shape", [(1,), (2,), (5,), ()]) + def test_cardinality(self, device, dtype, input_shape): + t_input_shape = (*input_shape, 2) + z = torch.randn((*input_shape, 2), dtype=dtype, device=device) + t = torch.randn(t_input_shape, dtype=dtype, device=device) + s = Se2(So2(torch.complex(z[..., 0], z[..., 1])), t) + theta = torch.rand((*input_shape, 3), dtype=dtype, device=device) + assert s.so2.z.shape == input_shape + assert s.t.shape == t_input_shape + assert (s * s).so2.z.shape == input_shape + assert (s * s).t.shape == t_input_shape + assert s.exp(theta).so2.z.shape == input_shape + assert s.exp(theta).t.shape == t_input_shape + assert s.log().shape == (*input_shape, 3) + if not any(input_shape): + expected_hat_shape = (3, 3) + else: + expected_hat_shape = (input_shape[0], 3, 3) + assert s.hat(theta).shape == expected_hat_shape + assert s.inverse().so2.z.shape == input_shape + assert s.inverse().t.shape == t_input_shape + + @pytest.mark.parametrize("batch_size", (1, 2, 5)) + def test_exception(self, device, dtype, batch_size): + with pytest.raises(ValueError): + r = So2.random(batch_size) + t1 = torch.randn((batch_size, 1), dtype=dtype, device=device) + t2 = torch.randn((batch_size, 3), dtype=dtype, device=device) + Se2(r, t1) + Se2(r, t2) + with pytest.raises(ValueError): + theta = torch.rand((batch_size, 2), dtype=dtype, device=device) + Se2.exp(theta) + with pytest.raises(ValueError): + v = torch.rand((batch_size, 2), dtype=dtype, device=device) + Se2.hat(v) + with pytest.raises(ValueError): + omega = torch.rand((4, 4), dtype=dtype, device=device) + Se2.vee(omega) + with pytest.raises(TypeError): + Se2.identity(1, device, dtype) * [1.0, 2.0, 1.0] + with pytest.raises(ValueError): + theta = torch.rand((batch_size, 2), dtype=dtype, device=device) + Se2.hat(theta) + with pytest.raises(Exception): + Se2.identity(batch_size=0) + with pytest.raises(Exception): + Se2.random(batch_size=0) + with pytest.raises(Exception): + x = torch.rand(5, dtype=dtype, device=device) + y = torch.rand(3, dtype=dtype, device=device) + Se2.trans(x, y) + + # TODO: implement me + def test_gradcheck(self, device): + pass + + # TODO: implement me + def test_jit(self, device, dtype): + pass + + # TODO: implement me + def test_module(self, device, dtype): + pass + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_init(self, device, dtype, batch_size): + s1 = Se2.random(batch_size, device, dtype) + s2 = Se2(s1.r, s1.t) + assert isinstance(s2, Se2) + self.assert_close(s1.r.z, s2.r.z) + self.assert_close(s1.t, s2.t) + + @pytest.mark.parametrize("batch_size", (1, 2, 5)) + def test_getitem(self, device, dtype, batch_size): + z = torch.rand(batch_size, dtype=torch.cfloat, device=device) + t = torch.rand((batch_size, 2), device=device, dtype=dtype) + s = Se2(So2(z), t) + for i in range(batch_size): + s1 = s[i] + self.assert_close(s1.r.z, z[i]) + self.assert_close(s1.t, t[i]) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_mul(self, device, dtype, batch_size): + s1 = Se2.identity(batch_size, device, dtype) + s2 = Se2.random(batch_size, device, dtype) + s1_pose_s2 = s1 * s2 + s2_pose_s2 = s2 * s2.inverse() + zeros_vec = torch.zeros(2, device=device, dtype=dtype) + if batch_size is not None: + zeros_vec = zeros_vec.repeat(batch_size, 1) + so2_expected = So2.identity(batch_size, device, dtype) + self.assert_close(s1_pose_s2.r.z, s2.r.z) + self.assert_close(s1_pose_s2.t, s2.t) + self.assert_close(s2_pose_s2.r.z.real, so2_expected.z.real) + self.assert_close(s2_pose_s2.r.z.imag, so2_expected.z.imag) + self.assert_close(s2_pose_s2.t, zeros_vec) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_mul_vector(self, device, dtype, batch_size): + s1 = Se2.identity(batch_size, device, dtype) + if batch_size is None: + shape = () + else: + shape = (batch_size,) + s2 = Se2(So2.identity(batch_size, device, dtype), Vector2.random(shape, device, dtype)) + s1_pose_s2 = s1 * s2 + s2_pose_s2 = s2 * s2.inverse() + zeros_vec = torch.zeros(2, device=device, dtype=dtype) + if batch_size is not None: + zeros_vec = zeros_vec.repeat(batch_size, 1) + so2_expected = So2.identity(batch_size, device, dtype) + self.assert_close(s1_pose_s2.r.z, s2.r.z) + self.assert_close(s1_pose_s2.t, s2.t) + self.assert_close(s2_pose_s2.r.z.real, so2_expected.z.real) + self.assert_close(s2_pose_s2.r.z.imag, so2_expected.z.imag) + self.assert_close(s2_pose_s2.t, zeros_vec) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_exp(self, device, dtype, batch_size): + t = self._make_rand_data(device, dtype, (batch_size, 2)) + theta = torch.zeros(batch_size if batch_size is not None else (), device=device, dtype=dtype) + z = torch.zeros((batch_size, 2) if batch_size is not None else (2,), device=device, dtype=dtype) + s = Se2.exp(torch.cat((t, theta[..., None]), -1)) + self.assert_close(s.r.z, So2.exp(theta).z) + self.assert_close(s.t, z) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_log(self, device, dtype, batch_size): + t = self._make_rand_data(device, dtype, (batch_size, 2)) + s = Se2(So2.identity(batch_size, device, dtype), t) + s.log() + zero_vec = torch.zeros(3, device=device, dtype=dtype) + if batch_size is not None: + zero_vec = zero_vec.repeat(batch_size, 1) + self.assert_close(s.log(), zero_vec) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_exp_log(self, device, dtype, batch_size): + a = self._make_rand_data(device, dtype, (batch_size, 3)) + b = Se2.exp(a).log() + self.assert_close(b, a, low_tolerance=True) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_hat(self, device, dtype, batch_size): + v = self._make_rand_data(device, dtype, (batch_size, 2)) + theta = self._make_rand_data(device, dtype, (batch_size, 1)) + s_hat = Se2.hat(torch.cat((v, theta), -1)) + self.assert_close(v, s_hat[..., 2, 0:2]) + self.assert_close(s_hat[..., 0:2, 0:2].squeeze(), So2.hat(theta).squeeze()) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_vee(self, device, dtype, batch_size): + omega = self._make_rand_data(device, dtype, input_shape=(batch_size, 3, 3)) + v = Se2.vee(omega) + self.assert_close(torch.stack((v[..., 0], v[..., 1]), -1), omega[..., 2, :2]) + self.assert_close(v[..., -1], omega[..., 0, 1]) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_hat_vee(self, device, dtype, batch_size): + a = self._make_rand_data(device, dtype, (batch_size, 3)) + omega_hat = Se2.hat(a) + b = Se2.vee(omega_hat) + self.assert_close(b, a) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_identity(self, device, dtype, batch_size): + s = Se2.random(batch_size) + s_pose_s = s * Se2.identity(batch_size) + self.assert_close(s_pose_s.so2.z.real, s.so2.z.real) + self.assert_close(s_pose_s.so2.z.imag, s.so2.z.imag) + self.assert_close(s.t, s_pose_s.t) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_matrix(self, device, dtype, batch_size): + theta = self._make_rand_data(device, dtype, (batch_size,)) + t = self._make_rand_data(device, dtype, (batch_size, 2)) + s = So2.exp(theta) + p1 = s * t + p2 = s.matrix() @ t[..., None] + self.assert_close(p1, p2.squeeze(-1)) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_from_matrix(self, device, dtype, batch_size): + theta = self._make_rand_data(device, dtype, (batch_size,)) + t = self._make_rand_data(device, dtype, (batch_size, 2)) + s = So2.exp(theta) + p1 = s * t + RT = torch.eye(3, device=device, dtype=dtype) + if batch_size is not None: + RT = RT.repeat(batch_size, 1, 1) + RT[..., :2, :2] = s.matrix() + p2 = Se2.from_matrix(RT) * t + self.assert_close(p1, p2) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_inverse(self, device, batch_size, dtype): + s = Se2.random(batch_size, device, dtype) + s_in_in = s.inverse().inverse() + self.assert_close(s_in_in.so2.z.real, s.so2.z.real) + self.assert_close(s_in_in.so2.z.imag, s.so2.z.imag) + self.assert_close(s_in_in.t, s.t) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_random(self, device, dtype, batch_size): + s = So2.random(batch_size=batch_size, device=device, dtype=dtype) + t = self._make_rand_data(device, dtype, (batch_size, 2)) + se2 = Se2(s, t) + se2_in_se2 = se2.inverse() * se2 + i = Se2.identity(batch_size=batch_size, device=device, dtype=dtype) + self.assert_close(se2_in_se2.so2.z.real, i.so2.z.real) + self.assert_close(se2_in_se2.so2.z.imag, i.so2.z.imag) + self.assert_close(se2_in_se2.t, i.t) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_trans(self, device, dtype, batch_size): + trans = self._make_rand_data(device, dtype, (batch_size, 2)) + x, y = trans[..., 0], trans[..., 1] + se2 = Se2.trans(x, y) + self.assert_close(se2.t, trans) + self.assert_close(se2.so2.matrix(), So2.identity(batch_size, device, dtype).matrix()) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_trans_x(self, device, dtype, batch_size): + x = self._make_rand_data(device, dtype, (batch_size, 1)).squeeze(-1) + zs = torch.zeros_like(x) + se2 = Se2.trans_x(x) + self.assert_close(se2.t, torch.stack((x, zs), -1)) + self.assert_close(se2.so2.matrix(), So2.identity(batch_size, device, dtype).matrix()) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_trans_y(self, device, dtype, batch_size): + y = self._make_rand_data(device, dtype, (batch_size, 1)).squeeze(-1) + zs = torch.zeros_like(y) + se2 = Se2.trans_y(y) + self.assert_close(se2.t, torch.stack((zs, y), -1)) + self.assert_close(se2.so2.matrix(), So2.identity(batch_size, device, dtype).matrix()) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_adjoint(self, device, dtype, batch_size): + x = Se2.random(batch_size) + y = Se2.random(batch_size) + self.assert_close(x.inverse().adjoint(), x.adjoint().inverse()) + self.assert_close((x * y).adjoint(), x.adjoint() @ y.adjoint()) diff --git a/tests/geometry/liegroup/test_se3.py b/tests/geometry/liegroup/test_se3.py new file mode 100644 index 0000000..333e686 --- /dev/null +++ b/tests/geometry/liegroup/test_se3.py @@ -0,0 +1,308 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.geometry.conversions import euler_from_quaternion, rotation_matrix_to_quaternion +from kornia.geometry.liegroup import Se3, So3 +from kornia.geometry.quaternion import Quaternion +from kornia.geometry.vector import Vector3 + +from testing.base import BaseTester + + +class TestSe3(BaseTester): + def _make_rand_se3d(self, device, dtype, batch_size) -> Se3: + q = Quaternion.random(batch_size, device, dtype) + t = self._make_rand_data(device, dtype, batch_size, dims=3) + return Se3(q, t) + + def _make_rand_se3d_vec(self, device, dtype, batch_size) -> Se3: + q = Quaternion.random(batch_size, device, dtype) + if batch_size is None: + shape = () + else: + shape = (batch_size,) + t = Vector3.random(shape, device, dtype) + return Se3(So3(q), t) + + def _make_rand_data(self, device, dtype, batch_size, dims): + shape = [] if batch_size is None else [batch_size] + return torch.rand([*shape, dims], device=device, dtype=dtype) + + def test_smoke(self, device, dtype): + q = Quaternion.from_coeffs(1.0, 0.0, 0.0, 0.0) + q = q.to(device, dtype) + t = torch.rand(1, 3, device=device, dtype=dtype) + s = Se3(So3(q), t) + assert isinstance(s, Se3) + assert isinstance(s.r, So3) + self.assert_close(s.r.q.data, q.data) + self.assert_close(s.t, t) + + @pytest.mark.parametrize("batch_size", (1, 2, 5)) + def test_cardinality(self, device, dtype, batch_size): + se: Se3 = self._make_rand_se3d(device, dtype, batch_size) + assert se.r.q.shape[0] == batch_size + + # TODO: implement me + def test_exception(self, device, dtype): + pass + + # TODO: implement me + def test_gradcheck(self, device): + pass + + # TODO: implement me + def test_jit(self, device, dtype): + pass + + # TODO: implement me + def test_module(self, device, dtype): + pass + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_init(self, device, dtype, batch_size): + s1: Se3 = self._make_rand_se3d(device, dtype, batch_size) + s2 = Se3(s1.r, s1.t) + assert isinstance(s2, Se3) + self.assert_close(s1.r.q.data, s2.r.q.data) + self.assert_close(s1.t, s2.t) + + @pytest.mark.parametrize("batch_size", (1, 2, 5)) + def test_getitem(self, device, dtype, batch_size): + q = Quaternion.random(batch_size, device, dtype) + t = torch.rand(batch_size, 3, device=device, dtype=dtype) + s = Se3(q, t) + for i in range(batch_size): + s1 = s[i] + self.assert_close(s1.r.q.data, q.data[i]) + self.assert_close(s1.t, t[i]) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_mul(self, device, dtype, batch_size): + s1 = Se3.identity(batch_size, device, dtype) + s2: Se3 = self._make_rand_se3d(device, dtype, batch_size) + s1s2 = s1 * s2 + s2s2inv = s2 * s2.inverse() + zeros_vec = torch.zeros(3, device=device, dtype=dtype) + if batch_size is not None: + zeros_vec = zeros_vec.repeat(batch_size, 1) + so3_expected = So3.identity(batch_size, device, dtype) + self.assert_close(s1s2.r.q.data, s2.r.q.data) + self.assert_close(s1s2.t, s2.t) + self.assert_close(s2s2inv.r.q.data, so3_expected.q.data) + self.assert_close(s2s2inv.t, zeros_vec) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_mul_point(self, device, dtype, batch_size): + world_pose_s1: Se3 = self._make_rand_se3d(device, dtype, batch_size) + world_pose_s2: Se3 = self._make_rand_se3d(device, dtype, batch_size) + pt_in_world = self._make_rand_data(device, dtype, batch_size, dims=3) + s1_pose_s2: Se3 = world_pose_s1.inverse() * world_pose_s2 + pt_in_s1 = world_pose_s1.inverse() * pt_in_world + pt_in_s2 = world_pose_s2.inverse() * pt_in_world + pt_in_s1_in_s2 = s1_pose_s2.inverse() * pt_in_s1 + pt_in_s2_in_s1 = s1_pose_s2 * pt_in_s2 + self.assert_close(pt_in_s1, pt_in_s2_in_s1) + self.assert_close(pt_in_s2, pt_in_s1_in_s2) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_mul_vector(self, device, dtype, batch_size): + world_pose_s1: Se3 = self._make_rand_se3d(device, dtype, batch_size) + world_pose_s2: Se3 = self._make_rand_se3d_vec(device, dtype, batch_size) + if batch_size is None: + shape = () + else: + shape = (batch_size,) + pt_in_world = Vector3.random(shape, device, dtype) + s1_pose_s2: Se3 = world_pose_s1.inverse() * world_pose_s2 + pt_in_s1 = world_pose_s1.inverse() * pt_in_world + pt_in_s2 = world_pose_s2.inverse() * pt_in_world + pt_in_s1_in_s2 = s1_pose_s2.inverse() * pt_in_s1 + pt_in_s2_in_s1 = s1_pose_s2 * pt_in_s2 + s3 = Se3.identity(batch_size, device, dtype) + s4: Se3 = self._make_rand_se3d_vec(device, dtype, batch_size) + s3s4 = s3 * s4 + s4s4inv = s4 * s4.inverse() + zeros_vec = torch.zeros(3, device=device, dtype=dtype) + if batch_size is not None: + zeros_vec = zeros_vec.repeat(batch_size, 1) + so3_expected = So3.identity(batch_size, device, dtype) + self.assert_close(pt_in_s1, pt_in_s2_in_s1) + self.assert_close(pt_in_s2, pt_in_s1_in_s2) + self.assert_close(s3s4.r.q.data, s4.r.q.data) + self.assert_close(s3s4.t, s4.t) + self.assert_close(s4s4inv.r.q.data, so3_expected.q.data) + self.assert_close(s4s4inv.t, zeros_vec) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_exp(self, device, dtype, batch_size): + omega = torch.zeros(3, device=device, dtype=dtype) + t = torch.rand(3, device=device, dtype=dtype) + if batch_size is not None: + omega = omega.repeat(batch_size, 1) + t = t.repeat(batch_size, 1) + s = Se3.exp(torch.cat((t, omega), -1)) + quat_expected = Quaternion.identity(batch_size, device, dtype) + self.assert_close(s.r.q.data, quat_expected.data) + self.assert_close(s.t, t) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_log(self, device, dtype, batch_size): + q = Quaternion.identity(batch_size, device, dtype) + t = self._make_rand_data(device, dtype, batch_size, dims=3) + s = Se3(So3(q), t) + zero_vec = torch.zeros(3, device=device, dtype=dtype) + if batch_size is not None: + zero_vec = zero_vec.repeat(batch_size, 1) + self.assert_close(s.log(), torch.cat((t, zero_vec), -1)) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_exp_log(self, device, dtype, batch_size): + a = self._make_rand_data(device, dtype, batch_size, dims=6) + b = Se3.exp(a).log() + self.assert_close(b, a) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_hat_vee(self, device, dtype, batch_size): + a = self._make_rand_data(device, dtype, batch_size, dims=6) + omega_hat = Se3.hat(a) + b = Se3.vee(omega_hat) + self.assert_close(b, a) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_matrix(self, device, dtype, batch_size): + q = Quaternion.random(batch_size, device, dtype) + t = self._make_rand_data(device, dtype, batch_size, dims=3) + rot = So3(q) + s = Se3(rot, t) + rot_mat = s.matrix() + assert rot_mat.shape[-2:] == (4, 4) + if batch_size is not None: + assert rot_mat.shape[0] == batch_size + self.assert_close(rot_mat[..., 0:3, 0:3], rot.matrix()) + self.assert_close(rot_mat[..., 0:3, 3], t) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_from_matrix(self, device, dtype, batch_size): + matrix = torch.tensor( + ((1.0, 0.0, 0.0, 0.0), (0.0, 0.0, -1.0, 0.0), (0.0, 1.0, 0.0, 0.0), (0.0, 0.0, 0.0, 1.0)), + device=device, + dtype=dtype, + ) + if batch_size is not None: + matrix = matrix.repeat(batch_size, 1, 1) + s = Se3.from_matrix(matrix) + self.assert_close(s.r.matrix(), matrix[..., 0:3, 0:3]) + self.assert_close(s.t, matrix[..., 0:3, 3]) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_from_qxyz(self, device, dtype, batch_size): + qxyz = self._make_rand_data(device, dtype, batch_size, dims=7) + s = Se3.from_qxyz(qxyz) + self.assert_close(s.r.q.data, qxyz[..., :4].data) + self.assert_close(s.t, qxyz[..., 4:]) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_inverse(self, device, dtype, batch_size): + q = Quaternion.random(batch_size, device, dtype) + rot = So3(q) + t = self._make_rand_data(device, dtype, batch_size, dims=3) + sinv = Se3(rot, t).inverse() + self.assert_close(sinv.r.inverse().q.data, q.data) + self.assert_close(sinv.t, sinv.r * (-1 * t)) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_rot_x(self, device, dtype, batch_size): + x = self._make_rand_data(device, dtype, batch_size, dims=1).squeeze(-1) + se3 = Se3.rot_x(x) + quat = rotation_matrix_to_quaternion(se3.so3.matrix()) + quat = Quaternion(quat) + roll, _, _ = euler_from_quaternion(*quat.coeffs) + self.assert_close(x, roll) + self.assert_close(se3.t, torch.zeros_like(se3.t)) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_rot_y(self, device, dtype, batch_size): + y = self._make_rand_data(device, dtype, batch_size, dims=1).squeeze(-1) + se3 = Se3.rot_y(y) + quat = rotation_matrix_to_quaternion(se3.so3.matrix()) + quat = Quaternion(quat) + _, pitch, _ = euler_from_quaternion(*quat.coeffs) + self.assert_close(y, pitch) + self.assert_close(se3.t, torch.zeros_like(se3.t)) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_rot_z(self, device, dtype, batch_size): + z = self._make_rand_data(device, dtype, batch_size, dims=1).squeeze(-1) + se3 = Se3.rot_z(z) + quat = rotation_matrix_to_quaternion(se3.so3.matrix()) + quat = Quaternion(quat) + _, _, yaw = euler_from_quaternion(*quat.coeffs) + self.assert_close(z, yaw) + self.assert_close(se3.t, torch.zeros_like(se3.t)) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_trans(self, device, dtype, batch_size): + trans = self._make_rand_data(device, dtype, batch_size, dims=3) + x, y, z = trans[..., 0], trans[..., 1], trans[..., 2] + se3 = Se3.trans(x, y, z) + self.assert_close(se3.t, trans) + self.assert_close(se3.so3.matrix(), So3.identity(batch_size, device, dtype).matrix()) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_trans_x(self, device, dtype, batch_size): + x = self._make_rand_data(device, dtype, batch_size, dims=1).squeeze(-1) + zs = torch.zeros_like(x) + se3 = Se3.trans_x(x) + self.assert_close(se3.t, torch.stack((x, zs, zs), -1)) + self.assert_close(se3.so3.matrix(), So3.identity(batch_size, device, dtype).matrix()) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_trans_y(self, device, dtype, batch_size): + y = self._make_rand_data(device, dtype, batch_size, dims=1).squeeze(-1) + zs = torch.zeros_like(y) + se3 = Se3.trans_y(y) + self.assert_close(se3.t, torch.stack((zs, y, zs), -1)) + self.assert_close(se3.so3.matrix(), So3.identity(batch_size, device, dtype).matrix()) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_trans_z(self, device, dtype, batch_size): + z = self._make_rand_data(device, dtype, batch_size, dims=1).squeeze(-1) + zs = torch.zeros_like(z) + se3 = Se3.trans_z(z) + self.assert_close(se3.t, torch.stack((zs, zs, z), -1)) + self.assert_close(se3.so3.matrix(), So3.identity(batch_size, device, dtype).matrix()) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_adjoint(self, device, dtype, batch_size): + x_data = self._make_rand_data(device, dtype, batch_size, dims=6) + y_data = self._make_rand_data(device, dtype, batch_size, dims=6) + x = Se3.exp(x_data) + y = Se3.exp(y_data) + self.assert_close(x.inverse().adjoint(), x.adjoint().inverse()) + self.assert_close((x * y).adjoint(), x.adjoint() @ y.adjoint()) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_random(self, device, dtype, batch_size): + s = Se3.random(batch_size=batch_size, device=device, dtype=dtype) + s_in_s = s.inverse() * s + i = Se3.identity(batch_size=batch_size, device=device, dtype=dtype) + self.assert_close(s_in_s.so3.q.data, i.so3.q.data) + self.assert_close(s_in_s.t, i.t) diff --git a/tests/geometry/liegroup/test_so2.py b/tests/geometry/liegroup/test_so2.py new file mode 100644 index 0000000..2fcb869 --- /dev/null +++ b/tests/geometry/liegroup/test_so2.py @@ -0,0 +1,241 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.geometry.liegroup import So2 +from kornia.geometry.vector import Vector2 + +from testing.base import BaseTester + + +class TestSo2(BaseTester): + def _make_rand_data(self, device, dtype, input_shape): + batch_size = input_shape[0] + shape = input_shape[1:] if batch_size is None else input_shape + return torch.rand(shape, device=device, dtype=dtype) + + @pytest.mark.parametrize("cdtype", (torch.cfloat, torch.cdouble)) + def test_smoke(self, device, cdtype): + z = torch.randn(2, 1, dtype=cdtype, device=device) + s = So2(z) + assert isinstance(s, So2) + self.assert_close(s.z.data, z.data) + + @pytest.mark.parametrize("input_shape", [(1,), (2,), (5,), ()]) + @pytest.mark.parametrize("cdtype", (torch.cfloat, torch.cdouble)) + def test_cardinality(self, device, dtype, input_shape, cdtype): + z = torch.randn(input_shape, dtype=cdtype, device=device) + s = So2(z) + theta = torch.rand(input_shape, dtype=dtype, device=device) + assert s.z.shape == input_shape + assert (s * s).z.shape == input_shape + assert s.exp(theta).z.shape == input_shape + assert s.log().shape == input_shape + if not any(input_shape): + expected_hat_shape = (2, 2) + else: + expected_hat_shape = (input_shape[0], 2, 2) + assert s.hat(theta).shape == expected_hat_shape + assert s.inverse().z.shape == input_shape + + @pytest.mark.parametrize("input_shape", [(1, 2, 2), (2, 2, 2), (5, 2, 2), (2, 2)]) + def test_matrix_cardinality(self, device, dtype, input_shape): + matrix = torch.rand(input_shape, dtype=dtype, device=device) + matrix[..., 0, 1] = -matrix[..., 1, 0] + matrix[..., 1, 1] = matrix[..., 0, 0] + s = So2.from_matrix(matrix) + assert s.matrix().shape == input_shape + + @pytest.mark.parametrize("batch_size", (1, 2, 5)) + @pytest.mark.parametrize("cdtype", (torch.cfloat, torch.cdouble)) + def test_exception(self, batch_size, device, dtype, cdtype): + with pytest.raises(ValueError): + z = torch.randn(batch_size, 2, dtype=cdtype, device=device) + assert So2(z) + with pytest.raises(TypeError): + assert So2.identity(1, device, dtype) * [1.0, 2.0, 1.0] + with pytest.raises(ValueError): + theta = torch.rand((2, 2), dtype=dtype, device=device) + assert So2.exp(theta) + with pytest.raises(ValueError): + theta = torch.rand((2, 2), dtype=dtype, device=device) + assert So2.hat(theta) + with pytest.raises(ValueError): + m = torch.rand((2, 2, 1), dtype=dtype, device=device) + assert So2.from_matrix(m) + with pytest.raises(ValueError): + m = torch.rand((2, 2, 1), dtype=dtype, device=device) + assert So2.from_matrix(m) + with pytest.raises(Exception): + assert So2.identity(batch_size=0) + + # TODO: implement me + def test_gradcheck(self, device): + pass + + # TODO: implement me + def test_jit(self, device, dtype): + pass + + # TODO: implement me + def test_module(self, device, dtype): + pass + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + @pytest.mark.parametrize("cdtype", (torch.cfloat, torch.cdouble)) + def test_init(self, device, dtype, batch_size, cdtype): + z1 = self._make_rand_data(device, cdtype, (batch_size,)) + z2 = self._make_rand_data(device, cdtype, (batch_size, 1)) + z3_real = self._make_rand_data(device, dtype, (batch_size,)) + z3_imag = self._make_rand_data(device, dtype, (batch_size,)) + z3 = torch.complex(z3_real, z3_imag) + s1 = So2(z1) + s2 = So2(s1.z) + assert isinstance(s2, So2) + self.assert_close(s1.z, s2.z) + self.assert_close(So2(z1).z, z1) + self.assert_close(So2(z2).z, z2) + self.assert_close(So2(z3).z, z3) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + @pytest.mark.parametrize("cdtype", (torch.cfloat, torch.cdouble)) + def test_getitem(self, device, batch_size, cdtype): + z = self._make_rand_data(device, cdtype, (batch_size,)) + s = So2(z) + n = 1 if batch_size is None else batch_size + for i in range(n): + if batch_size is None: + expected = s.z + actual = z + else: + expected = s[i].z.data.squeeze() + actual = z[i] + self.assert_close(expected, actual) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_mul(self, device, dtype, batch_size): + s1 = So2.identity(batch_size, device, dtype) + z = self._make_rand_data(device, dtype, (batch_size, 2)) + s2 = So2(torch.complex(z[..., 0], z[..., 1])) + t1 = self._make_rand_data(device, dtype, (batch_size, 2)) + t2 = self._make_rand_data(device, dtype, (2,)) + s1_pose_s2 = s1 * s2 + s2_pose_s2 = s2 * s2.inverse() + self.assert_close(s1_pose_s2.z.real, s2.z.real) + self.assert_close(s1_pose_s2.z.imag, s2.z.imag) + self.assert_close(s2_pose_s2.z.real, s1.z.real) + self.assert_close(s2_pose_s2.z.imag, s1.z.imag) + self.assert_close((s1 * t1), t1) + self.assert_close((So2.identity(device=device, dtype=dtype) * t2), t2) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_mul_vector(self, device, dtype, batch_size): + s1 = So2.identity(batch_size, device, dtype) + if batch_size is None: + shape = () + else: + shape = (batch_size,) + t1 = Vector2.random(shape, device, dtype) + t2 = Vector2.random(shape, device, dtype) + self.assert_close((s1 * t1), t1) + self.assert_close((So2.identity(device=device, dtype=dtype) * t2), t2) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_exp(self, device, dtype, batch_size): + theta = self._make_rand_data(device, dtype, (batch_size, 1)) + s = So2.exp(theta) + self.assert_close(s.z.real, theta.cos()) + self.assert_close(s.z.imag, theta.sin()) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + @pytest.mark.parametrize("cdtype", (torch.cfloat, torch.cdouble)) + def test_log(self, device, batch_size, cdtype): + z = self._make_rand_data(device, cdtype, (batch_size,)) + t = So2(z).log() + self.assert_close(t, z.imag.atan2(z.real)) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_exp_log(self, device, dtype, batch_size): + theta = self._make_rand_data(device, dtype, (batch_size, 1)) + self.assert_close(So2.exp(theta).log(), theta) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_hat(self, device, dtype, batch_size): + theta = self._make_rand_data(device, dtype, (batch_size,)) + m = So2.hat(theta) + o = torch.ones((2, 1), device=device, dtype=dtype) + self.assert_close((m @ o).reshape(-1, 2, 1), theta.reshape(-1, 1, 1).repeat(1, 2, 1)) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_vee(self, device, dtype, batch_size): + omega = self._make_rand_data(device, dtype, (batch_size, 2, 2)) + theta = So2.vee(omega) + self.assert_close(omega[..., 0, 1], theta) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_hat_vee(self, device, dtype, batch_size): + a = self._make_rand_data(device, dtype, (batch_size,)) + omega = So2.hat(a) + b = So2.vee(omega) + self.assert_close(b, a) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_matrix(self, device, dtype, batch_size): + theta = self._make_rand_data(device, dtype, (batch_size,)) + t = self._make_rand_data(device, dtype, (batch_size, 2)) + s = So2.exp(theta) + p1 = s * t + p2 = s.matrix() @ t[..., None] + self.assert_close(p1, p2.squeeze(-1)) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_from_matrix(self, device, dtype, batch_size): + matrix = torch.eye(2, device=device, dtype=dtype) + if batch_size is not None: + matrix = matrix.repeat(batch_size, 1, 1) + one = torch.ones((batch_size,), device=device, dtype=dtype) + zero = torch.zeros((batch_size,), device=device, dtype=dtype) + else: + one = torch.tensor(1, device=device, dtype=dtype) + zero = torch.tensor(0, device=device, dtype=dtype) + s = So2.from_matrix(matrix) + self.assert_close(s.z.real, one) + self.assert_close(s.z.imag, zero) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + @pytest.mark.parametrize("cdtype", (torch.cfloat, torch.cdouble)) + def test_inverse(self, device, batch_size, cdtype): + z = self._make_rand_data(device, cdtype, (batch_size,)) + s = So2(z) + s_in_in = s.inverse().inverse() + self.assert_close(s_in_in.z.real, z.real) + self.assert_close(s_in_in.z.imag, z.imag) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_random(self, device, dtype, batch_size): + s = So2.random(batch_size=batch_size, device=device, dtype=dtype) + s_in_s = s.inverse() * s + i = So2.identity(batch_size=batch_size, device=device, dtype=dtype) + self.assert_close(s_in_s.z.real, i.z.real) + self.assert_close(s_in_s.z.imag, i.z.imag) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_adjoint(self, device, dtype, batch_size): + s = So2.identity(batch_size, device=device, dtype=dtype) + self.assert_close(s.matrix(), s.adjoint()) diff --git a/tests/geometry/liegroup/test_so3.py b/tests/geometry/liegroup/test_so3.py new file mode 100644 index 0000000..5f204aa --- /dev/null +++ b/tests/geometry/liegroup/test_so3.py @@ -0,0 +1,286 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.geometry.conversions import euler_from_quaternion +from kornia.geometry.liegroup import So3 +from kornia.geometry.quaternion import Quaternion +from kornia.geometry.vector import Vector3 + +from testing.base import BaseTester + + +class TestSo3(BaseTester): + def _make_rand_data(self, device, dtype, batch_size, dims): + shape = [] if batch_size is None else [batch_size] + return torch.rand([*shape, dims], device=device, dtype=dtype) + + def test_smoke(self, device, dtype): + q = Quaternion.from_coeffs(1.0, 0.0, 0.0, 0.0) + q = q.to(device, dtype) + s = So3(q) + assert isinstance(s, So3) + self.assert_close(s.q.data, q.data) + + # TODO: implement me + def test_cardinality(self, device, dtype): + pass + + # TODO: implement me + def test_exception(self, device, dtype): + pass + + # TODO: implement me + def test_gradcheck(self, device): + pass + + # TODO: implement me + def test_jit(self, device, dtype): + pass + + # TODO: implement me + def test_module(self, device, dtype): + pass + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_init(self, device, dtype, batch_size): + q = Quaternion.identity(batch_size, device, dtype) + s1 = So3(q) + s2 = So3(s1.q) + assert isinstance(s2, So3) + self.assert_close(s1.q.data, s2.q.data) + + @pytest.mark.parametrize("batch_size", (1, 2, 5)) + def test_getitem(self, device, dtype, batch_size): + q = Quaternion.random(batch_size, device, dtype) + s = So3(q) + for i in range(batch_size): + s1 = s[i] + self.assert_close(s1.q.data, q.data[i]) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_mul(self, device, dtype, batch_size): + q1 = Quaternion.identity(batch_size, device, dtype) + q2 = Quaternion.random(batch_size, device, dtype) + t = self._make_rand_data(device, dtype, batch_size, dims=3) + s1 = So3(q1) + s2 = So3(q2) + self.assert_close((s1 * s2).q.data, s2.q.data) + self.assert_close((s2 * s2.inverse()).q.data, s1.q.data) + self.assert_close((s1 * t), t) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_mul_vec(self, device, dtype, batch_size): + q1 = Quaternion.identity(batch_size, device, dtype) + q2 = Quaternion.random(batch_size, device, dtype) + if batch_size is None: + shape = () + else: + shape = (batch_size,) + t = Vector3.random(shape, device, dtype) + s1 = So3(q1) + s2 = So3(q2) + self.assert_close((s1 * s2).q.data, s2.q.data) + self.assert_close((s2 * s2.inverse()).q.data, s1.q.data) + self.assert_close((s1 * t), t) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_unit_norm(self, device, dtype, batch_size): + q1 = Quaternion.random(batch_size, device, dtype) + q2 = Quaternion.random(batch_size, device, dtype) + s1 = So3(q1) + s2 = So3(q2) + s3 = s1 * s2 + s4 = s1.inverse() + s5 = s2.inverse() + s6 = s3.inverse() + + ones_vec = torch.tensor(1.0, device=device, dtype=dtype) + if batch_size is None: + self.assert_close(s1.q.norm(), ones_vec) + return + + for i in range(batch_size): + self.assert_close(s1[i].q.norm(), ones_vec) + self.assert_close(s2[i].q.norm(), ones_vec) + self.assert_close(s3[i].q.norm(), ones_vec) + self.assert_close(s4[i].q.norm(), ones_vec) + self.assert_close(s5[i].q.norm(), ones_vec) + self.assert_close(s6[i].q.norm(), ones_vec) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_exp(self, device, dtype, batch_size): + q = Quaternion.identity(batch_size, device, dtype) + s = So3(q) + zero_vec = 0 * self._make_rand_data(device, dtype, batch_size, dims=3) + self.assert_close(s.exp(zero_vec).q.data, q.data) # exp of zero vec is identity + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_log(self, device, dtype, batch_size): + q = Quaternion.identity(batch_size, device, dtype) + s = So3(q) + zero_vec = 0 * self._make_rand_data(device, dtype, batch_size, dims=3) + self.assert_close(s.log(), zero_vec) # log of identity quat is zero vec + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_exp_log(self, device, dtype, batch_size): + q = Quaternion.random(batch_size, device, dtype) + s = So3(q) + a = self._make_rand_data(device, dtype, batch_size, dims=3) + b = s.exp(a).log() + self.assert_close(b, a) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_hat(self, device, dtype, batch_size): + v = torch.tensor([1, 2, 3], device=device, dtype=dtype) + expected = v + if batch_size is not None: + v = v.repeat(batch_size, 1) + hat = So3.hat(v) + if batch_size is None: + hat = hat[None] + self.assert_close(hat.unique()[-3:], expected) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_vee(self, device, dtype, batch_size): + omega = torch.tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]], device=device, dtype=dtype) + expected = torch.tensor([8, 3, 4], device=device, dtype=dtype) + if batch_size is not None: + omega = omega.repeat(batch_size, 1, 1) + expected = expected.repeat(batch_size, 1) + self.assert_close(So3.vee(omega), expected) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_hat_vee(self, device, dtype, batch_size): + a = self._make_rand_data(device, dtype, batch_size, dims=3) + omega = So3.hat(a) + b = So3.vee(omega) + self.assert_close(b, a) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_matrix(self, device, dtype, batch_size): + q = Quaternion.random(batch_size, device, dtype) + r = So3(q).matrix() + if batch_size is None: + q = Quaternion(q.data[None]) + r = r[None] + for i in range(r.shape[0]): + q1 = q[i] + r1 = r[i, :, :] + pvec = torch.rand(3, device=device, dtype=dtype) + pquat = Quaternion(torch.cat([torch.tensor([0], device=device, dtype=dtype), pvec])) + qp_ = q1 * pquat * q1.inv() + rp_ = torch.matmul(r1, pvec) + self.assert_close(rp_, qp_.vec) # p_ = R*p = q*p*q_inv + self.assert_close(rp_.norm(), pvec.norm()) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_from_wxyz(self, device, dtype, batch_size): + wxyz = self._make_rand_data(device, dtype, batch_size, dims=4) + s = So3.from_wxyz(wxyz) + self.assert_close(s.q.data, wxyz) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_ortho(self, device, dtype, batch_size): + q = Quaternion.random(batch_size, device, dtype) + b_R_a = So3(q).matrix() + a_R_b = So3(q).inverse().matrix() + a_R_a = (So3(q) * So3(q).inverse()).matrix() + + eye_mat = torch.eye(3, device=device, dtype=dtype) + if batch_size is None: + eye_mat = eye_mat[None] + a_R_a = a_R_a[None] + a_R_b = a_R_b[None] + b_R_a = b_R_a[None] + if batch_size is not None: + eye_mat = eye_mat.repeat(batch_size, 1, 1) + + self.assert_close(a_R_a, eye_mat) + + for i in range(eye_mat.shape[0]): + self.assert_close(a_R_a[i, :, :], eye_mat[i]) + self.assert_close(a_R_b[i, :, :] @ b_R_a[i, :, :], eye_mat[i]) + self.assert_close(b_R_a[i, :, :] @ a_R_b[i, :, :], eye_mat[i]) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_inverse(self, device, dtype, batch_size): + q = Quaternion.random(batch_size, device, dtype) + self.assert_close(So3(q).inverse().inverse().q.data, q.data) + self.assert_close(So3(q).inverse().inverse().matrix(), So3(q).matrix()) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_rot_x(self, device, dtype, batch_size): + x = self._make_rand_data(device, dtype, batch_size, dims=1).squeeze(-1) + so3 = So3.rot_x(x) + roll, _, _ = euler_from_quaternion(*so3.q.coeffs) + self.assert_close(x, roll) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_rot_y(self, device, dtype, batch_size): + y = self._make_rand_data(device, dtype, batch_size, dims=1).squeeze(-1) + so3 = So3.rot_y(y) + _, pitch, _ = euler_from_quaternion(*so3.q.coeffs) + self.assert_close(y, pitch) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_rot_z(self, device, dtype, batch_size): + z = self._make_rand_data(device, dtype, batch_size, dims=1).squeeze(-1) + so3 = So3.rot_z(z) + _, _, yaw = euler_from_quaternion(*so3.q.coeffs) + self.assert_close(z, yaw) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_adjoint(self, device, dtype, batch_size): + q1 = Quaternion.random(batch_size, device, dtype) + q2 = Quaternion.random(batch_size, device, dtype) + x = So3(q1) + y = So3(q2) + self.assert_close(x.inverse().adjoint(), x.adjoint().inverse()) + self.assert_close((x * y).adjoint(), x.adjoint() @ y.adjoint()) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_random(self, device, dtype, batch_size): + s = So3.random(batch_size=batch_size, device=device, dtype=dtype) + s_in_s = s.inverse() * s + i = So3.identity(batch_size=batch_size, device=device, dtype=dtype) + self.assert_close(s_in_s.q.data, i.q.data) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_right_jacobian(self, device, dtype, batch_size): + vec = self._make_rand_data(device, dtype, batch_size, dims=3) + Jr = So3.right_jacobian(vec) + I = torch.eye(3, device=device, dtype=dtype).expand_as(Jr) # noqa: E741 + self.assert_close(vec[..., None], Jr @ vec[..., None]) + self.assert_close(Jr.transpose(-1, -2) @ Jr, I, atol=0.1, rtol=0.1) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_left_jacobian(self, device, dtype, batch_size): + vec = self._make_rand_data(device, dtype, batch_size, dims=3) + Jl = So3.left_jacobian(vec) + I = torch.eye(3, device=device, dtype=dtype).expand_as(Jl) # noqa: E741 + self.assert_close(vec[..., None], Jl @ vec[..., None]) + self.assert_close(Jl.transpose(-1, -2) @ Jl, I, atol=0.1, rtol=0.1) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_right_left_jacobian(self, device, dtype, batch_size): + vec = self._make_rand_data(device, dtype, batch_size, dims=3) + Jr = So3.right_jacobian(vec) + Jl = So3.left_jacobian(vec) + self.assert_close(Jl, Jr.transpose(-1, -2)) diff --git a/tests/geometry/solvers/test_homogeneous.py b/tests/geometry/solvers/test_homogeneous.py new file mode 100644 index 0000000..5eb94b3 --- /dev/null +++ b/tests/geometry/solvers/test_homogeneous.py @@ -0,0 +1,248 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.geometry import solvers + +from testing.base import BaseTester + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_rank3_matrix(null_vec: torch.Tensor, device, dtype) -> torch.Tensor: + """Build a 3x4 matrix whose null space is spanned by *null_vec*. + + Strategy: start from the 4x4 identity, set the last row to *null_vec* so + it becomes linearly dependent, then take the first three rows scaled by + small random factors. The result has rank 3 and its null vector is + proportional to *null_vec*. + """ + # Build the 4x4 matrix whose columns span the orthogonal complement of null_vec. + # We use a Gram-Schmidt orthonormalisation relative to null_vec. + n = null_vec.to(device=device, dtype=torch.float64) + n = n / n.norm() + + # Start from the standard basis and remove the component along n. + basis = torch.eye(4, device=device, dtype=torch.float64) + vecs = [] + for i in range(4): + v = basis[i] - (basis[i] @ n) * n + if v.norm() > 1e-6: + # Orthogonalise against already accepted vectors. + for u in vecs: + v = v - (v @ u) * u + if v.norm() > 1e-6: + vecs.append(v / v.norm()) + if len(vecs) == 3: + break + + A = torch.stack(vecs, dim=0).to(dtype) # (3, 4) + return A.unsqueeze(0) # (1, 3, 4) + + +class TestNullVector3x4(BaseTester): + # ------------------------------------------------------------------ + # Smoke + # ------------------------------------------------------------------ + + def test_smoke(self, device, dtype): + A = torch.rand(1, 3, 4, device=device, dtype=dtype) + v = solvers.null_vector_3x4(A) + assert v.shape == (1, 4) + + # ------------------------------------------------------------------ + # Cardinality / shape + # ------------------------------------------------------------------ + + @pytest.mark.parametrize("batch", [1, 2, 4, 8]) + def test_cardinality_batch(self, batch, device, dtype): + A = torch.rand(batch, 3, 4, device=device, dtype=dtype) + v = solvers.null_vector_3x4(A) + assert v.shape == (batch, 4) + + @pytest.mark.parametrize("extra", [(2, 3), (5,)]) + def test_cardinality_leading_dims(self, extra, device, dtype): + A = torch.rand(*extra, 3, 4, device=device, dtype=dtype) + v = solvers.null_vector_3x4(A) + assert v.shape == (*extra, 4) + + def test_cardinality_unbatched(self, device, dtype): + A = torch.rand(3, 4, device=device, dtype=dtype) + v = solvers.null_vector_3x4(A) + assert v.shape == (4,) + + # ------------------------------------------------------------------ + # Exception / input validation + # ------------------------------------------------------------------ + + def test_exception_wrong_rows(self, device, dtype): + with pytest.raises(Exception): + solvers.null_vector_3x4(torch.rand(1, 4, 4, device=device, dtype=dtype)) + + def test_exception_wrong_cols(self, device, dtype): + with pytest.raises(Exception): + solvers.null_vector_3x4(torch.rand(1, 3, 3, device=device, dtype=dtype)) + + def test_exception_1d(self, device, dtype): + with pytest.raises(Exception): + solvers.null_vector_3x4(torch.rand(4, device=device, dtype=dtype)) + + def test_exception_not_tensor(self, device, dtype): + with pytest.raises(Exception): + solvers.null_vector_3x4([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0]]) + + # ------------------------------------------------------------------ + # Known null vectors (hand-computed) + # ------------------------------------------------------------------ + + @pytest.mark.parametrize( + "A_data, expected_null", + [ + # Standard basis: last column is free → null vector is e4 = [0,0,0,1]. + ( + [[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0]], + [0.0, 0.0, 0.0, -1.0], # sign comes from cofactor; direction only + ), + # A = [[1,0,0,1],[0,1,0,1],[0,0,1,1]] → null = [1,1,1,-1]. + ( + [[1.0, 0.0, 0.0, 1.0], [0.0, 1.0, 0.0, 1.0], [0.0, 0.0, 1.0, 1.0]], + [1.0, 1.0, 1.0, -1.0], + ), + # A = [[2,0,0,0],[0,3,0,0],[0,0,5,0]] → null = [0,0,0,1] (up to sign/scale). + ( + [[2.0, 0.0, 0.0, 0.0], [0.0, 3.0, 0.0, 0.0], [0.0, 0.0, 5.0, 0.0]], + [0.0, 0.0, 0.0, -30.0], + ), + ], + ) + def test_known_null_vectors(self, A_data, expected_null, device, dtype): + A = torch.tensor([A_data], device=device, dtype=dtype) # (1, 3, 4) + expected = torch.tensor([expected_null], device=device, dtype=dtype) # (1, 4) + v = solvers.null_vector_3x4(A) + # Compare direction (ratio of corresponding components, ignoring global sign). + # Normalise both and check |cos θ| ≈ 1. + v_n = v / v.norm(dim=-1, keepdim=True).clamp(min=1e-8) + e_n = expected / expected.norm(dim=-1, keepdim=True).clamp(min=1e-8) + cos_theta = (v_n * e_n).sum(dim=-1).abs() + self.assert_close(cos_theta, torch.ones_like(cos_theta), atol=1e-4, rtol=1e-4) + + # ------------------------------------------------------------------ + # Residual: A @ v must be (close to) zero + # ------------------------------------------------------------------ + + @pytest.mark.parametrize( + "null_vec", + [ + [1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0], + [1.0, 1.0, 1.0, 1.0], + [1.0, -2.0, 3.0, -4.0], + [0.5, 0.5, -0.5, 0.5], + ], + ) + def test_residual_near_zero(self, null_vec, device, dtype): + """A @ null_vector_3x4(A) must be (close to) zero for any rank-3 A.""" + nv = torch.tensor(null_vec, dtype=torch.float64) + A = _make_rank3_matrix(nv, device=device, dtype=dtype) # (1, 3, 4) + v = solvers.null_vector_3x4(A) # (1, 4) + residual = (A @ v.unsqueeze(-1)).squeeze(-1) # (1, 3) + atol = {torch.float16: 1e-1, torch.bfloat16: 1e-1, torch.float32: 1e-4}.get(dtype, 1e-8) + self.assert_close(residual, torch.zeros_like(residual), atol=atol, rtol=0.0) + + def test_residual_random_batch(self, device, dtype): + """Batch of random rank-3 matrices: residual should vanish.""" + torch.manual_seed(42) + B = 16 + null_vecs = torch.randn(B, 4) + As = torch.cat([_make_rank3_matrix(null_vecs[i], device=device, dtype=dtype) for i in range(B)], dim=0) + vs = solvers.null_vector_3x4(As) + residuals = (As @ vs.unsqueeze(-1)).squeeze(-1) # (B, 3) + atol = {torch.float16: 5e-1, torch.bfloat16: 5e-1, torch.float32: 1e-3}.get(dtype, 1e-7) + self.assert_close(residuals, torch.zeros_like(residuals), atol=atol, rtol=0.0) + + # ------------------------------------------------------------------ + # Direction recovery: null vector matches the known one up to scale + # ------------------------------------------------------------------ + + @pytest.mark.parametrize("seed", [0, 7, 42]) + def test_direction_recovery(self, seed, device, dtype): + """The returned vector must be collinear with the true null vector.""" + torch.manual_seed(seed) + null_vec = torch.randn(4) + A = _make_rank3_matrix(null_vec, device=device, dtype=dtype) + v = solvers.null_vector_3x4(A).squeeze(0) # (4,) + + v_n = v / v.norm().clamp(min=1e-8) + nv_t = null_vec.to(device=device, dtype=dtype) + nv_n = nv_t / nv_t.norm().clamp(min=1e-8) + + cos_theta = (v_n * nv_n).sum().abs() + atol = {torch.float16: 1e-1, torch.bfloat16: 1e-1, torch.float32: 5e-4}.get(dtype, 1e-8) + self.assert_close(cos_theta, cos_theta.new_ones(()), atol=atol, rtol=0.0) + + # ------------------------------------------------------------------ + # Dtype and device consistency + # ------------------------------------------------------------------ + + def test_output_dtype_matches_input(self, device, dtype): + A = torch.rand(2, 3, 4, device=device, dtype=dtype) + v = solvers.null_vector_3x4(A) + assert v.dtype == dtype + + def test_output_device_matches_input(self, device, dtype): + A = torch.rand(2, 3, 4, device=device, dtype=dtype) + v = solvers.null_vector_3x4(A) + assert v.device == A.device + + # ------------------------------------------------------------------ + # Gradcheck + # ------------------------------------------------------------------ + + def test_gradcheck(self, device): + # Use a well-conditioned rank-3 matrix to keep gradients stable. + nv = torch.tensor([1.0, -2.0, 3.0, -4.0]) + A = _make_rank3_matrix(nv, device=device, dtype=torch.float64) + A = A.detach().requires_grad_(True) + self.gradcheck(solvers.null_vector_3x4, (A,), raise_exception=True, fast_mode=True) + + # ------------------------------------------------------------------ + # Dynamo / torch.compile + # ------------------------------------------------------------------ + + def test_dynamo(self, device, dtype, torch_optimizer): + A = torch.rand(4, 3, 4, device=device, dtype=dtype) + op = solvers.null_vector_3x4 + op_compiled = torch_optimizer(op) + self.assert_close(op_compiled(A), op(A), atol=1e-5, rtol=1e-5) + + # ------------------------------------------------------------------ + # Module (no nn.Module wrapper — use function call form) + # ------------------------------------------------------------------ + + def test_module(self, device, dtype): + # null_vector_3x4 is a plain function; verify it is importable from the + # top-level kornia.geometry.solvers namespace. + import kornia.geometry.solvers as s + + assert hasattr(s, "null_vector_3x4") + A = torch.rand(1, 3, 4, device=device, dtype=dtype) + v = s.null_vector_3x4(A) + assert v.shape == (1, 4) diff --git a/tests/geometry/solvers/test_polynomial_solver.py b/tests/geometry/solvers/test_polynomial_solver.py new file mode 100644 index 0000000..899920f --- /dev/null +++ b/tests/geometry/solvers/test_polynomial_solver.py @@ -0,0 +1,372 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia.geometry.solvers as solver + +from testing.base import BaseTester + + +class TestQuadraticSolver(BaseTester): + def test_smoke(self, device, dtype): + coeffs = torch.rand(1, 3, device=device, dtype=dtype) + roots = solver.solve_quadratic(coeffs) + assert roots.shape == (1, 2) + + @pytest.mark.parametrize("batch_size", [1, 2, 4, 7]) + def test_shape(self, batch_size, device, dtype): + B: int = batch_size + coeffs = torch.rand(B, 3, device=device, dtype=dtype) + roots = solver.solve_quadratic(coeffs) + assert roots.shape == (B, 2) + + @pytest.mark.parametrize( + "coeffs, expected_solutions", + [ + (torch.tensor([[1.0, 4.0, 4.0]]), torch.tensor([[-2.0, -2.0]])), # zero discriminant + (torch.tensor([[1.0, -5.0, 6.0]]), torch.tensor([[3.0, 2.0]])), + (torch.tensor([[1.0, 2.0, 3.0]]), torch.tensor([[0.0, 0.0]])), # negative discriminant + ], + ) + def test_solve_quadratic(self, coeffs, expected_solutions, device, dtype): + roots = solver.solve_quadratic(coeffs) + self.assert_close(roots[0], expected_solutions[0]) + + def gradcheck(self, device): + coeffs = torch.rand(1, 3, device=device, dtype=torch.float64, requires_grad=True) + assert self.gradcheck(solver.solve_quadratic, (coeffs), raise_exception=True, fast_mode=True) + + +class TestCubicSolver(BaseTester): + def test_smoke(self, device, dtype): + coeffs = torch.rand(1, 4, device=device, dtype=dtype) + roots = solver.solve_cubic(coeffs) + assert roots.shape == (1, 3) + + @pytest.mark.parametrize("batch_size", [1, 2, 4, 7]) + def test_shape(self, batch_size, device, dtype): + B: int = batch_size + coeffs = torch.rand(B, 4, device=device, dtype=dtype) + roots = solver.solve_cubic(coeffs) + assert roots.shape == (B, 3) + + @pytest.mark.parametrize( + "coeffs, expected_solutions", + [ + (torch.tensor([[2.0, 3.0, -11.0, -6.0]]), torch.tensor([[2.0, -3.0, -0.5]])), + (torch.tensor([[1.0, 0.0, 4.0, 4.0]]), torch.tensor([[-0.847, 0.0, 0.0]])), + (torch.tensor([[2.0, -6.0, 6.0, -2.0]]), torch.tensor([[1.0, 1.0, 1.0]])), + (torch.tensor([[0.0, 0.0, 1.0, -1.0]]), torch.tensor([[1.0, 0.0, 0.0]])), # handle first order + (torch.tensor([[0.0, 1.0, -5.0, 6.0]]), torch.tensor([[3.0, 2.0, 0.0]])), # handle second order + ], + ) + def test_solve_quadratic_in_cubic(self, coeffs, expected_solutions, device, dtype): + roots = solver.solve_cubic(coeffs) + self.assert_close(roots[0], expected_solutions[0], rtol=1e-3, atol=1e-3) + + def gradcheck(self, device): + coeffs = torch.rand(1, 4, device=device, dtype=torch.float64, requires_grad=True) + assert self.gradcheck(solver.solve_cubic, (coeffs), raise_exception=True, fast_mode=True) + + +class TestMultiplyDegOnePoly(BaseTester): + def test_smoke(self, device, dtype): + a = torch.rand(1, 4, device=device, dtype=dtype) + b = torch.rand(1, 4, device=device, dtype=dtype) + out_poly = solver.multiply_deg_one_poly(a, b) + assert out_poly.shape == (1, 10) + + @pytest.mark.parametrize("batch_size", [1, 2, 4, 8]) + def test_shape(self, batch_size, device, dtype): + B: int = batch_size + a = torch.rand(B, 4, device=device, dtype=dtype) + b = torch.rand(B, 4, device=device, dtype=dtype) + out_poly = solver.multiply_deg_one_poly(a, b) + assert out_poly.shape == (B, 10) + + @pytest.mark.parametrize( + "a_coeffs, b_coeffs, expected_coeffs", + [ + # Case 1: (x + 2y + 3z + 4) * (5x + 6y + 7z + 8) + ( + torch.tensor([[1.0, 2.0, 3.0, 4.0]]), + torch.tensor([[5.0, 6.0, 7.0, 8.0]]), + torch.tensor([[5.0, 16.0, 22.0, 28.0, 12.0, 32.0, 40.0, 21.0, 52.0, 32.0]]), + ), + # Case 2: Squaring a polynomial (x - y + 2z - 3)^2 + ( + torch.tensor([[1.0, -1.0, 2.0, -3.0]]), + torch.tensor([[1.0, -1.0, 2.0, -3.0]]), + torch.tensor([[1.0, -2.0, 4.0, -6.0, 1.0, -4.0, 6.0, 4.0, -12.0, 9.0]]), + ), + # Case 3: Multiplying by zero + ( + torch.tensor([[1.0, 1.0, 1.0, 1.0]]), + torch.tensor([[0.0, 0.0, 0.0, 0.0]]), + torch.tensor([[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]), + ), + # Case 4: Only constant terms (10) * (5) + ( + torch.tensor([[0.0, 0.0, 0.0, 10.0]]), + torch.tensor([[0.0, 0.0, 0.0, 5.0]]), + torch.tensor([[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 50.0]]), + ), + ], + ) + def test_values(self, a_coeffs, b_coeffs, expected_coeffs, device, dtype): + # Move tensor data to the target device and dtype + a = a_coeffs.to(device, dtype) + b = b_coeffs.to(device, dtype) + expected = expected_coeffs.to(device, dtype) + + # Compute the result + result = solver.multiply_deg_one_poly(a, b) + + # Compare result with expected values + self.assert_close(result, expected, rtol=1e-4, atol=1e-4) + + +class TestMultiplyDegTwoOnePoly(BaseTester): + def test_smoke(self, device, dtype): + a = torch.rand(1, 10, device=device, dtype=dtype) + b = torch.rand(1, 4, device=device, dtype=dtype) + out_poly = solver.multiply_deg_two_one_poly(a, b) + assert out_poly.shape == (1, 20) + + @pytest.mark.parametrize("batch_size", [1, 2, 4, 8]) + def test_shape(self, batch_size, device, dtype): + B: int = batch_size + a = torch.rand(B, 10, device=device, dtype=dtype) + b = torch.rand(B, 4, device=device, dtype=dtype) + out_poly = solver.multiply_deg_two_one_poly(a, b) + assert out_poly.shape == (B, 20) + + @pytest.mark.parametrize( + "a_coeffs, b_coeffs, expected_coeffs", + [ + # Case 1: (x^2 + 2y) * (3x + 4) = 3x^3 + 4x^2 + 6xy + 8y + ( + torch.tensor([[1.0, 0, 0, 0, 0, 0, 2.0, 0, 0, 0]]), + torch.tensor([[3.0, 0, 0, 4.0]]), + torch.tensor([[3.0, 0, 0, 0, 0, 4.0, 0, 0, 0, 6.0, 0, 0, 0, 0, 0, 8.0, 0, 0, 0, 0]]), + ), + # Case 2: (xy + z^2) * (y + z) = xy^2 + xyz + yz^2 + z^3 + ( + torch.tensor([[0, 1.0, 0, 0, 0, 0, 0, 1.0, 0, 0]]), + torch.tensor([[0, 1.0, 1.0, 0]]), + torch.tensor([[0, 0, 0, 1.0, 0, 0, 0, 0, 1.0, 0, 0, 0, 0, 1.0, 0, 0, 1.0, 0, 0, 0]]), + ), + # Case 3: Multiplying a complex polynomial by a constant: (x^2+y) * 5 + ( + torch.tensor([[1.0, 0, 0, 0, 0, 0, 1.0, 0, 0, 0]]), + torch.tensor([[0, 0, 0, 5.0]]), + # Expected: 5x^2 + 5y + torch.tensor([[0, 0, 0, 0, 0, 5.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5.0, 0, 0, 0, 0]]), + ), + # Case 4: Multiplication by zero + ( + torch.tensor([[1.0, 1, 1, 1, 1, 1, 1, 1, 1, 1]]), + torch.tensor([[0, 0, 0, 0]]), + torch.zeros(1, 20), # Expect all coefficients to be zero + ), + ], + ) + def test_values(self, a_coeffs, b_coeffs, expected_coeffs, device, dtype): + a = a_coeffs.to(device, dtype) + b = b_coeffs.to(device, dtype) + expected = expected_coeffs.to(device, dtype) + result = solver.multiply_deg_two_one_poly(a, b) + self.assert_close(result, expected, rtol=1e-4, atol=1e-4) + + +class TestDeterminantToPolynomial(BaseTester): + def test_smoke(self, device, dtype): + A = torch.rand(1, 3, 13, device=device, dtype=dtype) + poly = solver.determinant_to_polynomial(A) + assert poly.shape == (1, 11) + + @pytest.mark.parametrize("batch_size", [1, 2, 8]) + def test_shape(self, batch_size, device, dtype): + B: int = batch_size + A = torch.rand(B, 3, 13, device=device, dtype=dtype) + poly = solver.determinant_to_polynomial(A) + assert poly.shape == (B, 11) + + @pytest.mark.parametrize( + "A_in, expected_poly_coeffs", + [ + # Case 1: An all-zero input should result in an all-zero polynomial. + ( + torch.zeros(1, 3, 13), + torch.zeros(1, 11), + ), + # Case 2: A sparse input designed to activate only the first term of cs[:, 10]. + # A[0,0,0]=2, A[0,1,4]=3, A[0,2,8]=5 -> term is 2*3*5 = 30. + ( + torch.zeros(1, 3, 13).index_put( + (torch.tensor([0, 0, 0]), torch.tensor([0, 1, 2]), torch.tensor([0, 4, 8])), + torch.tensor([2.0, 3.0, 5.0]), + ), + torch.zeros(1, 11).index_put((torch.tensor([0]), torch.tensor([10])), torch.tensor([30.0])), + ), + # Case 3: A sparse input designed to activate only one negative term in cs[:, 0]. + # A[0,0,7]=2, A[0,1,3]=3, A[0,2,12]=5 -> term is -A[0,7]*A[1,3]*A[2,12] = -30 + ( + torch.zeros(1, 3, 13).index_put( + (torch.tensor([0, 0, 0]), torch.tensor([0, 1, 2]), torch.tensor([7, 3, 12])), + torch.tensor([2.0, 3.0, 5.0]), + ), + torch.zeros(1, 11).index_put((torch.tensor([0]), torch.tensor([0])), torch.tensor([-30.0])), + ), + ], + ) + def test_values(self, A_in, expected_poly_coeffs, device, dtype): + # Move tensor data to the target device and dtype + A = A_in.to(device, dtype) + expected = expected_poly_coeffs.to(device, dtype) + + # Compute the result + result = solver.determinant_to_polynomial(A) + + # Compare result with expected values + self.assert_close(result, expected, rtol=1e-5, atol=1e-5) + + +class TestQuarticSolver(BaseTester): + def test_smoke(self, device, dtype): + coeffs = torch.rand(1, 5, device=device, dtype=dtype) + roots = solver.solve_quartic(coeffs) + assert roots.shape == (1, 4) + + @pytest.mark.parametrize("batch_size", [1, 2, 4, 7]) + def test_shape(self, batch_size, device, dtype): + B: int = batch_size + coeffs = torch.rand(B, 5, device=device, dtype=dtype) + roots = solver.solve_quartic(coeffs) + assert roots.shape == (B, 4) + + @pytest.mark.parametrize( + "coeffs, expected_solutions", + [ + # Case 1: Distinct Real Roots + # x^4 - 10x^3 + 35x^2 - 50x + 24 = 0 -> Roots: 1, 2, 3, 4 + ( + torch.tensor([[1.0, -10.0, 35.0, -50.0, 24.0]]), + torch.tensor([[1.0, 2.0, 3.0, 4.0]]), + ), + # Case 2: Biquadratic (Symmetric) + # x^4 - 5x^2 + 4 = 0 -> Roots: 1, -1, 2, -2 + ( + torch.tensor([[1.0, 0.0, -5.0, 0.0, 4.0]]), + torch.tensor([[-2.0, -1.0, 1.0, 2.0]]), + ), + # Case 3: Double Roots + # (x-2)^2 * (x-3) * (x+1) -> Roots: -1, 2, 2, 3 + ( + torch.tensor([[1.0, -6.0, 9.0, 4.0, -12.0]]), + torch.tensor([[-1.0, 2.0, 2.0, 3.0]]), + ), + # Case 4: Cubic Fallback (a=0) + # 0x^4 + x^3 - 6x^2 + 11x - 6 = 0 -> Roots: 1, 2, 3. Last col 0. + ( + torch.tensor([[0.0, 1.0, -6.0, 11.0, -6.0]]), + torch.tensor([[1.0, 2.0, 3.0, 0.0]]), + ), + # Case 5: Degenerate / All Zeros + # x^4 = 0 -> Roots: 0, 0, 0, 0 + ( + torch.tensor([[1.0, 0.0, 0.0, 0.0, 0.0]]), + torch.tensor([[0.0, 0.0, 0.0, 0.0]]), + ), + # Case 6: Complex Roots (Should be 0s per contract) + # x^4 + 1 = 0 -> Roots: +/- sqrt(i) ... all complex -> 0, 0, 0, 0 + ( + torch.tensor([[1.0, 0.0, 0.0, 0.0, 1.0]]), + torch.tensor([[0.0, 0.0, 0.0, 0.0]]), + ), + # Case 7: Mixed Real/Complex + # x^4 - 1 = 0 -> Roots: 1, -1, i, -i -> Real: 1, -1. Others 0. + ( + torch.tensor([[1.0, 0.0, 0.0, 0.0, -1.0]]), + torch.tensor([[-1.0, 1.0, 0.0, 0.0]]), + ), + ], + ) + def test_solve_quartic(self, coeffs, expected_solutions, device, dtype): + coeffs = coeffs.to(device, dtype) + expected_solutions = expected_solutions.to(device, dtype) + + roots = solver.solve_quartic(coeffs) + + # Sort roots to ensure order-invariant comparison + # We sort both expected and actual to match this behavior. + roots_sorted, _ = torch.sort(roots, dim=-1) + expected_sorted, _ = torch.sort(expected_solutions, dim=-1) + + self.assert_close(roots_sorted, expected_sorted, rtol=1e-3, atol=1e-3) + + def test_random(self, device, dtype): + # Generate random roots and construct coefficients to ensure valid solutions exist + B = 10 + true_roots = torch.randn(B, 4, device=device, dtype=dtype) + + # Sort true roots for comparison later + true_roots_sorted, _ = torch.sort(true_roots, dim=-1) + + r1, r2, r3, r4 = true_roots.unbind(-1) + + # Construct polynomial coefficients from roots + # (x-r1)(x-r2)(x-r3)(x-r4) = 0 + a = torch.ones(B, device=device, dtype=dtype) + b = -(r1 + r2 + r3 + r4) + c = r1 * r2 + r1 * r3 + r1 * r4 + r2 * r3 + r2 * r4 + r3 * r4 + d = -(r1 * r2 * r3 + r1 * r2 * r4 + r1 * r3 * r4 + r2 * r3 * r4) + e = r1 * r2 * r3 * r4 + + coeffs = torch.stack([a, b, c, d, e], dim=-1) + computed_roots = solver.solve_quartic(coeffs) + + computed_roots_sorted, _ = torch.sort(computed_roots, dim=-1) + + # 1. Check Residuals (Equation satisfaction) + residuals = ( + coeffs[:, 0:1] * computed_roots**4 + + coeffs[:, 1:2] * computed_roots**3 + + coeffs[:, 2:3] * computed_roots**2 + + coeffs[:, 3:4] * computed_roots + + coeffs[:, 4:5] + ) + self.assert_close(residuals, torch.zeros_like(residuals), atol=1e-3, rtol=1e-3) + + # 2. Check Root Matching (Stronger Test) + # Since we synthesized the coefficients from real roots, we expect + # to recover exactly those roots (no complex outputs). + self.assert_close(computed_roots_sorted, true_roots_sorted, atol=1e-3, rtol=1e-3) + + def test_gradcheck(self, device): + # Use a specific polynomial with distinct roots to ensure gradient stability + # x^4 - 10x^3 + 35x^2 - 50x + 24 = 0 + # Avoid double roots for gradcheck as gradients are undefined/infinite there. + coeffs = torch.tensor( + [[1.0, -10.0, 35.0, -50.0, 24.0]], + device=device, + dtype=torch.float64, + requires_grad=True, + ) + self.gradcheck(solver.solve_quartic, (coeffs,), raise_exception=True, fast_mode=True) diff --git a/tests/geometry/subpix/test_dsnt.py b/tests/geometry/subpix/test_dsnt.py new file mode 100644 index 0000000..feb8177 --- /dev/null +++ b/tests/geometry/subpix/test_dsnt.py @@ -0,0 +1,117 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestRenderGaussian2d(BaseTester): + @pytest.fixture() + def gaussian(self, device, dtype): + # For a standard gaussian on 5 points [-1, -0.5, 0, 0.5, 1] with std=0.25 + # The equation is exp( -x^2 / (2 * std^2) ) -> exp( -x^2 * 8 ) + # x=0 -> exp(0) = 1.0 + # x=0.5 -> exp(-2) ≈ 0.135335 + # x=1.0 -> exp(-8) ≈ 0.000335 + + vec = torch.tensor([0.00033546, 0.13533528, 1.00000000, 0.13533528, 0.00033546], device=device, dtype=dtype) + + # Create 2D from 1D (Outer Product) + grid = vec.unsqueeze(1) * vec.unsqueeze(0) + + # Normalize sum to 1 + return grid / grid.sum() + + def test_normalized_coordinates(self, gaussian, device, dtype): + mean = torch.tensor([0.0, 0.0], dtype=dtype, device=device) + std = torch.tensor([0.25, 0.25], dtype=dtype, device=device) + + actual = kornia.geometry.subpix.render_gaussian2d(mean.view(1, 2), std.view(1, 2), (5, 5), True) + + self.assert_close(actual[0], gaussian, rtol=1e-5, atol=1e-5) + + def test_pixel_coordinates(self, gaussian, device, dtype): + mean = torch.tensor([2.0, 2.0], dtype=dtype, device=device) + std = torch.tensor([0.5, 0.5], dtype=dtype, device=device) + + actual = kornia.geometry.subpix.render_gaussian2d(mean.view(1, 2), std.view(1, 2), (5, 5), False) + + self.assert_close(actual[0], gaussian, rtol=1e-5, atol=1e-5) + + def test_dynamo(self, device, dtype, torch_optimizer): + mean = torch.tensor([0.0, 0.0], dtype=dtype, device=device) + std = torch.tensor([0.25, 0.25], dtype=dtype, device=device) + + op = kornia.geometry.subpix.render_gaussian2d + op_optimized = torch_optimizer(op) + + res_orig = op(mean.view(1, 2), std.view(1, 2), (5, 5), True) + res_opt = op_optimized(mean.view(1, 2), std.view(1, 2), (5, 5), True) + + self.assert_close(res_orig, res_opt) + + +class TestSpatialSoftmax2d(BaseTester): + @pytest.fixture(params=[torch.ones(1, 1, 5, 7), torch.randn(2, 3, 16, 16)]) + def input(self, request, device, dtype): + return request.param.to(device, dtype) + + def test_forward(self, input): + actual = kornia.geometry.subpix.spatial_softmax2d(input) + assert actual.lt(0).sum().item() == 0, "expected no negative values" + sums = actual.sum(-1).sum(-1) + self.assert_close(sums, torch.ones_like(sums)) + + def test_dynamo(self, input, torch_optimizer): + op = kornia.geometry.subpix.spatial_softmax2d + op_optimized = torch_optimizer(op) + + self.assert_close(op(input), op_optimized(input)) + + +class TestSpatialExpectation2d(BaseTester): + @pytest.fixture( + params=[ + ( + torch.tensor([[[[0.0, 0.0, 1.0], [0.0, 0.0, 0.0]]]]), + torch.tensor([[[1.0, -1.0]]]), + torch.tensor([[[2.0, 0.0]]]), + ) + ] + ) + def example(self, request, device, dtype): + input, expected_norm, expected_px = request.param + return input.to(device, dtype), expected_norm.to(device, dtype), expected_px.to(device, dtype) + + def test_forward(self, example): + input, expected_norm, expected_px = example + actual_norm = kornia.geometry.subpix.spatial_expectation2d(input, True) + self.assert_close(actual_norm, expected_norm) + actual_px = kornia.geometry.subpix.spatial_expectation2d(input, False) + self.assert_close(actual_px, expected_px) + + @pytest.mark.skip("After the op be optimized the results are not the same") + def test_dynamo(self, dtype, device, torch_optimizer): + data = torch.tensor([[[[0.0, 0.0, 1.0], [0.0, 0.0, 0.0]]]], device=device, dtype=dtype) + op = kornia.geometry.subpix.spatial_expectation2d + op_optimized = torch_optimizer(op) + + self.assert_close(op(data, True), op_optimized(data, True)) diff --git a/tests/geometry/subpix/test_iterative_quad_interp_pyhesaff.py b/tests/geometry/subpix/test_iterative_quad_interp_pyhesaff.py new file mode 100644 index 0000000..1d4aee9 --- /dev/null +++ b/tests/geometry/subpix/test_iterative_quad_interp_pyhesaff.py @@ -0,0 +1,196 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Reference tests for conv_quad_interp3d. + +These tests verify that the subpixel localisation implemented in +``conv_quad_interp3d`` agrees with the single-step C++ HessAff +``localizeKeypoint`` formula (ported here in pure Python/NumPy). + +A synthetic 3-scale Gaussian response map is used as input so that the +expected peak location is known analytically and no external dependencies +are needed. +""" + +from __future__ import annotations + +import numpy as np +import pytest +import torch + +from kornia.geometry.subpix.nms import nms3d +from kornia.geometry.subpix.spatial_soft_argmax import conv_quad_interp3d + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_3d_gauss_response( + H: int, W: int, cx: float, cy: float, cs: float, sigma_xy: float = 1.5, sigma_s: float = 1.0 +) -> torch.Tensor: + """Build a synthetic (1, 1, 3, H, W) Gaussian response with known peak. + + The analytic peak is at (cx, cy) in x/y and cs in the scale dimension + (0-indexed, expected value in (0, 2) so that it is interior to the 3 slices). + """ + xs = np.arange(W, dtype=np.float64) + ys = np.arange(H, dtype=np.float64) + xx, yy = np.meshgrid(xs, ys) + resp = np.zeros((3, H, W), dtype=np.float32) + for d in range(3): + resp[d] = np.exp(-((xx - cx) ** 2 + (yy - cy) ** 2) / (2 * sigma_xy**2) - (d - cs) ** 2 / (2 * sigma_s**2)) + return torch.from_numpy(resp).unsqueeze(0).unsqueeze(0) + + +# --------------------------------------------------------------------------- +# Reference formula: Python port of C++ localizeKeypoint (one iteration) +# --------------------------------------------------------------------------- + + +def _localizeKeypoint_ref( + low: np.ndarray, cur: np.ndarray, high: np.ndarray, r: int, c: int +) -> tuple[float, float, float, bool]: + """Single step of the C++ HessianDetector::localizeKeypoint. + + Solves ``H * [shift_x, shift_y, shift_s]^T = -[dx, dy, ds]^T`` and returns + ``(shift_x, shift_y, shift_s, is_valid)``. The coordinate convention matches + the C++ code: x = column direction, y = row direction, s = scale direction. + """ + c000 = cur[r, c] + dx = 0.5 * (cur[r, c + 1] - cur[r, c - 1]) + dy = 0.5 * (cur[r + 1, c] - cur[r - 1, c]) + ds = 0.5 * (high[r, c] - low[r, c]) + + dxx = cur[r, c - 1] - 2 * c000 + cur[r, c + 1] + dyy = cur[r - 1, c] - 2 * c000 + cur[r + 1, c] + dss = low[r, c] - 2 * c000 + high[r, c] + dxy = 0.25 * (cur[r + 1, c + 1] - cur[r + 1, c - 1] - cur[r - 1, c + 1] + cur[r - 1, c - 1]) + dxs = 0.25 * (high[r, c + 1] - high[r, c - 1] - low[r, c + 1] + low[r, c - 1]) + dys = 0.25 * (high[r + 1, c] - high[r - 1, c] - low[r + 1, c] + low[r - 1, c]) + + A = np.array([[dxx, dxy, dxs], [dxy, dyy, dys], [dxs, dys, dss]], dtype=np.float64) + b = np.array([-dx, -dy, -ds], dtype=np.float64) + try: + x = np.linalg.solve(A, b) + except np.linalg.LinAlgError: + return 0.0, 0.0, 0.0, False + return float(x[0]), float(x[1]), float(x[2]), True + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestIterativeQuadInterp3dVsRefFormula: + """Compare our function against a direct Python port of the C++ formula.""" + + def _run(self, cx: float, cy: float, cs: float, device: torch.device, dtype: torch.dtype) -> None: + resp_t = _make_3d_gauss_response(19, 19, cx, cy, cs).to(device=device, dtype=dtype) + + assert nms3d(resp_t, (3, 3, 3), True).sum().item() >= 1, "no NMS peak found" + + # Our function — one iteration keeps the initial integer position + coords, _ = conv_quad_interp3d(resp_t, n_iters=1, strict_maxima_bonus=0) + + # Reference: apply the C++ formula to the numpy arrays + resp_np = resp_t.cpu().float().numpy()[0, 0] # (3, H, W) + low_np, cur_np, high_np = resp_np[0], resp_np[1], resp_np[2] + h_peak, w_peak = round(cy), round(cx) + ref_sx, ref_sy, ref_ss, valid = _localizeKeypoint_ref(low_np, cur_np, high_np, h_peak, w_peak) + assert valid, "reference formula failed to solve" + + d_peak = 1 # middle scale + ours_x = coords[0, 0, 1, d_peak, h_peak, w_peak].item() + ours_y = coords[0, 0, 2, d_peak, h_peak, w_peak].item() + ours_s = coords[0, 0, 0, d_peak, h_peak, w_peak].item() + + tol = 1e-3 + assert abs(ours_x - (w_peak + ref_sx)) < tol, f"x mismatch: ours={ours_x:.5f} ref={w_peak + ref_sx:.5f}" + assert abs(ours_y - (h_peak + ref_sy)) < tol, f"y mismatch: ours={ours_y:.5f} ref={h_peak + ref_sy:.5f}" + assert abs(ours_s - (d_peak + ref_ss)) < tol, f"s mismatch: ours={ours_s:.5f} ref={d_peak + ref_ss:.5f}" + + @pytest.mark.parametrize("cx,cy,cs", [(9.4, 8.7, 1.3), (5.3, 4.8, 1.2), (9.0, 9.0, 1.0), (7.7, 8.2, 1.4)]) + def test_single_step_matches_cpp_formula_float32(self, cx: float, cy: float, cs: float) -> None: + self._run(cx, cy, cs, torch.device("cpu"), torch.float32) + + @pytest.mark.parametrize("cx,cy,cs", [(9.4, 8.7, 1.3), (5.3, 4.8, 1.2)]) + def test_single_step_matches_cpp_formula_float64(self, cx: float, cy: float, cs: float) -> None: + self._run(cx, cy, cs, torch.device("cpu"), torch.float64) + + +class TestIterativeQuadInterp3dAccuracy: + """Verify that conv_quad_interp3d accurately recovers subpixel positions.""" + + def _run(self, cx: float, cy: float, cs: float, device: torch.device, dtype: torch.dtype) -> None: + resp_t = _make_3d_gauss_response(19, 19, cx, cy, cs).to(device=device, dtype=dtype) + + nms_mask = nms3d(resp_t, (3, 3, 3), True) + assert nms_mask.sum().item() >= 1, "no NMS peak" + + coords, _ = conv_quad_interp3d(resp_t, strict_maxima_bonus=0) + d_peak = 1 + h_peak, w_peak = round(cy), round(cx) + ours_x = coords[0, 0, 1, d_peak, h_peak, w_peak].item() + ours_y = coords[0, 0, 2, d_peak, h_peak, w_peak].item() + + tol = 0.05 + assert abs(ours_x - cx) < tol, f"x error {abs(ours_x - cx):.4f} for blob at x={cx}" + assert abs(ours_y - cy) < tol, f"y error {abs(ours_y - cy):.4f} for blob at y={cy}" + + @pytest.mark.parametrize("cx,cy,cs", [(9.4, 8.7, 1.3), (5.3, 4.8, 1.2), (8.4, 9.3, 1.3)]) + def test_subpixel_accuracy_float32(self, cx: float, cy: float, cs: float) -> None: + self._run(cx, cy, cs, torch.device("cpu"), torch.float32) + + @pytest.mark.parametrize("cx,cy,cs", [(9.4, 8.7, 1.3), (5.3, 4.8, 1.2)]) + def test_subpixel_accuracy_float64(self, cx: float, cy: float, cs: float) -> None: + self._run(cx, cy, cs, torch.device("cpu"), torch.float64) + + def test_integer_peak_exact(self) -> None: + """Blob at an integer position — recovered coords should be very close to integer.""" + cx, cy, cs = 9.0, 9.0, 1.0 + resp_t = _make_3d_gauss_response(19, 19, cx, cy, cs) + coords, _ = conv_quad_interp3d(resp_t, strict_maxima_bonus=0) + d_peak = 1 + h_peak, w_peak = round(cy), round(cx) + ours_x = coords[0, 0, 1, d_peak, h_peak, w_peak].item() + ours_y = coords[0, 0, 2, d_peak, h_peak, w_peak].item() + assert abs(ours_x - cx) < 1e-3 + assert abs(ours_y - cy) < 1e-3 + + def test_multiple_peaks(self) -> None: + """Two blobs at different subpixel positions — both should be recovered.""" + H, W = 32, 32 + blobs = [(8.3, 7.8, 1.3), (22.6, 21.1, 1.2)] + resp_np = np.zeros((3, H, W), dtype=np.float32) + xs = np.arange(W, dtype=np.float64) + ys = np.arange(H, dtype=np.float64) + xx, yy = np.meshgrid(xs, ys) + for bx, by, bs in blobs: + for d in range(3): + resp_np[d] += np.exp(-((xx - bx) ** 2 + (yy - by) ** 2) / (2 * 1.5**2) - (d - bs) ** 2 / (2 * 1.0**2)) + resp_t = torch.from_numpy(resp_np).unsqueeze(0).unsqueeze(0) + + coords, _ = conv_quad_interp3d(resp_t, strict_maxima_bonus=0) + d_peak = 1 + for bx, by, _ in blobs: + h_int, w_int = round(by), round(bx) + ours_x = coords[0, 0, 1, d_peak, h_int, w_int].item() + ours_y = coords[0, 0, 2, d_peak, h_int, w_int].item() + assert abs(ours_x - bx) < 0.1, f"blob ({bx},{by}): x ours={ours_x:.3f}" + assert abs(ours_y - by) < 0.1, f"blob ({bx},{by}): y ours={ours_y:.3f}" diff --git a/tests/geometry/subpix/test_nms.py b/tests/geometry/subpix/test_nms.py new file mode 100644 index 0000000..a678699 --- /dev/null +++ b/tests/geometry/subpix/test_nms.py @@ -0,0 +1,273 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestNMS2d(BaseTester): + def test_shape(self, device): + inp = torch.ones(1, 3, 4, 4, device=device) + nms = kornia.geometry.subpix.NonMaximaSuppression2d((3, 3)).to(device) + assert nms(inp).shape == inp.shape + + def test_shape_batch(self, device): + inp = torch.ones(4, 3, 4, 4, device=device) + nms = kornia.geometry.subpix.NonMaximaSuppression2d((3, 3)).to(device) + assert nms(inp).shape == inp.shape + + def test_shape_5x5(self, device): + inp = torch.ones(1, 2, 10, 10, device=device) + nms = kornia.geometry.subpix.NonMaximaSuppression2d((5, 5)).to(device) + assert nms(inp).shape == inp.shape + + def test_shape_7x7(self, device): + inp = torch.ones(1, 2, 14, 14, device=device) + nms = kornia.geometry.subpix.NonMaximaSuppression2d((7, 7)).to(device) + assert nms(inp).shape == inp.shape + + def test_nms_5x5_single_peak(self, device): + # A single isolated peak should be preserved; everything else zeroed. + inp = torch.zeros(1, 1, 15, 15, device=device) + inp[0, 0, 7, 7] = 1.0 + nms = kornia.geometry.subpix.NonMaximaSuppression2d((5, 5)).to(device) + out = nms(inp) + assert out[0, 0, 7, 7].item() == pytest.approx(1.0) + assert out.sum().item() == pytest.approx(1.0) + + def test_nms_5x5_suppress_close_neighbor(self, device): + # 5x5 kernel has radius 2 (checks ±2 pixels). Two peaks separated by exactly 2 pixels + # are inside each other's window; only the higher one survives. + inp = torch.zeros(1, 1, 20, 20, device=device) + inp[0, 0, 8, 8] = 2.0 + inp[0, 0, 8, 10] = 1.0 # distance 2 — inside the 5x5 neighbourhood of (8,8) + nms = kornia.geometry.subpix.NonMaximaSuppression2d((5, 5)).to(device) + out = nms(inp) + assert out[0, 0, 8, 8].item() == pytest.approx(2.0) + assert out[0, 0, 8, 10].item() == pytest.approx(0.0) + + def test_nms_5x5_keep_far_peaks(self, device): + # Two peaks separated by 5 pixels (outside a 5x5 ±2 window): both survive. + inp = torch.zeros(1, 1, 20, 20, device=device) + inp[0, 0, 4, 4] = 2.0 + inp[0, 0, 4, 9] = 1.0 # distance 5 — outside the 5x5 neighbourhood + nms = kornia.geometry.subpix.NonMaximaSuppression2d((5, 5)).to(device) + out = nms(inp) + assert out[0, 0, 4, 4].item() == pytest.approx(2.0) + assert out[0, 0, 4, 9].item() == pytest.approx(1.0) + + def test_nms_5x5_matches_3x3_on_well_separated_peaks(self, device): + # When peaks are far apart, 3x3 and 5x5 NMS should agree. + inp = torch.zeros(1, 1, 30, 30, device=device) + inp[0, 0, 5, 5] = 3.0 + inp[0, 0, 20, 20] = 2.0 + nms3 = kornia.geometry.subpix.NonMaximaSuppression2d((3, 3)).to(device) + nms5 = kornia.geometry.subpix.NonMaximaSuppression2d((5, 5)).to(device) + out3 = nms3(inp) + out5 = nms5(inp) + # Both NMS variants should detect the same two peaks (peaks are far from each other). + assert (out3 > 0).equal(out5 > 0) + + def test_nms_7x7_single_peak(self, device): + # A single isolated peak should be preserved. + inp = torch.zeros(1, 1, 20, 20, device=device) + inp[0, 0, 10, 10] = 1.0 + nms = kornia.geometry.subpix.NonMaximaSuppression2d((7, 7)).to(device) + out = nms(inp) + assert out[0, 0, 10, 10].item() == pytest.approx(1.0) + assert out.sum().item() == pytest.approx(1.0) + + def test_nms_7x7_suppress_close_neighbor(self, device): + # 7x7 kernel has radius 3 (checks ±3 pixels). Two peaks separated by exactly 3 pixels + # are inside each other's window; only the higher one survives. + inp = torch.zeros(1, 1, 25, 25, device=device) + inp[0, 0, 10, 10] = 2.0 + inp[0, 0, 10, 13] = 1.0 # distance 3 — inside the 7x7 neighbourhood + nms = kornia.geometry.subpix.NonMaximaSuppression2d((7, 7)).to(device) + out = nms(inp) + assert out[0, 0, 10, 10].item() == pytest.approx(2.0) + assert out[0, 0, 10, 13].item() == pytest.approx(0.0) + + def test_nms_7x7_keep_far_peaks(self, device): + # Two peaks separated by 7 pixels (outside 7x7 ±3 window): both survive. + inp = torch.zeros(1, 1, 30, 30, device=device) + inp[0, 0, 5, 5] = 2.0 + inp[0, 0, 5, 12] = 1.0 # distance 7 + nms = kornia.geometry.subpix.NonMaximaSuppression2d((7, 7)).to(device) + out = nms(inp) + assert out[0, 0, 5, 5].item() == pytest.approx(2.0) + assert out[0, 0, 5, 12].item() == pytest.approx(1.0) + + def test_gradcheck_5x5(self, device): + img = torch.rand(1, 2, 7, 7, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.subpix.nms2d, (img, (5, 5)), nondet_tol=1e-4) + + def test_gradcheck_7x7(self, device): + img = torch.rand(1, 2, 9, 9, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.subpix.nms2d, (img, (7, 7)), nondet_tol=1e-4) + + def test_nms(self, device): + inp = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.1, 1.0, 0.0, 1.0, 1.0, 0.0], + [0.0, 0.7, 1.1, 0.0, 1.0, 2.0, 0.0], + [0.0, 0.8, 1.0, 0.0, 1.0, 1.0, 0.0], + ] + ] + ], + device=device, + ).float() + + expected = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0, 0, 0.0, 0, 0.0, 0.0], + [0.0, 0, 1.1, 0.0, 0.0, 2.0, 0.0], + [0.0, 0, 0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + ).float() + nms = kornia.geometry.subpix.NonMaximaSuppression2d((3, 3)).to(device) + scores = nms(inp) + self.assert_close(scores, expected, atol=1e-4, rtol=1e-3) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 2, 5, 4 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.subpix.nms2d, (img, (3, 3)), nondet_tol=1e-4) + + +class TestNMS3d(BaseTester): + def test_shape(self, device): + inp = torch.ones(1, 1, 3, 4, 4, device=device) + nms = kornia.geometry.subpix.NonMaximaSuppression3d((3, 3, 3)).to(device) + assert nms(inp).shape == inp.shape + + def test_shape_batch(self, device): + inp = torch.ones(4, 1, 3, 4, 4, device=device) + nms = kornia.geometry.subpix.NonMaximaSuppression3d((3, 3, 3)).to(device) + assert nms(inp).shape == inp.shape + + def test_nms(self, device): + inp = torch.tensor( + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 1.0, 2.0, 1.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + ] + ] + ] + ).to(device) + + expected = torch.tensor( + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 2.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + ] + ] + ] + ).to(device) + nms = kornia.geometry.subpix.NonMaximaSuppression3d((3, 3, 3)).to(device) + scores = nms(inp) + self.assert_close(scores, expected, atol=1e-4, rtol=1e-3) + + def test_gradcheck(self, device): + batch_size, channels, depth, height, width = 1, 1, 4, 5, 4 + img = torch.rand(batch_size, channels, depth, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.subpix.nms3d, (img, (3, 3, 3)), nondet_tol=1e-4) + + +class TestNMS3dMinMax(BaseTester): + def test_shapes(self, device): + inp = torch.randn(1, 1, 5, 10, 10, device=device) + max_mask, min_mask = kornia.geometry.subpix.nms3d_minmax(inp) + assert max_mask.shape == inp.shape + assert min_mask.shape == inp.shape + assert max_mask.dtype == torch.bool + assert min_mask.dtype == torch.bool + + def test_consistent_with_nms3d(self, device): + """nms3d_minmax must match nms3d(x) and nms3d(-x) exactly.""" + inp = torch.randn(2, 3, 7, 12, 12, device=device) + max_mask, min_mask = kornia.geometry.subpix.nms3d_minmax(inp) + max_ref = kornia.geometry.subpix.nms3d(inp, (3, 3, 3), mask_only=True) + min_ref = kornia.geometry.subpix.nms3d(-inp, (3, 3, 3), mask_only=True) + assert max_mask.equal(max_ref), "max mask mismatch" + assert min_mask.equal(min_ref), "min mask mismatch" + + def test_no_overlap(self, device): + """A voxel cannot be both a strict local maximum and minimum.""" + inp = torch.randn(1, 1, 5, 10, 10, device=device) + max_mask, min_mask = kornia.geometry.subpix.nms3d_minmax(inp) + assert not (max_mask & min_mask).any() + + def test_gradcheck(self, device): + # nms3d_minmax is not differentiable (bool masks), so we just check it runs. + inp = torch.randn(1, 1, 5, 7, 7, device=device) + kornia.geometry.subpix.nms3d_minmax(inp) diff --git a/tests/geometry/subpix/test_spatial_softargmax.py b/tests/geometry/subpix/test_spatial_softargmax.py new file mode 100644 index 0000000..b56275f --- /dev/null +++ b/tests/geometry/subpix/test_spatial_softargmax.py @@ -0,0 +1,793 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +from torch.nn.functional import mse_loss + +import kornia +from kornia.geometry.subpix.spatial_soft_argmax import ( + _get_center_kernel2d, + _get_center_kernel3d, + conv_quad_interp3d, +) + +from testing.base import BaseTester + + +class TestCenterKernel2d(BaseTester): + def test_smoke(self, device, dtype): + kernel = _get_center_kernel2d(3, 4, device=device).to(dtype=dtype) + assert kernel.shape == (2, 2, 3, 4) + + def test_odd(self, device, dtype): + kernel = _get_center_kernel2d(3, 3, device=device).to(dtype=dtype) + expected = torch.tensor( + [ + [ + [[0.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + ], + [ + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]], + ], + ], + device=device, + dtype=dtype, + ) + self.assert_close(kernel, expected, atol=1e-4, rtol=1e-4) + + def test_even(self, device, dtype): + kernel = _get_center_kernel2d(2, 2, device=device).to(dtype=dtype) + expected = torch.ones(2, 2, 2, 2, device=device, dtype=dtype) * 0.25 + expected[0, 1] = 0 + expected[1, 0] = 0 + self.assert_close(kernel, expected, atol=1e-4, rtol=1e-4) + + +class TestCenterKernel3d(BaseTester): + def test_smoke(self, device, dtype): + kernel = _get_center_kernel3d(6, 3, 4, device=device).to(dtype=dtype) + assert kernel.shape == (3, 3, 6, 3, 4) + + def test_odd(self, device, dtype): + kernel = _get_center_kernel3d(3, 5, 7, device=device).to(dtype=dtype) + expected = torch.zeros(3, 3, 3, 5, 7, device=device, dtype=dtype) + expected[0, 0, 1, 2, 3] = 1.0 + expected[1, 1, 1, 2, 3] = 1.0 + expected[2, 2, 1, 2, 3] = 1.0 + self.assert_close(kernel, expected, atol=1e-4, rtol=1e-4) + + def test_even(self, device, dtype): + kernel = _get_center_kernel3d(2, 4, 3, device=device).to(dtype=dtype) + expected = torch.zeros(3, 3, 2, 4, 3, device=device, dtype=dtype) + expected[0, 0, :, 1:3, 1] = 0.25 + expected[1, 1, :, 1:3, 1] = 0.25 + expected[2, 2, :, 1:3, 1] = 0.25 + self.assert_close(kernel, expected, atol=1e-4, rtol=1e-4) + + +class TestSpatialSoftArgmax2d(BaseTester): + def test_smoke(self, device, dtype): + sample = torch.zeros(1, 1, 2, 3, device=device, dtype=dtype) + m = kornia.geometry.subpix.SpatialSoftArgmax2d() + assert m(sample).shape == (1, 1, 2) + + def test_smoke_batch(self, device, dtype): + sample = torch.zeros(2, 1, 2, 3, device=device, dtype=dtype) + m = kornia.geometry.subpix.SpatialSoftArgmax2d() + assert m(sample).shape == (2, 1, 2) + + def test_top_left_normalized(self, device, dtype): + sample = torch.zeros(1, 1, 2, 3, device=device, dtype=dtype) + sample[..., 0, 0] = 1e16 + + coord = kornia.geometry.subpix.spatial_soft_argmax2d(sample, normalized_coordinates=True) + self.assert_close(coord[..., 0].item(), -1.0, atol=1e-4, rtol=1e-4) + self.assert_close(coord[..., 1].item(), -1.0, atol=1e-4, rtol=1e-4) + + def test_top_left(self, device, dtype): + sample = torch.zeros(1, 1, 2, 3, device=device, dtype=dtype) + sample[..., 0, 0] = 1e16 + + coord = kornia.geometry.subpix.spatial_soft_argmax2d(sample, normalized_coordinates=False) + self.assert_close(coord[..., 0].item(), 0.0, atol=1e-4, rtol=1e-4) + self.assert_close(coord[..., 1].item(), 0.0, atol=1e-4, rtol=1e-4) + + def test_bottom_right_normalized(self, device, dtype): + sample = torch.zeros(1, 1, 2, 3, device=device, dtype=dtype) + sample[..., -1, -1] = 1e16 + + coord = kornia.geometry.subpix.spatial_soft_argmax2d(sample, normalized_coordinates=True) + self.assert_close(coord[..., 0].item(), 1.0, atol=1e-4, rtol=1e-4) + self.assert_close(coord[..., 1].item(), 1.0, atol=1e-4, rtol=1e-4) + + def test_bottom_right(self, device, dtype): + sample = torch.zeros(1, 1, 2, 3, device=device, dtype=dtype) + sample[..., -1, -1] = 1e16 + + coord = kornia.geometry.subpix.spatial_soft_argmax2d(sample, normalized_coordinates=False) + self.assert_close(coord[..., 0].item(), 2.0, atol=1e-4, rtol=1e-4) + self.assert_close(coord[..., 1].item(), 1.0, atol=1e-4, rtol=1e-4) + + def test_batch2_n2(self, device, dtype): + sample = torch.zeros(2, 2, 2, 3, device=device, dtype=dtype) + sample[0, 0, 0, 0] = 1e16 # top-left + sample[0, 1, 0, -1] = 1e16 # top-right + sample[1, 0, -1, 0] = 1e16 # bottom-left + sample[1, 1, -1, -1] = 1e16 # bottom-right + + coord = kornia.geometry.subpix.spatial_soft_argmax2d(sample) + self.assert_close(coord[0, 0, 0].item(), -1.0, atol=1e-4, rtol=1e-4) # top-left + self.assert_close(coord[0, 0, 1].item(), -1.0, atol=1e-4, rtol=1e-4) + self.assert_close(coord[0, 1, 0].item(), 1.0, atol=1e-4, rtol=1e-4) # top-right + self.assert_close(coord[0, 1, 1].item(), -1.0, atol=1e-4, rtol=1e-4) + self.assert_close(coord[1, 0, 0].item(), -1.0, atol=1e-4, rtol=1e-4) # bottom-left + self.assert_close(coord[1, 0, 1].item(), 1.0, atol=1e-4, rtol=1e-4) + self.assert_close(coord[1, 1, 0].item(), 1.0, atol=1e-4, rtol=1e-4) # bottom-right + self.assert_close(coord[1, 1, 1].item(), 1.0, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + sample = torch.rand(2, 3, 3, 2, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.subpix.spatial_soft_argmax2d, (sample)) + + def test_end_to_end(self, device, dtype): + sample = torch.full((1, 2, 7, 7), 1.0, requires_grad=True, device=device, dtype=dtype) + target = torch.as_tensor([[[0.0, 0.0], [1.0, 1.0]]], device=device, dtype=dtype) + std = torch.tensor([1.0, 1.0], device=device, dtype=dtype) + + hm = kornia.geometry.subpix.spatial_softmax2d(sample) + self.assert_close( + hm.sum(-1).sum(-1), torch.tensor([[1.0, 1.0]], device=device, dtype=dtype), atol=1e-4, rtol=1e-4 + ) + + pred = kornia.geometry.subpix.spatial_expectation2d(hm) + self.assert_close( + pred, torch.as_tensor([[[0.0, 0.0], [0.0, 0.0]]], device=device, dtype=dtype), atol=1e-4, rtol=1e-4 + ) + + loss1 = mse_loss(pred, target, size_average=None, reduce=None, reduction="none").mean(-1, keepdim=False) + expected_loss1 = torch.as_tensor([[0.0, 1.0]], device=device, dtype=dtype) + self.assert_close(loss1, expected_loss1, atol=1e-4, rtol=1e-4) + + target_hm = kornia.geometry.subpix.render_gaussian2d(target, std, sample.shape[-2:]).contiguous() + + loss2 = kornia.losses.js_div_loss_2d(hm, target_hm, reduction="none") + expected_loss2 = torch.as_tensor([[0.0087, 0.0818]], device=device, dtype=dtype) + self.assert_close(loss2, expected_loss2, rtol=0, atol=1e-3) + + loss = (loss1 + loss2).mean() + loss.backward() + + def test_dynamo(self, device, dtype, torch_optimizer): + data = torch.rand((2, 3, 7, 7), dtype=dtype, device=device) + op = kornia.geometry.subpix.spatial_soft_argmax2d + op_optimized = torch_optimizer(op) + + self.assert_close(op(data), op_optimized(data)) + + +class TestConvSoftArgmax2d(BaseTester): + def test_smoke(self, device, dtype): + sample = torch.zeros(1, 1, 3, 3, device=device, dtype=dtype) + m = kornia.geometry.subpix.ConvSoftArgmax2d((3, 3)) + assert m(sample).shape == (1, 1, 2, 3, 3) + + def test_smoke_batch(self, device, dtype): + sample = torch.zeros(2, 5, 3, 3, device=device, dtype=dtype) + m = kornia.geometry.subpix.ConvSoftArgmax2d() + assert m(sample).shape == (2, 5, 2, 3, 3) + + def test_smoke_with_val(self, device, dtype): + sample = torch.zeros(1, 1, 3, 3, device=device, dtype=dtype) + m = kornia.geometry.subpix.ConvSoftArgmax2d((3, 3), output_value=True) + coords, val = m(sample) + assert coords.shape == (1, 1, 2, 3, 3) + assert val.shape == (1, 1, 3, 3) + + def test_smoke_batch_with_val(self, device, dtype): + sample = torch.zeros(2, 5, 3, 3, device=device, dtype=dtype) + m = kornia.geometry.subpix.ConvSoftArgmax2d((3, 3), output_value=True) + coords, val = m(sample) + assert coords.shape == (2, 5, 2, 3, 3) + assert val.shape == (2, 5, 3, 3) + + def test_gradcheck(self, device): + sample = torch.rand(2, 3, 5, 5, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.subpix.conv_soft_argmax2d, (sample), nondet_tol=1e-8) + + def test_cold_diag(self, device, dtype): + sample = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + softargmax = kornia.geometry.subpix.ConvSoftArgmax2d( + (3, 3), (2, 2), (0, 0), temperature=0.05, normalized_coordinates=False, output_value=True + ) + expected_val = torch.tensor([[[[1.0, 0.0], [0.0, 1.0]]]], device=device, dtype=dtype) + expected_coord = torch.tensor( + [[[[[1.0, 3.0], [1.0, 3.0]], [[1.0, 1.0], [3.0, 3.0]]]]], device=device, dtype=dtype + ) + coords, val = softargmax(sample) + self.assert_close(val, expected_val, atol=1e-4, rtol=1e-4) + self.assert_close(coords, expected_coord, atol=1e-4, rtol=1e-4) + + def test_hot_diag(self, device, dtype): + sample = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + softargmax = kornia.geometry.subpix.ConvSoftArgmax2d( + (3, 3), (2, 2), (0, 0), temperature=10.0, normalized_coordinates=False, output_value=True + ) + expected_val = torch.tensor([[[[0.1214, 0.0], [0.0, 0.1214]]]], device=device, dtype=dtype) + expected_coord = torch.tensor( + [[[[[1.0, 3.0], [1.0, 3.0]], [[1.0, 1.0], [3.0, 3.0]]]]], device=device, dtype=dtype + ) + coords, val = softargmax(sample) + self.assert_close(val, expected_val, atol=1e-4, rtol=1e-4) + self.assert_close(coords, expected_coord, atol=1e-4, rtol=1e-4) + + def test_cold_diag_norm(self, device, dtype): + sample = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + softargmax = kornia.geometry.subpix.ConvSoftArgmax2d( + (3, 3), (2, 2), (0, 0), temperature=0.05, normalized_coordinates=True, output_value=True + ) + expected_val = torch.tensor([[[[1.0, 0.0], [0.0, 1.0]]]], device=device, dtype=dtype) + expected_coord = torch.tensor( + [[[[[-0.5, 0.5], [-0.5, 0.5]], [[-0.5, -0.5], [0.5, 0.5]]]]], device=device, dtype=dtype + ) + coords, val = softargmax(sample) + self.assert_close(val, expected_val, atol=1e-4, rtol=1e-4) + self.assert_close(coords, expected_coord, atol=1e-4, rtol=1e-4) + + def test_hot_diag_norm(self, device, dtype): + sample = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ], + device=device, + dtype=dtype, + ) + softargmax = kornia.geometry.subpix.ConvSoftArgmax2d( + (3, 3), (2, 2), (0, 0), temperature=10.0, normalized_coordinates=True, output_value=True + ) + expected_val = torch.tensor([[[[0.1214, 0.0], [0.0, 0.1214]]]], device=device, dtype=dtype) + expected_coord = torch.tensor( + [[[[[-0.5, 0.5], [-0.5, 0.5]], [[-0.5, -0.5], [0.5, 0.5]]]]], device=device, dtype=dtype + ) + coords, val = softargmax(sample) + self.assert_close(val, expected_val, atol=1e-4, rtol=1e-4) + self.assert_close(coords, expected_coord, atol=1e-4, rtol=1e-4) + + +class TestConvSoftArgmax3d(BaseTester): + def test_smoke(self, device, dtype): + sample = torch.zeros(1, 1, 3, 3, 3, device=device, dtype=dtype) + m = kornia.geometry.subpix.ConvSoftArgmax3d((3, 3, 3), output_value=False) + assert m(sample).shape == (1, 1, 3, 3, 3, 3) + + def test_smoke_with_val(self, device, dtype): + sample = torch.zeros(1, 1, 3, 3, 3, device=device, dtype=dtype) + m = kornia.geometry.subpix.ConvSoftArgmax3d((3, 3, 3), output_value=True) + coords, val = m(sample) + assert coords.shape == (1, 1, 3, 3, 3, 3) + assert val.shape == (1, 1, 3, 3, 3) + + def test_gradcheck(self, device): + sample = torch.rand(1, 2, 3, 5, 5, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.subpix.conv_soft_argmax3d, (sample), nondet_tol=1e-8) + + def test_cold_diag(self, device, dtype): + sample = torch.tensor( + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ] + ], + device=device, + dtype=dtype, + ) + softargmax = kornia.geometry.subpix.ConvSoftArgmax3d( + (1, 3, 3), (1, 2, 2), (0, 0, 0), temperature=0.05, normalized_coordinates=False, output_value=True + ) + expected_val = torch.tensor([[[[[1.0, 0.0], [0.0, 1.0]]]]], device=device, dtype=dtype) + expected_coord = torch.tensor( + [[[[[[0.0, 0.0], [0.0, 0.0]]], [[[1.0, 3.0], [1.0, 3.0]]], [[[1.0, 1.0], [3.0, 3.0]]]]]], + device=device, + dtype=dtype, + ) + coords, val = softargmax(sample) + self.assert_close(val, expected_val, atol=1e-4, rtol=1e-4) + self.assert_close(coords, expected_coord, atol=1e-4, rtol=1e-4) + + def test_hot_diag(self, device, dtype): + sample = torch.tensor( + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ] + ], + device=device, + dtype=dtype, + ) + softargmax = kornia.geometry.subpix.ConvSoftArgmax3d( + (1, 3, 3), (1, 2, 2), (0, 0, 0), temperature=10.0, normalized_coordinates=False, output_value=True + ) + expected_val = torch.tensor([[[[[0.1214, 0.0], [0.0, 0.1214]]]]], device=device, dtype=dtype) + expected_coord = torch.tensor( + [[[[[[0.0, 0.0], [0.0, 0.0]]], [[[1.0, 3.0], [1.0, 3.0]]], [[[1.0, 1.0], [3.0, 3.0]]]]]], + device=device, + dtype=dtype, + ) + coords, val = softargmax(sample) + self.assert_close(val, expected_val, atol=1e-4, rtol=1e-4) + self.assert_close(coords, expected_coord, atol=1e-4, rtol=1e-4) + + def test_cold_diag_norm(self, device, dtype): + sample = torch.tensor( + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ] + ], + device=device, + dtype=dtype, + ) + softargmax = kornia.geometry.subpix.ConvSoftArgmax3d( + (1, 3, 3), (1, 2, 2), (0, 0, 0), temperature=0.05, normalized_coordinates=True, output_value=True + ) + expected_val = torch.tensor([[[[[1.0, 0.0], [0.0, 1.0]]]]], device=device, dtype=dtype) + expected_coord = torch.tensor( + [[[[[[-1.0, -1.0], [-1.0, -1.0]]], [[[-0.5, 0.5], [-0.5, 0.5]]], [[[-0.5, -0.5], [0.5, 0.5]]]]]], + device=device, + dtype=dtype, + ) + coords, val = softargmax(sample) + self.assert_close(val, expected_val, atol=1e-4, rtol=1e-4) + self.assert_close(coords, expected_coord, atol=1e-4, rtol=1e-4) + + def test_hot_diag_norm(self, device, dtype): + sample = torch.tensor( + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ] + ] + ], + device=device, + dtype=dtype, + ) + softargmax = kornia.geometry.subpix.ConvSoftArgmax3d( + (1, 3, 3), (1, 2, 2), (0, 0, 0), temperature=10.0, normalized_coordinates=True, output_value=True + ) + expected_val = torch.tensor([[[[[0.1214, 0.0], [0.0, 0.1214]]]]], device=device, dtype=dtype) + expected_coord = torch.tensor( + [[[[[[-1.0, -1.0], [-1.0, -1.0]]], [[[-0.5, 0.5], [-0.5, 0.5]]], [[[-0.5, -0.5], [0.5, 0.5]]]]]], + device=device, + dtype=dtype, + ) + coords, val = softargmax(sample) + self.assert_close(val, expected_val, atol=1e-4, rtol=1e-4) + self.assert_close(coords, expected_coord, atol=1e-4, rtol=1e-4) + + +class TestConvQuadInterp3dModule(BaseTester): + def test_smoke(self, device, dtype): + sample = torch.randn(2, 3, 3, 4, 4, device=device, dtype=dtype) + nms = kornia.geometry.ConvQuadInterp3d(1) + coord, val = nms(sample) + assert coord.shape == (2, 3, 3, 3, 4, 4) + assert val.shape == (2, 3, 3, 4, 4) + + def test_gradcheck(self, device): + sample = torch.rand(1, 1, 3, 5, 5, device=device, dtype=torch.float64) + sample[0, 0, 1, 2, 2] += 20.0 + self.gradcheck(kornia.geometry.ConvQuadInterp3d(strict_maxima_bonus=0), (sample), atol=1e-3, rtol=1e-3) + + def test_diag(self, device, dtype): + sample = torch.tensor( + [ + [ + [ + [0.0, 0.0, 0.0, 0, 0], + [0.0, 0.0, 0.0, 0, 0.0], + [0.0, 0, 0.0, 0, 0.0], + [0.0, 0.0, 0, 0, 0.0], + [0.0, 0.0, 0.0, 0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0, 0], + [0.0, 0.0, 1, 0, 0.0], + [0.0, 1, 1.2, 1.1, 0.0], + [0.0, 0.0, 1.0, 0, 0.0], + [0.0, 0.0, 0.0, 0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0, 0], + [0.0, 0.0, 0.0, 0, 0.0], + [0.0, 0, 0.0, 0, 0.0], + [0.0, 0.0, 0, 0, 0.0], + [0.0, 0.0, 0.0, 0, 0.0], + ], + ] + ], + device=device, + dtype=dtype, + ) + sample = kornia.filters.gaussian_blur2d(sample, (5, 5), (0.5, 0.5)).unsqueeze(0) + softargmax = kornia.geometry.ConvQuadInterp3d(10) + expected_val = torch.tensor( + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0, 0], + [0.0, 0.0, 0.0, 0, 0.0], + [0.0, 0, 0.0, 0, 0.0], + [0.0, 0.0, 0, 0, 0.0], + [0.0, 0.0, 0.0, 0, 0.0], + ], + [ + [2.2504e-04, 2.3146e-02, 1.6808e-01, 2.3188e-02, 2.3628e-04], + [2.3146e-02, 1.8118e-01, 7.4338e-01, 1.8955e-01, 2.5413e-02], + [1.6807e-01, 7.4227e-01, 1.1086e01, 8.0414e-01, 1.8482e-01], + [2.3146e-02, 1.8118e-01, 7.4338e-01, 1.8955e-01, 2.5413e-02], + [2.2504e-04, 2.3146e-02, 1.6808e-01, 2.3188e-02, 2.3628e-04], + ], + [ + [0.0, 0.0, 0.0, 0, 0], + [0.0, 0.0, 0.0, 0, 0.0], + [0.0, 0, 0.0, 0, 0.0], + [0.0, 0.0, 0, 0, 0.0], + [0.0, 0.0, 0.0, 0, 0.0], + ], + ] + ] + ], + device=device, + dtype=dtype, + ) + expected_coord = torch.tensor( + [ + [ + [ + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0], + ], + [ + [2.0, 2.0, 2.0, 2.0, 2.0], + [2.0, 2.0, 2.0, 2.0, 2.0], + [2.0, 2.0, 2.0, 2.0, 2.0], + [2.0, 2.0, 2.0, 2.0, 2.0], + [2.0, 2.0, 2.0, 2.0, 2.0], + ], + ], + [ + [ + [0.0, 1.0, 2.0, 3.0, 4.0], + [0.0, 1.0, 2.0, 3.0, 4.0], + [0.0, 1.0, 2.0, 3.0, 4.0], + [0.0, 1.0, 2.0, 3.0, 4.0], + [0.0, 1.0, 2.0, 3.0, 4.0], + ], + [ + [0.0, 1.0, 2.0, 3.0, 4.0], + [0.0, 1.0, 2.0, 3.0, 4.0], + [0.0, 1.0, 2.0495, 3.0, 4.0], + [0.0, 1.0, 2.0, 3.0, 4.0], + [0.0, 1.0, 2.0, 3.0, 4.0], + ], + [ + [0.0, 1.0, 2.0, 3.0, 4.0], + [0.0, 1.0, 2.0, 3.0, 4.0], + [0.0, 1.0, 2.0, 3.0, 4.0], + [0.0, 1.0, 2.0, 3.0, 4.0], + [0.0, 1.0, 2.0, 3.0, 4.0], + ], + ], + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [1.0, 1.0, 1.0, 1.0, 1.0], + [2.0, 2.0, 2.0, 2.0, 2.0], + [3.0, 3.0, 3.0, 3.0, 3.0], + [4.0, 4.0, 4.0, 4.0, 4.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [1.0, 1.0, 1.0, 1.0, 1.0], + [2.0, 2.0, 2.0, 2.0, 2.0], + [3.0, 3.0, 3.0, 3.0, 3.0], + [4.0, 4.0, 4.0, 4.0, 4.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [1.0, 1.0, 1.0, 1.0, 1.0], + [2.0, 2.0, 2.0, 2.0, 2.0], + [3.0, 3.0, 3.0, 3.0, 3.0], + [4.0, 4.0, 4.0, 4.0, 4.0], + ], + ], + ] + ] + ], + device=device, + dtype=dtype, + ) + coords, val = softargmax(sample) + self.assert_close(val, expected_val, atol=1e-4, rtol=1e-4) + self.assert_close(coords, expected_coord, atol=1e-4, rtol=1e-4) + + +class TestConvQuadInterp3d(BaseTester): + def test_smoke(self, device, dtype): + sample = torch.randn(2, 3, 3, 4, 4, device=device, dtype=dtype) + op = kornia.geometry.subpix.ConvQuadInterp3d(n_iters=3, strict_maxima_bonus=1) + coord, val = op(sample) + assert coord.shape == (2, 3, 3, 3, 4, 4) + assert val.shape == (2, 3, 3, 4, 4) + + def test_exception(self, device, dtype): + with pytest.raises(TypeError): + conv_quad_interp3d("not_a_tensor") + with pytest.raises(ValueError): + conv_quad_interp3d(torch.randn(3, 4, 4, device=device, dtype=dtype)) + + def test_cardinality(self, device, dtype): + for B, C, D, H, W in [(1, 1, 3, 5, 5), (2, 4, 3, 8, 6)]: + sample = torch.randn(B, C, D, H, W, device=device, dtype=dtype) + coord, val = conv_quad_interp3d(sample) + assert coord.shape == (B, C, 3, D, H, W) + assert val.shape == (B, C, D, H, W) + + def test_gradcheck(self, device): + sample = torch.rand(1, 1, 3, 5, 5, device=device, dtype=torch.float64) + sample[0, 0, 1, 2, 2] += 20.0 + self.gradcheck( + kornia.geometry.subpix.ConvQuadInterp3d(strict_maxima_bonus=0, n_iters=1), + (sample,), + atol=1e-3, + rtol=1e-3, + ) + + def test_dynamo(self, device, dtype, torch_optimizer): + sample = torch.rand(1, 1, 3, 5, 5, device=device, dtype=dtype) + sample[0, 0, 1, 2, 2] += 20.0 + op = kornia.geometry.subpix.ConvQuadInterp3d(strict_maxima_bonus=0, n_iters=1) + op_opt = torch_optimizer(op) + self.assert_close(op(sample)[0], op_opt(sample)[0]) + self.assert_close(op(sample)[1], op_opt(sample)[1]) + + def test_peak_at_center(self, device, dtype): + # A clear peak at scale=1, h=2, w=2 should return coords close to (1, 2, 2). + sample = torch.zeros(1, 1, 3, 5, 5, device=device, dtype=dtype) + sample[0, 0, 1, 2, 2] = 10.0 + coord, _val = conv_quad_interp3d(sample, strict_maxima_bonus=0) + # coords_max layout: dim2 = [scale, x(width), y(height)] + assert coord[0, 0, 0, 1, 2, 2].item() == pytest.approx(1.0, abs=1e-3) # scale + assert coord[0, 0, 1, 1, 2, 2].item() == pytest.approx(2.0, abs=1e-3) # x + assert coord[0, 0, 2, 1, 2, 2].item() == pytest.approx(2.0, abs=1e-3) # y + + def test_subpixel_shift(self, device, dtype): + # Peak shifted slightly — subpixel offset should be non-zero. + # Use D=5 so the center scale index 2 has valid ±1 neighbours. + sample = torch.zeros(1, 1, 5, 7, 7, device=device, dtype=dtype) + # Place a Gaussian-shaped peak slightly off-center (center at d=2, h=3, w=3). + for dd in range(-1, 2): + for dh in range(-1, 2): + for dw in range(-1, 2): + dist2 = (dd - 0.2) ** 2 + (dh - 0.1) ** 2 + (dw + 0.15) ** 2 + sample[0, 0, 2 + dd, 3 + dh, 3 + dw] += float(torch.exp(torch.tensor(-dist2 * 4))) + coord, _ = conv_quad_interp3d(sample, strict_maxima_bonus=0) + # The refined x (width) coord at the integer peak (d=2, h=3, w=3) should shift toward -0.15. + x_coord = coord[0, 0, 1, 2, 3, 3].item() + assert x_coord < 3.0 # shift in negative x direction + + def test_no_keypoints(self, device, dtype): + # Flat input — no NMS maxima, output should equal input coords/values. + sample = torch.ones(1, 1, 3, 4, 4, device=device, dtype=dtype) + coord, val = conv_quad_interp3d(sample, strict_maxima_bonus=0) + assert coord.shape == (1, 1, 3, 3, 4, 4) + assert val.shape == (1, 1, 3, 4, 4) + + def test_convergence_within_iters(self, device, dtype): + # With a clear symmetric peak a single iteration should converge. + sample = torch.zeros(1, 1, 3, 5, 5, device=device, dtype=dtype) + sample[0, 0, 1, 2, 2] = 5.0 + coord1, _ = conv_quad_interp3d(sample, n_iters=1, strict_maxima_bonus=0) + coord5, _ = conv_quad_interp3d(sample, n_iters=5, strict_maxima_bonus=0) + self.assert_close(coord1, coord5, atol=1e-5, rtol=1e-5) + + +class TestAdaptiveQuadInterp3d(BaseTester): + def test_smoke(self, device, dtype): + for mode in ("patch", "conv", "auto"): + x = torch.randn(1, 1, 3, 8, 8, device=device, dtype=dtype) + coords, vals = kornia.geometry.subpix.AdaptiveQuadInterp3d(mode=mode)(x) + assert coords.shape == (1, 1, 3, 3, 8, 8) + assert vals.shape == (1, 1, 3, 8, 8) + + def test_invalid_mode(self, device, dtype): + with pytest.raises(ValueError, match="mode must be one of"): + kornia.geometry.subpix.AdaptiveQuadInterp3d(mode="bogus") + + def test_patch_conv_agree(self, device, dtype): + """patch and conv backends must produce numerically identical results.""" + torch.manual_seed(7) + x = torch.randn(1, 1, 5, 16, 16, device=device, dtype=dtype) + coords_p, vals_p = kornia.geometry.subpix.AdaptiveQuadInterp3d(mode="patch", strict_maxima_bonus=0)(x) + coords_c, vals_c = kornia.geometry.subpix.AdaptiveQuadInterp3d(mode="conv", strict_maxima_bonus=0)(x) + from kornia.geometry.subpix import nms3d + + mask = nms3d(x, (3, 3, 3), True) + b, c, d, h, w = torch.where(mask) + self.assert_close(coords_p[b, c, :, d, h, w], coords_c[b, c, :, d, h, w], atol=1e-5, rtol=1e-5) + self.assert_close(vals_p[b, c, d, h, w], vals_c[b, c, d, h, w], atol=1e-5, rtol=1e-5) + + def test_auto_dispatches(self, device, dtype): + """auto mode must use conv on CUDA, patch on CPU (verified via result equality).""" + x = torch.randn(1, 1, 3, 8, 8, device=device, dtype=dtype) + auto = kornia.geometry.subpix.AdaptiveQuadInterp3d(mode="auto", strict_maxima_bonus=0) + expected_mode = "conv" if x.is_cuda else "patch" + ref = kornia.geometry.subpix.AdaptiveQuadInterp3d(mode=expected_mode, strict_maxima_bonus=0) + self.assert_close(auto(x)[0], ref(x)[0], atol=1e-5, rtol=1e-5) + self.assert_close(auto(x)[1], ref(x)[1], atol=1e-5, rtol=1e-5) + + def test_gradcheck(self, device): + x = torch.zeros(1, 1, 3, 5, 5, device=device, dtype=torch.float64) + x[0, 0, 1, 2, 2] = 5.0 + self.gradcheck( + kornia.geometry.subpix.AdaptiveQuadInterp3d(mode="patch", strict_maxima_bonus=0), + (x,), + atol=1e-3, + rtol=1e-3, + ) + + def test_dynamo(self, device, dtype, torch_optimizer): + x = torch.rand(1, 1, 3, 5, 5, device=device, dtype=dtype) + x[0, 0, 1, 2, 2] += 20.0 + op = kornia.geometry.subpix.AdaptiveQuadInterp3d(mode="patch", strict_maxima_bonus=0) + op_opt = torch_optimizer(op) + self.assert_close(op(x)[0], op_opt(x)[0]) + + def test_conv_matches_iterative(self, device, dtype): + """conv_quad_interp3d and iterative_quad_interp3d must give identical results at NMS maxima. + + Regression test for the c000_safe normalisation bug: dividing the Hessian by |centre| + before the det-threshold check made conv accept poorly-conditioned positions that + iterative correctly rejected, causing up to ~1 px coordinate divergence. + """ + from kornia.geometry.subpix.nms import nms3d_minmax + from kornia.geometry.subpix.spatial_soft_argmax import conv_quad_interp3d, iterative_quad_interp3d + + torch.manual_seed(7) + B, C, D, H, W = 1, 1, 5, 32, 32 + # Synthetic DoG-like response: mix of positive and negative small-amplitude blobs + x = torch.zeros(B, C, D, H, W, device=device, dtype=dtype) + for d0, h0, w0, sign, amp in [(2, 8, 10, 1, 0.008), (2, 22, 16, -1, 0.005), (3, 14, 24, 1, 0.012)]: + dd = torch.arange(D, device=device, dtype=dtype) - d0 + dh = torch.arange(H, device=device, dtype=dtype) - h0 + dw = torch.arange(W, device=device, dtype=dtype) - w0 + x[0, 0] += ( + sign + * amp + * ( + torch.exp(-0.5 * dd**2).view(D, 1, 1) + * torch.exp(-0.5 * (dh / 2.5) ** 2).view(1, H, 1) + * torch.exp(-0.5 * (dw / 2.5) ** 2).view(1, 1, W) + ) + ) + + max_mask, _ = nms3d_minmax(x) + coord_conv, _ = conv_quad_interp3d( + x, n_iters=5, strict_maxima_bonus=0.0, precomputed_nms_mask=max_mask, dilation_radius=3 + ) + coord_iter, _ = iterative_quad_interp3d(x, n_iters=5, strict_maxima_bonus=0.0) + + d_idx, h_idx, w_idx = torch.where(max_mask.view(D, H, W)) + assert len(d_idx) > 0, "No NMS maxima found — check the synthetic input" + diff = (coord_conv[0, 0, :, d_idx, h_idx, w_idx] - coord_iter[0, 0, :, d_idx, h_idx, w_idx]).abs().max() + self.assert_close(diff, torch.zeros_like(diff), atol=1e-5, rtol=0) diff --git a/tests/geometry/test_bbox.py b/tests/geometry/test_bbox.py new file mode 100644 index 0000000..6a94452 --- /dev/null +++ b/tests/geometry/test_bbox.py @@ -0,0 +1,306 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch + +import kornia +from kornia.geometry.bbox import ( + infer_bbox_shape, + infer_bbox_shape3d, + nms, + transform_bbox, + validate_bbox, + validate_bbox3d, +) + +from testing.base import BaseTester + + +class TestBbox2D(BaseTester): + def test_smoke(self, device, dtype): + # Sample two points of the rectangle + points = torch.rand(1, 4, device=device, dtype=dtype) + + # Fill according missing points + bbox = torch.zeros(1, 4, 2, device=device, dtype=dtype) + bbox[0, 0] = points[0][:2] + bbox[0, 1, 0] = points[0][2] + bbox[0, 1, 1] = points[0][1] + bbox[0, 2] = points[0][2:] + bbox[0, 3, 0] = points[0][0] + bbox[0, 3, 1] = points[0][3] + + # Validate + assert validate_bbox(bbox) + + def test_bounding_boxes_dim_inferring(self, device, dtype): + boxes = torch.tensor([[[1.0, 1.0], [3.0, 1.0], [3.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype) + h, w = infer_bbox_shape(boxes) + assert (h, w) == (2, 3) + + def test_bounding_boxes_dim_inferring_batch(self, device, dtype): + boxes = torch.tensor( + [[[1.0, 1.0], [3.0, 1.0], [3.0, 2.0], [1.0, 2.0]], [[2.0, 2.0], [4.0, 2.0], [4.0, 3.0], [2.0, 3.0]]], + device=device, + dtype=dtype, + ) + h, w = infer_bbox_shape(boxes) + assert (h.unique().item(), w.unique().item()) == (2, 3) + + def test_gradcheck(self, device): + boxes = torch.tensor([[[1.0, 1.0], [3.0, 1.0], [3.0, 2.0], [1.0, 2.0]]], device=device, dtype=torch.float64) + self.gradcheck(infer_bbox_shape, (boxes,)) + + def test_dynamo(self, device, dtype, torch_optimizer): + # Define script + op = infer_bbox_shape + op_optimized = torch_optimizer(op) + # Define input + boxes = torch.tensor([[[1.0, 1.0], [3.0, 1.0], [3.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype) + # Run + expected = op(boxes) + actual = op_optimized(boxes) + # Compare + self.assert_close(actual, expected) + + def test_jit(self, device, dtype): + # Test with valid rectangular box + boxes = torch.tensor([[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0]]], device=device, dtype=dtype) + + # JIT compile the validate_bbox function + scripted_fn = torch.jit.script(validate_bbox) + + # Test with valid box + self.assert_close(scripted_fn(boxes), validate_bbox(boxes)) + + # Test with non-rectangular box + boxes_invalid = torch.tensor([[[0.0, 0.0], [2.0, 0.0], [3.0, 1.0], [1.0, 1.0]]], device=device, dtype=dtype) + self.assert_close(scripted_fn(boxes_invalid), validate_bbox(boxes_invalid)) + + # Test with invalid shape + boxes_wrong_shape = torch.rand(1, 3, 2, device=device, dtype=dtype) + self.assert_close(scripted_fn(boxes_wrong_shape), validate_bbox(boxes_wrong_shape)) + + +class TestTransformBoxes2D(BaseTester): + def test_transform_boxes(self, device, dtype): + boxes = torch.tensor([[139.2640, 103.0150, 397.3120, 410.5225]], device=device, dtype=dtype) + + expected = torch.tensor([[114.6880, 103.0150, 372.7360, 410.5225]], device=device, dtype=dtype) + + trans_mat = torch.tensor([[[-1.0, 0.0, 512.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + + out = transform_bbox(trans_mat, boxes, restore_coordinates=True) + self.assert_close(out, expected, atol=1e-4, rtol=1e-4) + + def test_transform_multiple_boxes(self, device, dtype): + boxes = torch.tensor( + [ + [139.2640, 103.0150, 397.3120, 410.5225], + [1.0240, 80.5547, 512.0000, 512.0000], + [165.2053, 262.1440, 510.6347, 508.9280], + [119.8080, 144.2067, 257.0240, 410.1292], + ], + device=device, + dtype=dtype, + ) + + boxes = boxes.repeat(2, 1, 1) # 2 x 4 x 4 two images 4 boxes each + + expected = torch.tensor( + [ + [ + [114.6880, 103.0150, 372.7360, 410.5225], + [0.0000, 80.5547, 510.9760, 512.0000], + [1.3652, 262.1440, 346.7947, 508.9280], + [254.9760, 144.2067, 392.1920, 410.1292], + ], + [ + [139.2640, 103.0150, 397.3120, 410.5225], + [1.0240, 80.5547, 512.0000, 512.0000], + [165.2053, 262.1440, 510.6347, 508.9280], + [119.8080, 144.2067, 257.0240, 410.1292], + ], + ], + device=device, + dtype=dtype, + ) + + trans_mat = torch.tensor( + [ + [[-1.0, 0.0, 512.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], + [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], + ], + device=device, + dtype=dtype, + ) + + out = transform_bbox(trans_mat, boxes, restore_coordinates=True) + self.assert_close(out, expected, atol=1e-4, rtol=1e-4) + + def test_transform_boxes_wh(self, device, dtype): + boxes = torch.tensor( + [ + [139.2640, 103.0150, 258.0480, 307.5075], + [1.0240, 80.5547, 510.9760, 431.4453], + [165.2053, 262.1440, 345.4293, 246.7840], + [119.8080, 144.2067, 137.2160, 265.9225], + ], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [ + [114.6880, 103.0150, 258.0480, 307.5075], + [0.0000, 80.5547, 510.9760, 431.4453], + [1.3654, 262.1440, 345.4293, 246.7840], + [254.9760, 144.2067, 137.2160, 265.9225], + ], + device=device, + dtype=dtype, + ) + + trans_mat = torch.tensor([[[-1.0, 0.0, 512.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + + out = transform_bbox(trans_mat, boxes, mode="xywh", restore_coordinates=True) + self.assert_close(out, expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + boxes = torch.tensor( + [ + [139.2640, 103.0150, 258.0480, 307.5075], + [1.0240, 80.5547, 510.9760, 431.4453], + [165.2053, 262.1440, 345.4293, 246.7840], + [119.8080, 144.2067, 137.2160, 265.9225], + ], + device=device, + dtype=torch.float64, + ) + + trans_mat = torch.tensor( + [[[-1.0, 0.0, 512.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=torch.float64 + ) + + self.gradcheck(transform_bbox, (trans_mat, boxes, "xyxy", True)) + + def test_dynamo(self, device, dtype, torch_optimizer): + boxes = torch.tensor([[139.2640, 103.0150, 258.0480, 307.5075]], device=device, dtype=dtype) + trans_mat = torch.tensor([[[-1.0, 0.0, 512.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + args = (boxes, trans_mat) + op = kornia.geometry.transform_points + op_optimized = torch_optimizer(op) + self.assert_close(op(*args), op_optimized(*args)) + + +class TestBbox3D(BaseTester): + def test_smoke(self, device, dtype): + # Sample two points of the 3d rect + points = torch.rand(1, 6, device=device, dtype=dtype) + + # Fill according missing points + bbox = torch.zeros(1, 8, 3, device=device, dtype=dtype) + bbox[0, 0] = points[0][:3] + bbox[0, 1, 0] = points[0][3] + bbox[0, 1, 1] = points[0][1] + bbox[0, 1, 2] = points[0][2] + bbox[0, 2, 0] = points[0][3] + bbox[0, 2, 1] = points[0][4] + bbox[0, 2, 2] = points[0][2] + bbox[0, 3, 0] = points[0][0] + bbox[0, 3, 1] = points[0][4] + bbox[0, 3, 2] = points[0][2] + bbox[0, 4, 0] = points[0][0] + bbox[0, 4, 1] = points[0][1] + bbox[0, 4, 2] = points[0][5] + bbox[0, 5, 0] = points[0][3] + bbox[0, 5, 1] = points[0][1] + bbox[0, 5, 2] = points[0][5] + bbox[0, 6] = points[0][3:] + bbox[0, 7, 0] = points[0][0] + bbox[0, 7, 1] = points[0][4] + bbox[0, 7, 2] = points[0][5] + + # Validate + assert validate_bbox3d(bbox) + + def test_bounding_boxes_dim_inferring(self, device, dtype): + boxes = torch.tensor( + [ + [[0, 1, 2], [10, 1, 2], [10, 21, 2], [0, 21, 2], [0, 1, 32], [10, 1, 32], [10, 21, 32], [0, 21, 32]], + [[3, 4, 5], [43, 4, 5], [43, 54, 5], [3, 54, 5], [3, 4, 65], [43, 4, 65], [43, 54, 65], [3, 54, 65]], + ], + device=device, + dtype=dtype, + ) # 2x8x3 + d, h, w = infer_bbox_shape3d(boxes) + + self.assert_close(d, torch.tensor([31.0, 61.0], device=device, dtype=dtype)) + self.assert_close(h, torch.tensor([21.0, 51.0], device=device, dtype=dtype)) + self.assert_close(w, torch.tensor([11.0, 41.0], device=device, dtype=dtype)) + + def test_gradcheck(self, device): + boxes = torch.tensor( + [ + [ + [0.0, 1.0, 2.0], + [10, 1, 2], + [10, 21, 2], + [0, 21, 2], + [0, 1, 32], + [10, 1, 32], + [10, 21, 32], + [0, 21, 32], + ] + ], + device=device, + dtype=torch.float64, + ) + self.gradcheck(infer_bbox_shape3d, (boxes,)) + + def test_dynamo(self, device, dtype, torch_optimizer): + # Define script + op = infer_bbox_shape3d + op_script = torch_optimizer(op) + + boxes = torch.tensor( + [[[0, 0, 1], [3, 0, 1], [3, 2, 1], [0, 2, 1], [0, 0, 3], [3, 0, 3], [3, 2, 3], [0, 2, 3]]], + device=device, + dtype=dtype, + ) # 1x8x3 + + actual = op_script(boxes) + expected = op(boxes) + self.assert_close(actual, expected) + + +class TestNMS(BaseTester): + def test_smoke(self, device, dtype): + boxes = torch.tensor( + [ + [10.0, 10.0, 20.0, 20.0], + [15.0, 5.0, 15.0, 25.0], + [100.0, 100.0, 200.0, 200.0], + [100.0, 100.0, 200.0, 200.0], + ], + device=device, + dtype=dtype, + ) + scores = torch.tensor([0.9, 0.8, 0.7, 0.9], device=device, dtype=dtype) + expected = torch.tensor([0, 3, 1], device=device, dtype=torch.long) + actual = nms(boxes, scores, iou_threshold=0.8) + self.assert_close(actual, expected) diff --git a/tests/geometry/test_boxes.py b/tests/geometry/test_boxes.py new file mode 100644 index 0000000..60652e1 --- /dev/null +++ b/tests/geometry/test_boxes.py @@ -0,0 +1,933 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from functools import partial + +import pytest +import torch + +from kornia.geometry.boxes import Boxes, Boxes3D + +from testing.base import BaseTester + + +class TestBoxes2D(BaseTester): + def test_smoke(self, device, dtype): + def _create_tensor_box(): + # Sample two points of the rectangle + points = torch.rand(1, 4, device=device, dtype=dtype) + + # Fill according missing points + tensor_boxes = torch.zeros(1, 4, 2, device=device, dtype=dtype) + tensor_boxes[0, 0] = points[0][:2] + tensor_boxes[0, 1, 0] = points[0][2] + tensor_boxes[0, 1, 1] = points[0][1] + tensor_boxes[0, 2] = points[0][2:] + tensor_boxes[0, 3, 0] = points[0][0] + tensor_boxes[0, 3, 1] = points[0][3] + return tensor_boxes + + # Validate + assert Boxes(_create_tensor_box()) # Validate 1 box + + # 2 boxes without batching (N, 4, 2) where N=2 + two_boxes = torch.cat([_create_tensor_box(), _create_tensor_box()]) + assert Boxes(two_boxes) + + # 2 boxes in batch (B, 1, 4, 2) where B=2 + batched_bbox = torch.stack([_create_tensor_box(), _create_tensor_box()]) + assert Boxes(batched_bbox) + + def test_get_boxes_shape(self, device, dtype): + box = Boxes(torch.tensor([[[1.0, 1.0], [3.0, 2.0], [1.0, 2.0], [3.0, 1.0]]], device=device, dtype=dtype)) + t_boxes = torch.tensor( + [[[1.0, 1.0], [3.0, 1.0], [1.0, 2.0], [3.0, 2.0]], [[5.0, 4.0], [2.0, 2.0], [5.0, 2.0], [2.0, 4.0]]], + device=device, + dtype=dtype, + ) # (2, 4, 2) + boxes = Boxes(t_boxes) + boxes_batch = Boxes(t_boxes[None]) # (1, 2, 4, 2) + + # Single box + h, w = box.get_boxes_shape() + assert (h.item(), w.item()) == (2, 3) + + # Boxes + h, w = boxes.get_boxes_shape() + assert h.ndim == 1 + assert w.ndim == 1 + assert len(h) == 2 + assert len(w) == 2 + self.assert_close(h, torch.as_tensor([2.0, 3.0], device=device, dtype=dtype)) + self.assert_close(w, torch.as_tensor([3.0, 4.0], device=device, dtype=dtype)) + + # Box batch + h, w = boxes_batch.get_boxes_shape() + assert h.ndim == 2 + assert w.ndim == 2 + assert h.shape == (1, 2) + assert w.shape == (1, 2) + self.assert_close(h, torch.as_tensor([[2.0, 3.0]], device=device, dtype=dtype)) + self.assert_close(w, torch.as_tensor([[3.0, 4.0]], device=device, dtype=dtype)) + + def test_get_boxes_shape_batch(self, device, dtype): + t_box1 = torch.tensor([[[1.0, 1.0], [3.0, 2.0], [3.0, 1.0], [1.0, 2.0]]], device=device, dtype=dtype) + t_box2 = torch.tensor([[[5.0, 2.0], [2.0, 2.0], [5.0, 4.0], [2.0, 4.0]]], device=device, dtype=dtype) + batched_boxes = Boxes(torch.stack([t_box1, t_box2])) + + h, w = batched_boxes.get_boxes_shape() + assert h.ndim == 2 + assert w.ndim == 2 + assert h.shape == (2, 1) + assert w.shape == (2, 1) + self.assert_close(h, torch.as_tensor([[2], [3]], device=device, dtype=dtype)) + self.assert_close(w, torch.as_tensor([[3], [4]], device=device, dtype=dtype)) + + @pytest.mark.parametrize("shape", [(1, 4), (1, 1, 4)]) + def test_from_tensor(self, shape, device, dtype): + box_xyxy = torch.as_tensor([[1, 2, 3, 4]], device=device, dtype=dtype).view(*shape) + box_xyxy_plus = torch.as_tensor([[1, 2, 2, 3]], device=device, dtype=dtype).view(*shape) + box_xywh = torch.as_tensor([[1, 2, 2, 2]], device=device, dtype=dtype).view(*shape) + box_vertices = torch.as_tensor([[[1, 2], [3, 2], [3, 4], [1, 4]]], device=device, dtype=dtype).view(*shape, 2) + box_vertices_plus = torch.as_tensor([[[1, 2], [2, 2], [2, 3], [1, 3]]], device=device, dtype=dtype).view( + *shape, 2 + ) + + expected_box = torch.as_tensor([[[1, 2], [2, 2], [2, 3], [1, 3]]], device=device, dtype=dtype).view(*shape, 2) + + boxes_xyxy = Boxes.from_tensor(box_xyxy, mode="xyxy").data + boxes_xyxy_plus = Boxes.from_tensor(box_xyxy_plus, mode="xyxy_plus").data + boxes_xywh = Boxes.from_tensor(box_xywh, mode="xywh").data + box_vertices = Boxes.from_tensor(box_vertices, mode="vertices").data + boxes_vertices_plus = Boxes.from_tensor(box_vertices_plus, mode="vertices_plus").data + + assert boxes_xyxy.shape == expected_box.shape + self.assert_close(boxes_xyxy, expected_box) + + assert boxes_xyxy_plus.shape == expected_box.shape + self.assert_close(boxes_xyxy_plus, expected_box) + + assert boxes_xywh.shape == expected_box.shape + self.assert_close(boxes_xywh, expected_box) + + assert box_vertices.shape == expected_box.shape + self.assert_close(box_vertices, expected_box) + + assert boxes_vertices_plus.shape == expected_box.shape + self.assert_close(boxes_vertices_plus, expected_box) + + @pytest.mark.parametrize("shape", [(1, 4), (1, 1, 4)]) + def test_from_invalid_tensor(self, shape, device, dtype): + box_xyxy = torch.as_tensor([[1, 2, -3, 4]], device=device, dtype=dtype).view(*shape) # Invalid width + box_xyxy_plus = torch.as_tensor([[1, 2, 0, 3]], device=device, dtype=dtype).view(*shape) # Invalid height + + try: + Boxes.from_tensor(box_xyxy, mode="xyxy") + raise AssertionError("Boxes.from_tensor should have raised any exception") + except ValueError: + pass + + try: + Boxes.from_tensor(box_xyxy_plus, mode="xyxy_plus") + raise AssertionError("Boxes.from_tensor should have raised any exception") + except ValueError: + pass + + @pytest.mark.parametrize("shape", [(1, 4), (1, 1, 4)]) + def test_boxes_to_tensor(self, shape, device, dtype): + # quadrilateral with randomized vertices to reflect possible transforms. + box = Boxes(torch.as_tensor([[[2, 2], [2, 3], [1, 3], [1, 2]]], device=device, dtype=dtype).view(*shape, 2)) + + expected_box_xyxy = torch.as_tensor([[1, 2, 3, 4]], device=device, dtype=dtype).view(*shape) + expected_box_xyxy_plus = torch.as_tensor([[1, 2, 2, 3]], device=device, dtype=dtype).view(*shape) + expected_box_xywh = torch.as_tensor([[1, 2, 2, 2]], device=device, dtype=dtype).view(*shape) + expected_vertices = torch.as_tensor([[[1, 2], [3, 2], [3, 4], [1, 4]]], device=device, dtype=dtype).view( + *shape, 2 + ) + expected_vertices_plus = torch.as_tensor([[[1, 2], [2, 2], [2, 3], [1, 3]]], device=device, dtype=dtype).view( + *shape, 2 + ) + + boxes_xyxy = box.to_tensor(mode="xyxy") + boxes_xyxy_plus = box.to_tensor(mode="xyxy_plus") + boxes_xywh = box.to_tensor(mode="xywh") + boxes_vertices = box.to_tensor(mode="vertices") + boxes_vertices_plus = box.to_tensor(mode="vertices_plus") + + assert boxes_xyxy.shape == expected_box_xyxy.shape + self.assert_close(boxes_xyxy, expected_box_xyxy) + + assert boxes_xyxy_plus.shape == expected_box_xyxy_plus.shape + self.assert_close(boxes_xyxy_plus, expected_box_xyxy_plus) + + assert boxes_xywh.shape == expected_box_xywh.shape + self.assert_close(boxes_xywh, expected_box_xywh) + + assert boxes_vertices.shape == expected_vertices.shape + self.assert_close(boxes_vertices, expected_vertices) + + assert boxes_vertices_plus.shape == expected_vertices_plus.shape + self.assert_close(boxes_vertices_plus, expected_vertices_plus) + + @pytest.mark.parametrize("mode", ["xyxy", "xyxy_plus", "xywh", "vertices", "vertices_plus"]) + def test_boxes_list_to_tensor_list(self, mode, device, dtype): + src_1 = [ + torch.as_tensor([[[1, 2], [1, 3], [2, 2], [2, 3]]], device=device, dtype=dtype), + torch.as_tensor( + [[[1, 2], [1, 3], [2, 2], [2, 3]], [[1, 2], [1, 3], [2, 2], [2, 3]]], device=device, dtype=dtype + ), + ] + src_2 = [ + torch.as_tensor([[1, 1, 5, 5]], device=device, dtype=dtype), + torch.as_tensor([[1, 1, 5, 5], [1, 1, 5, 5]], device=device, dtype=dtype), + ] + src = src_1 if mode in ["vertices", "vertices_plus"] else src_2 + box = Boxes.from_tensor(src, mode=mode) + out = box.to_tensor(mode) + assert out[0].shape == src[0].shape + assert out[1].shape == src[1].shape + + def test_boxes_to_mask(self, device, dtype): + t_box1 = torch.tensor( + [[[1.0, 1.0], [3.0, 1.0], [3.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype + ) # (1, 4, 2) + t_box2 = torch.tensor( + [[[2.0, 2.0], [4.0, 2.0], [4.0, 5.0], [2.0, 4.0]]], device=device, dtype=dtype + ) # (1, 4, 2) + box1, box2 = Boxes(t_box1), Boxes(t_box2) + two_boxes = Boxes(torch.cat([t_box1, t_box2])) # (2, 4, 2) + batched_boxes = Boxes(torch.stack([t_box1, t_box2])) # (2, 1, 4, 2) + + height, width = 7, 5 + + expected_mask1 = torch.tensor( + [ + [ + [0, 0, 0, 0, 0], + [0, 1, 1, 1, 0], + [0, 1, 1, 1, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + ] + ], + device=device, + dtype=dtype, + ) + + expected_mask2 = torch.tensor( + [ + [ + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 1, 1, 1], + [0, 0, 1, 1, 1], + [0, 0, 1, 1, 1], + [0, 0, 1, 1, 1], + [0, 0, 0, 0, 0], + ] + ], + device=device, + dtype=dtype, + ) + expected_two_masks = torch.cat([expected_mask1, expected_mask2]) + expected_batched_masks = torch.stack([expected_mask1, expected_mask2]) + + mask1 = box1.to_mask(height, width) + mask2 = box2.to_mask(height, width) + two_masks = two_boxes.to_mask(height, width) + batched_masks = batched_boxes.to_mask(height, width) + + assert mask1.shape == expected_mask1.shape + self.assert_close(mask1, expected_mask1) + + assert mask2.shape == expected_mask2.shape + self.assert_close(mask2, expected_mask2) + + assert two_masks.shape == expected_two_masks.shape + self.assert_close(two_masks, expected_two_masks) + + assert batched_masks.shape == expected_batched_masks.shape + self.assert_close(batched_masks, expected_batched_masks) + + def test_to(self, device, dtype): + boxes = Boxes.from_tensor(torch.as_tensor([[1, 2, 3, 4]], device="cpu", dtype=torch.float32)) + assert boxes.to(device=device).data.device == device + assert boxes.to(dtype=dtype).data.dtype == dtype + + boxes_moved = boxes.to(device, dtype) + assert boxes_moved is boxes # to is an inplace op. + assert boxes_moved.data.device == device, boxes_moved.data.dtype == dtype + + def test_gradcheck(self, device): + def apply_boxes_method(tensor: torch.Tensor, method: str, **kwargs): + boxes = Boxes(tensor) + result = getattr(boxes, method)(**kwargs) + return result.data if isinstance(result, Boxes) else result + + t_boxes1 = torch.tensor([[[1.0, 1.0], [3.0, 1.0], [3.0, 2.0], [1.0, 2.0]]], device=device, dtype=torch.float64) + + t_boxes2 = t_boxes1.detach().clone() + t_boxes3 = t_boxes1.detach().clone() + t_boxes4 = t_boxes1.detach().clone() + t_boxes_xyxy = torch.tensor([[1.0, 3.0, 5.0, 6.0]]) + t_boxes_xyxy1 = t_boxes_xyxy.detach().clone() + + self.gradcheck(partial(apply_boxes_method, method="to_tensor"), (t_boxes2,)) + self.gradcheck(partial(apply_boxes_method, method="to_tensor", mode="xyxy_plus"), (t_boxes3,)) + self.gradcheck(partial(apply_boxes_method, method="to_tensor", mode="vertices_plus"), (t_boxes4,)) + self.gradcheck(partial(apply_boxes_method, method="get_boxes_shape"), (t_boxes1,)) + self.gradcheck(lambda x: Boxes.from_tensor(x, mode="xyxy_plus").data, (t_boxes_xyxy,)) + self.gradcheck(lambda x: Boxes.from_tensor(x, mode="xywh").data, (t_boxes_xyxy1,)) + + def test_compute_area(self): + # Rectangle + box_1 = [[0.0, 0.0], [100.0, 0.0], [100.0, 50.0], [0.0, 50.0]] + # Trapezoid + box_2 = [[0.0, 0.0], [60.0, 0.0], [40.0, 50.0], [20.0, 50.0]] + # Parallelogram + box_3 = [[0.0, 0.0], [100.0, 0.0], [120.0, 50.0], [20.0, 50.0]] + # Random quadrilateral + box_4 = [ + [50.0, 50.0], + [150.0, 250.0], + [0.0, 500.0], + [27.0, 80], + ] + # Random quadrilateral + box_5 = [ + [0.0, 0.0], + [150.0, 0.0], + [150.0, 150.0], + [0.0, 0.5], + ] + # Rectangle with minus coordinates + box_6 = [[-500.0, -500.0], [-300.0, -500.0], [-300.0, -300.0], [-500.0, -300.0]] + + expected_values = [5000.0, 2000.0, 5000.0, 31925.0, 11287.5, 40000.0] + box_coordinates = torch.tensor([box_1, box_2, box_3, box_4, box_5, box_6]) + computed_areas = Boxes(box_coordinates).compute_area().tolist() + computed_areas_w_batch = Boxes(box_coordinates.reshape(2, 3, 4, 2)).compute_area().tolist() + flattened_computed_areas_w_batch = [area for batch in computed_areas_w_batch for area in batch] + assert all( + computed_area == expected_area for computed_area, expected_area in zip(computed_areas, expected_values) + ) + assert all( + computed_area == expected_area + for computed_area, expected_area in zip(flattened_computed_areas_w_batch, expected_values) + ) + + +class TestTransformBoxes2D(BaseTester): + def test_transform_boxes(self, device, dtype): + # Define boxes in XYXY format for simplicity. + boxes_xyxy = torch.tensor([[139.2640, 103.0150, 398.3120, 411.5225]], device=device, dtype=dtype) + expected_boxes_xyxy = torch.tensor([[372.7360, 103.0150, 115.6880, 411.5225]], device=device, dtype=dtype) + + boxes = Boxes.from_tensor(boxes_xyxy) + expected_boxes = Boxes.from_tensor(expected_boxes_xyxy, validate_boxes=False) + + trans_mat = torch.tensor([[[-1.0, 0.0, 512.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + + transformed_boxes = boxes.transform_boxes(trans_mat) + self.assert_close(transformed_boxes.data, expected_boxes.data, atol=1e-4, rtol=1e-4) + # inplace check + assert transformed_boxes is not boxes + + def test_transform_boxes_(self, device, dtype): + # Define boxes in XYXY format for simplicity. + boxes_xyxy = torch.tensor([[139.2640, 103.0150, 398.3120, 411.5225]], device=device, dtype=dtype) + expected_boxes_xyxy = torch.tensor([[372.7360, 103.0150, 115.6880, 411.5225]], device=device, dtype=dtype) + + boxes = Boxes.from_tensor(boxes_xyxy) + expected_boxes = Boxes.from_tensor(expected_boxes_xyxy, validate_boxes=False) + + trans_mat = torch.tensor([[[-1.0, 0.0, 512.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + + transformed_boxes = boxes.transform_boxes_(trans_mat) + self.assert_close(transformed_boxes.data, expected_boxes.data, atol=1e-4, rtol=1e-4) + # inplace check + assert transformed_boxes is boxes + + def test_transform_multiple_boxes(self, device, dtype): + # Define boxes in XYXY format for simplicity. + boxes_xyxy = torch.tensor( + [ + [139.2640, 103.0150, 398.3120, 411.5225], + [1.0240, 80.5547, 513.0000, 513.0000], + [165.2053, 262.1440, 511.6347, 509.9280], + [119.8080, 144.2067, 258.0240, 411.1292], + ], + device=device, + dtype=dtype, + ).repeat(2, 1, 1) # 2 x 4 x 4 two images 4 boxes each + + expected_boxes_xyxy = torch.tensor( + [ + [ + [372.7360, 103.0150, 115.6880, 411.5225], + [510.9760, 80.5547, 1.0000, 513.0000], + [346.7947, 262.1440, 2.3653, 509.9280], + [392.1920, 144.2067, 255.9760, 411.1292], + ], + [ + [139.2640, 103.0150, 398.3120, 411.5225], + [1.0240, 80.5547, 513.0000, 513.0000], + [165.2053, 262.1440, 511.6347, 509.9280], + [119.8080, 144.2067, 258.0240, 411.1292], + ], + ], + device=device, + dtype=dtype, + ) + + trans_mat = torch.tensor( + [ + [[-1.0, 0.0, 512.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], + [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], + ], + device=device, + dtype=dtype, + ) + + boxes = Boxes.from_tensor(boxes_xyxy) + expected_boxes = Boxes.from_tensor(expected_boxes_xyxy, validate_boxes=False) + + out = boxes.transform_boxes(trans_mat) + self.assert_close(out.data, expected_boxes.data, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + # Define boxes in XYXY format for simplicity. + boxes_xyxy = torch.tensor( + [ + [139.2640, 103.0150, 258.0480, 307.5075], + [1.0240, 80.5547, 510.9760, 431.4453], + [165.2053, 262.1440, 345.4293, 546.7840], + [119.8080, 144.2067, 137.2160, 265.9225], + ], + device=device, + dtype=torch.float64, + ) + boxes = Boxes.from_tensor(boxes_xyxy) + + trans_mat = torch.tensor( + [[[-1.0, 0.0, 512.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=torch.float64 + ) + + def _wrapper_transform_boxes(quadrilaterals, M): + boxes = Boxes(quadrilaterals) + boxes = boxes.transform_boxes(M) + return boxes.data + + self.gradcheck(_wrapper_transform_boxes, (boxes.data, trans_mat)) + + +class TestBbox3D(BaseTester): + def test_smoke(self, device, dtype): + def _create_tensor_box(): + # Sample two points of the 3d rect + points = torch.rand(1, 6, device=device, dtype=dtype) + + # Fill according missing points + tensor_boxes = torch.zeros(1, 8, 3, device=device, dtype=dtype) + tensor_boxes[0, 0] = points[0][:3] + tensor_boxes[0, 1, 0] = points[0][3] + tensor_boxes[0, 1, 1] = points[0][1] + tensor_boxes[0, 1, 2] = points[0][2] + tensor_boxes[0, 2, 0] = points[0][3] + tensor_boxes[0, 2, 1] = points[0][4] + tensor_boxes[0, 2, 2] = points[0][2] + tensor_boxes[0, 3, 0] = points[0][0] + tensor_boxes[0, 3, 1] = points[0][4] + tensor_boxes[0, 3, 2] = points[0][2] + tensor_boxes[0, 4, 0] = points[0][0] + tensor_boxes[0, 4, 1] = points[0][1] + tensor_boxes[0, 4, 2] = points[0][5] + tensor_boxes[0, 5, 0] = points[0][3] + tensor_boxes[0, 5, 1] = points[0][1] + tensor_boxes[0, 5, 2] = points[0][5] + tensor_boxes[0, 6] = points[0][3:] + tensor_boxes[0, 7, 0] = points[0][0] + tensor_boxes[0, 7, 1] = points[0][4] + tensor_boxes[0, 7, 2] = points[0][5] + return tensor_boxes + + # Validate + assert Boxes3D(_create_tensor_box()) # Validate 1 box + + # 2 boxes without batching (N, 8, 3) where N=2 + two_boxes = torch.cat([_create_tensor_box(), _create_tensor_box()]) + assert Boxes3D(two_boxes) + + # 2 boxes in batch (B, 1, 8, 3) where B=2 + batched_bbox = torch.stack([_create_tensor_box(), _create_tensor_box()]) + assert Boxes3D(batched_bbox) + + def test_get_boxes_shape(self, device, dtype): + box = Boxes3D( + torch.tensor( + [[[0, 1, 2], [0, 1, 32], [10, 21, 2], [0, 21, 2], [10, 1, 32], [10, 21, 32], [10, 1, 2], [0, 21, 32]]], + device=device, + dtype=dtype, + ) + ) # 1x8x3 + t_boxes = torch.tensor( + [ + [[0, 21, 32], [0, 1, 2], [10, 1, 2], [0, 21, 2], [0, 1, 32], [10, 21, 2], [10, 1, 32], [10, 21, 32]], + [[3, 4, 5], [3, 4, 65], [43, 54, 5], [3, 54, 5], [43, 4, 5], [43, 4, 65], [43, 54, 65], [3, 54, 65]], + ], + device=device, + dtype=dtype, + ) # 2x8x3 + boxes = Boxes3D(t_boxes) + boxes_batch = Boxes3D(t_boxes[None]) # (1, 2, 8, 3) + + # Single box + d, h, w = box.get_boxes_shape() + assert (d.item(), h.item(), w.item()) == (31.0, 21.0, 11.0) + + # Boxes + d, h, w = boxes.get_boxes_shape() + assert h.ndim == 1 + assert w.ndim == 1 + assert len(d) == 2 + assert len(h) == 2 + assert len(w) == 2 + self.assert_close(d, torch.as_tensor([31.0, 61.0], device=device, dtype=dtype)) + self.assert_close(h, torch.as_tensor([21.0, 51.0], device=device, dtype=dtype)) + self.assert_close(w, torch.as_tensor([11.0, 41.0], device=device, dtype=dtype)) + + # Box batch + d, h, w = boxes_batch.get_boxes_shape() + assert h.ndim == 2 + assert w.ndim == 2 + assert h.shape == (1, 2) + assert w.shape == (1, 2) + self.assert_close(d, torch.as_tensor([[31.0, 61.0]], device=device, dtype=dtype)) + self.assert_close(h, torch.as_tensor([[21.0, 51.0]], device=device, dtype=dtype)) + self.assert_close(w, torch.as_tensor([[11.0, 41.0]], device=device, dtype=dtype)) + + def test_get_boxes_shape_batch(self, device, dtype): + t_box1 = torch.tensor( + [[[0, 1, 2], [0, 1, 32], [10, 21, 2], [0, 21, 2], [10, 1, 32], [10, 21, 32], [10, 1, 2], [0, 21, 32]]], + device=device, + dtype=dtype, + ) + t_box2 = torch.tensor( + [[[3, 4, 5], [3, 4, 65], [43, 54, 5], [3, 54, 5], [43, 4, 5], [43, 4, 65], [43, 54, 65], [3, 54, 65]]], + device=device, + dtype=dtype, + ) + batched_boxes = Boxes3D(torch.stack([t_box1, t_box2])) + + d, h, w = batched_boxes.get_boxes_shape() + assert d.ndim == 2 + assert h.ndim == 2 + assert w.ndim == 2 + assert d.shape == (2, 1) + assert h.shape == (2, 1) + assert w.shape == (2, 1) + self.assert_close(d, torch.as_tensor([[31.0], [61.0]], device=device, dtype=dtype)) + self.assert_close(h, torch.as_tensor([[21.0], [51.0]], device=device, dtype=dtype)) + self.assert_close(w, torch.as_tensor([[11.0], [41.0]], device=device, dtype=dtype)) + + @pytest.mark.parametrize("shape", [(1, 6), (1, 1, 6)]) + def test_from_tensor(self, shape, device, dtype): + box_xyzxyz = torch.as_tensor([[1, 2, 3, 4, 5, 6]], device=device, dtype=dtype).view(*shape) + box_xyzxyz_plus = torch.as_tensor([[1, 2, 3, 3, 4, 5]], device=device, dtype=dtype).view(*shape) + box_xyzwhd = torch.as_tensor([[1, 2, 3, 3, 3, 3]], device=device, dtype=dtype).view(*shape) + + expected_box = torch.as_tensor( + [[[1, 2, 3], [3, 2, 3], [3, 4, 3], [1, 4, 3], [1, 2, 5], [3, 2, 5], [3, 4, 5], [1, 4, 5]]], # Front # Back + device=device, + dtype=dtype, + ).view(*shape[:-1], 8, 3) + + kornia_xyzxyz = Boxes3D.from_tensor(box_xyzxyz, mode="xyzxyz").data + kornia_xyzxyz_plus = Boxes3D.from_tensor(box_xyzxyz_plus, mode="xyzxyz_plus").data + kornia_xyzwhd = Boxes3D.from_tensor(box_xyzwhd, mode="xyzwhd").data + + assert kornia_xyzxyz.shape == expected_box.shape + self.assert_close(kornia_xyzxyz, expected_box) + + assert kornia_xyzxyz_plus.shape == expected_box.shape + self.assert_close(kornia_xyzxyz_plus, expected_box) + + assert kornia_xyzwhd.shape == expected_box.shape + self.assert_close(kornia_xyzwhd, expected_box) + + @pytest.mark.parametrize("shape", [(1, 6), (1, 1, 6)]) + def test_from_invalid_tensor(self, shape, device, dtype): + box_xyzxyz = torch.as_tensor([[1, 2, 3, 4, -5, 6]], device=device, dtype=dtype).view(*shape) + box_xyzxyz_plus = torch.as_tensor([[1, 2, 3, 0, 6, 4]], device=device, dtype=dtype).view(*shape) + + try: + Boxes3D.from_tensor(box_xyzxyz, mode="xyzxyz") + raise AssertionError("Boxes3D.from_tensor should have raised any exception") + except ValueError: + pass + + try: + Boxes3D.from_tensor(box_xyzxyz_plus, mode="xyzxyz_plus") + raise AssertionError("Boxes3D.from_tensor should have raised any exception") + except ValueError: + pass + + @pytest.mark.parametrize("shape", [(1, 6), (1, 1, 6)]) + def test_boxes_to_tensor(self, shape, device, dtype): + # Hexahedron with randomized vertices to reflect possible transforms. + box = Boxes3D( + torch.as_tensor( + [[[2, 2, 1], [1, 2, 1], [2, 3, 2], [1, 3, 2], [2, 2, 2], [1, 3, 1], [2, 3, 1], [1, 2, 2]]], + device=device, + dtype=dtype, + ).view(*shape[:-1], 8, 3) + ) + + expected_box_xyzxyz = torch.as_tensor([[1, 2, 1, 3, 4, 3]], device=device, dtype=dtype).view(*shape) + expected_box_xyzxyz_plus = torch.as_tensor([[1, 2, 1, 2, 3, 2]], device=device, dtype=dtype).view(*shape) + expected_box_xyzwhd = torch.as_tensor([[1, 2, 1, 2, 2, 2]], device=device, dtype=dtype).view(*shape) + expected_vertices = torch.as_tensor( + [[[1, 2, 1], [3, 2, 1], [3, 4, 1], [1, 4, 1], [1, 2, 3], [3, 2, 3], [3, 4, 3], [1, 4, 3]]], # Front # Back + device=device, + dtype=dtype, + ).view(*shape[:-1], 8, 3) + expected_vertices_plus = torch.as_tensor( + [[[1, 2, 1], [2, 2, 1], [2, 3, 1], [1, 3, 1], [1, 2, 2], [2, 2, 2], [2, 3, 2], [1, 3, 2]]], # Front # Back + device=device, + dtype=dtype, + ).view(*shape[:-1], 8, 3) + + kornia_xyzxyz = box.to_tensor(mode="xyzxyz") + kornia_xyzxyz_plus = box.to_tensor(mode="xyzxyz_plus") + kornia_xyzwhd = box.to_tensor(mode="xyzwhd") + kornia_vertices = box.to_tensor(mode="vertices") + kornia_vertices_plus = box.to_tensor(mode="vertices_plus") + + assert kornia_xyzxyz.shape == expected_box_xyzxyz.shape + self.assert_close(kornia_xyzxyz, expected_box_xyzxyz) + + assert kornia_xyzxyz_plus.shape == expected_box_xyzxyz_plus.shape + self.assert_close(kornia_xyzxyz_plus, expected_box_xyzxyz_plus) + + assert kornia_xyzwhd.shape == expected_box_xyzwhd.shape + self.assert_close(kornia_xyzwhd, expected_box_xyzwhd) + + assert kornia_vertices.shape == expected_vertices.shape + self.assert_close(kornia_vertices, expected_vertices) + + assert kornia_vertices_plus.shape == expected_vertices_plus.shape + self.assert_close(kornia_vertices_plus, expected_vertices_plus) + + def test_bbox_to_mask(self, device, dtype): + t_box1 = torch.tensor( + [ + [ + [1.0, 1.0, 1.0], + [3.0, 1.0, 1.0], + [3.0, 2.0, 1.0], + [1.0, 2.0, 1.0], # Front + [1.0, 1.0, 2.0], + [3.0, 1.0, 2.0], + [3.0, 2.0, 2.0], + [1.0, 2.0, 2.0], # Back + ] + ], + device=device, + dtype=dtype, + ) # (1, 8, 3) + t_box2 = torch.tensor( + [ + [ + [2.0, 2.0, 1.0], + [4.0, 2.0, 1.0], + [4.0, 5.0, 1.0], + [4.0, 2.0, 1.0], # Front + [2.0, 2.0, 1.0], + [4.0, 2.0, 1.0], + [4.0, 5.0, 1.0], + [4.0, 5.0, 1.0], # Back + ] + ], + device=device, + dtype=dtype, + ) # (1, 8, 3) + + box1, box2 = Boxes3D(t_box1), Boxes3D(t_box2) + two_boxes = Boxes3D(torch.cat([t_box1, t_box2])) # (2, 8, 3) + batched_boxes = Boxes3D(torch.stack([t_box1, t_box2])) # (2, 1, 8, 3) + + depth, height, width = 3, 7, 5 + + expected_mask1 = torch.tensor( + [ + [ + [ # Depth 0 + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + ], + [ # Depth 1 + [0, 0, 0, 0, 0], + [0, 1, 1, 1, 0], + [0, 1, 1, 1, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + ], + [ # Depth 2 + [0, 0, 0, 0, 0], + [0, 1, 1, 1, 0], + [0, 1, 1, 1, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + ], + ] + ], + device=device, + dtype=dtype, + ) + + expected_mask2 = torch.tensor( + [ + [ + [ # Depth 0 + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + ], + [ # Depth 1 + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 1, 1, 1], + [0, 0, 1, 1, 1], + [0, 0, 1, 1, 1], + [0, 0, 1, 1, 1], + [0, 0, 0, 0, 0], + ], + [ # Depth 2 + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + ], + ] + ], + device=device, + dtype=dtype, + ) + expected_two_masks = torch.cat([expected_mask1, expected_mask2]) + expected_batched_masks = torch.stack([expected_mask1, expected_mask2]) + + mask1 = box1.to_mask(depth, height, width) + mask2 = box2.to_mask(depth, height, width) + two_masks = two_boxes.to_mask(depth, height, width) + batched_masks = batched_boxes.to_mask(depth, height, width) + + assert mask1.shape == expected_mask1.shape + self.assert_close(mask1, expected_mask1) + + assert mask2.shape == expected_mask2.shape + self.assert_close(mask2, expected_mask2) + + assert two_masks.shape == expected_two_masks.shape + self.assert_close(two_masks, expected_two_masks) + + assert batched_masks.shape == expected_batched_masks.shape + self.assert_close(batched_masks, expected_batched_masks) + + def test_to(self, device, dtype): + boxes = Boxes3D.from_tensor(torch.as_tensor([[1, 2, 3, 4, 5, 6]], device="cpu", dtype=torch.float32)) + assert boxes.to(device=device).data.device == device + assert boxes.to(dtype=dtype).data.dtype == dtype + + boxes_moved = boxes.to(device, dtype) + assert boxes_moved is boxes # to is an inplace op. + assert boxes_moved.data.device == device, boxes_moved.data.dtype == dtype + + def test_gradcheck(self, device): + # Uncomment when enabling gradient checks + # def apply_boxes_method(tensor: torch.Tensor, method: str, **kwargs): + # boxes = Boxes3D(tensor) + # result = getattr(boxes, method)(**kwargs) + # return result.data if isinstance(result, Boxes3D) else result + + # t_boxes1 = torch.tensor( + # [ + # [ + # [0.0, 1.0, 2.0], + # [10, 1, 2], + # [10, 21, 2], + # [0, 21, 2], + # [0, 1, 32], + # [10, 1, 32], + # [10, 21, 32], + # [0, 21, 32], + # ] + # ], + # device=device, + # dtype=torch.float64, + # ) + + # Uncomment when enabling gradient checks + # t_boxes2 = tensor_to_gradcheck_var(t_boxes1.detach().clone()) + # t_boxes3 = tensor_to_gradcheck_var(t_boxes1.detach().clone()) + # t_boxes4 = tensor_to_gradcheck_var(t_boxes1.detach().clone()) + t_boxes_xyzxyz = torch.tensor([[1.0, 3.0, 8.0, 5.0, 6.0, 12.0]], device=device, dtype=torch.float64) + t_boxes_xyzxyz1 = t_boxes_xyzxyz.detach().clone() + + # Gradient checks for Boxes3D.to_tensor (and Boxes3D.get_boxes_shape) are disable since the is a bug + # in their gradient. See https://github.com/kornia/kornia/issues/1396. + # assert gradcheck(partial(apply_boxes_method, method='to_tensor'), (t_boxes2,), raise_exception=True) + # assert gradcheck( + # partial(apply_boxes_method, method='to_tensor', mode='xyzxyz_plus'), (t_boxes3,), raise_exception=True + # ) + # assert gradcheck( + # partial(apply_boxes_method, method='to_tensor', mode='vertices_plus'), (t_boxes4,), raise_exception=True + # ) + # assert gradcheck(partial(apply_boxes_method, method='get_boxes_shape'), (t_boxes1,), raise_exception=True) + self.gradcheck(lambda x: Boxes3D.from_tensor(x, mode="xyzxyz_plus").data, (t_boxes_xyzxyz,)) + self.gradcheck(lambda x: Boxes3D.from_tensor(x, mode="xyzwhd").data, (t_boxes_xyzxyz1,)) + + +class TestTransformBoxes3D(BaseTester): + def test_transform_boxes(self, device, dtype): + # Define boxes in XYZXYZ format with integer coordinates (TF32-safe on CUDA). + boxes_xyzxyz = torch.tensor([[140, 104, 284, 398, 412, 454]], device=device, dtype=dtype) + expected_boxes_xyzxyz = torch.tensor([[372, 104, 569, 116, 412, 908]], device=device, dtype=dtype) + + boxes = Boxes3D.from_tensor(boxes_xyzxyz) + expected_boxes = Boxes3D.from_tensor(expected_boxes_xyzxyz, validate_boxes=False) + + trans_mat = torch.tensor( + [[[-1.0, 0.0, 0.0, 512.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 2.0, 1.0], [0.0, 0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) + + transformed_boxes = boxes.transform_boxes(trans_mat) + self.assert_close(transformed_boxes.data, expected_boxes.data, atol=1e-4, rtol=1e-4) + # inplace check + assert transformed_boxes is not boxes + + def test_transform_boxes_(self, device, dtype): + # Define boxes in XYZXYZ format with integer coordinates (TF32-safe on CUDA). + boxes_xyzxyz = torch.tensor([[140, 104, 284, 398, 412, 454]], device=device, dtype=dtype) + expected_boxes_xyzxyz = torch.tensor([[372, 104, 569, 116, 412, 908]], device=device, dtype=dtype) + + boxes = Boxes3D.from_tensor(boxes_xyzxyz) + expected_boxes = Boxes3D.from_tensor(expected_boxes_xyzxyz, validate_boxes=False) + + trans_mat = torch.tensor( + [[[-1.0, 0.0, 0.0, 512.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 2.0, 1.0], [0.0, 0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) + + transformed_boxes = boxes.transform_boxes_(trans_mat) + self.assert_close(transformed_boxes.data, expected_boxes.data, atol=1e-4, rtol=1e-4) + # inplace check + assert transformed_boxes is boxes + + def test_transform_multiple_boxes(self, device, dtype): + # Define boxes in XYZXYZ format with integer coordinates (TF32-safe on CUDA). + boxes_xyzxyz = torch.tensor( + [ + [140, 104, 284, 398, 412, 454], + [2, 81, 470, 512, 513, 513], + [166, 263, 43, 512, 510, 786], + [120, 145, 235, 258, 412, 387], + ], + device=device, + dtype=dtype, + ).repeat(2, 1, 1) # 2 x 4 x 4 two images 4 boxes each + + expected_boxes_xyzxyz = torch.tensor( + [ + [ + [372, 104, 569, 116, 412, 908], + [510, 81, 941, 2, 513, 1026], + [346, 263, 87, 2, 510, 1572], + [392, 145, 471, 256, 412, 774], + ], + [ + [140, 104, 284, 398, 412, 454], + [2, 81, 470, 512, 513, 513], + [166, 263, 43, 512, 510, 786], + [120, 145, 235, 258, 412, 387], + ], + ], + device=device, + dtype=dtype, + ) + + trans_mat = torch.tensor( + [ + [[-1.0, 0.0, 0.0, 512.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 2.0, 1.0], [0.0, 0.0, 0.0, 1.0]], + [[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]], + ], + device=device, + dtype=dtype, + ) + + boxes = Boxes3D.from_tensor(boxes_xyzxyz) + expected_boxes = Boxes3D.from_tensor(expected_boxes_xyzxyz, validate_boxes=False) + + out = boxes.transform_boxes(trans_mat) + self.assert_close(out.data, expected_boxes.data, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + # Define boxes in XYZXYZ format for simplicity. + boxes_xyzxyz = torch.tensor( + [ + [139.2640, 103.0150, 283.162, 397.3120, 410.5225, 453.185], + [1.0240, 80.5547, 469.50, 512.0000, 512.0000, 512.0], + [165.2053, 262.1440, 42.98, 510.6347, 508.9280, 784.443], + [119.8080, 144.2067, 234.21, 257.0240, 410.1292, 386.14], + ], + device=device, + dtype=torch.float64, + ) + boxes = Boxes3D.from_tensor(boxes_xyzxyz) + + trans_mat = torch.tensor( + [[[-1.0, 0.0, 0.0, 512.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 2.0, 1.0], [0.0, 0.0, 0.0, 1.0]]], + device=device, + dtype=torch.float64, + ) + + def _wrapper_transform_boxes(hexahedrons, M): + boxes = Boxes3D(hexahedrons) + boxes = boxes.transform_boxes(M) + return boxes.data + + self.gradcheck(_wrapper_transform_boxes, (boxes.data, trans_mat)) diff --git a/tests/geometry/test_conversions.py b/tests/geometry/test_conversions.py new file mode 100644 index 0000000..92ae7d8 --- /dev/null +++ b/tests/geometry/test_conversions.py @@ -0,0 +1,1369 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +from functools import partial + +import numpy as np +import pytest +import torch + +import kornia +from kornia.core._compat import torch_version +from kornia.core.ops import eye_like +from kornia.geometry.conversions import ( + ARKitQTVecs_to_ColmapQTVecs, + Rt_to_matrix4x4, + axis_angle_to_rotation_matrix, + camtoworld_graphics_to_vision_4x4, + camtoworld_graphics_to_vision_Rt, + camtoworld_to_worldtocam_Rt, + camtoworld_vision_to_graphics_4x4, + camtoworld_vision_to_graphics_Rt, + euler_from_quaternion, + matrix4x4_to_Rt, + quaternion_from_euler, + worldtocam_to_camtoworld_Rt, +) +from kornia.geometry.quaternion import Quaternion + +from testing.base import BaseTester, assert_close + + +@pytest.fixture() +def atol(device, dtype): + """Lower tolerance for cuda-float16 only.""" + if "cuda" in device.type and dtype == torch.float16: + return 1.0e-3 + return 1.0e-4 + + +@pytest.fixture() +def rtol(device, dtype): + """Lower tolerance for cuda-float16 only.""" + if "cuda" in device.type and dtype == torch.float16: + return 1.0e-3 + return 1.0e-4 + + +class TestAngleAxisToQuaternion(BaseTester): + # based on: + # https://github.com/ceres-solver/ceres-solver/blob/master/internal/ceres/rotation_test.cc#L271 + + def test_smoke(self, device, dtype): + axis_angle = torch.zeros(3, dtype=dtype, device=device) + quaternion = kornia.geometry.conversions.axis_angle_to_quaternion(axis_angle) + assert quaternion.shape == (4,) + + @pytest.mark.parametrize("batch_size", (1, 3, 8)) + def test_smoke_batch(self, batch_size, device, dtype): + axis_angle = torch.zeros(batch_size, 3, device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.axis_angle_to_quaternion(axis_angle) + assert quaternion.shape == (batch_size, 4) + + def test_zero_angle(self, device, dtype, atol, rtol): + axis_angle = torch.tensor((0.0, 0.0, 0.0), device=device, dtype=dtype) + expected = torch.tensor((1.0, 0.0, 0.0, 0.0), device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.axis_angle_to_quaternion(axis_angle) + self.assert_close(quaternion, expected, atol=atol, rtol=rtol) + + def test_small_angle_x(self, device, dtype, atol, rtol): + theta = 1.0e-2 + axis_angle = torch.tensor((theta, 0.0, 0.0), device=device, dtype=dtype) + expected = torch.tensor((np.cos(theta / 2.0), np.sin(theta / 2.0), 0.0, 0.0), device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.axis_angle_to_quaternion(axis_angle) + self.assert_close(quaternion, expected, atol=atol, rtol=rtol) + + def test_small_angle_y(self, device, dtype, atol, rtol): + theta = 1.0e-2 + axis_angle = torch.tensor((0.0, theta, 0.0), device=device, dtype=dtype) + expected = torch.tensor((np.cos(theta / 2.0), 0.0, np.sin(theta / 2.0), 0.0), device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.axis_angle_to_quaternion(axis_angle) + self.assert_close(quaternion, expected, atol=atol, rtol=rtol) + + def test_small_angle_z(self, device, dtype, atol, rtol): + theta = 1.0e-2 + axis_angle = torch.tensor((0.0, 0.0, theta), device=device, dtype=dtype) + expected = torch.tensor((np.cos(theta / 2.0), 0.0, 0.0, np.sin(theta / 2.0)), device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.axis_angle_to_quaternion(axis_angle) + self.assert_close(quaternion, expected, atol=atol, rtol=rtol) + + def test_x_rotation(self, device, dtype, atol, rtol): + half_sqrt2 = 0.5 * np.sqrt(2.0) + axis_angle = torch.tensor((kornia.pi / 2.0, 0.0, 0.0), device=device, dtype=dtype) + expected = torch.tensor((half_sqrt2, half_sqrt2, 0.0, 0.0), device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.axis_angle_to_quaternion(axis_angle) + self.assert_close(quaternion, expected, atol=atol, rtol=rtol) + + def test_y_rotation(self, device, dtype, atol, rtol): + half_sqrt2 = 0.5 * np.sqrt(2.0) + axis_angle = torch.tensor((0.0, kornia.pi / 2.0, 0.0), device=device, dtype=dtype) + expected = torch.tensor((half_sqrt2, 0.0, half_sqrt2, 0.0), device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.axis_angle_to_quaternion(axis_angle) + self.assert_close(quaternion, expected, atol=atol, rtol=rtol) + + def test_z_rotation(self, device, dtype, atol, rtol): + half_sqrt2 = 0.5 * np.sqrt(2.0) + axis_angle = torch.tensor((0.0, 0.0, kornia.pi / 2.0), device=device, dtype=dtype) + expected = torch.tensor((half_sqrt2, 0.0, 0.0, half_sqrt2), device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.axis_angle_to_quaternion(axis_angle) + self.assert_close(quaternion, expected, atol=atol, rtol=rtol) + + def test_gradcheck(self, device): + dtype = torch.float64 + eps = torch.finfo(dtype).eps + axis_angle = torch.tensor((0.0, 0.0, 0.0), device=device, dtype=dtype) + eps + # evaluate function gradient + self.gradcheck(partial(kornia.geometry.conversions.axis_angle_to_quaternion), (axis_angle,)) + + +class TestQuaternionToAngleAxis(BaseTester): + def test_smoke(self, device, dtype): + quaternion = torch.zeros(4, device=device, dtype=dtype) + axis_angle = kornia.geometry.conversions.quaternion_to_axis_angle(quaternion) + assert axis_angle.shape == (3,) + + @pytest.mark.parametrize("batch_size", (1, 3, 8)) + def test_smoke_batch(self, batch_size, device, dtype): + quaternion = torch.zeros(batch_size, 4, device=device, dtype=dtype) + axis_angle = kornia.geometry.conversions.quaternion_to_axis_angle(quaternion) + assert axis_angle.shape == (batch_size, 3) + + def test_unit_quaternion(self, device, dtype, atol, rtol): + quaternion = torch.tensor((1.0, 0.0, 0.0, 0.0), device=device, dtype=dtype) + expected = torch.tensor((0.0, 0.0, 0.0), device=device, dtype=dtype) + axis_angle = kornia.geometry.conversions.quaternion_to_axis_angle(quaternion) + self.assert_close(axis_angle, expected, atol=atol, rtol=rtol) + + def test_x_rotation(self, device, dtype, atol, rtol): + quaternion = torch.tensor((0.0, 1.0, 0.0, 0.0), device=device, dtype=dtype) + expected = torch.tensor((kornia.pi, 0.0, 0.0), device=device, dtype=dtype) + axis_angle = kornia.geometry.conversions.quaternion_to_axis_angle(quaternion) + self.assert_close(axis_angle, expected, atol=atol, rtol=rtol) + + def test_y_rotation(self, device, dtype, atol, rtol): + quaternion = torch.tensor((0.0, 0.0, 1.0, 0.0), device=device, dtype=dtype) + expected = torch.tensor((0.0, kornia.pi, 0.0), device=device, dtype=dtype) + axis_angle = kornia.geometry.conversions.quaternion_to_axis_angle(quaternion) + self.assert_close(axis_angle, expected, atol=atol, rtol=rtol) + + def test_z_rotation(self, device, dtype, atol, rtol): + quaternion = torch.tensor((np.sqrt(3.0) / 2.0, 0.0, 0.0, 0.5), device=device, dtype=dtype) + expected = torch.tensor((0.0, 0.0, kornia.pi / 3.0), device=device, dtype=dtype) + axis_angle = kornia.geometry.conversions.quaternion_to_axis_angle(quaternion) + self.assert_close(axis_angle, expected, atol=atol, rtol=rtol) + + def test_small_angle_x(self, device, dtype, atol, rtol): + theta = 1.0e-2 + quaternion = torch.tensor((np.cos(theta / 2.0), np.sin(theta / 2.0), 0.0, 0.0), device=device, dtype=dtype) + expected = torch.tensor((theta, 0.0, 0.0), device=device, dtype=dtype) + axis_angle = kornia.geometry.conversions.quaternion_to_axis_angle(quaternion) + self.assert_close(axis_angle, expected, atol=atol, rtol=rtol) + + def test_small_angle_y(self, device, dtype, atol, rtol): + theta = 1.0e-2 + quaternion = torch.tensor((np.cos(theta / 2), 0.0, np.sin(theta / 2), 0.0), device=device, dtype=dtype) + expected = torch.tensor((0.0, theta, 0.0), device=device, dtype=dtype) + axis_angle = kornia.geometry.conversions.quaternion_to_axis_angle(quaternion) + self.assert_close(axis_angle, expected, atol=atol, rtol=rtol) + + def test_small_angle_z(self, device, dtype, atol, rtol): + theta = 1.0e-2 + quaternion = torch.tensor((np.cos(theta / 2), 0.0, 0.0, np.sin(theta / 2)), device=device, dtype=dtype) + expected = torch.tensor((0.0, 0.0, theta), device=device, dtype=dtype) + axis_angle = kornia.geometry.conversions.quaternion_to_axis_angle(quaternion) + self.assert_close(axis_angle, expected, atol=atol, rtol=rtol) + + def test_gradcheck(self, device): + dtype = torch.float64 + eps = torch.finfo(dtype).eps + quaternion = torch.tensor((1.0, 0.0, 0.0, 0.0), device=device, dtype=dtype) + eps + # evaluate function gradient + self.gradcheck(partial(kornia.geometry.conversions.quaternion_to_axis_angle), (quaternion,)) + + +class TestRotationMatrixToQuaternion(BaseTester): + @pytest.mark.parametrize("batch_size", (1, 3, 8)) + def test_smoke_batch(self, batch_size, device, dtype): + matrix = torch.zeros(batch_size, 3, 3, device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.rotation_matrix_to_quaternion(matrix) + assert quaternion.shape == (batch_size, 4) + + def test_identity(self, device, dtype, atol, rtol): + matrix = torch.tensor(((1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, 1.0)), device=device, dtype=dtype) + expected = torch.tensor((1.0, 0.0, 0.0, 0.0), device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.rotation_matrix_to_quaternion(matrix) + self.assert_close(quaternion, expected, atol=atol, rtol=rtol) + + def test_rot_x_45(self, device, dtype, atol, rtol): + matrix = torch.tensor(((1.0, 0.0, 0.0), (0.0, 0.0, -1.0), (0.0, 1.0, 0.0)), device=device, dtype=dtype) + pi_half2 = torch.cos(kornia.pi / 4.0).to(device=device, dtype=dtype) + expected = torch.tensor((pi_half2, pi_half2, 0.0, 0.0), device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.rotation_matrix_to_quaternion(matrix) + self.assert_close(quaternion, expected, atol=atol, rtol=rtol) + + def test_back_and_forth(self, device, dtype, atol, rtol): + eps = torch.finfo(dtype).eps + matrix = torch.tensor(((1.0, 0.0, 0.0), (0.0, 0.0, -1.0), (0.0, 1.0, 0.0)), device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.rotation_matrix_to_quaternion(matrix, eps=eps) + matrix_hat = kornia.geometry.conversions.quaternion_to_rotation_matrix(quaternion) + self.assert_close(matrix, matrix_hat, atol=atol, rtol=rtol) + + def test_corner_case(self, device, dtype, atol, rtol): + eps = torch.finfo(dtype).eps + matrix = torch.tensor( + ( + (-0.7799533010, -0.5432914495, 0.3106555045), + (0.0492402576, -0.5481169224, -0.8349509239), + (0.6238971353, -0.6359263659, 0.4542570710), + ), + device=device, + dtype=dtype, + ) + quaternion_true = torch.tensor( + (0.177614107728004, 0.280136495828629, -0.440902262926102, 0.834015488624573), device=device, dtype=dtype + ) + quaternion = kornia.geometry.conversions.rotation_matrix_to_quaternion(matrix, eps=eps) + torch.set_printoptions(precision=10) + self.assert_close(quaternion_true, quaternion, atol=atol, rtol=rtol) + + def test_cond1_180_rot_x(self, device, dtype, atol, rtol): + # 180° rotation around X: trace < 0, m00 > m11 and m00 > m22 → activates cond_1 branch. + # R_x(π) = diag(1, -1, -1); expected quaternion (w,x,y,z) = (0, 1, 0, 0). + eps = torch.finfo(dtype).eps + matrix = torch.tensor(((1.0, 0.0, 0.0), (0.0, -1.0, 0.0), (0.0, 0.0, -1.0)), device=device, dtype=dtype) + expected = torch.tensor((0.0, 1.0, 0.0, 0.0), device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.rotation_matrix_to_quaternion(matrix, eps=eps) + self.assert_close(quaternion.abs(), expected.abs(), atol=atol, rtol=rtol) + # Round-trip: convert back and verify the rotation matrix is recovered. + mat_back = kornia.geometry.conversions.quaternion_to_rotation_matrix(quaternion) + self.assert_close(mat_back, matrix, atol=atol, rtol=rtol) + + def test_cond2_180_rot_y(self, device, dtype, atol, rtol): + # 180° rotation around Y: trace < 0, m11 > m22 and m00 not dominant → activates cond_2 branch. + # R_y(π) = diag(-1, 1, -1); expected quaternion (w,x,y,z) = (0, 0, 1, 0). + eps = torch.finfo(dtype).eps + matrix = torch.tensor(((-1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, -1.0)), device=device, dtype=dtype) + expected = torch.tensor((0.0, 0.0, 1.0, 0.0), device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.rotation_matrix_to_quaternion(matrix, eps=eps) + self.assert_close(quaternion.abs(), expected.abs(), atol=atol, rtol=rtol) + mat_back = kornia.geometry.conversions.quaternion_to_rotation_matrix(quaternion) + self.assert_close(mat_back, matrix, atol=atol, rtol=rtol) + + def test_all_four_branches_in_batch(self, device, dtype, atol, rtol): + # Batch of 4 rotation matrices that each activate a different internal branch. + # Verify consistency via round-trip: R → q → R must recover the original rotation. + eps = torch.finfo(dtype).eps + identity = torch.eye(3, device=device, dtype=dtype) # trace > 0 → trace_positive_cond + rot_x_180 = torch.tensor(((1.0, 0.0, 0.0), (0.0, -1.0, 0.0), (0.0, 0.0, -1.0)), device=device, dtype=dtype) + rot_y_180 = torch.tensor(((-1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, -1.0)), device=device, dtype=dtype) + rot_z_180 = torch.tensor(((-1.0, 0.0, 0.0), (0.0, -1.0, 0.0), (0.0, 0.0, 1.0)), device=device, dtype=dtype) + batch = torch.stack([identity, rot_x_180, rot_y_180, rot_z_180]) # (4, 3, 3) + quaternions = kornia.geometry.conversions.rotation_matrix_to_quaternion(batch, eps=eps) + mats_back = kornia.geometry.conversions.quaternion_to_rotation_matrix(quaternions) + self.assert_close(mats_back, batch, atol=atol, rtol=rtol) + + def test_gradcheck(self, device): + dtype = torch.float64 + eps = torch.finfo(dtype).eps + matrix = torch.eye(3, device=device, dtype=dtype) + # evaluate function gradient + self.gradcheck(partial(kornia.geometry.conversions.rotation_matrix_to_quaternion, eps=eps), (matrix,)) + + def test_dynamo(self, device, dtype, torch_optimizer): + quaternion = torch.tensor((0.0, 0.0, 1.0), device=device, dtype=dtype) + op = kornia.geometry.conversions.quaternion_log_to_exp + op_optimized = torch_optimizer(op) + + actual = op_optimized(quaternion) + expected = op(quaternion) + + self.assert_close(actual, expected) + + +class TestQuaternionToRotationMatrix(BaseTester): + @pytest.mark.parametrize("batch_dims", ((), (1,), (3,), (8,), (1, 1), (5, 6))) + def test_smoke_batch(self, batch_dims, device, dtype): + quaternion = torch.zeros(*batch_dims, 4, device=device, dtype=dtype) + matrix = kornia.geometry.conversions.quaternion_to_rotation_matrix(quaternion) + assert matrix.shape == (*batch_dims, 3, 3) + + def test_unit_quaternion(self, device, dtype, atol, rtol): + quaternion = torch.tensor((1.0, 0.0, 0.0, 0.0), device=device, dtype=dtype) + expected = torch.tensor(((1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, 1.0)), device=device, dtype=dtype) + matrix = kornia.geometry.conversions.quaternion_to_rotation_matrix(quaternion) + self.assert_close(matrix, expected, atol=atol, rtol=rtol) + + def test_x_rotation(self, device, dtype, atol, rtol): + quaternion = torch.tensor((0.0, 1.0, 0.0, 0.0), device=device, dtype=dtype) + expected = torch.tensor(((1.0, 0.0, 0.0), (0.0, -1.0, 0.0), (0.0, 0.0, -1.0)), device=device, dtype=dtype) + matrix = kornia.geometry.conversions.quaternion_to_rotation_matrix(quaternion) + self.assert_close(matrix, expected, atol=atol, rtol=rtol) + + def test_y_rotation(self, device, dtype, atol, rtol): + quaternion = torch.tensor((0.0, 0.0, 1.0, 0.0), device=device, dtype=dtype) + expected = torch.tensor(((-1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, -1.0)), device=device, dtype=dtype) + matrix = kornia.geometry.conversions.quaternion_to_rotation_matrix(quaternion) + self.assert_close(matrix, expected, atol=atol, rtol=rtol) + + def test_z_rotation(self, device, dtype, atol, rtol): + quaternion = torch.tensor((0.0, 0.0, 0.0, 1.0), device=device, dtype=dtype) + expected = torch.tensor(((-1.0, 0.0, 0.0), (0.0, -1.0, 0.0), (0.0, 0.0, 1.0)), device=device, dtype=dtype) + matrix = kornia.geometry.conversions.quaternion_to_rotation_matrix(quaternion) + self.assert_close(matrix, expected, atol=atol, rtol=rtol) + + def test_gradcheck(self, device): + quaternion = torch.tensor((0.0, 0.0, 0.0, 1.0), device=device, dtype=torch.float64) + # evaluate function gradient + self.gradcheck(partial(kornia.geometry.conversions.quaternion_to_rotation_matrix), (quaternion,)) + + def test_dynamo(self, device, dtype, torch_optimizer): + quaternion = torch.tensor((0.0, 0.0, 1.0, 0.0), device=device, dtype=dtype) + op = kornia.geometry.conversions.quaternion_to_rotation_matrix + op_optimized = torch_optimizer(op) + + actual = op_optimized(quaternion) + expected = op(quaternion) + + self.assert_close(actual, expected) + + +class TestQuaternionLogToExp(BaseTester): + @pytest.mark.parametrize("batch_size", (1, 3, 8)) + def test_smoke_batch(self, batch_size, device, dtype): + quaternion_log = torch.zeros(batch_size, 3, device=device, dtype=dtype) + quaternion_exp = kornia.geometry.conversions.quaternion_log_to_exp(quaternion_log) + assert quaternion_exp.shape == (batch_size, 4) + + def test_unit_quaternion(self, device, dtype, atol, rtol): + eps = torch.finfo(dtype).eps + quaternion_log = torch.tensor((0.0, 0.0, 0.0), device=device, dtype=dtype) + expected = torch.tensor((1.0, 0.0, 0.0, 0.0), device=device, dtype=dtype) + quaternion_exp = kornia.geometry.conversions.quaternion_log_to_exp(quaternion_log, eps=eps) + self.assert_close(quaternion_exp, expected, atol=atol, rtol=rtol) + + def test_pi_quaternion_x(self, device, dtype, atol, rtol): + eps = torch.finfo(dtype).eps + one = torch.tensor(1.0, device=device, dtype=dtype) + quaternion_log = torch.tensor((1.0, 0.0, 0.0), device=device, dtype=dtype) + expected = torch.tensor((torch.cos(one), torch.sin(one), 0.0, 0.0), device=device, dtype=dtype) + quaternion_exp = kornia.geometry.conversions.quaternion_log_to_exp(quaternion_log, eps=eps) + self.assert_close(quaternion_exp, expected, atol=atol, rtol=rtol) + + def test_pi_quaternion_y(self, device, dtype, atol, rtol): + eps = torch.finfo(dtype).eps + one = torch.tensor(1.0, device=device, dtype=dtype) + quaternion_log = torch.tensor((0.0, 1.0, 0.0), device=device, dtype=dtype) + expected = torch.tensor((torch.cos(one), 0.0, torch.sin(one), 0.0), device=device, dtype=dtype) + quaternion_exp = kornia.geometry.conversions.quaternion_log_to_exp(quaternion_log, eps=eps) + self.assert_close(quaternion_exp, expected, atol=atol, rtol=rtol) + + def test_pi_quaternion_z(self, device, dtype, atol, rtol): + eps = torch.finfo(dtype).eps + one = torch.tensor(1.0, device=device, dtype=dtype) + quaternion_log = torch.tensor((0.0, 0.0, 1.0), device=device, dtype=dtype) + expected = torch.tensor((torch.cos(one), 0.0, 0.0, torch.sin(one)), device=device, dtype=dtype) + quaternion_exp = kornia.geometry.conversions.quaternion_log_to_exp(quaternion_log, eps=eps) + self.assert_close(quaternion_exp, expected, atol=atol, rtol=rtol) + + def test_back_and_forth(self, device, dtype, atol, rtol): + eps = torch.finfo(dtype).eps + quaternion_log = torch.tensor((1.0, 0.0, 0.0), device=device, dtype=dtype) + + quaternion_exp = kornia.geometry.conversions.quaternion_log_to_exp(quaternion_log, eps=eps) + quaternion_log_hat = kornia.geometry.conversions.quaternion_exp_to_log(quaternion_exp, eps=eps) + self.assert_close(quaternion_log, quaternion_log_hat, atol=atol, rtol=rtol) + + def test_gradcheck(self, device): + dtype = torch.float64 + eps = torch.finfo(dtype).eps + quaternion = torch.tensor((0.0, 0.0, 1.0), device=device, dtype=dtype) + # evaluate function gradient + self.gradcheck(partial(kornia.geometry.conversions.quaternion_log_to_exp, eps=eps), (quaternion,)) + + def test_dynamo(self, device, dtype, torch_optimizer): + quaternion = torch.tensor((0.0, 0.0, 1.0), device=device, dtype=dtype) + op = kornia.geometry.conversions.quaternion_log_to_exp + op_optimized = torch_optimizer(op) + + actual = op_optimized(quaternion) + expected = op(quaternion) + + self.assert_close(actual, expected) + + +class TestQuaternionExpToLog(BaseTester): + @pytest.mark.parametrize("batch_size", (1, 3, 8)) + def test_smoke_batch(self, batch_size, device, dtype): + eps = torch.finfo(dtype).eps + quaternion_exp = torch.zeros(batch_size, 4, device=device, dtype=dtype) + quaternion_log = kornia.geometry.conversions.quaternion_exp_to_log(quaternion_exp, eps=eps) + assert quaternion_log.shape == (batch_size, 3) + + def test_unit_quaternion(self, device, dtype, atol, rtol): + eps = torch.finfo(dtype).eps + quaternion_exp = torch.tensor((1.0, 0.0, 0.0, 0.0), device=device, dtype=dtype) + expected = torch.tensor((0.0, 0.0, 0.0), device=device, dtype=dtype) + quaternion_log = kornia.geometry.conversions.quaternion_exp_to_log(quaternion_exp, eps=eps) + self.assert_close(quaternion_log, expected, atol=atol, rtol=rtol) + + def test_pi_quaternion_x(self, device, dtype, atol, rtol): + eps = torch.finfo(dtype).eps + quaternion_exp = torch.tensor((0.0, 1.0, 0.0, 0.0), device=device, dtype=dtype) + expected = torch.tensor((kornia.pi / 2.0, 0.0, 0.0), device=device, dtype=dtype) + quaternion_log = kornia.geometry.conversions.quaternion_exp_to_log(quaternion_exp, eps=eps) + self.assert_close(quaternion_log, expected, atol=atol, rtol=rtol) + + def test_pi_quaternion_y(self, device, dtype, atol, rtol): + eps = torch.finfo(dtype).eps + quaternion_exp = torch.tensor((0.0, 0.0, 1.0, 0.0), device=device, dtype=dtype) + expected = torch.tensor((0.0, kornia.pi / 2.0, 0.0), device=device, dtype=dtype) + quaternion_log = kornia.geometry.conversions.quaternion_exp_to_log(quaternion_exp, eps=eps) + self.assert_close(quaternion_log, expected, atol=atol, rtol=rtol) + + def test_pi_quaternion_z(self, device, dtype, atol, rtol): + eps = torch.finfo(dtype).eps + quaternion_exp = torch.tensor((0.0, 0.0, 0.0, 1.0), device=device, dtype=dtype) + expected = torch.tensor((0.0, 0.0, kornia.pi / 2.0), device=device, dtype=dtype) + quaternion_log = kornia.geometry.conversions.quaternion_exp_to_log(quaternion_exp, eps=eps) + self.assert_close(quaternion_log, expected, atol=atol, rtol=rtol) + + def test_back_and_forth(self, device, dtype, atol, rtol): + eps = torch.finfo(dtype).eps + quaternion_exp = torch.tensor((0.0, 1.0, 0.0, 0.0), device=device, dtype=dtype) + quaternion_log = kornia.geometry.conversions.quaternion_exp_to_log(quaternion_exp, eps=eps) + quaternion_exp_hat = kornia.geometry.conversions.quaternion_log_to_exp(quaternion_log, eps=eps) + self.assert_close(quaternion_exp, quaternion_exp_hat, atol=atol, rtol=rtol) + + def test_gradcheck(self, device): + dtype = torch.float64 + eps = torch.finfo(dtype).eps + quaternion = torch.tensor((0.0, 1.0, 0.0, 0.0), device=device, dtype=dtype) + # evaluate function gradient + self.gradcheck(partial(kornia.geometry.conversions.quaternion_exp_to_log, eps=eps), (quaternion,)) + + def test_dynamo(self, device, dtype, torch_optimizer): + quaternion = torch.tensor((0.0, 0.0, 1.0, 0.0), device=device, dtype=dtype) + op = kornia.geometry.conversions.quaternion_exp_to_log + op_optimized = torch_optimizer(op) + + actual = op_optimized(quaternion) + expected = op(quaternion) + + self.assert_close(actual, expected) + + +class TestAngleAxisToRotationMatrix(BaseTester): + @pytest.mark.parametrize("batch_size", (1, 2, 5)) + def test_rand_axis_angle_gradcheck(self, batch_size, device, atol, rtol): + dtype = torch.float64 + # generate input data + axis_angle = torch.rand(batch_size, 3, device=device, dtype=dtype) + eye_batch = eye_like(3, axis_angle) + + # apply transform + rotation_matrix = kornia.geometry.conversions.axis_angle_to_rotation_matrix(axis_angle) + + rotation_matrix_eye = torch.matmul(rotation_matrix, rotation_matrix.transpose(-2, -1)) + self.assert_close(rotation_matrix_eye, eye_batch, atol=atol, rtol=rtol) + + # evaluate function gradient + self.gradcheck(kornia.geometry.conversions.axis_angle_to_rotation_matrix, (axis_angle,)) + + def test_axis_angle_to_rotation_matrix(self, device, dtype, atol, rtol): + rmat_1 = torch.tensor( + ( + (-0.30382753, -0.95095137, -0.05814062), + (-0.71581715, 0.26812278, -0.64476041), + (0.62872461, -0.15427791, -0.76217038), + ), + device=device, + dtype=dtype, + ) + rvec_1 = torch.tensor((1.50485376, -2.10737739, 0.7214174), device=device, dtype=dtype) + + rmat_2 = torch.tensor( + ( + (0.6027768, -0.79275544, -0.09054801), + (-0.67915707, -0.56931658, 0.46327563), + (-0.41881476, -0.21775548, -0.88157628), + ), + device=device, + dtype=dtype, + ) + rvec_2 = torch.tensor((-2.44916812, 1.18053411, 0.4085298), device=device, dtype=dtype) + rmat = torch.stack((rmat_2, rmat_1), dim=0) + rvec = torch.stack((rvec_2, rvec_1), dim=0) + + self.assert_close(kornia.geometry.conversions.axis_angle_to_rotation_matrix(rvec), rmat, atol=atol, rtol=rtol) + + +class TestRotationMatrixToAngleAxis(BaseTester): + @pytest.mark.parametrize("batch_size", (1, 2, 5)) + def test_rand_quaternion_gradcheck(self, batch_size, device, dtype, atol, rtol): + # generate input data + quaternion = torch.rand(batch_size, 4, device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.normalize_quaternion(quaternion + 1e-6) + rotation_matrix = kornia.geometry.conversions.quaternion_to_rotation_matrix(quaternion=quaternion) + + eye_batch = eye_like(3, rotation_matrix) + rotation_matrix_eye = torch.matmul(rotation_matrix, rotation_matrix.transpose(-2, -1)) + # This didn't pass with atol=0.001, rtol=0.001 for float16 Cuda 11.2 GeForce 1080 Ti + self.assert_close(rotation_matrix_eye, eye_batch, atol=atol * 10.0, rtol=rtol * 10.0) + + @pytest.mark.parametrize("batch_size", [4]) + def test_gradcheck(self, batch_size, device): + dtype = torch.float64 + quaternion = torch.rand(batch_size, 4, device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.normalize_quaternion(quaternion + 1e-6) + rotation_matrix = kornia.geometry.conversions.quaternion_to_rotation_matrix(quaternion=quaternion) + # evaluate function gradient + self.gradcheck(kornia.geometry.conversions.rotation_matrix_to_axis_angle, (rotation_matrix,)) + + def test_rotation_matrix_to_axis_angle(self, device, dtype, atol, rtol): + rmat_1 = torch.tensor( + ( + (-0.30382753, -0.95095137, -0.05814062), + (-0.71581715, 0.26812278, -0.64476041), + (0.62872461, -0.15427791, -0.76217038), + ), + device=device, + dtype=dtype, + ) + rvec_1 = torch.tensor((1.50485376, -2.10737739, 0.7214174), device=device, dtype=dtype) + + rmat_2 = torch.tensor( + ( + (0.6027768, -0.79275544, -0.09054801), + (-0.67915707, -0.56931658, 0.46327563), + (-0.41881476, -0.21775548, -0.88157628), + ), + device=device, + dtype=dtype, + ) + rvec_2 = torch.tensor((-2.44916812, 1.18053411, 0.4085298), device=device, dtype=dtype) + rmat = torch.stack((rmat_2, rmat_1), dim=0) + rvec = torch.stack((rvec_2, rvec_1), dim=0) + + self.assert_close(kornia.geometry.conversions.rotation_matrix_to_axis_angle(rmat), rvec, atol=atol, rtol=rtol) + + +class TestRadDegConversions(BaseTester): + def test_pi(self): + self.assert_close(kornia.constants.pi.item(), 3.141592) + + @pytest.mark.parametrize("batch_shape", [(2, 3), (1, 2, 3), (2, 3, 3), (5, 5, 3)]) + def test_rad2deg(self, batch_shape, device, dtype): + # generate input data + x_rad = kornia.constants.pi * torch.rand(batch_shape, device=device, dtype=dtype) + + # convert radians/degrees + x_deg = kornia.geometry.conversions.rad2deg(x_rad) + x_deg_to_rad = kornia.geometry.conversions.deg2rad(x_deg) + + # compute error + self.assert_close(x_rad, x_deg_to_rad) + + @pytest.mark.parametrize("batch_shape", [(2, 3), (1, 2, 3), (2, 3, 3), (5, 5, 3)]) + def test_rad2deg_gradcheck(self, batch_shape, device): + dtype = torch.float64 + x_rad = torch.rand(batch_shape, device=device, dtype=dtype) + # evaluate function gradient + self.gradcheck(kornia.geometry.conversions.rad2deg, (x_rad,)) + + @pytest.mark.parametrize("batch_shape", [(2, 3), (1, 2, 3), (2, 3, 3), (5, 5, 3)]) + def test_deg2rad(self, batch_shape, device, dtype, atol, rtol): + # generate input data + x_deg = 180.0 * torch.rand(batch_shape, device=device, dtype=dtype) + + # convert radians/degrees + x_rad = kornia.geometry.conversions.deg2rad(x_deg) + x_rad_to_deg = kornia.geometry.conversions.rad2deg(x_rad) + + self.assert_close(x_deg, x_rad_to_deg, atol=atol, rtol=rtol) + + @pytest.mark.parametrize("batch_shape", [(2, 3), (1, 2, 3), (2, 3, 3), (5, 5, 3)]) + def test_deg2rad_gradcheck(self, batch_shape, device): + x_deg = 180.0 * torch.rand(batch_shape, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.conversions.deg2rad, (x_deg,)) + + +class TestPolCartConversions(BaseTester): + def test_smoke(self, device, dtype): + x = torch.ones(1, 1, 1, 1, device=device, dtype=dtype) + assert kornia.geometry.conversions.pol2cart(x, x) is not None + assert kornia.geometry.conversions.cart2pol(x, x) is not None + + @pytest.mark.parametrize("batch_shape", [(2, 3), (1, 2, 3), (2, 3, 3), (5, 5, 3)]) + def test_pol2cart(self, batch_shape, device, dtype): + # generate input data + rho = torch.rand(batch_shape, dtype=dtype) + phi = kornia.constants.pi * torch.rand(batch_shape, dtype=dtype) + rho = rho.to(device) + phi = phi.to(device) + + # convert pol/cart + x_pol2cart, y_pol2cart = kornia.geometry.conversions.pol2cart(rho, phi) + rho_pol2cart, phi_pol2cart = kornia.geometry.conversions.cart2pol(x_pol2cart, y_pol2cart, 0) + + self.assert_close(rho, rho_pol2cart) + self.assert_close(phi, phi_pol2cart) + + @pytest.mark.parametrize("batch_shape", [(2, 3)]) + def test_gradcheck(self, batch_shape, device): + rho = torch.rand(batch_shape, dtype=torch.float64, device=device) + phi = kornia.constants.pi * torch.rand(batch_shape, dtype=torch.float64, device=device) + self.gradcheck(kornia.geometry.conversions.pol2cart, (rho, phi)) + self.gradcheck(kornia.geometry.conversions.cart2pol, (rho, phi)) + + @pytest.mark.parametrize("batch_shape", [(2, 3), (1, 2, 3), (2, 3, 3), (5, 5, 3)]) + def test_cart2pol(self, batch_shape, device, dtype): + # generate input data + x = torch.rand(batch_shape, dtype=dtype) + y = torch.rand(batch_shape, dtype=dtype) + x = x.to(device) + y = y.to(device) + + # convert cart/pol + rho_cart2pol, phi_cart2pol = kornia.geometry.conversions.cart2pol(x, y, 0) + x_cart2pol, y_cart2pol = kornia.geometry.conversions.pol2cart(rho_cart2pol, phi_cart2pol) + + self.assert_close(x, x_cart2pol) + self.assert_close(y, y_cart2pol) + + +class TestConvertPointsToHomogeneous(BaseTester): + def test_convert_points(self, device, dtype): + # generate input data + points_h = torch.tensor( + [[1.0, 2.0, 1.0], [0.0, 1.0, 2.0], [2.0, 1.0, 0.0], [-1.0, -2.0, -1.0], [0.0, 1.0, -2.0]], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [ + [1.0, 2.0, 1.0, 1.0], + [0.0, 1.0, 2.0, 1.0], + [2.0, 1.0, 0.0, 1.0], + [-1.0, -2.0, -1.0, 1.0], + [0.0, 1.0, -2.0, 1.0], + ], + device=device, + dtype=dtype, + ) + + # to euclidean + points = kornia.geometry.conversions.convert_points_to_homogeneous(points_h) + self.assert_close(points, expected, atol=1e-4, rtol=1e-4) + + def test_convert_points_batch(self, device, dtype): + # generate input data + points_h = torch.tensor([[[2.0, 1.0, 0.0]], [[0.0, 1.0, 2.0]], [[0.0, 1.0, -2.0]]], device=device, dtype=dtype) + + expected = torch.tensor( + [[[2.0, 1.0, 0.0, 1.0]], [[0.0, 1.0, 2.0, 1.0]], [[0.0, 1.0, -2.0, 1.0]]], device=device, dtype=dtype + ) + + # to euclidean + points = kornia.geometry.conversions.convert_points_to_homogeneous(points_h) + self.assert_close(points, expected, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_shape", [(2, 3), (1, 2, 3), (2, 3, 3), (5, 5, 3)]) + def test_gradcheck(self, batch_shape, device): + points_h = torch.rand(batch_shape, device=device, dtype=torch.float64) + + # evaluate function gradient + self.gradcheck(kornia.geometry.conversions.convert_points_to_homogeneous, (points_h,)) + + def test_dynamo(self, device, dtype, torch_optimizer): + points_h = torch.zeros(1, 2, 3, device=device, dtype=dtype) + + op = kornia.geometry.conversions.convert_points_to_homogeneous + op_optimized = torch_optimizer(op) + + actual = op_optimized(points_h) + expected = op(points_h) + + self.assert_close(actual, expected) + + +class TestConvertAtoH(BaseTester): + def test_convert_points(self, device, dtype): + # generate input data + A = torch.tensor([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]], device=device, dtype=dtype).view(1, 2, 3) + + expected = torch.tensor([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], device=device, dtype=dtype).view( + 1, 3, 3 + ) + + # to euclidean + H = kornia.geometry.conversions.convert_affinematrix_to_homography(A) + self.assert_close(H, expected) + + @pytest.mark.parametrize("batch_shape", [(10, 2, 3), (16, 2, 3)]) + def test_gradcheck(self, batch_shape, device): + points_h = torch.rand(batch_shape, device=device, dtype=torch.float64) + + # evaluate function gradient + self.gradcheck(kornia.geometry.conversions.convert_affinematrix_to_homography, (points_h,)) + + def test_dynamo(self, device, dtype, torch_optimizer): + points_h = torch.zeros(1, 2, 3, device=device, dtype=dtype) + + op = kornia.geometry.conversions.convert_affinematrix_to_homography + op_optimized = torch_optimizer(op) + + actual = op_optimized(points_h) + expected = op(points_h) + + self.assert_close(actual, expected) + + +class TestConvertPointsFromHomogeneous(BaseTester): + @pytest.mark.parametrize("batch_shape", [(2, 3), (1, 2, 3), (2, 3, 3), (5, 5, 3)]) + def test_cardinality(self, device, dtype, batch_shape): + points_h = torch.rand(batch_shape, device=device, dtype=dtype) + points = kornia.geometry.conversions.convert_points_from_homogeneous(points_h) + assert points.shape == points.shape[:-1] + (2,) + + def test_points(self, device, dtype): + # generate input data + points_h = torch.tensor( + [[1.0, 2.0, 1.0], [0.0, 1.0, 2.0], [2.0, 1.0, 0.0], [-1.0, -2.0, -1.0], [0.0, 1.0, -2.0]], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [[1.0, 2.0], [0.0, 0.5], [2.0, 1.0], [1.0, 2.0], [0.0, -0.5]], device=device, dtype=dtype + ) + + # to euclidean + points = kornia.geometry.conversions.convert_points_from_homogeneous(points_h) + self.assert_close(points, expected, atol=1e-4, rtol=1e-4) + + def test_points_batch(self, device, dtype): + # generate input data + points_h = torch.tensor([[[2.0, 1.0, 0.0]], [[0.0, 1.0, 2.0]], [[0.0, 1.0, -2.0]]], device=device, dtype=dtype) + + expected = torch.tensor([[[2.0, 1.0]], [[0.0, 0.5]], [[0.0, -0.5]]], device=device, dtype=dtype) + + # to euclidean + points = kornia.geometry.conversions.convert_points_from_homogeneous(points_h) + self.assert_close(points, expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + points_h = torch.ones(1, 10, 3, device=device, dtype=torch.float64) + + # evaluate function gradient + self.gradcheck(kornia.geometry.conversions.convert_points_from_homogeneous, (points_h,)) + + def test_gradcheck_zvec_zeros(self, device): + # generate input data + points_h = torch.tensor([[1.0, 2.0, 0.0], [0.0, 1.0, 0.1], [2.0, 1.0, 0.1]], device=device, dtype=torch.float64) + + # evaluate function gradient + self.gradcheck(kornia.geometry.conversions.convert_points_from_homogeneous, (points_h,), eps=1e-8) + + def test_dynamo(self, device, dtype, torch_optimizer): + points_h = torch.zeros(1, 2, 3, device=device, dtype=dtype) + + op = kornia.geometry.conversions.convert_points_from_homogeneous + op_optimized = torch_optimizer(op) + + actual = op_optimized(points_h) + expected = op(points_h) + + self.assert_close(actual, expected) + + +class TestNormalizePixelCoordinates(BaseTester): + def test_tensor_bhw2(self, device, dtype, atol, rtol): + eps = torch.finfo(dtype).eps + height, width = 3, 4 + grid = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=False, device=device).to( + dtype=dtype + ) + + expected = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=True, device=device).to( + dtype=dtype + ) + + grid_norm = kornia.geometry.conversions.normalize_pixel_coordinates(grid, height, width, eps=eps) + + self.assert_close(grid_norm, expected, atol=atol, rtol=rtol) + + def test_list(self, device, dtype, atol, rtol): + eps = torch.finfo(dtype).eps + height, width = 3, 4 + grid = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=False, device=device).to( + dtype=dtype + ) + grid = grid.contiguous().view(-1, 2) + + expected = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=True, device=device).to( + dtype=dtype + ) + expected = expected.contiguous().view(-1, 2) + + grid_norm = kornia.geometry.conversions.normalize_pixel_coordinates(grid, height, width, eps=eps) + + self.assert_close(grid_norm, expected, atol=atol, rtol=rtol) + + def test_dynamo(self, device, dtype, torch_optimizer): + if device == torch.device("cpu"): + pytest.skip("NormalizePixelCoordinates not working on CPU with dynamo!") + + op = kornia.geometry.conversions.normalize_pixel_coordinates + op_optimized = torch_optimizer(op) + + height, width = 3, 4 + grid = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=True, device=device).to( + dtype=dtype + ) + + actual = op_optimized(grid, height, width) + expected = op(grid, height, width) + + self.assert_close(actual, expected) + + +class TestDenormalizePixelCoordinates(BaseTester): + def test_tensor_bhw2(self, device, dtype): + height, width = 3, 4 + grid = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=True, device=device).to( + dtype=dtype + ) + + expected = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=False, device=device).to( + dtype=dtype + ) + + grid_norm = kornia.geometry.conversions.denormalize_pixel_coordinates(grid, height, width) + + self.assert_close(grid_norm, expected, atol=1e-4, rtol=1e-4) + + def test_list(self, device, dtype): + height, width = 3, 4 + grid = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=True, device=device).to( + dtype=dtype + ) + grid = grid.contiguous().view(-1, 2) + + expected = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=False, device=device).to( + dtype=dtype + ) + expected = expected.contiguous().view(-1, 2) + + grid_norm = kornia.geometry.conversions.denormalize_pixel_coordinates(grid, height, width) + + self.assert_close(grid_norm, expected, atol=1e-4, rtol=1e-4) + + def test_dynamo(self, device, dtype, torch_optimizer): + if device == torch.device("cpu"): + pytest.xfail("DenormalizePixelCoordinates not working on CPU with dynamo!") + + op = kornia.geometry.conversions.denormalize_pixel_coordinates + op_optimized = torch_optimizer(op) + + height, width = 3, 4 + grid = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=True, device=device).to( + dtype=dtype + ) + + actual = op_optimized(grid, height, width) + expected = op(grid, height, width) + + self.assert_close(actual, expected) + + +class TestProjectPoints(BaseTester): + def test_smoke(self, device, dtype): + point_3d = torch.zeros(1, 3, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(1, -1, -1) + point_2d = kornia.geometry.camera.project_points(point_3d, camera_matrix) + assert point_2d.shape == (1, 2) + + def test_smoke_batch(self, device, dtype): + point_3d = torch.zeros(2, 3, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(2, -1, -1) + point_2d = kornia.geometry.camera.project_points(point_3d, camera_matrix) + assert point_2d.shape == (2, 2) + + def test_smoke_batch_multi(self, device, dtype): + point_3d = torch.zeros(2, 4, 3, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(2, 4, -1, -1) + point_2d = kornia.geometry.camera.project_points(point_3d, camera_matrix) + assert point_2d.shape == (2, 4, 2) + + def test_project_and_unproject(self, device, dtype): + point_3d = torch.tensor([[10.0, 2.0, 30.0]], device=device, dtype=dtype) + depth = point_3d[..., -1:] + camera_matrix = torch.tensor( + [[[2746.0, 0.0, 991.0], [0.0, 2748.0, 619.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype + ) + point_2d = kornia.geometry.camera.project_points(point_3d, camera_matrix) + point_3d_hat = kornia.geometry.camera.unproject_points(point_2d, depth, camera_matrix) + self.assert_close(point_3d, point_3d_hat, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + # TODO: point [0, 0, 0] crashes + points_3d = torch.ones(1, 3, device=device, dtype=torch.float64) + camera_matrix = torch.eye(3, device=device, dtype=torch.float64).expand(1, -1, -1) + + # evaluate function gradient + self.gradcheck(kornia.geometry.camera.project_points, (points_3d, camera_matrix)) + + def test_dynamo(self, device, dtype, torch_optimizer): + points_3d = torch.zeros(1, 3, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(1, -1, -1) + op = kornia.geometry.camera.project_points + op_optimized = torch_optimizer(op) + + actual = op_optimized(points_3d, camera_matrix) + expected = op(points_3d, camera_matrix) + + self.assert_close(actual, expected) + + +class TestDenormalizePointsWithIntrinsics(BaseTester): + def test_smoke(self, device, dtype): + points_2d = torch.zeros(1, 2, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(1, -1, -1) + points_norm = kornia.geometry.conversions.denormalize_points_with_intrinsics(points_2d, camera_matrix) + assert points_norm.shape == (1, 2) + + def test_smoke_batch(self, device, dtype): + points_2d = torch.zeros(2, 2, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(2, -1, -1) + points_norm = kornia.geometry.conversions.denormalize_points_with_intrinsics(points_2d, camera_matrix) + assert points_norm.shape == (2, 2) + + def test_smoke_batch_n(self, device, dtype): + points_2d = torch.zeros(2, 9, 2, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(2, -1, -1) + points_norm = kornia.geometry.conversions.denormalize_points_with_intrinsics(points_2d, camera_matrix) + assert points_norm.shape == (2, 9, 2) + + def test_toy(self, device, dtype): + point_2d = torch.tensor([[1.0, 1.0]], device=device, dtype=dtype) + camera_matrix = torch.tensor( + [[64.0, 0.0, 128.0], [0.0, 64.0, 128.0], [0.0, 0.0, 1.0]], device=device, dtype=dtype + ) + op = kornia.geometry.conversions.denormalize_points_with_intrinsics + expected = torch.tensor([[192.0, 192.0]], device=device, dtype=dtype) + self.assert_close(op(point_2d, camera_matrix), expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + points_2d = torch.zeros(1, 2, device=device, dtype=torch.float64) + camera_matrix = torch.eye(3, device=device, dtype=torch.float64).expand(1, -1, -1) + + # evaluate function gradient + self.gradcheck(kornia.geometry.conversions.denormalize_points_with_intrinsics, (points_2d, camera_matrix)) + + def test_dynamo(self, device, dtype, torch_optimizer): + points_2d = torch.zeros(1, 2, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(1, -1, -1) + op = kornia.geometry.conversions.denormalize_points_with_intrinsics + op_optimized = torch_optimizer(op) + + actual = op_optimized(points_2d, camera_matrix) + expected = op(points_2d, camera_matrix) + + self.assert_close(actual, expected) + + +class TestNormalizePointsWithIntrinsics(BaseTester): + def test_smoke(self, device, dtype): + points_2d = torch.zeros(1, 2, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(1, -1, -1) + points_norm = kornia.geometry.conversions.normalize_points_with_intrinsics(points_2d, camera_matrix) + assert points_norm.shape == (1, 2) + + def test_smoke_batch(self, device, dtype): + points_2d = torch.zeros(2, 2, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(2, -1, -1) + points_norm = kornia.geometry.conversions.normalize_points_with_intrinsics(points_2d, camera_matrix) + assert points_norm.shape == (2, 2) + + def test_smoke_batch_n(self, device, dtype): + points_2d = torch.zeros(2, 10, 2, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(2, -1, -1) + points_norm = kornia.geometry.conversions.normalize_points_with_intrinsics(points_2d, camera_matrix) + assert points_norm.shape == (2, 10, 2) + + def test_norm_unnorm(self, device, dtype): + point_2d = torch.tensor([[128.0, 128.0]], device=device, dtype=dtype) + camera_matrix = torch.tensor( + [[64.0, 0.0, 128.0], [0.0, 64.0, 128.0], [0.0, 0.0, 1.0]], device=device, dtype=dtype + ) + op = kornia.geometry.conversions.normalize_points_with_intrinsics + back = kornia.geometry.conversions.denormalize_points_with_intrinsics + point_2d_norm = op(point_2d, camera_matrix) + point_2d_hat = back(point_2d_norm, camera_matrix) + self.assert_close(point_2d, point_2d_hat, atol=1e-4, rtol=1e-4) + + def test_toy(self, device, dtype): + point_2d = torch.tensor([[192.0, 192.0]], device=device, dtype=dtype) + camera_matrix = torch.tensor( + [[64.0, 0.0, 128.0], [0.0, 64.0, 128.0], [0.0, 0.0, 1.0]], device=device, dtype=dtype + ) + op = kornia.geometry.conversions.normalize_points_with_intrinsics + out = op(point_2d, camera_matrix) + expected = torch.tensor([[1.0, 1.0]], device=device, dtype=dtype) + self.assert_close(out, expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + points_2d = torch.zeros(1, 2, device=device, dtype=torch.float64) + camera_matrix = torch.eye(3, device=device, dtype=torch.float64).expand(1, -1, -1) + + # evaluate function gradient + self.gradcheck(kornia.geometry.conversions.normalize_points_with_intrinsics, (points_2d, camera_matrix)) + + def test_dynamo(self, device, dtype, torch_optimizer): + points_2d = torch.zeros(1, 2, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).expand(1, -1, -1) + op = kornia.geometry.conversions.normalize_points_with_intrinsics + op_optimized = torch_optimizer(op) + + actual = op_optimized(points_2d, camera_matrix) + expected = op(points_2d, camera_matrix) + + self.assert_close(actual, expected) + + +class TestRt2Extrinsics(BaseTester): + @pytest.mark.parametrize("batch_size", [1, 2, 3]) + def test_everything(self, batch_size, device, dtype): + # generate input data + R = torch.rand(batch_size, 3, 3, dtype=dtype, device=device) + t = torch.rand(batch_size, 3, 1, dtype=dtype, device=device) + + Rt = Rt_to_matrix4x4(R, t) + assert Rt.shape == (batch_size, 4, 4) + + R2, t2 = matrix4x4_to_Rt(Rt) + assert R2.shape == (batch_size, 3, 3) + assert t2.shape == (batch_size, 3, 1) + + self.assert_close(R, R2, rtol=1e-4, atol=1e-5) + self.assert_close(t, t2, rtol=1e-4, atol=1e-5) + + @pytest.mark.parametrize("batch_size", [5]) + def test_gradcheck(self, batch_size, device): + R = torch.rand(batch_size, 3, 3, dtype=torch.float64, device=device) + t = torch.rand(batch_size, 3, 1, dtype=torch.float64, device=device) + self.gradcheck(kornia.geometry.conversions.Rt_to_matrix4x4, (R, t)) + + +class TestCamtoworldGraphicsToVision(BaseTester): + @pytest.mark.parametrize("batch_size", [1, 2, 3]) + def test_everything(self, batch_size, device, dtype): + # generate input data + t_vis = torch.tensor([2, 3, 4], device=device, dtype=dtype).view(1, 3, 1).repeat(batch_size, 1, 1) + angles = torch.tensor([0, kornia.pi / 2.0, 0.0], device=device, dtype=dtype)[None] + R_vis = kornia.geometry.axis_angle_to_rotation_matrix(angles).repeat(batch_size, 1, 1) + K_vis = Rt_to_matrix4x4(R_vis, t_vis) + K_graf = camtoworld_vision_to_graphics_4x4(K_vis) + + expected = torch.tensor( + [[0, 0, -1, 2], [0, -1, 0, 3], [-1, 0, 0, 4], [0, 0, 0, 1]], device=device, dtype=dtype + )[None].repeat(batch_size, 1, 1) + + self.assert_close(K_graf, expected, rtol=1e-4, atol=1e-5) + R_graf, t_graf = camtoworld_vision_to_graphics_Rt(R_vis, t_vis) + expected_R = torch.tensor([[0, 0, -1], [0, -1, 0], [-1, 0, 0]], device=device, dtype=dtype)[None].repeat( + batch_size, 1, 1 + ) + expected_t = torch.tensor([2, 3, 4], device=device, dtype=dtype).reshape(1, 3, 1).repeat(batch_size, 1, 1) + + self.assert_close(t_graf, expected_t, rtol=1e-4, atol=1e-5) + self.assert_close(R_graf, expected_R, rtol=1e-4, atol=1e-5) + + Kvis_back = camtoworld_graphics_to_vision_4x4(K_graf) + self.assert_close(Kvis_back, K_vis, rtol=1e-4, atol=1e-5) + + R_vis_back, t_vis_back = camtoworld_graphics_to_vision_Rt(R_graf, t_graf) + self.assert_close(R_vis_back, R_vis, rtol=1e-4, atol=1e-5) + self.assert_close(t_vis_back, t_vis, rtol=1e-4, atol=1e-5) + + @pytest.mark.parametrize("batch_size", [4]) + def test_gradcheck(self, batch_size, device): + t_vis = torch.tensor([2, 3, 4], device=device, dtype=torch.float64).view(1, 3, 1).repeat(batch_size, 1, 1) + angles = torch.tensor([0, kornia.pi / 2.0, 0.0], device=device, dtype=torch.float64)[None] + R_vis = kornia.geometry.axis_angle_to_rotation_matrix(angles).repeat(batch_size, 1, 1) + K_vis = Rt_to_matrix4x4(R_vis, t_vis) + self.gradcheck(camtoworld_graphics_to_vision_4x4, (K_vis,)) + self.gradcheck(camtoworld_vision_to_graphics_4x4, (K_vis,)) + + +class TestCamtoworldRtToPoseRt(BaseTester): + @pytest.mark.parametrize("batch_size", [1, 2, 3]) + def test_everything(self, batch_size, device, dtype): + # generate input data + t = torch.tensor([2, 3, 4], device=device, dtype=dtype).view(1, 3, 1).repeat(batch_size, 1, 1) + angles = torch.tensor([0, kornia.pi / 2.0, 0.0], device=device, dtype=dtype)[None] + R = kornia.geometry.axis_angle_to_rotation_matrix(angles).repeat(batch_size, 1, 1) + + Rp, tp = camtoworld_to_worldtocam_Rt(R, t) + + expected_Rp = torch.tensor([[0, 0, -1], [0, 1, 0], [1, 0, 0]], device=device, dtype=dtype)[None].repeat( + batch_size, 1, 1 + ) + expected_tp = torch.tensor([4, -3, -2], device=device, dtype=dtype).view(1, 3, 1).repeat(batch_size, 1, 1) + self.assert_close(Rp, expected_Rp, rtol=1e-4, atol=1e-5) + self.assert_close(tp, expected_tp, rtol=1e-4, atol=1e-5) + + Rback, tback = worldtocam_to_camtoworld_Rt(Rp, tp) + self.assert_close(Rback, R, rtol=1e-4, atol=1e-5) + self.assert_close(tback, t, rtol=1e-4, atol=1e-5) + + @pytest.mark.parametrize("batch_size", [4]) + def test_gradcheck(self, batch_size, device): + t = torch.tensor([2, 3, 4], device=device, dtype=torch.float64).view(1, 3, 1).repeat(batch_size, 1, 1) + angles = torch.tensor([0, kornia.pi / 2.0, 0.0], device=device, dtype=torch.float64)[None] + R = kornia.geometry.axis_angle_to_rotation_matrix(angles).repeat(batch_size, 1, 1) + self.gradcheck(camtoworld_to_worldtocam_Rt, (R, t)) + self.gradcheck(worldtocam_to_camtoworld_Rt, (R, t)) + + +class TestCARKitToColmap(BaseTester): + def test_everything(self, device, dtype): + # generate input data + t = torch.tensor([1, 0, 0], device=device, dtype=dtype).view(1, 3, 1) + ang_deg = torch.tensor([45, 60.0, 0.0], device=device, dtype=dtype)[None] + ang_rad = kornia.geometry.conversions.deg2rad(ang_deg) + qvec = kornia.geometry.axis_angle_to_quaternion(ang_rad) + + q_colmap, t_colmap = ARKitQTVecs_to_ColmapQTVecs(qvec, t) + + angles_colmap = kornia.geometry.conversions.quaternion_to_axis_angle(q_colmap) + angles_colmap = kornia.geometry.conversions.rad2deg(angles_colmap) + expected_angles = torch.tensor([[116.8870620728, 0.0, -71.7524719238]], device=device, dtype=dtype) + expected_t = torch.tensor([[[-0.5256], [0.3558], [0.7727]]], device=device, dtype=dtype) + + self.assert_close(angles_colmap, expected_angles, rtol=1e-4, atol=1e-5) + self.assert_close(t_colmap, expected_t, rtol=1e-4, atol=1e-5) + + +class TestEulerFromQuaternion(BaseTester): + def test_smoke(self, device, dtype): + q = Quaternion.random(batch_size=1) + q = q.to(device, dtype) + roll, pitch, yaw = euler_from_quaternion(q.w, q.x, q.y, q.z) + assert roll.shape == pitch.shape + assert pitch.shape == yaw.shape + + @pytest.mark.parametrize("batch_size", ((1, 3, 4))) + def test_cardinality(self, device, dtype, batch_size): + q = Quaternion.random(batch_size=batch_size) + q = q.to(device, dtype) + roll, pitch, yaw = euler_from_quaternion(q.w, q.x, q.y, q.z) + assert roll.shape[0] == batch_size + assert pitch.shape[0] == batch_size + assert yaw.shape[0] == batch_size + + def test_exception(self, device, dtype): + q = Quaternion.random(batch_size=2) + q = q.to(device, dtype) + with pytest.raises(Exception): + euler_from_quaternion(q.w, torch.rand(1), q.y, q.z) + + def test_gradcheck(self, device): + q = Quaternion.random(batch_size=1).to(device, torch.float64) + self.gradcheck(euler_from_quaternion, (q.w, q.x, q.y, q.z)) + + @pytest.mark.skipif( + torch_version() in {"2.0.1", "2.1.2", "2.2.2", "2.3.1"} and sys.version_info.minor == 8, + reason="Not working on 2.0", + ) + def test_dynamo(self, device, dtype, torch_optimizer): + q = Quaternion.random(batch_size=1) + q = q.to(device, dtype) + op = euler_from_quaternion + op_optimized = torch_optimizer(op) + self.assert_close(op(q.w, q.x, q.y, q.z), op_optimized(q.w, q.x, q.y, q.z)) + + def test_forth_and_back(self, device, dtype): + q = Quaternion.random(batch_size=2) + q = q.to(device, dtype) + roll, pitch, yaw = euler_from_quaternion(q.w, q.x, q.y, q.z) + qw, qx, qy, qz = quaternion_from_euler(roll, pitch, yaw) + # TODO: check hwo to prevent getting inverted angles sometimes + self.assert_close(q.w.abs(), qw.abs()) + self.assert_close(q.x.abs(), qx.abs()) + self.assert_close(q.y.abs(), qy.abs()) + self.assert_close(q.z.abs(), qz.abs()) + + +class TestQuaternionFromEuler(BaseTester): + def test_smoke(self, device, dtype): + roll, pitch, yaw = torch.rand(3, device=device, dtype=dtype) + qw, qx, qy, qz = quaternion_from_euler(roll, pitch, yaw) + assert qw.shape == qx.shape + assert qx.shape == qy.shape + assert qy.shape == qz.shape + + @pytest.mark.parametrize("batch_size", ((1, 3, 4))) + def test_cardinality(self, device, dtype, batch_size): + roll, pitch, yaw = torch.rand(3, batch_size, device=device, dtype=dtype) + qw, qx, qy, qz = quaternion_from_euler(roll, pitch, yaw) + assert qw.shape[0] == batch_size + assert qx.shape[0] == batch_size + assert qy.shape[0] == batch_size + assert qz.shape[0] == batch_size + + def test_exception(self, device, dtype): + _, pitch, yaw = torch.rand(3, 2, device=device, dtype=dtype) + with pytest.raises(Exception): + quaternion_from_euler(torch.rand(1), pitch, yaw) + + def test_gradcheck(self, device): + roll, pitch, yaw = torch.rand(3, 2, device=device, dtype=torch.float64, requires_grad=True) + self.gradcheck(quaternion_from_euler, (roll, pitch, yaw)) + + def test_dynamo(self, device, dtype, torch_optimizer): + roll, pitch, yaw = torch.rand(3, 2, device=device, dtype=dtype) + + op = quaternion_from_euler + op_optimized = torch_optimizer(op) + + actual = op_optimized(roll, pitch, yaw) + expected = op(roll, pitch, yaw) + + self.assert_close(actual[0], expected[0]) + self.assert_close(actual[1], expected[1]) + self.assert_close(actual[2], expected[2]) + + def test_forth_and_back(self, device, dtype): + roll, pitch, yaw = torch.rand(3, 2, device=device, dtype=dtype) + qw, qx, qy, qz = quaternion_from_euler(roll, pitch, yaw) + roll_new, pitch_new, yaw_new = euler_from_quaternion(qw, qx, qy, qz) + self.assert_close(roll, roll_new) + self.assert_close(pitch, pitch_new) + self.assert_close(yaw, yaw_new) + + def test_values(self, device, dtype): + # num_samples = 5 + # data = 2 * torch.rand(3, num_samples, device=device, dtype=dtype) - 1 + # roll, pitch, yaw = torch.pi * data + roll = torch.tensor( + [2.6518599987, 0.0612506270, 1.2417907715, 2.8829660416, -1.9961174726], device=device, dtype=dtype + ) + + pitch = torch.tensor( + [2.3267219067, -2.7309591770, -1.4011553526, -2.1962766647, 2.1454355717], device=device, dtype=dtype + ) + + yaw = torch.tensor( + [-0.8856627345, 0.2605336905, 0.4579202533, -1.3095731735, 0.6096843481], device=device, dtype=dtype + ) + + euler_expected = torch.tensor( + [ + [-0.4897327125, 0.8148705959, 2.2559301853], + [-3.0803420544, -0.4106334746, -2.8810589314], + [1.2417914867, -1.4011553526, 0.4579201937], + [-0.2586266696, -0.9453159571, 1.8320195675], + [1.1454752684, 0.9961569905, -2.5319085121], + ], + device=device, + dtype=dtype, + ) + + qw, qx, qy, qz = quaternion_from_euler(roll, pitch, yaw) + euler = euler_from_quaternion(qw, qx, qy, qz) + euler = torch.stack(euler, -1) + + self.assert_close(euler, euler_expected, 1e-4, 1e-4) + + # this test is passing: pip install transforms3d + # import transforms3d as tf3 + # out = [tf3.euler.euler2quat(roll[i], pitch[i], yaw[i]) for i in range(num_samples)] + # out = torch.tensor(out, device=device, dtype=dtype) + # self.assert_close(torch.stack((qw, qx, qy, qz), -1), out) + + # out = [tf3.euler.quat2euler((qw[i], qx[i], qy[i], qz[i])) for i in range(num_samples)] + # out = torch.tensor(out, device=device, dtype=dtype) + + +@pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) +def test_vector_to_skew_symmetric_matrix(batch_size, device, dtype): + if batch_size is None: + vector = torch.rand(3, device=device, dtype=dtype) + else: + vector = torch.rand((batch_size, 3), device=device, dtype=dtype) + skew_symmetric_matrix = kornia.geometry.conversions.vector_to_skew_symmetric_matrix(vector) + assert skew_symmetric_matrix.shape[-1] == 3 + assert skew_symmetric_matrix.shape[-2] == 3 + z = torch.zeros_like(vector[..., 0]) + assert_close(skew_symmetric_matrix[..., 0, 0], z) + assert_close(skew_symmetric_matrix[..., 1, 1], z) + assert_close(skew_symmetric_matrix[..., 2, 2], z) + assert_close(skew_symmetric_matrix[..., 0, 1], -vector[..., 2]) + assert_close(skew_symmetric_matrix[..., 1, 0], vector[..., 2]) + assert_close(skew_symmetric_matrix[..., 0, 2], vector[..., 1]) + assert_close(skew_symmetric_matrix[..., 2, 0], -vector[..., 1]) + assert_close(skew_symmetric_matrix[..., 1, 2], -vector[..., 0]) + assert_close(skew_symmetric_matrix[..., 2, 1], vector[..., 0]) + + +class TestAxisAngleToRotationMatrix: + def test_identity_rotation(self): + aa = torch.zeros(1, 3, dtype=torch.float64, requires_grad=True) + R = axis_angle_to_rotation_matrix(aa) + Id = torch.eye(3, dtype=torch.float64).unsqueeze(0) + assert torch.allclose(R, Id, atol=1e-6) + + def test_90deg_x_axis(self): + aa = torch.tensor([[torch.pi / 2, 0.0, 0.0]], dtype=torch.float64) + R = axis_angle_to_rotation_matrix(aa).squeeze(0) + expected = torch.tensor( + [ + [1.0, 0.0, 0.0], + [0.0, 0.0, -1.0], + [0.0, 1.0, 0.0], + ], + dtype=torch.float64, + ) + assert torch.allclose(R, expected, atol=1e-6) + + def test_180deg_y_axis(self): + aa = torch.tensor([[0.0, torch.pi, 0.0]], dtype=torch.float64) + R = axis_angle_to_rotation_matrix(aa).squeeze(0) + expected = torch.tensor( + [ + [-1.0, 0.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 0.0, -1.0], + ], + dtype=torch.float64, + ) + assert torch.allclose(R, expected, atol=1e-6) + + def test_batched_input(self): + aa = torch.tensor( + [ + [0.0, 0.0, 0.0], + [torch.pi / 2, 0.0, 0.0], + [0.0, torch.pi, 0.0], + ], + dtype=torch.float64, + ) + R = axis_angle_to_rotation_matrix(aa) + assert R.shape == (3, 3, 3) diff --git a/tests/geometry/test_depth.py b/tests/geometry/test_depth.py new file mode 100644 index 0000000..327900d --- /dev/null +++ b/tests/geometry/test_depth.py @@ -0,0 +1,504 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestDepthTo3d(BaseTester): + def test_smoke(self, device, dtype): + depth = torch.rand(1, 1, 3, 4, device=device, dtype=dtype) + camera_matrix = torch.rand(1, 3, 3, device=device, dtype=dtype) + + points3d = kornia.geometry.depth.depth_to_3d(depth, camera_matrix) + assert points3d.shape == (1, 3, 3, 4) + + @pytest.mark.parametrize("batch_size", [2, 4, 5]) + def test_shapes(self, batch_size, device, dtype): + depth = torch.rand(batch_size, 1, 3, 4, device=device, dtype=dtype) + camera_matrix = torch.rand(batch_size, 3, 3, device=device, dtype=dtype) + + points3d = kornia.geometry.depth.depth_to_3d(depth, camera_matrix) + assert points3d.shape == (batch_size, 3, 3, 4) + + @pytest.mark.parametrize("batch_size", [1, 2, 4, 5]) + def test_shapes_broadcast(self, batch_size, device, dtype): + depth = torch.rand(batch_size, 1, 3, 4, device=device, dtype=dtype) + camera_matrix = torch.rand(1, 3, 3, device=device, dtype=dtype) + + points3d = kornia.geometry.depth.depth_to_3d(depth, camera_matrix) + assert points3d.shape == (batch_size, 3, 3, 4) + + def test_depth_to_3d_v2(self, device, dtype): + depth = torch.rand(5, 1, 3, 4, device=device, dtype=dtype) + camera_matrix = torch.rand(5, 3, 3, device=device, dtype=dtype) + + points3d = kornia.geometry.depth.depth_to_3d(depth, camera_matrix) + + # TODO: implement me with batch + # Permute the depth tensor to match the expected input shape for depth_to_3d_v2. + depth = torch.permute(depth, (1, 0, 2, 3)) + points3d_v2 = kornia.geometry.depth.depth_to_3d_v2(depth[0], camera_matrix) + # Align the output format of depth_to_3d with depth_to_3d_v2 by reordering dimensions. + self.assert_close(points3d.permute(0, 2, 3, 1), points3d_v2) + + def test_depth_to_3d_v2_cached_grid(self, device, dtype): + # Passing a pre-computed xyz_grid must not raise "Boolean value of Tensor + # is ambiguous" (regression for the `xyz_grid or ...` truthiness bug). + depth = torch.rand(2, 3, 4, device=device, dtype=dtype).add_(0.1) + camera_matrix = torch.eye(3, device=device, dtype=dtype).unsqueeze(0).expand(2, -1, -1).contiguous() + grid = kornia.geometry.unproject_meshgrid(3, 4, camera_matrix, device=device, dtype=dtype) + out_cached = kornia.geometry.depth.depth_to_3d_v2(depth, camera_matrix, xyz_grid=grid) + out_uncached = kornia.geometry.depth.depth_to_3d_v2(depth, camera_matrix) + self.assert_close(out_cached, out_uncached) + + def test_unproject_meshgrid(self, device, dtype): + # TODO: implement me with batch + camera_matrix = torch.eye(3, device=device, dtype=dtype).repeat(2, 1, 1) + grid = kornia.geometry.unproject_meshgrid(3, 4, camera_matrix, device=device, dtype=dtype) + assert grid.shape == (2, 3, 4, 3) + # test for now that the grid is correct and have homogeneous coords + self.assert_close(grid[..., 2], torch.ones_like(grid[..., 2])) + + def test_unproject_denormalized(self, device, dtype): + # this is for default normalize_points=False + depth = 2 * torch.tensor( + [[[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]]]], device=device, dtype=dtype + ) + + camera_matrix = torch.tensor([[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + + points3d_expected = torch.tensor( + [ + [ + [[0.0, 2.0, 4.0], [0.0, 2.0, 4.0], [0.0, 2.0, 4.0], [0.0, 2.0, 4.0]], + [[0.0, 0.0, 0.0], [2.0, 2.0, 2.0], [4.0, 4.0, 4.0], [6.0, 6.0, 6.0]], + [[2.0, 2.0, 2.0], [2.0, 2.0, 2.0], [2.0, 2.0, 2.0], [2.0, 2.0, 2.0]], + ] + ], + device=device, + dtype=dtype, + ) + + points3d = kornia.geometry.depth.depth_to_3d(depth, camera_matrix) # default is normalize_points=False + self.assert_close(points3d, points3d_expected, atol=1e-4, rtol=1e-4) + + def test_unproject_normalized(self, device, dtype): + # this is for normalize_points=True + depth = 2 * torch.tensor( + [[[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]]]], device=device, dtype=dtype + ) + + camera_matrix = torch.tensor([[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + + points3d_expected = torch.tensor( + [ + [ + [ + [0.0000, 1.4142, 1.7889], + [0.0000, 1.1547, 1.6330], + [0.0000, 0.8165, 1.3333], + [0.0000, 0.6030, 1.0690], + ], + [ + [0.0000, 0.0000, 0.0000], + [1.4142, 1.1547, 0.8165], + [1.7889, 1.6330, 1.3333], + [1.8974, 1.8091, 1.6036], + ], + [ + [2.0000, 1.4142, 0.8944], + [1.4142, 1.1547, 0.8165], + [0.8944, 0.8165, 0.6667], + [0.6325, 0.6030, 0.5345], + ], + ] + ], + device=device, + dtype=dtype, + ) + + points3d = kornia.geometry.depth.depth_to_3d(depth, camera_matrix, normalize_points=True) + self.assert_close(points3d, points3d_expected, atol=1e-4, rtol=1e-4) + + def test_unproject_and_project(self, device, dtype): + depth = 2 * torch.tensor( + [[[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]]]], device=device, dtype=dtype + ) + + camera_matrix = torch.tensor([[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + + points3d = kornia.geometry.depth.depth_to_3d(depth, camera_matrix) + points2d = kornia.geometry.camera.project_points(points3d.permute(0, 2, 3, 1), camera_matrix[:, None, None]) + points2d_expected = kornia.geometry.create_meshgrid(4, 3, False, device=device).to(dtype=dtype) + self.assert_close(points2d, points2d_expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + # generate input data + depth = torch.rand(1, 1, 3, 4, device=device, dtype=torch.float64) + + camera_matrix = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + + # evaluate function gradient + self.gradcheck(kornia.geometry.depth.depth_to_3d, (depth, camera_matrix)) + + +class TestDepthToNormals(BaseTester): + def test_smoke(self, device, dtype): + depth = torch.rand(1, 1, 3, 4, device=device, dtype=dtype) + camera_matrix = torch.rand(1, 3, 3, device=device, dtype=dtype) + + points3d = kornia.geometry.depth.depth_to_normals(depth, camera_matrix) + assert points3d.shape == (1, 3, 3, 4) + + @pytest.mark.parametrize("batch_size", [2, 4, 5]) + def test_shapes(self, batch_size, device, dtype): + depth = torch.rand(batch_size, 1, 3, 4, device=device, dtype=dtype) + camera_matrix = torch.rand(batch_size, 3, 3, device=device, dtype=dtype) + + points3d = kornia.geometry.depth.depth_to_normals(depth, camera_matrix) + assert points3d.shape == (batch_size, 3, 3, 4) + + @pytest.mark.parametrize("batch_size", [2, 4, 5]) + def test_shapes_broadcast(self, batch_size, device, dtype): + depth = torch.rand(batch_size, 1, 3, 4, device=device, dtype=dtype) + camera_matrix = torch.rand(1, 3, 3, device=device, dtype=dtype) + + points3d = kornia.geometry.depth.depth_to_normals(depth, camera_matrix) + assert points3d.shape == (batch_size, 3, 3, 4) + + def test_simple(self, device, dtype): + # this is for default normalize_points=False + depth = 2 * torch.tensor( + [[[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]]]], device=device, dtype=dtype + ) + + camera_matrix = torch.tensor([[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + + normals_expected = torch.tensor( + [ + [ + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]], + ] + ], + device=device, + dtype=dtype, + ) + + normals = kornia.geometry.depth.depth_to_normals(depth, camera_matrix) # default is normalize_points=False + self.assert_close(normals, normals_expected, rtol=1e-3, atol=1e-3) + + def test_simple_normalized(self, device, dtype): + # this is for default normalize_points=False + depth = 2 * torch.tensor( + [[[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]]]], device=device, dtype=dtype + ) + + camera_matrix = torch.tensor([[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + + normals_expected = torch.tensor( + [ + [ + [ + [0.3432, 0.4861, 0.7628], + [0.2873, 0.4260, 0.6672], + [0.2284, 0.3683, 0.5596], + [0.1695, 0.2980, 0.4496], + ], + [ + [0.3432, 0.2873, 0.2363], + [0.4861, 0.4260, 0.3785], + [0.8079, 0.7261, 0.6529], + [0.8948, 0.8237, 0.7543], + ], + [ + [0.8743, 0.8253, 0.6019], + [0.8253, 0.7981, 0.6415], + [0.5432, 0.5807, 0.5105], + [0.4129, 0.4824, 0.4784], + ], + ] + ], + device=device, + dtype=dtype, + ) + + normals = kornia.geometry.depth.depth_to_normals(depth, camera_matrix, normalize_points=True) + self.assert_close(normals, normals_expected, rtol=1e-3, atol=1e-3) + + def test_gradcheck(self, device): + # generate input data + depth = torch.rand(1, 1, 3, 4, device=device, dtype=torch.float64) + + camera_matrix = torch.rand(1, 3, 3, device=device, dtype=torch.float64) + + # evaluate function gradient + self.gradcheck(kornia.geometry.depth.depth_to_normals, (depth, camera_matrix)) + + +class TestWarpFrameDepth(BaseTester): + def test_smoke(self, device, dtype): + image_src = torch.rand(1, 3, 3, 4, device=device, dtype=dtype) + depth_dst = torch.rand(1, 1, 3, 4, device=device, dtype=dtype) + src_trans_dst = torch.rand(1, 4, 4, device=device, dtype=dtype) + camera_matrix = torch.rand(1, 3, 3, device=device, dtype=dtype) + + image_dst = kornia.geometry.depth.warp_frame_depth(image_src, depth_dst, src_trans_dst, camera_matrix) + assert image_dst.shape == (1, 3, 3, 4) + + @pytest.mark.parametrize("batch_size", [2, 4, 5]) + @pytest.mark.parametrize("num_features", [1, 3, 5]) + def test_shape(self, batch_size, num_features, device, dtype): + image_src = torch.rand(batch_size, num_features, 3, 4, device=device, dtype=dtype) + depth_dst = torch.rand(batch_size, 1, 3, 4, device=device, dtype=dtype) + src_trans_dst = torch.rand(batch_size, 4, 4, device=device, dtype=dtype) + camera_matrix = torch.rand(batch_size, 3, 3, device=device, dtype=dtype) + + image_dst = kornia.geometry.depth.warp_frame_depth(image_src, depth_dst, src_trans_dst, camera_matrix) + assert image_dst.shape == (batch_size, num_features, 3, 4) + + def test_translation(self, device, dtype): + # this is for normalize_points=False + image_src = torch.tensor( + [[[[1.0, 2.0, 3.0], [1.0, 2.0, 3.0], [1.0, 2.0, 3.0], [1.0, 2.0, 3.0]]]], device=device, dtype=dtype + ) + + depth_dst = torch.tensor( + [[[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]]]], device=device, dtype=dtype + ) + + src_trans_dst = torch.tensor( + [[[1.0, 0.0, 0.0, 1.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) + + h, w = image_src.shape[-2:] + camera_matrix = torch.tensor( + [[[1.0, 0.0, w / 2], [0.0, 1.0, h / 2], [0.0, 0.0, 1.0]]], device=device, dtype=dtype + ) + + image_dst_expected = torch.tensor( + [[[[2.0, 3.0, 0.0], [2.0, 3.0, 0.0], [2.0, 3.0, 0.0], [2.0, 3.0, 0.0]]]], device=device, dtype=dtype + ) + + image_dst = kornia.geometry.depth.warp_frame_depth( + image_src, depth_dst, src_trans_dst, camera_matrix + ) # default is normalize_points=False + self.assert_close(image_dst, image_dst_expected, rtol=1e-3, atol=1e-3) + + def test_translation_normalized(self, device, dtype): + # this is for normalize_points=True + image_src = torch.tensor( + [[[[1.0, 2.0, 3.0], [1.0, 2.0, 3.0], [1.0, 2.0, 3.0], [1.0, 2.0, 3.0]]]], device=device, dtype=dtype + ) + + depth_dst = torch.tensor( + [[[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]]]], device=device, dtype=dtype + ) + + src_trans_dst = torch.tensor( + [[[1.0, 0.0, 0.0, 1.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) + + h, w = image_src.shape[-2:] + camera_matrix = torch.tensor( + [[[1.0, 0.0, w / 2], [0.0, 1.0, h / 2], [0.0, 0.0, 1.0]]], device=device, dtype=dtype + ) + + image_dst_expected = torch.tensor( + [ + [ + [ + [0.9223, 0.0000, 0.0000], + [2.8153, 1.5000, 0.0000], + [2.8028, 2.6459, 0.0000], + [2.8153, 1.5000, 0.0000], + ] + ] + ], + device=device, + dtype=dtype, + ) + + image_dst = kornia.geometry.depth.warp_frame_depth( + image_src, depth_dst, src_trans_dst, camera_matrix, normalize_points=True + ) + self.assert_close(image_dst, image_dst_expected, rtol=1e-3, atol=1e-3) + + def test_gradcheck(self, device): + dtype = torch.float64 + image_src = torch.rand(1, 3, 3, 4, device=device, dtype=dtype) + + depth_dst = torch.rand(1, 1, 3, 4, device=device, dtype=dtype) + + src_trans_dst = torch.rand(1, 4, 4, device=device, dtype=dtype) + + camera_matrix = torch.rand(1, 3, 3, device=device, dtype=dtype) + + # evaluate function gradient + self.gradcheck(kornia.geometry.depth.warp_frame_depth, (image_src, depth_dst, src_trans_dst, camera_matrix)) + + +class TestDepthFromDisparity(BaseTester): + def test_smoke(self, device, dtype): + disparity = 2 * torch.tensor( + [[[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]]]], device=device, dtype=dtype + ) + + baseline = torch.tensor([1.0], device=device, dtype=dtype) + focal = torch.tensor([1.0], device=device, dtype=dtype) + + depth_expected = torch.tensor( + [ + [ + [ + [0.5000, 0.5000, 0.5000], + [0.5000, 0.5000, 0.5000], + [0.5000, 0.5000, 0.5000], + [0.5000, 0.5000, 0.5000], + ] + ] + ], + device=device, + dtype=dtype, + ) + + depth = kornia.geometry.depth.depth_from_disparity(disparity, baseline, focal) + self.assert_close(depth, depth_expected, rtol=1e-3, atol=1e-3) + + @pytest.mark.parametrize("batch_size", [2, 4, 5]) + def test_cardinality(self, batch_size, device, dtype): + disparity = torch.rand(batch_size, 1, 3, 4, device=device, dtype=dtype) + baseline = torch.rand(1, device=device, dtype=dtype) + focal = torch.rand(1, device=device, dtype=dtype) + + points3d = kornia.geometry.depth.depth_from_disparity(disparity, baseline, focal) + assert points3d.shape == (batch_size, 1, 3, 4) + + @pytest.mark.parametrize("shape", [(1, 1, 3, 4), (4, 1, 3, 4), (4, 3, 4), (1, 3, 4), (3, 4)]) + def test_shapes(self, shape, device, dtype): + disparity = torch.randn(shape, device=device, dtype=dtype) + baseline = torch.rand(1, device=device, dtype=dtype) + focal = torch.rand(1, device=device, dtype=dtype) + + points3d = kornia.geometry.depth.depth_from_disparity(disparity, baseline, focal) + assert points3d.shape == shape + + def test_gradcheck(self, device): + # generate input data + disparity = torch.rand(1, 1, 3, 4, device=device, dtype=torch.float64) + + baseline = torch.rand(1, device=device, dtype=torch.float64) + + focal = torch.rand(1, device=device, dtype=torch.float64) + + # evaluate function gradient + self.gradcheck(kornia.geometry.depth.depth_from_disparity, (disparity, baseline, focal)) + + +class TestDepthFromPlaneEquation(BaseTester): + def test_smoke(self, device, dtype): + B = 2 + N = 10 + plane_normals = torch.randn(B, 3, device=device, dtype=dtype) + plane_offsets = torch.randn(B, 1, device=device, dtype=dtype) + points_uv = torch.randn(B, N, 2, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).unsqueeze(0).repeat(B, 1, 1) + + depth = kornia.geometry.depth.depth_from_plane_equation(plane_normals, plane_offsets, points_uv, camera_matrix) + assert depth.shape == (B, N), f"Expected depth shape to be ({B}, {N}), but got {depth.shape}" + + @pytest.mark.parametrize("batch_size", [1, 2, 4]) + def test_shapes(self, batch_size, device, dtype): + B = batch_size + N = 10 + plane_normals = torch.randn(B, 3, device=device, dtype=dtype) + plane_offsets = torch.randn(B, 1, device=device, dtype=dtype) + points_uv = torch.randn(B, N, 2, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype).unsqueeze(0).repeat(B, 1, 1) + + depth = kornia.geometry.depth.depth_from_plane_equation(plane_normals, plane_offsets, points_uv, camera_matrix) + assert depth.shape == (B, N), f"Expected depth shape to be ({B}, {N}), but got {depth.shape}" + + @pytest.mark.parametrize("batch_size", [1, 2, 4]) + def test_shapes_broadcast(self, batch_size, device, dtype): + B = batch_size + N = 10 + plane_normals = torch.randn(1, 3, device=device, dtype=dtype) # Broadcasting plane normals + plane_offsets = torch.randn(1, 1, device=device, dtype=dtype) + points_uv = torch.randn(B, N, 2, device=device, dtype=dtype) + camera_matrix = torch.eye(3, device=device, dtype=dtype) + + depth = kornia.geometry.depth.depth_from_plane_equation( + plane_normals.expand(B, -1), plane_offsets.expand(B, -1), points_uv, camera_matrix.expand(B, -1, -1) + ) + assert depth.shape == (B, N), f"Expected depth shape to be ({B}, {N}), but got {depth.shape}" + + def test_simple(self, device, dtype): + """Test the function with a simple plane equation to verify numerical correctness. + + Plane equation: z = 2 (plane normal [0, 0, 1], offset 2) + Expected depth for any point is 2. + """ + # Define plane parameters + plane_normals = torch.tensor([[0.0, 0.0, 1.0]], device=device, dtype=dtype) # Shape: (B, 3) + plane_offsets = torch.tensor([[2.0]], device=device, dtype=dtype) + + # Define pixel coordinates + points_uv = torch.tensor( + [[[0.0, 0.0], [1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]], + device=device, + dtype=dtype, + ) # Shape: (B, N, 2) + + # Camera intrinsic matrix (identity) + camera_matrix = torch.eye(3, device=device, dtype=dtype).unsqueeze(0) # Shape: (B, 3, 3) + + # Expected depth values + depth_expected = torch.tensor([[2.0, 2.0, 2.0, 2.0]], device=device, dtype=dtype) # Shape: (B, N) + + # Compute depth + depth = kornia.geometry.depth.depth_from_plane_equation(plane_normals, plane_offsets, points_uv, camera_matrix) + + # Assert that the computed depth matches the expected depth + self.assert_close(depth, depth_expected, rtol=1e-6, atol=1e-6) + + def test_gradcheck(self, device): + B = 2 + N = 5 + plane_normals = torch.rand(B, 3, device=device, dtype=torch.float64, requires_grad=True) + plane_offsets = torch.rand(B, 1, device=device, dtype=torch.float64, requires_grad=True) + points_uv = torch.rand(B, N, 2, device=device, dtype=torch.float64, requires_grad=True) + camera_matrix = torch.eye(3, device=device, dtype=torch.float64).unsqueeze(0).repeat(B, 1, 1) + camera_matrix.requires_grad_() + + # Perform gradient check + self.gradcheck( + kornia.geometry.depth.depth_from_plane_equation, + (plane_normals, plane_offsets, points_uv, camera_matrix), + eps=1e-6, + atol=1e-4, + ) diff --git a/tests/geometry/test_depth_warper.py b/tests/geometry/test_depth_warper.py new file mode 100644 index 0000000..c1c6b23 --- /dev/null +++ b/tests/geometry/test_depth_warper.py @@ -0,0 +1,192 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia +from kornia.geometry.conversions import normalize_pixel_coordinates + +from testing.base import BaseTester + + +class TestDepthWarper(BaseTester): + eps = 1e-6 + + def _create_pinhole_pair(self, batch_size, device, dtype): + # prepare data + fx, fy = 1.0, 1.0 + height, width = 3, 5 + cx, cy = width / 2, height / 2 + tx, ty, tz = 0, 0, 0 + + # create pinhole cameras + pinhole_src = kornia.geometry.camera.PinholeCamera.from_parameters( + fx, fy, cx, cy, height, width, tx, ty, tz, batch_size, device=device, dtype=dtype + ) + pinhole_dst = kornia.geometry.camera.PinholeCamera.from_parameters( + fx, fy, cx, cy, height, width, tx, ty, tz, batch_size, device=device, dtype=dtype + ) + return pinhole_src, pinhole_dst + + @pytest.mark.parametrize("batch_size", (1, 2)) + def test_compute_projection_matrix(self, batch_size, device, dtype): + height, width = 3, 5 # output shape + pinhole_src, pinhole_dst = self._create_pinhole_pair(batch_size, device, dtype) + pinhole_dst.tx += 1.0 # apply offset to tx + + # create warper + warper = kornia.geometry.depth.DepthWarper(pinhole_dst, height, width) + assert warper._dst_proj_src is None + + # initialize projection matrices + warper.compute_projection_matrix(pinhole_src) + assert warper._dst_proj_src is not None + + # retrieve computed projection matrix and compare to expected + dst_proj_src = warper._dst_proj_src + dst_proj_src_expected = torch.eye(4, device=device, dtype=dtype)[None].repeat(batch_size, 1, 1) # Bx4x4 + dst_proj_src_expected[..., 0, -2] += pinhole_src.cx + dst_proj_src_expected[..., 1, -2] += pinhole_src.cy + dst_proj_src_expected[..., 0, -1] += 1.0 # offset to x-axis + self.assert_close(dst_proj_src, dst_proj_src_expected) + + @pytest.mark.parametrize("batch_size", (1, 2)) + def test_warp_grid_offset_x1_depth1(self, batch_size, device, dtype): + height, width = 3, 5 # output shape + pinhole_src, pinhole_dst = self._create_pinhole_pair(batch_size, device, dtype) + pinhole_dst.tx += 1.0 # apply offset to tx + + # initialize depth to one + depth_src = torch.ones(batch_size, 1, height, width, device=device, dtype=dtype) + + # create warper, initialize projection matrices and warp grid + warper = kornia.geometry.depth.DepthWarper(pinhole_dst, height, width) + warper.compute_projection_matrix(pinhole_src) + + grid_warped = warper.warp_grid(depth_src) + assert grid_warped.shape == (batch_size, height, width, 2) + + # normalize base meshgrid + grid = warper.grid[..., :2].to(device=device, dtype=dtype) + grid_norm = normalize_pixel_coordinates(grid, height, width) + + # check offset in x-axis + self.assert_close(grid_warped[..., -2, 0], grid_norm[..., -1, 0].repeat(batch_size, 1), atol=1e-4, rtol=1e-4) + # check that y-axis remain the same + self.assert_close(grid_warped[..., -1, 1], grid_norm[..., -1, 1].repeat(batch_size, 1), rtol=1e-4, atol=1e-4) + + @pytest.mark.parametrize("batch_size", (1, 2)) + def test_warp_grid_offset_x1y1_depth1(self, batch_size, device, dtype): + height, width = 3, 5 # output shape + pinhole_src, pinhole_dst = self._create_pinhole_pair(batch_size, device, dtype) + pinhole_dst.tx += 1.0 # apply offset to tx + pinhole_dst.ty += 1.0 # apply offset to ty + + # initialize depth to one + depth_src = torch.ones(batch_size, 1, height, width, device=device, dtype=dtype) + + # create warper, initialize projection matrices and warp grid + warper = kornia.geometry.depth.DepthWarper(pinhole_dst, height, width) + warper.compute_projection_matrix(pinhole_src) + + grid_warped = warper.warp_grid(depth_src) + assert grid_warped.shape == (batch_size, height, width, 2) + + # normalize base meshgrid + grid = warper.grid[..., :2].to(device=device, dtype=dtype) + grid_norm = normalize_pixel_coordinates(grid, height, width) + + # check offset in x-axis + self.assert_close(grid_warped[..., -2, 0], grid_norm[..., -1, 0].repeat(batch_size, 1), atol=1e-4, rtol=1e-4) + # check that y-axis remain the same + self.assert_close( + grid_warped[..., -2, :, 1], grid_norm[..., -1, :, 1].repeat(batch_size, 1), rtol=1e-4, atol=1e-4 + ) + + @pytest.mark.parametrize("batch_size", (1, 2)) + def test_warp_tensor_offset_x1y1(self, batch_size, device, dtype): + channels, height, width = 3, 3, 5 # output shape + pinhole_src, pinhole_dst = self._create_pinhole_pair(batch_size, device, dtype) + pinhole_dst.tx += 1.0 # apply offset to tx + pinhole_dst.ty += 1.0 # apply offset to ty + + # initialize depth to one + depth_src = torch.ones(batch_size, 1, height, width, device=device, dtype=dtype) + + # create warper, initialize projection matrices and warp grid + warper = kornia.geometry.depth.DepthWarper(pinhole_dst, height, width) + warper.compute_projection_matrix(pinhole_src) + + # create patch to warp + patch_dst = ( + torch.arange(float(height * width), device=device, dtype=dtype) + .view(1, 1, height, width) + .expand(batch_size, channels, -1, -1) + ) + + # warpd source patch by depth + patch_src = warper(depth_src, patch_dst) + + # compare patches + self.assert_close(patch_dst[..., 1:, 1:], patch_src[..., :2, :4], atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_size", (1, 2)) + def test_compute_projection(self, batch_size, device, dtype): + height, width = 3, 5 # output shape + pinhole_src, pinhole_dst = self._create_pinhole_pair(batch_size, device, dtype) + + # create warper, initialize projection matrices and warp grid + warper = kornia.geometry.depth.DepthWarper(pinhole_dst, height, width) + warper.compute_projection_matrix(pinhole_src) + + # test compute_projection + xy_projected = warper._compute_projection(0.0, 0.0, 1.0) + assert xy_projected.shape == (batch_size, 2) + + @pytest.mark.parametrize("batch_size", (1, 2)) + def test_compute_subpixel_step(self, batch_size, device, dtype): + height, width = 3, 5 # output shape + pinhole_src, pinhole_dst = self._create_pinhole_pair(batch_size, device, dtype) + + # create warper, initialize projection matrices and warp grid + warper = kornia.geometry.depth.DepthWarper(pinhole_dst, height, width) + warper.compute_projection_matrix(pinhole_src) + + # test compute_subpixel_step + subpixel_step = warper.compute_subpixel_step() + self.assert_close(subpixel_step.item(), 0.1715, rtol=1e-3, atol=1e-3) + + @pytest.mark.parametrize("batch_size", (1, 2)) + def test_gradcheck(self, batch_size, device): + dtype = torch.float64 + # prepare data + channels, height, width = 3, 3, 5 # output shape + pinhole_src, pinhole_dst = self._create_pinhole_pair(batch_size, device, dtype) + + # initialize depth to one + depth_src = torch.ones(batch_size, 1, height, width, device=device, dtype=dtype) + + # create patch to warp + img_dst = torch.ones(batch_size, channels, height, width, device=device, dtype=dtype) + + # evaluate function gradient + self.gradcheck(kornia.geometry.depth.depth_warp, (pinhole_dst, pinhole_src, depth_src, img_dst, height, width)) + + # TODO(edgar): we should include a test showing some kind of occlusion + # def test_warp_with_occlusion(self): + # pass diff --git a/tests/geometry/test_grid.py b/tests/geometry/test_grid.py new file mode 100644 index 0000000..fb1ad86 --- /dev/null +++ b/tests/geometry/test_grid.py @@ -0,0 +1,87 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import assert_close + + +def test_create_meshgrid(device, dtype): + height, width = 4, 6 + normalized_coordinates = False + + # create the meshgrid and verify shape + grid = kornia.geometry.create_meshgrid(height, width, normalized_coordinates, device=device, dtype=dtype) + + assert grid.device == device + assert grid.dtype == dtype + assert grid.shape == (1, height, width, 2) + + # check grid corner values + assert tuple(grid[0, 0, 0].cpu().numpy()) == (0.0, 0.0) + assert tuple(grid[0, height - 1, width - 1].cpu().numpy()) == (width - 1, height - 1) + + +def test_normalize_pixel_grid(device, dtype): + if device.type == "cuda" and dtype == torch.float16: + pytest.skip('"inverse_cuda" not implemented for "Half"') + + # generate input data + height, width = 2, 4 + + # create points grid + grid_norm = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=True, device=device, dtype=dtype) + + assert grid_norm.device == device + assert grid_norm.dtype == dtype + grid_norm = torch.unsqueeze(grid_norm, dim=0) + + grid_pix = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=False, device=device, dtype=dtype) + + assert grid_pix.device == device + assert grid_pix.dtype == dtype + grid_pix = torch.unsqueeze(grid_pix, dim=0) + + # grid from pixel space to normalized + norm_trans_pix = kornia.geometry.conversions.normal_transform_pixel( + height, width, device=device, dtype=dtype + ) # 1x3x3 + pix_trans_norm = torch.inverse(norm_trans_pix) # 1x3x3 + # transform grids + grid_pix_to_norm = kornia.geometry.linalg.transform_points(norm_trans_pix, grid_pix) + grid_norm_to_pix = kornia.geometry.linalg.transform_points(pix_trans_norm, grid_norm) + assert_close(grid_pix, grid_norm_to_pix) + assert_close(grid_norm, grid_pix_to_norm) + + +def test_create_meshgrid3d(device, dtype): + depth, height, width = 5, 4, 6 + normalized_coordinates = False + + # create the meshgrid and verify shape + grid = kornia.geometry.create_meshgrid3d(depth, height, width, normalized_coordinates, device=device, dtype=dtype) + + assert grid.device == device + assert grid.dtype == dtype + assert grid.shape == (1, depth, height, width, 3) + + # check grid corner values + assert tuple(grid[0, 0, 0, 0].cpu().numpy()) == (0.0, 0.0, 0.0) + assert tuple(grid[0, depth - 1, height - 1, width - 1].cpu().numpy()) == (depth - 1, width - 1, height - 1) diff --git a/tests/geometry/test_homography.py b/tests/geometry/test_homography.py new file mode 100644 index 0000000..5935666 --- /dev/null +++ b/tests/geometry/test_homography.py @@ -0,0 +1,494 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys + +import pytest +import torch + +import kornia +from kornia.core._compat import torch_version_le +from kornia.geometry.homography import ( + find_homography_dlt, + find_homography_dlt_iterated, + find_homography_lines_dlt, + find_homography_lines_dlt_iterated, + line_segment_transfer_error_one_way, + oneway_transfer_error, + sample_is_valid_for_homography, + symmetric_transfer_error, +) + +from testing.base import BaseTester +from testing.geometry.create import create_random_homography + + +class TestSampleValidation(BaseTester): + def test_good(self, device, dtype): + pts1 = torch.tensor([[0.0, 0.0], [0.0, 1.0], [1.0, 1.0], [1.0, 0.0]], device=device, dtype=dtype)[None] + mask = sample_is_valid_for_homography(pts1, pts1) + expected = torch.tensor([True], device=device, dtype=torch.bool) + assert torch.equal(mask, expected) + + def test_bad(self, device, dtype): + pts1 = torch.tensor([[0.0, 0.0], [1.0, 0.0], [0.0, 1.0], [1.0, 1.0]], device=device, dtype=dtype)[None] + + pts2 = torch.tensor([[0.0, 0.0], [0.0, 1.0], [1.0, 0.0], [1.0, 1.0]], device=device, dtype=dtype)[None] + mask = sample_is_valid_for_homography(pts1, pts2) + expected = torch.tensor([False], device=device, dtype=torch.bool) + assert torch.equal(mask, expected) + + def test_batch(self, device, dtype): + batch_size = 5 + pts1 = torch.rand(batch_size, 4, 2, device=device, dtype=dtype) + pts2 = torch.rand(batch_size, 4, 2, device=device, dtype=dtype) + mask = sample_is_valid_for_homography(pts1, pts2) + assert mask.shape == torch.Size([batch_size]) + + +class TestOneWayError(BaseTester): + def test_smoke(self, device, dtype): + pts1 = torch.rand(1, 6, 2, device=device, dtype=dtype) + pts2 = torch.rand(1, 6, 2, device=device, dtype=dtype) + H = create_random_homography(pts1, 3) + assert oneway_transfer_error(pts1, pts2, H).shape == (1, 6) + + def test_batch(self, device, dtype): + batch_size = 5 + pts1 = torch.rand(batch_size, 3, 2, device=device, dtype=dtype) + pts2 = torch.rand(batch_size, 3, 2, device=device, dtype=dtype) + H = create_random_homography(pts1, 3) + assert oneway_transfer_error(pts1, pts2, H).shape == (batch_size, 3) + + def test_gradcheck(self, device): + # generate input data + batch_size, num_points, num_dims = 2, 3, 2 + points1 = torch.rand(batch_size, num_points, num_dims, device=device, dtype=torch.float64, requires_grad=True) + points2 = torch.rand(batch_size, num_points, num_dims, device=device, dtype=torch.float64) + H = create_random_homography(points1, 3) + self.gradcheck(oneway_transfer_error, (points1, points2, H)) + + def test_shift(self, device, dtype): + pts1 = torch.zeros(3, 2, device=device, dtype=dtype)[None] + pts2 = torch.tensor([[1.0, 0.0], [2.0, 0.0], [2.0, 2.0]], device=device, dtype=dtype)[None] + H = torch.tensor([[1.0, 0.0, 1.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], dtype=dtype, device=device)[None] + expected = torch.tensor([0.0, 1.0, 5.0], device=device, dtype=dtype)[None] + self.assert_close(oneway_transfer_error(pts1, pts2, H), expected, atol=1e-4, rtol=1e-4) + + +class TestLineSegmentOneWayError(BaseTester): + def test_smoke(self, device, dtype): + ls1 = torch.rand(1, 6, 2, 2, device=device, dtype=dtype) + ls2 = torch.rand(1, 6, 2, 2, device=device, dtype=dtype) + H = create_random_homography(ls1, 3) + assert line_segment_transfer_error_one_way(ls1, ls2, H).shape == (1, 6) + + def test_batch(self, device, dtype): + batch_size = 5 + ls1 = torch.rand(batch_size, 3, 2, 2, device=device, dtype=dtype) + ls2 = torch.rand(batch_size, 3, 2, 2, device=device, dtype=dtype) + H = create_random_homography(ls1, 3) + assert line_segment_transfer_error_one_way(ls1, ls2, H).shape == (batch_size, 3) + + def test_gradcheck(self, device): + # generate input data + batch_size, num_points, num_dims = 2, 3, 2 + ls1 = torch.rand(batch_size, num_points, num_dims, 2, device=device, dtype=torch.float64, requires_grad=True) + ls2 = torch.rand(batch_size, num_points, num_dims, 2, device=device, dtype=torch.float64) + H = create_random_homography(ls1, 3) + self.gradcheck(line_segment_transfer_error_one_way, (ls1, ls2, H)) + + def test_shift(self, device, dtype): + pts1 = torch.zeros(3, 2, device=device, dtype=dtype)[None] + pts1_end = torch.ones(3, 2, device=device, dtype=dtype)[None] + ls1 = torch.stack([pts1, pts1_end], dim=2) + + pts2 = torch.tensor([[1.0, 0.0], [2.0, 0.0], [2.0, 2.0]], device=device, dtype=dtype)[None] + pts2_end = pts2 + torch.ones(3, 2, device=device, dtype=dtype)[None] + ls2 = torch.stack([pts2, pts2_end], dim=2) + H = torch.tensor([[1.0, 0.0, 1.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], dtype=dtype, device=device)[None] + expected = torch.tensor([0.0, 1.0, 1.0], device=device, dtype=dtype)[None] + self.assert_close(line_segment_transfer_error_one_way(ls1, ls2, H), expected, atol=1e-4, rtol=1e-4) + + +class TestSymmetricTransferError(BaseTester): + def test_smoke(self, device, dtype): + pts1 = torch.rand(1, 6, 2, device=device, dtype=dtype) + pts2 = torch.rand(1, 6, 2, device=device, dtype=dtype) + H = create_random_homography(pts1, 3) + assert symmetric_transfer_error(pts1, pts2, H).shape == (1, 6) + + def test_batch(self, device, dtype): + batch_size = 5 + pts1 = torch.rand(batch_size, 3, 2, device=device, dtype=dtype) + pts2 = torch.rand(batch_size, 3, 2, device=device, dtype=dtype) + H = create_random_homography(pts1, 3) + assert symmetric_transfer_error(pts1, pts2, H).shape == (batch_size, 3) + + def test_gradcheck(self, device): + # generate input data + batch_size, num_points, num_dims = 2, 3, 2 + points1 = torch.rand(batch_size, num_points, num_dims, device=device, dtype=torch.float64, requires_grad=True) + points2 = torch.rand(batch_size, num_points, num_dims, device=device, dtype=torch.float64) + H = create_random_homography(points1, 3) + self.gradcheck(symmetric_transfer_error, (points1, points2, H)) + + def test_shift(self, device, dtype): + pts1 = torch.zeros(3, 2, device=device, dtype=dtype)[None] + pts2 = torch.tensor([[1.0, 0.0], [2.0, 0.0], [2.0, 2.0]], device=device, dtype=dtype)[None] + H = torch.tensor([[1.0, 0.0, 1.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], dtype=dtype, device=device)[None] + expected = torch.tensor([0.0, 2.0, 10.0], device=device, dtype=dtype)[None] + self.assert_close(symmetric_transfer_error(pts1, pts2, H), expected, atol=1e-4, rtol=1e-4) + + +class TestFindHomographyDLT(BaseTester): + def test_smoke(self, device, dtype): + points1 = torch.rand(1, 4, 2, device=device, dtype=dtype) + points2 = torch.rand(1, 4, 2, device=device, dtype=dtype) + weights = torch.ones(1, 4, device=device, dtype=dtype) + H = find_homography_dlt(points1, points2, weights) + assert H.shape == (1, 3, 3) + + def test_nocrash(self, device, dtype): + points1 = torch.rand(1, 4, 2, device=device, dtype=dtype) + points2 = torch.rand(1, 4, 2, device=device, dtype=dtype) + weights = torch.ones(1, 4, device=device, dtype=dtype) + points1[0, 0, 0] = float("nan") + H = find_homography_dlt(points1, points2, weights) + assert H.shape == (1, 3, 3) + + def test_nocrash_lu(self, device, dtype): + points1 = torch.rand(1, 4, 2, device=device, dtype=dtype) + points2 = torch.rand(1, 4, 2, device=device, dtype=dtype) + weights = torch.ones(1, 4, device=device, dtype=dtype) + points1[0, 0, 0] = float("nan") + H = find_homography_dlt(points1, points2, weights, "lu") + assert H.shape == (1, 3, 3) + + @pytest.mark.parametrize("batch_size, num_points", [(1, 4), (2, 5), (3, 6)]) + def test_shape(self, batch_size, num_points, device, dtype): + B, N = batch_size, num_points + points1 = torch.rand(B, N, 2, device=device, dtype=dtype) + points2 = torch.rand(B, N, 2, device=device, dtype=dtype) + weights = torch.ones(B, N, device=device, dtype=dtype) + H = find_homography_dlt(points1, points2, weights) + assert H.shape == (B, 3, 3) + + @pytest.mark.parametrize("batch_size, num_points", [(1, 4), (2, 5), (3, 6)]) + def test_shape_noweights(self, batch_size, num_points, device, dtype): + B, N = batch_size, num_points + points1 = torch.rand(B, N, 2, device=device, dtype=dtype) + points2 = torch.rand(B, N, 2, device=device, dtype=dtype) + H = find_homography_dlt(points1, points2, None) + assert H.shape == (B, 3, 3) + + @pytest.mark.parametrize("batch_size, num_points", [(1, 4), (2, 5), (3, 6)]) + def test_points_noweights(self, batch_size, num_points, device, dtype): + B, N = batch_size, num_points + points1 = torch.rand(B, N, 2, device=device, dtype=dtype) + points2 = torch.rand(B, N, 2, device=device, dtype=dtype) + weights = torch.ones(B, N, device=device, dtype=dtype) + H_noweights = find_homography_dlt(points1, points2, None) + H_withweights = find_homography_dlt(points1, points2, weights) + assert H_noweights.shape == (B, 3, 3) + assert H_withweights.shape == (B, 3, 3) + self.assert_close(H_noweights, H_withweights, rtol=1e-3, atol=1e-4) + + def test_scaled_fixed_points(self, device, dtype): + points1 = torch.tensor([[[0.0, 0.0], [0.0, 1.0], [1.0, 0.0], [1.0, 1.0]]], device=device, dtype=dtype) + points2 = points1 * 100 + H = find_homography_dlt(points1, points2, None, "lu") + assert not torch.isnan(H).any() + + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_clean_points_svd(self, batch_size, device, dtype): + # generate input data + points_src = torch.rand(batch_size, 10, 2, device=device, dtype=dtype) + H = kornia.core.ops.eye_like(3, points_src) + H = H * 0.3 * torch.rand_like(H) + H = H / H[:, 2:3, 2:3] + + points_dst = kornia.geometry.transform_points(H, points_src) + weights = torch.ones(batch_size, 10, device=device, dtype=dtype) + + # compute transform from source to target + dst_homo_src = find_homography_dlt(points_src, points_dst, weights, "svd") + rtol = 1e-3 + atol = 1e-4 + if dtype not in (torch.float32, torch.float64): + rtol = 3e-3 + atol = 1e-3 + elif device.type == "cuda" and dtype == torch.float32: + atol = 2e-3 + self.assert_close(kornia.geometry.transform_points(dst_homo_src, points_src), points_dst, rtol=rtol, atol=atol) + + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_clean_points_lu(self, batch_size, device, dtype): + # generate input data + points_src = torch.rand(batch_size, 10, 2, device=device, dtype=dtype) + H = kornia.core.ops.eye_like(3, points_src) + H = H * 0.3 * torch.rand_like(H) + H = H / H[:, 2:3, 2:3] + + points_dst = kornia.geometry.transform_points(H, points_src) + weights = torch.ones(batch_size, 10, device=device, dtype=dtype) + + # compute transform from source to target + dst_homo_src = find_homography_dlt(points_src, points_dst, weights, "lu") + rtol = 1e-3 + atol = 1e-4 + if dtype not in (torch.float32, torch.float64): + rtol = 3e-3 + atol = 1e-3 + elif device.type == "cuda" and dtype == torch.float32: + atol = 2e-3 + self.assert_close(kornia.geometry.transform_points(dst_homo_src, points_src), points_dst, rtol=rtol, atol=atol) + + @pytest.mark.grad() + def test_gradcheck(self, device): + points_src = torch.rand(1, 10, 2, device=device, dtype=torch.float64, requires_grad=True) + points_dst = torch.rand_like(points_src) + weights = torch.ones_like(points_src)[..., 0] + + self.gradcheck(find_homography_dlt, (points_src, points_dst, weights), rtol=1e-6, atol=1e-6) + + @pytest.mark.grad() + def test_gradcheck_lu(self, device): + points_src = torch.rand(1, 10, 2, device=device, dtype=torch.float64, requires_grad=True) + + points_dst = torch.rand_like(points_src) + weights = torch.ones_like(points_src)[..., 0] + self.gradcheck(find_homography_dlt, (points_src, points_dst, weights, "lu"), rtol=1e-6, atol=1e-6) + + +class TestFindHomographyFromLinesDLT(BaseTester): + def test_smoke(self, device, dtype): + points1st = torch.rand(1, 4, 2, device=device, dtype=dtype) + points1end = torch.rand(1, 4, 2, device=device, dtype=dtype) + points2st = torch.rand(1, 4, 2, device=device, dtype=dtype) + points2end = torch.rand(1, 4, 2, device=device, dtype=dtype) + weights = torch.ones(1, 4, device=device, dtype=dtype) + ls1 = torch.stack([points1st, points1end], dim=2) + ls2 = torch.stack([points2st, points2end], dim=2) + H = find_homography_lines_dlt(ls1, ls2, weights) + assert H.shape == (1, 3, 3) + + def test_smoke2(self, device, dtype): + points1st = torch.rand(4, 2, device=device, dtype=dtype) + points1end = torch.rand(4, 2, device=device, dtype=dtype) + points2st = torch.rand(4, 2, device=device, dtype=dtype) + points2end = torch.rand(4, 2, device=device, dtype=dtype) + ls1 = torch.stack([points1st, points1end], dim=1) + ls2 = torch.stack([points2st, points2end], dim=1) + H = find_homography_lines_dlt(ls1, ls2, None) + assert H.shape == (1, 3, 3) + + def test_nocrash(self, device, dtype): + points1st = torch.rand(1, 4, 2, device=device, dtype=dtype) + points1end = torch.rand(1, 4, 2, device=device, dtype=dtype) + points2st = torch.rand(1, 4, 2, device=device, dtype=dtype) + points2end = torch.rand(1, 4, 2, device=device, dtype=dtype) + weights = torch.ones(1, 4, device=device, dtype=dtype) + points1st[0, 0, 0] = float("nan") + ls1 = torch.stack([points1st, points1end], dim=2) + ls2 = torch.stack([points2st, points2end], dim=2) + H = find_homography_lines_dlt(ls1, ls2, weights) + assert H.shape == (1, 3, 3) + + @pytest.mark.parametrize("batch_size, num_points", [(1, 4), (2, 5), (3, 6)]) + def test_shape(self, batch_size, num_points, device, dtype): + B, N = batch_size, num_points + points1st = torch.rand(B, N, 2, device=device, dtype=dtype) + points1end = torch.rand(B, N, 2, device=device, dtype=dtype) + points2st = torch.rand(B, N, 2, device=device, dtype=dtype) + points2end = torch.rand(B, N, 2, device=device, dtype=dtype) + weights = torch.ones(B, N, device=device, dtype=dtype) + ls1 = torch.stack([points1st, points1end], dim=2) + ls2 = torch.stack([points2st, points2end], dim=2) + H = find_homography_lines_dlt(ls1, ls2, weights) + assert H.shape == (B, 3, 3) + + @pytest.mark.parametrize("batch_size, num_points", [(1, 4), (2, 5), (3, 6)]) + def test_shape_noweights(self, batch_size, num_points, device, dtype): + B, N = batch_size, num_points + points1st = torch.rand(B, N, 2, device=device, dtype=dtype) + points1end = torch.rand(B, N, 2, device=device, dtype=dtype) + points2st = torch.rand(B, N, 2, device=device, dtype=dtype) + points2end = torch.rand(B, N, 2, device=device, dtype=dtype) + ls1 = torch.stack([points1st, points1end], dim=2) + ls2 = torch.stack([points2st, points2end], dim=2) + H = find_homography_lines_dlt(ls1, ls2, None) + assert H.shape == (B, 3, 3) + + @pytest.mark.parametrize("batch_size, num_points", [(1, 4), (2, 5), (3, 6)]) + def test_points_noweights(self, batch_size, num_points, device, dtype): + B, N = batch_size, num_points + points1st = torch.rand(B, N, 2, device=device, dtype=dtype) + points1end = torch.rand(B, N, 2, device=device, dtype=dtype) + points2st = torch.rand(B, N, 2, device=device, dtype=dtype) + points2end = torch.rand(B, N, 2, device=device, dtype=dtype) + weights = torch.ones(B, N, device=device, dtype=dtype) + ls1 = torch.stack([points1st, points1end], dim=2) + ls2 = torch.stack([points2st, points2end], dim=2) + H_noweights = find_homography_lines_dlt(ls1, ls2, None) + H_withweights = find_homography_lines_dlt(ls1, ls2, weights) + assert H_noweights.shape == (B, 3, 3) + assert H_withweights.shape == (B, 3, 3) + # On CUDA with float32, A^T A and A^T diag(1) A use different cuBLAS call sequences, + # so TF32 rounding accumulates differently. Use the same relaxed tolerance as + # test_clean_points does for this device+dtype combination. + rtol, atol = 1e-3, 1e-4 + if device.type == "cuda" and dtype == torch.float32: + rtol, atol = 5e-3, 5e-3 + self.assert_close(H_noweights, H_withweights, rtol=rtol, atol=atol) + + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_clean_points(self, batch_size, device, dtype): + # generate input data + points_src_st = torch.rand(batch_size, 10, 2, device=device, dtype=dtype) + points_src_end = torch.rand(batch_size, 10, 2, device=device, dtype=dtype) + + H = kornia.core.ops.eye_like(3, points_src_st) + H = H * 0.3 * torch.rand_like(H) + H = H / H[:, 2:3, 2:3] + points_dst_st = kornia.geometry.transform_points(H, points_src_st) + points_dst_end = kornia.geometry.transform_points(H, points_src_end) + + ls1 = torch.stack([points_src_st, points_src_end], axis=2) + ls2 = torch.stack([points_dst_st, points_dst_end], axis=2) + # compute transform from source to target + dst_homo_src = find_homography_lines_dlt(ls1, ls2, None) + rtol = 1e-3 + atol = 1e-4 + if dtype not in (torch.float32, torch.float64): + rtol = 5e-3 + atol = 1e-3 + elif device.type == "cuda" and dtype == torch.float32: + atol = 5e-3 + self.assert_close( + kornia.geometry.transform_points(dst_homo_src, points_src_st), points_dst_st, rtol=rtol, atol=atol + ) + + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_clean_points_iter(self, batch_size, device, dtype): + # generate input data + points_src_st = torch.rand(batch_size, 10, 2, device=device, dtype=dtype) + points_src_end = torch.rand(batch_size, 10, 2, device=device, dtype=dtype) + + H = kornia.core.ops.eye_like(3, points_src_st) + H = H * 0.3 * torch.rand_like(H) + H = H / H[:, 2:3, 2:3] + points_dst_st = kornia.geometry.transform_points(H, points_src_st) + points_dst_end = kornia.geometry.transform_points(H, points_src_end) + + ls1 = torch.stack([points_src_st, points_src_end], axis=2) + ls2 = torch.stack([points_dst_st, points_dst_end], axis=2) + # compute transform from source to target + dst_homo_src = find_homography_lines_dlt_iterated(ls1, ls2, None, 5) + rtol = 1e-3 + atol = 1e-4 + if dtype not in (torch.float32, torch.float64): + rtol = 5e-3 + atol = 1e-3 + elif device.type == "cuda" and dtype == torch.float32: + atol = 5e-3 + self.assert_close( + kornia.geometry.transform_points(dst_homo_src, points_src_st), points_dst_st, rtol=rtol, atol=atol + ) + + @pytest.mark.grad() + def test_gradcheck(self, device): + points_src_st = torch.rand(1, 10, 2, device=device, dtype=torch.float64, requires_grad=True) + points_src_end = torch.rand(1, 10, 2, device=device, dtype=torch.float64, requires_grad=True) + + points_dst_st = torch.rand_like(points_src_st) + points_dst_end = torch.rand_like(points_src_end) + weights = torch.ones_like(points_src_st)[..., 0] + ls1 = torch.stack([points_src_st, points_src_end], axis=2) + ls2 = torch.stack([points_dst_st, points_dst_end], axis=2) + + self.gradcheck(find_homography_lines_dlt, (ls1, ls2, weights), rtol=1e-6, atol=1e-6) + + +class TestFindHomographyDLTIter(BaseTester): + def test_smoke(self, device, dtype): + points1 = torch.rand(1, 4, 2, device=device, dtype=dtype) + points2 = torch.rand(1, 4, 2, device=device, dtype=dtype) + weights = torch.ones(1, 4, device=device, dtype=dtype) + H = find_homography_dlt_iterated(points1, points2, weights, 5) + assert H.shape == (1, 3, 3) + + @pytest.mark.parametrize("batch_size, num_points", [(1, 4), (2, 5), (3, 6)]) + @pytest.mark.skipif( + sys.platform == "darwin" and torch_version_le(1, 9, 1), reason="Known bug in torch 1.9.1 on macos" + ) + def test_shape(self, batch_size, num_points, device, dtype): + B, N = batch_size, num_points + points1 = torch.rand(B, N, 2, device=device, dtype=dtype) + points2 = torch.rand(B, N, 2, device=device, dtype=dtype) + weights = torch.ones(B, N, device=device, dtype=dtype) + H = find_homography_dlt_iterated(points1, points2, weights, 5) + assert H.shape == (B, 3, 3) + + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_clean_points(self, batch_size, device, dtype): + # generate input data + points_src = torch.rand(batch_size, 10, 2, device=device, dtype=dtype) + H = kornia.core.ops.eye_like(3, points_src) + H = H * 0.3 * torch.rand_like(H) + H = H / H[:, 2:3, 2:3] + + points_dst = kornia.geometry.transform_points(H, points_src) + weights = torch.ones(batch_size, 10, device=device, dtype=dtype) + + # compute transform from source to target + dst_homo_src = find_homography_dlt_iterated(points_src, points_dst, weights, 10) + + atol = 2e-3 if (device.type == "cuda" and dtype == torch.float32) else 1e-4 + self.assert_close(kornia.geometry.transform_points(dst_homo_src, points_src), points_dst, rtol=1e-3, atol=atol) + + @pytest.mark.grad() + def test_gradcheck(self, device): + torch.manual_seed(0) + points_src = torch.rand(1, 10, 2, device=device, dtype=torch.float64, requires_grad=True) + points_dst = torch.rand_like(points_src) + weights = torch.ones_like(points_src)[..., 0] + self.gradcheck(find_homography_dlt_iterated, (points_src, points_dst, weights), rtol=1e-6, atol=1e-6) + + @pytest.mark.grad() + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_dirty_points_and_gradcheck(self, batch_size, device, dtype): + # generate input data + points_src = torch.rand(batch_size, 10, 2, device=device, dtype=dtype) + H = kornia.core.ops.eye_like(3, points_src) + H = H * (1 + torch.rand_like(H)) + H = H / H[:, 2:3, 2:3] + + points_src = 100.0 * torch.rand(batch_size, 20, 2, device=device, dtype=dtype) + points_dst = kornia.geometry.transform_points(H, points_src) + + # making last point an outlier + points_dst[:, -1, :] += 20 + + weights = torch.ones(batch_size, 20, device=device, dtype=dtype) + + # compute transform from source to target + dst_homo_src = find_homography_dlt_iterated(points_src, points_dst, weights, 0.5, 10) + + self.assert_close( + kornia.geometry.transform_points(dst_homo_src, points_src[:, :-1]), points_dst[:, :-1], rtol=1e-3, atol=1e-3 + ) diff --git a/tests/geometry/test_keypoints.py b/tests/geometry/test_keypoints.py new file mode 100644 index 0000000..81c82bb --- /dev/null +++ b/tests/geometry/test_keypoints.py @@ -0,0 +1,367 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.geometry.keypoints import Keypoints, Keypoints3D, VideoKeypoints + +from testing.base import BaseTester + + +class TestKeypoints(BaseTester): + def test_smoke(self, device, dtype): + data = torch.rand(10, 2, device=device, dtype=dtype) + kp = Keypoints(data) + assert isinstance(kp, Keypoints) + + def test_cardinality(self, device, dtype): + data = torch.rand(10, 2, device=device, dtype=dtype) + kp = Keypoints(data) + assert kp.shape == (10, 2) + + def test_batched(self, device, dtype): + data = torch.rand(3, 10, 2, device=device, dtype=dtype) + kp = Keypoints(data) + assert kp.shape == (3, 10, 2) + assert kp._is_batched is True + + def test_unbatched(self, device, dtype): + data = torch.rand(10, 2, device=device, dtype=dtype) + kp = Keypoints(data) + assert kp._is_batched is False + + def test_device_dtype(self, device, dtype): + data = torch.rand(5, 2, device=device, dtype=dtype) + kp = Keypoints(data) + assert kp.device == device + assert kp.dtype == dtype + + def test_from_tensor(self, device, dtype): + data = torch.rand(5, 2, device=device, dtype=dtype) + kp = Keypoints.from_tensor(data) + assert kp.shape == data.shape + + def test_to_tensor(self, device, dtype): + data = torch.rand(5, 2, device=device, dtype=dtype) + kp = Keypoints(data) + out = kp.to_tensor() + assert out.shape == data.shape + self.assert_close(out, data) + + def test_clone(self, device, dtype): + data = torch.rand(5, 2, device=device, dtype=dtype) + kp = Keypoints(data) + kp2 = kp.clone() + self.assert_close(kp.data, kp2.data) + kp2._data[0, 0] = 999.0 + assert not torch.allclose(kp.data, kp2.data) + + def test_getitem(self, device, dtype): + data = torch.rand(10, 2, device=device, dtype=dtype) + kp = Keypoints(data) + kp2 = kp[:5] + assert kp2.shape == (5, 2) + + def test_setitem(self, device, dtype): + data = torch.rand(10, 2, device=device, dtype=dtype) + kp = Keypoints(data) + new_data = torch.zeros(5, 2, device=device, dtype=dtype) + new_kp = Keypoints(new_data) + kp[:5] = new_kp + self.assert_close(kp.data[:5], new_data) + + def test_transform_keypoints(self, device, dtype): + # Use batched keypoints (B, N, 2) with batched M (B, 3, 3) + data = torch.tensor([[[1.0, 0.0], [0.0, 1.0]]], device=device, dtype=dtype) # (1, 2, 2) + kp = Keypoints(data) + M = torch.eye(3, device=device, dtype=dtype).unsqueeze(0) # (1, 3, 3) + M[0, 0, 2] = 2.0 # translate x by 2 + M[0, 1, 2] = 3.0 # translate y by 3 + kp_t = kp.transform_keypoints(M) + expected = torch.tensor([[[3.0, 3.0], [2.0, 4.0]]], device=device, dtype=dtype) + self.assert_close(kp_t.data, expected) + + def test_transform_keypoints_inplace(self, device, dtype): + data = torch.tensor([[[1.0, 0.0]]], device=device, dtype=dtype) # (1, 1, 2) + kp = Keypoints(data) + M = torch.eye(3, device=device, dtype=dtype).unsqueeze(0) # (1, 3, 3) + M[0, 0, 2] = 1.0 + kp.transform_keypoints_(M) + expected = torch.tensor([[[2.0, 0.0]]], device=device, dtype=dtype) + self.assert_close(kp.data, expected) + + def test_transform_keypoints_batched(self, device, dtype): + data = torch.ones(2, 4, 2, device=device, dtype=dtype) + kp = Keypoints(data) + M = torch.eye(3, device=device, dtype=dtype).unsqueeze(0).expand(2, -1, -1).clone() + M[:, 0, 2] = 5.0 + kp_t = kp.transform_keypoints(M) + assert kp_t.shape == (2, 4, 2) + self.assert_close(kp_t.data[..., 0], torch.full((2, 4), 6.0, device=device, dtype=dtype)) + + def test_pad(self, device, dtype): + data = torch.zeros(2, 4, 2, device=device, dtype=dtype) + kp = Keypoints(data) + padding = torch.tensor([[1.0, 0.0, 2.0, 0.0], [0.0, 0.0, 3.0, 0.0]], device=device, dtype=dtype) + kp.pad(padding) + # x += left_pad, y += top_pad + self.assert_close(kp.data[0, :, 0], torch.full((4,), 1.0, device=device, dtype=dtype)) + self.assert_close(kp.data[1, :, 0], torch.zeros(4, device=device, dtype=dtype)) + self.assert_close(kp.data[0, :, 1], torch.full((4,), 2.0, device=device, dtype=dtype)) + + def test_unpad(self, device, dtype): + data = torch.ones(2, 4, 2, device=device, dtype=dtype) * 5.0 + kp = Keypoints(data) + padding = torch.tensor([[1.0, 0.0, 2.0, 0.0], [0.0, 0.0, 0.0, 0.0]], device=device, dtype=dtype) + kp.unpad(padding) + self.assert_close(kp.data[0, :, 0], torch.full((4,), 4.0, device=device, dtype=dtype)) + self.assert_close(kp.data[0, :, 1], torch.full((4,), 3.0, device=device, dtype=dtype)) + + def test_index_put(self, device, dtype): + data = torch.zeros(10, 2, device=device, dtype=dtype) + kp = Keypoints(data) + new_vals = torch.ones(3, 2, device=device, dtype=dtype) + idx = (torch.tensor([0, 1, 2], device=device),) + kp2 = kp.index_put(idx, new_vals) + self.assert_close(kp2.data[:3], new_vals) + + def test_index_put_inplace(self, device, dtype): + data = torch.zeros(10, 2, device=device, dtype=dtype) + kp = Keypoints(data) + new_vals = torch.ones(3, 2, device=device, dtype=dtype) + idx = (torch.tensor([0, 1, 2], device=device),) + kp.index_put(idx, new_vals, inplace=True) + self.assert_close(kp.data[:3], new_vals) + + def test_type(self, device, dtype): + if device.type == "mps": + pytest.skip("MPS does not support float64") + data = torch.rand(5, 2, device=device, dtype=torch.float32) + kp = Keypoints(data) + kp.type(torch.float64) + assert kp.dtype == torch.float64 + + def test_exception(self, device, dtype): + with pytest.raises(TypeError): + Keypoints("not a tensor") + + with pytest.raises(ValueError): + Keypoints(torch.tensor([1, 2, 3], dtype=torch.int32)) + + with pytest.raises(ValueError): + Keypoints(torch.rand(3, 3, device=device, dtype=dtype)) + + with pytest.raises(ValueError): + Keypoints(torch.rand(3, 4, 2, 2, device=device, dtype=dtype)) + + def test_transform_exception(self, device, dtype): + kp = Keypoints(torch.rand(5, 2, device=device, dtype=dtype)) + with pytest.raises(ValueError): + kp.transform_keypoints(torch.eye(4, device=device, dtype=dtype)) + + def test_pad_exception(self, device, dtype): + kp = Keypoints(torch.rand(2, 4, 2, device=device, dtype=dtype)) + with pytest.raises(RuntimeError): + kp.pad(torch.zeros(2, 3, device=device, dtype=dtype)) + + def test_int_input_raises_by_default(self, device, dtype): + with pytest.raises(ValueError): + Keypoints(torch.ones(5, 2, device=device, dtype=torch.int32)) + + def test_int_input_converted_when_not_raising(self, device, dtype): + data = torch.ones(5, 2, device=device, dtype=torch.int32) + kp = Keypoints(data, raise_if_not_floating_point=False) + assert kp.dtype == torch.float32 + + def test_gradcheck(self, device): + data = torch.rand(1, 5, 2, device=device, dtype=torch.float64, requires_grad=True) + M = torch.eye(3, device=device, dtype=torch.float64).unsqueeze(0) + M[0, 0, 2] = 1.0 + + def fn(x): + return Keypoints(x).transform_keypoints(M).data + + self.gradcheck(fn, (data,)) + + def test_dynamo(self, device, dtype, torch_optimizer): + data = torch.rand(1, 5, 2, device=device, dtype=dtype) + M = torch.eye(3, device=device, dtype=dtype).unsqueeze(0) + + def fn(x): + return Keypoints(x).transform_keypoints(M).data + + op = torch_optimizer(fn) + self.assert_close(op(data), fn(data)) + + def test_smoke_jit(self, device, dtype): + pass # Keypoints is not a nn.Module, jit test not applicable + + def test_module(self, device, dtype): + pass # Keypoints is not a nn.Module + + +class TestVideoKeypoints(BaseTester): + def test_smoke(self, device, dtype): + data = torch.rand(2, 5, 10, 2, device=device, dtype=dtype) + vkp = VideoKeypoints.from_tensor(data) + assert isinstance(vkp, VideoKeypoints) + + def test_cardinality(self, device, dtype): + B, T, N = 2, 5, 10 + data = torch.rand(B, T, N, 2, device=device, dtype=dtype) + vkp = VideoKeypoints.from_tensor(data) + assert vkp.temporal_channel_size == T + out = vkp.to_tensor() + assert out.shape == (B, T, N, 2) + + def test_to_tensor_roundtrip(self, device, dtype): + data = torch.rand(2, 4, 8, 2, device=device, dtype=dtype) + vkp = VideoKeypoints.from_tensor(data) + out = vkp.to_tensor() + self.assert_close(out, data) + + def test_clone(self, device, dtype): + data = torch.rand(2, 4, 8, 2, device=device, dtype=dtype) + vkp = VideoKeypoints.from_tensor(data) + vkp2 = vkp.clone() + self.assert_close(vkp.to_tensor(), vkp2.to_tensor()) + assert vkp2.temporal_channel_size == vkp.temporal_channel_size + + def test_transform_keypoints(self, device, dtype): + B, T, N = 1, 3, 5 + data = torch.ones(B, T, N, 2, device=device, dtype=dtype) + vkp = VideoKeypoints.from_tensor(data) + # After from_tensor, internal shape is (B*T, N, 2); need M with batch size B*T or 1 + M = torch.eye(3, device=device, dtype=dtype).unsqueeze(0) # (1, 3, 3) broadcasts + out = vkp.transform_keypoints(M) + assert isinstance(out, VideoKeypoints) + assert out.temporal_channel_size == T + + def test_exception(self, device, dtype): + with pytest.raises(ValueError): + VideoKeypoints.from_tensor(torch.rand(5, 2, device=device, dtype=dtype)) + + with pytest.raises(ValueError): + VideoKeypoints.from_tensor(torch.rand(2, 5, 10, 3, device=device, dtype=dtype)) + + def test_gradcheck(self, device): + pass # VideoKeypoints ops not differentiable through from_tensor reshape + + def test_dynamo(self, device, dtype, torch_optimizer): + pass # VideoKeypoints uses reshape; not straightforward to dynamo + + def test_smoke_jit(self, device, dtype): + pass + + def test_module(self, device, dtype): + pass + + def test_exception_in_base(self, device, dtype): + pass + + +class TestKeypoints3D(BaseTester): + def test_smoke(self, device, dtype): + data = torch.rand(10, 3, device=device, dtype=dtype) + kp = Keypoints3D(data) + assert isinstance(kp, Keypoints3D) + + def test_cardinality(self, device, dtype): + data = torch.rand(10, 3, device=device, dtype=dtype) + kp = Keypoints3D(data) + assert kp.shape == (10, 3) + + def test_batched(self, device, dtype): + data = torch.rand(3, 10, 3, device=device, dtype=dtype) + kp = Keypoints3D(data) + assert kp.shape == (3, 10, 3) + assert kp._is_batched is True + + def test_unbatched(self, device, dtype): + data = torch.rand(10, 3, device=device, dtype=dtype) + kp = Keypoints3D(data) + assert kp._is_batched is False + + def test_from_tensor(self, device, dtype): + data = torch.rand(5, 3, device=device, dtype=dtype) + kp = Keypoints3D.from_tensor(data) + assert kp.shape == data.shape + + def test_to_tensor(self, device, dtype): + data = torch.rand(5, 3, device=device, dtype=dtype) + kp = Keypoints3D(data) + out = kp.to_tensor() + self.assert_close(out, data) + + def test_clone(self, device, dtype): + data = torch.rand(5, 3, device=device, dtype=dtype) + kp = Keypoints3D(data) + kp2 = kp.clone() + self.assert_close(kp.data, kp2.data) + kp2._data[0, 0] = 999.0 + assert not torch.allclose(kp.data, kp2.data) + + def test_getitem(self, device, dtype): + data = torch.rand(10, 3, device=device, dtype=dtype) + kp = Keypoints3D(data) + kp2 = kp[:5] + assert kp2.shape == (5, 3) + + def test_setitem(self, device, dtype): + data = torch.rand(10, 3, device=device, dtype=dtype) + kp = Keypoints3D(data) + new_data = torch.zeros(5, 3, device=device, dtype=dtype) + new_kp = Keypoints3D(new_data) + kp[:5] = new_kp + self.assert_close(kp.data[:5], new_data) + + def test_not_implemented(self, device, dtype): + kp = Keypoints3D(torch.rand(5, 3, device=device, dtype=dtype)) + with pytest.raises(NotImplementedError): + kp.pad(torch.zeros(1, 6, device=device, dtype=dtype)) + with pytest.raises(NotImplementedError): + kp.unpad(torch.zeros(1, 6, device=device, dtype=dtype)) + with pytest.raises(NotImplementedError): + kp.transform_keypoints(torch.eye(4, device=device, dtype=dtype)) + + def test_exception(self, device, dtype): + with pytest.raises(TypeError): + Keypoints3D("not a tensor") + + with pytest.raises(ValueError): + Keypoints3D(torch.tensor([1, 2, 3], dtype=torch.int32)) + + with pytest.raises(ValueError): + Keypoints3D(torch.rand(3, 2, device=device, dtype=dtype)) + + def test_int_input_converted_when_not_raising(self, device, dtype): + data = torch.ones(5, 3, device=device, dtype=torch.int32) + kp = Keypoints3D(data, raise_if_not_floating_point=False) + assert kp.data.dtype == torch.float32 + + def test_gradcheck(self, device): + pass # Keypoints3D transform ops are NotImplemented + + def test_dynamo(self, device, dtype, torch_optimizer): + pass + + def test_smoke_jit(self, device, dtype): + pass + + def test_module(self, device, dtype): + pass diff --git a/tests/geometry/test_linalg.py b/tests/geometry/test_linalg.py new file mode 100644 index 0000000..a0d7ef1 --- /dev/null +++ b/tests/geometry/test_linalg.py @@ -0,0 +1,379 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia +import kornia.geometry.linalg as kgl + +from testing.base import BaseTester +from testing.geometry.create import create_random_homography +from testing.geometry.linalg import euler_angles_to_rotation_matrix, identity_matrix + + +class TestTransformPoints(BaseTester): + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + @pytest.mark.parametrize("num_points", [2, 3, 5]) + @pytest.mark.parametrize("num_dims", [2, 3]) + def test_transform_points(self, batch_size, num_points, num_dims, device, dtype): + # generate input data + eye_size = num_dims + 1 + points_src = torch.rand(batch_size, num_points, num_dims, device=device, dtype=dtype) + + dst_homo_src = create_random_homography(points_src, eye_size) + dst_homo_src = dst_homo_src.to(device) + + # transform the points from dst to ref + points_dst = kgl.transform_points(dst_homo_src, points_src) + + # transform the points from ref to dst + src_homo_dst = torch.inverse(dst_homo_src) + points_dst_to_src = kgl.transform_points(src_homo_dst, points_dst) + + # projected should be equal as initial + atol = 1e-3 if (device.type == "cuda" and dtype == torch.float32) else 1e-4 + self.assert_close(points_src, points_dst_to_src, atol=atol, rtol=1e-4) + + def test_gradcheck(self, device): + # generate input data + batch_size, num_points, num_dims = 2, 3, 2 + eye_size = num_dims + 1 + points_src = torch.rand(batch_size, num_points, num_dims, device=device, dtype=torch.float64) + dst_homo_src = create_random_homography(points_src, eye_size) + # evaluate function gradient + self.gradcheck(kornia.geometry.transform_points, (dst_homo_src, points_src)) + + def test_dynamo(self, device, dtype, torch_optimizer): + points = torch.ones(1, 2, 2, device=device, dtype=dtype) + transform = kornia.core.ops.eye_like(3, points) + op = kornia.geometry.transform_points + op_script = torch_optimizer(op) + actual = op_script(transform, points) + expected = op(transform, points) + self.assert_close(actual, expected, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("trans_dtype", [torch.float16, torch.float32, torch.float64]) + @pytest.mark.parametrize("points_dtype", [torch.float16, torch.float32, torch.float64]) + def test_mixed_dtypes(self, device, trans_dtype, points_dtype): + # Regression test for https://github.com/kornia/kornia/issues/3705 + points_src = torch.rand(2, 3, 2, device=device, dtype=points_dtype) + trans = kornia.core.ops.eye_like(3, points_src).to(trans_dtype) + out = kgl.transform_points(trans, points_src) + assert out.dtype == points_dtype + self.assert_close(out.to(torch.float32), points_src.to(torch.float32), atol=1e-2, rtol=1e-2) + + +class TestComposeTransforms(BaseTester): + def test_smoke(self, device, dtype): + batch_size = 2 + trans_01 = identity_matrix(batch_size=batch_size, device=device, dtype=dtype) + trans_12 = identity_matrix(batch_size=batch_size, device=device, dtype=dtype) + + to_check_1 = kornia.geometry.compose_transformations(trans_01, trans_12) + to_check_2 = kornia.geometry.compose_transformations(trans_01[0], trans_12[0]) + + assert to_check_1.shape == (batch_size, 4, 4) + assert to_check_2.shape == (4, 4) + + def test_exception(self, device, dtype): + to_check_1 = torch.rand((7, 4, 4, 3), device=device, dtype=dtype) + to_check_2 = torch.rand((5, 10, 10), device=device, dtype=dtype) + to_check_3 = torch.rand((6, 4, 4), device=device, dtype=dtype) + to_check_4 = torch.rand((4, 4), device=device, dtype=dtype) + to_check_5 = torch.rand((3, 3), device=device, dtype=dtype) + + # Testing if exception is thrown when both inputs have shape (3, 3) + with pytest.raises(ValueError): + _ = kornia.geometry.compose_transformations(to_check_5, to_check_5) + + # Testing if exception is thrown when both inputs have shape (5, 10, 10) + with pytest.raises(ValueError): + _ = kornia.geometry.compose_transformations(to_check_2, to_check_2) + + # Testing if exception is thrown when one input has shape (6, 4, 4) + # whereas the other input has shape (4, 4) + with pytest.raises(ValueError): + _ = kornia.geometry.compose_transformations(to_check_3, to_check_4) + + # Testing if exception is thrown when one input has shape (7, 4, 4, 3) + # whereas the other input has shape (4, 4) + with pytest.raises(ValueError): + _ = kornia.geometry.compose_transformations(to_check_1, to_check_4) + + def test_translation_4x4(self, device, dtype): + offset = 10 + trans_01 = identity_matrix(batch_size=1, device=device, dtype=dtype)[0] + trans_12 = identity_matrix(batch_size=1, device=device, dtype=dtype)[0] + trans_12[..., :3, -1] += offset # add offset to translation vector + + trans_02 = kgl.compose_transformations(trans_01, trans_12) + self.assert_close(trans_02, trans_12, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_translation_Bx4x4(self, batch_size, device, dtype): + offset = 10 + trans_01 = identity_matrix(batch_size, device=device, dtype=dtype) + trans_12 = identity_matrix(batch_size, device=device, dtype=dtype) + trans_12[..., :3, -1] += offset # add offset to translation vector + + trans_02 = kgl.compose_transformations(trans_01, trans_12) + self.assert_close(trans_02, trans_12, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_gradcheck(self, batch_size, device): + trans_01 = identity_matrix(batch_size, device=device, dtype=torch.float64) + trans_12 = identity_matrix(batch_size, device=device, dtype=torch.float64) + + self.gradcheck(kgl.compose_transformations, (trans_01, trans_12)) + + +class TestInverseTransformation(BaseTester): + def test_smoke(self, device, dtype): + batch_size = 2 + trans_01 = identity_matrix(batch_size=batch_size, device=device, dtype=dtype) + + to_check_1 = kornia.geometry.inverse_transformation(trans_01) + to_check_2 = kornia.geometry.inverse_transformation(trans_01[0]) + + assert to_check_1.shape == (batch_size, 4, 4) + assert to_check_2.shape == (4, 4) + + def test_exception(self, device, dtype): + to_check_1 = torch.rand((7, 4, 4, 3), device=device, dtype=dtype) + to_check_2 = torch.rand((5, 10, 10), device=device, dtype=dtype) + to_check_3 = torch.rand((3, 3), device=device, dtype=dtype) + + # Testing if exception is thrown when the input has shape (7, 4, 4, 3) + with pytest.raises(ValueError): + _ = kornia.geometry.inverse_transformation(to_check_1) + + # Testing if exception is thrown when the input has shape (5, 10, 10) + with pytest.raises(ValueError): + _ = kornia.geometry.inverse_transformation(to_check_2) + + # Testing if exception is thrown when the input has shape (3, 3) + with pytest.raises(ValueError): + _ = kornia.geometry.inverse_transformation(to_check_3) + + def test_translation_4x4(self, device, dtype): + offset = 10 + trans_01 = identity_matrix(batch_size=1, device=device, dtype=dtype)[0] + trans_01[..., :3, -1] += offset # add offset to translation vector + + trans_10 = kgl.inverse_transformation(trans_01) + trans_01_hat = kgl.inverse_transformation(trans_10) + self.assert_close(trans_01, trans_01_hat, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_translation_Bx4x4(self, batch_size, device, dtype): + offset = 10 + trans_01 = identity_matrix(batch_size, device=device, dtype=dtype) + trans_01[..., :3, -1] += offset # add offset to translation vector + + trans_10 = kgl.inverse_transformation(trans_01) + trans_01_hat = kgl.inverse_transformation(trans_10) + self.assert_close(trans_01, trans_01_hat, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_rotation_translation_Bx4x4(self, batch_size, device, dtype): + offset = 10 + x, y, z = 0, 0, kornia.pi + ones = torch.ones(batch_size, device=device, dtype=dtype) + rmat_01 = euler_angles_to_rotation_matrix(x * ones, y * ones, z * ones) + + trans_01 = identity_matrix(batch_size, device=device, dtype=dtype) + trans_01[..., :3, -1] += offset # add offset to translation vector + trans_01[..., :3, :3] = rmat_01[..., :3, :3] + + trans_10 = kgl.inverse_transformation(trans_01) + trans_01_hat = kgl.inverse_transformation(trans_10) + self.assert_close(trans_01, trans_01_hat, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_gradcheck(self, batch_size, device): + trans_01 = identity_matrix(batch_size, device=device, dtype=torch.float64) + self.gradcheck(kgl.inverse_transformation, (trans_01,)) + + +class TestRelativeTransformation(BaseTester): + def test_smoke(self, device, dtype): + batch_size = 2 + trans_01 = identity_matrix(batch_size=batch_size, device=device, dtype=dtype) + trans_02 = identity_matrix(batch_size=batch_size, device=device, dtype=dtype) + + to_check_1 = kornia.geometry.relative_transformation(trans_01, trans_02) + to_check_2 = kornia.geometry.relative_transformation(trans_01[0], trans_02[0]) + + assert to_check_1.shape == (batch_size, 4, 4) + assert to_check_2.shape == (4, 4) + + def test_exception(self, device, dtype): + to_check_1 = torch.rand((7, 4, 4, 3), device=device, dtype=dtype) + to_check_2 = torch.rand((5, 10, 10), device=device, dtype=dtype) + to_check_3 = torch.rand((6, 4, 4), device=device, dtype=dtype) + to_check_4 = torch.rand((4, 4), device=device, dtype=dtype) + to_check_5 = torch.rand((3, 3), device=device, dtype=dtype) + + # Testing if exception is thrown when both inputs have shape (3, 3) + with pytest.raises(ValueError): + _ = kornia.geometry.relative_transformation(to_check_5, to_check_5) + + # Testing if exception is thrown when both inputs have shape (5, 10, 10) + with pytest.raises(ValueError): + _ = kornia.geometry.relative_transformation(to_check_2, to_check_2) + + # Testing if exception is thrown when one input has shape (6, 4, 4) + # whereas the other input has shape (4, 4) + with pytest.raises(ValueError): + _ = kornia.geometry.relative_transformation(to_check_3, to_check_4) + + # Testing if exception is thrown when one input has shape (7, 4, 4, 3) + # whereas the other input has shape (4, 4) + with pytest.raises(ValueError): + _ = kornia.geometry.relative_transformation(to_check_1, to_check_4) + + def test_translation_4x4(self, device, dtype): + offset = 10.0 + trans_01 = identity_matrix(batch_size=1, device=device, dtype=dtype)[0] + trans_02 = identity_matrix(batch_size=1, device=device, dtype=dtype)[0] + trans_02[..., :3, -1] += offset # add offset to translation vector + + trans_12 = kgl.relative_transformation(trans_01, trans_02) + trans_02_hat = kgl.compose_transformations(trans_01, trans_12) + self.assert_close(trans_02_hat, trans_02, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_rotation_translation_Bx4x4(self, batch_size, device, dtype): + offset = 10.0 + x, y, z = 0.0, 0.0, kornia.pi + ones = torch.ones(batch_size, device=device, dtype=dtype) + rmat_02 = euler_angles_to_rotation_matrix(x * ones, y * ones, z * ones) + + trans_01 = identity_matrix(batch_size, device=device, dtype=dtype) + trans_02 = identity_matrix(batch_size, device=device, dtype=dtype) + trans_02[..., :3, -1] += offset # add offset to translation vector + trans_02[..., :3, :3] = rmat_02[..., :3, :3] + + trans_12 = kgl.relative_transformation(trans_01, trans_02) + trans_02_hat = kgl.compose_transformations(trans_01, trans_12) + self.assert_close(trans_02_hat, trans_02, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_gradcheck(self, batch_size, device): + trans_01 = identity_matrix(batch_size, device=device, dtype=torch.float64) + trans_02 = identity_matrix(batch_size, device=device, dtype=torch.float64) + + self.gradcheck(kgl.relative_transformation, (trans_01, trans_02)) + + +class TestPointsLinesDistances(BaseTester): + def test_smoke(self, device, dtype): + pts = torch.rand(1, 1, 2, device=device, dtype=dtype) + lines = torch.rand(1, 1, 3, device=device, dtype=dtype) + distances = kgl.point_line_distance(pts, lines) + assert distances.shape == (1, 1) + + # homogeneous + pts = torch.rand(1, 1, 3, device=device, dtype=dtype) + lines = torch.rand(1, 1, 3, device=device, dtype=dtype) + distances = kgl.point_line_distance(pts, lines) + assert distances.shape == (1, 1) + + @pytest.mark.parametrize( + "batch_size, sample_size", [(1, 1), (2, 1), (4, 1), (7, 1), (1, 3), (2, 3), (4, 3), (7, 3)] + ) + def test_shape(self, batch_size, sample_size, device, dtype): + B, N = batch_size, sample_size + pts = torch.rand(B, N, 2, device=device, dtype=dtype) + lines = torch.rand(B, N, 3, device=device, dtype=dtype) + distances = kgl.point_line_distance(pts, lines) + assert distances.shape == (B, N) + + @pytest.mark.parametrize( + "batch_size, extra_dim_size", [(1, 1), (2, 1), (4, 1), (7, 1), (1, 3), (2, 3), (4, 3), (7, 3)] + ) + def test_shapes(self, batch_size, extra_dim_size, device, dtype): + B, T, N = batch_size, extra_dim_size, 3 + pts = torch.rand(B, T, N, 2, device=device, dtype=dtype) + lines = torch.rand(B, T, N, 3, device=device, dtype=dtype) + distances = kgl.point_line_distance(pts, lines) + assert distances.shape == (B, T, N) + + def test_functional(self, device): + if device.type == "mps": + pytest.skip("MPS does not support float64") + pts = torch.tensor([1.0, 0], device=device, dtype=torch.float64).view(1, 1, 2).tile(1, 6, 1) + lines = torch.tensor( + [[0.0, 1.0, 0.0], [0.0, 1.0, 1.0], [1.0, 0.0, 0.0], [1.0, 0.0, 1.0], [1.0, 1.0, 0.0], [1.0, 1.0, 1.0]], + device=device, + dtype=torch.float64, + ).view(1, 6, 3) + distances = kgl.point_line_distance(pts, lines) + distances_expected = torch.tensor( + [ + 0.0, + 1.0, + 1.0, + 2.0, + torch.sqrt(torch.tensor(2, dtype=torch.float64)) / 2, + torch.sqrt(torch.tensor(2, dtype=torch.float64)), + ], + device=device, + ).view(1, 6) + self.assert_close(distances, distances_expected, rtol=1e-6, atol=1e-6) + + def test_gradcheck(self, device): + pts = torch.rand(2, 3, 2, device=device, requires_grad=True, dtype=torch.float64) + lines = torch.rand(2, 3, 3, device=device, requires_grad=True, dtype=torch.float64) + self.gradcheck(kgl.point_line_distance, (pts, lines)) + + +class TestEuclideanDistance(BaseTester): + def test_smoke(self, device, dtype): + pt1 = torch.tensor([0, 0, 0], device=device, dtype=dtype) + pt2 = torch.tensor([1, 0, 0], device=device, dtype=dtype) + dst = kgl.euclidean_distance(pt1, pt2) + self.assert_close(dst, torch.tensor(1.0, device=device, dtype=dtype)) + + @pytest.mark.parametrize("shape", [(2,), (3,), (1, 2), (2, 3)]) + def test_cardinality(self, device, dtype, shape): + pt1 = torch.rand(shape, device=device, dtype=dtype) + pt2 = torch.rand(shape, device=device, dtype=dtype) + dst = kgl.euclidean_distance(pt1, pt2) + assert len(dst.shape) == len(shape) - 1 + + def test_exception(self, device, dtype): + pt1 = torch.tensor([0, 0, 0], device=device, dtype=dtype) + pt2 = torch.rand(1, 2, device=device, dtype=dtype) + with pytest.raises(Exception): + kgl.euclidean_distance(pt1, pt2) + + def test_gradcheck(self, device): + pt1 = torch.rand(2, 3, device=device, dtype=torch.float64, requires_grad=True) + pt2 = torch.rand(2, 3, device=device, dtype=torch.float64, requires_grad=True) + self.gradcheck(kgl.euclidean_distance, (pt1, pt2)) + + def test_dynamo(self, device, dtype, torch_optimizer): + pt1 = torch.rand(2, 3, device=device, dtype=dtype) + pt2 = torch.rand(2, 3, device=device, dtype=dtype) + op = kgl.euclidean_distance + op_optimized = torch_optimizer(op) + self.assert_close(op(pt1, pt2), op_optimized(pt1, pt2)) + + def test_module(self, device, dtype): + pass diff --git a/tests/geometry/test_line.py b/tests/geometry/test_line.py new file mode 100644 index 0000000..dd981ef --- /dev/null +++ b/tests/geometry/test_line.py @@ -0,0 +1,242 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +from torch.autograd import gradcheck + +from kornia.geometry.line import ParametrizedLine, fit_line +from kornia.geometry.plane import Hyperplane + +from testing.base import BaseTester, assert_close + + +class TestParametrizedLine(BaseTester): + def test_smoke(self, device, dtype): + p0 = torch.tensor([0.0, 0.0], device=device, dtype=dtype) + d0 = torch.tensor([1.0, 0.0], device=device, dtype=dtype) + l0 = ParametrizedLine(p0, d0) + self.assert_close(l0.origin, p0) + self.assert_close(l0.direction, d0) + assert l0.dim() == 2 + + def test_through(self, device, dtype): + p0 = torch.tensor([-1.0, -1.0], device=device, dtype=dtype) + p1 = torch.tensor([1.0, 1.0], device=device, dtype=dtype) + l1 = ParametrizedLine.through(p0, p1) + direction_expected = torch.tensor([0.7071, 0.7071], device=device, dtype=dtype) + self.assert_close(l1.origin, p0) + self.assert_close(l1.direction, direction_expected) + + def test_point_at(self, device, dtype): + p0 = torch.tensor([0.0, 0.0], device=device, dtype=dtype) + p1 = torch.tensor([1.0, 0.0], device=device, dtype=dtype) + l1 = ParametrizedLine.through(p0, p1) + self.assert_close(l1.point_at(0.0), torch.tensor([0.0, 0.0], device=device, dtype=dtype)) + self.assert_close(l1.point_at(0.5), torch.tensor([0.5, 0.0], device=device, dtype=dtype)) + self.assert_close(l1.point_at(1.0), torch.tensor([1.0, 0.0], device=device, dtype=dtype)) + + def test_projection1(self, device, dtype): + p0 = torch.tensor([0.0, 0.0], device=device, dtype=dtype) + p1 = torch.tensor([1.0, 0.0], device=device, dtype=dtype) + p2 = torch.tensor([0.5, 0.5], device=device, dtype=dtype) + p3_expected = torch.tensor([0.5, 0.0], device=device, dtype=dtype) + l1 = ParametrizedLine.through(p0, p1) + p3 = l1.projection(p2) + self.assert_close(p3, p3_expected) + + def test_projection2(self, device, dtype): + p0 = torch.tensor([0.0, 0.0], device=device, dtype=dtype) + p1 = torch.tensor([0.0, 1.0], device=device, dtype=dtype) + p2 = torch.tensor([0.5, 0.5], device=device, dtype=dtype) + p3_expected = torch.tensor([0.0, 0.5], device=device, dtype=dtype) + l1 = ParametrizedLine.through(p0, p1) + p3 = l1.projection(p2) + self.assert_close(p3, p3_expected) + + def test_projection(self, device, dtype): + p0 = torch.tensor([0.0, 0.0], device=device, dtype=dtype) + p1 = torch.tensor([1.0, 0.0], device=device, dtype=dtype) + l1 = ParametrizedLine.through(p0, p1) + point = torch.tensor([1.0, 2.0], device=device, dtype=dtype) + point_projection = torch.tensor([1.0, 0.0], device=device, dtype=dtype) + self.assert_close(l1.projection(point), point_projection) + + def test_distance(self, device, dtype): + p0 = torch.tensor([0.0, 0.0], device=device, dtype=dtype) + p1 = torch.tensor([1.0, 0.0], device=device, dtype=dtype) + l1 = ParametrizedLine.through(p0, p1) + point = torch.tensor([1.0, 4.0], device=device, dtype=dtype) + distance_expected = torch.tensor(4.0, device=device, dtype=dtype) + self.assert_close(l1.distance(point), distance_expected) + + def test_squared_distance(self, device, dtype): + p0 = torch.tensor([0.0, 0.0], device=device, dtype=dtype) + p1 = torch.tensor([1.0, 0.0], device=device, dtype=dtype) + l1 = ParametrizedLine.through(p0, p1) + point = torch.tensor([1.0, 4.0], device=device, dtype=dtype) + distance_expected = torch.tensor(16.0, device=device, dtype=dtype) + self.assert_close(l1.squared_distance(point), distance_expected) + + def test_instersect_plane(self, device, dtype): + p0 = torch.tensor([0.0, 0.0, 0.0], device=device, dtype=dtype) + p1 = torch.tensor([1.0, 0.0, 0.0], device=device, dtype=dtype) + l1 = ParametrizedLine.through(p0, p1) + + v0 = torch.tensor([3.0, 0.0, 1.0], device=device, dtype=dtype) + v1 = torch.tensor([3.0, 1.0, 0.0], device=device, dtype=dtype) + v2 = torch.tensor([3.0, 0.0, -1.0], device=device, dtype=dtype) + pl0 = Hyperplane.through(v0, v1, v2) + + lmbda, point = l1.intersect(pl0) + + expected_point = torch.tensor([3.0, 0.0, 0.0], device=device, dtype=dtype) + expected_lambda = torch.tensor(3.0, device=device, dtype=dtype) + + self.assert_close(lmbda, expected_lambda) + self.assert_close(point, expected_point) + + @pytest.mark.skip(reason="not implemented yet") + def test_cardinality(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_jit(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_exception(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_module(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_gradcheck(self, device): + pass + + +class TestFitLine(BaseTester): + @pytest.mark.parametrize("B", (1, 2)) + @pytest.mark.parametrize("D", (2, 3, 4)) + def test_smoke(self, device, dtype, B, D): + N: int = 10 # num points + points = torch.ones(B, N, D, device=device, dtype=dtype) + line = fit_line(points) + assert isinstance(line, ParametrizedLine) + assert line.origin.shape == (B, D) + assert line.direction.shape == (B, D) + + assert_close(line.origin, line[0]) + assert_close(line.direction, line[1]) + + origin, direction = fit_line(points) + assert_close(line.origin, origin) + assert_close(line.direction, direction) + + def test_fit_line2(self, device, dtype): + p0 = torch.tensor([0.0, 0.0], device=device, dtype=dtype) + p1 = torch.tensor([1.0, 1.0], device=device, dtype=dtype) + + l1 = ParametrizedLine.through(p0, p1) + num_points: int = 10 + + pts = [] + for t in torch.linspace(-10, 10, num_points): + p2 = l1.point_at(t) + pts.append(p2) + pts = torch.stack(pts) + + line_est = fit_line(pts[None]) + dir_exp = torch.tensor([0.7071, 0.7071], device=device, dtype=dtype) + # NOTE: for some reason the result in c[u/cuda differs + angle_est = torch.nn.functional.cosine_similarity(line_est.direction, dir_exp, -1) + + angle_exp = torch.tensor([1.0], device=device, dtype=dtype) + self.assert_close(angle_est.abs(), angle_exp) + + def test_fit_line3(self, device, dtype): + p0 = torch.tensor([0.0, 0.0, 0.0], device=device, dtype=dtype) + p1 = torch.tensor([1.0, 1.0, 1.0], device=device, dtype=dtype) + + l1 = ParametrizedLine.through(p0, p1) + num_points: int = 10 + + pts = [] + for t in torch.linspace(-10, 10, num_points): + p2 = l1.point_at(t) + pts.append(p2) + pts = torch.stack(pts) + + line_est = fit_line(pts[None]) + dir_exp = torch.tensor([0.7071, 0.7071, 0.7071], device=device, dtype=dtype) + # NOTE: result differs with the sign between cpu/cuda + angle_est = torch.nn.functional.cosine_similarity(line_est.direction, dir_exp, -1) + angle_exp = torch.tensor([1.0], device=device, dtype=dtype) + self.assert_close(angle_est.abs(), angle_exp) + + def test_fit_line_weighted(self, device, dtype): + p0 = torch.tensor([0.0, 0.0], device=device, dtype=dtype) + p1 = torch.tensor([1.0, 1.0], device=device, dtype=dtype) + l1 = ParametrizedLine.through(p0, p1) + + num_points = 20 + ts = torch.linspace(-10, 10, num_points) + points = torch.stack([l1.point_at(t) for t in ts]) + + noise = torch.randn_like(points) * 0.05 + points_noisy = points + noise + + distances = torch.norm(points, dim=1) + weights = torch.exp(-distances * 0.2) + weights = weights / weights.max() + + line_est = fit_line(points_noisy[None], weights=weights[None]) + + expected_dir = torch.tensor([0.7071, 0.7071], device=device, dtype=dtype) + expected_dir = expected_dir / expected_dir.norm() + + angle_est = torch.nn.functional.cosine_similarity(line_est.direction, expected_dir, dim=-1) + + assert angle_est.abs() > 0.998 + + @pytest.mark.skip(reason="numerical do not match with analytical") + def test_gradcheck(self, device): + def proxy_func(pts, weights): + line = fit_line(pts, weights) + return line.projection(pts[:, 0].T) + + pts = torch.rand(1, 3, 2, device=device, dtype=torch.float64, requires_grad=True) + weights = torch.rand(1, 3, device=device, dtype=torch.float64, requires_grad=False) + assert gradcheck(proxy_func, (pts, weights), raise_exception=True) + + @pytest.mark.skip(reason="not implemented yet") + def test_cardinality(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_jit(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_exception(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_module(self, device, dtype): + pass diff --git a/tests/geometry/test_normalize.py b/tests/geometry/test_normalize.py new file mode 100644 index 0000000..4dc7877 --- /dev/null +++ b/tests/geometry/test_normalize.py @@ -0,0 +1,98 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +import torch.nn.functional as F + + +# Reference implementations used in the test suite: +def batched_dot_product(x: torch.Tensor, y: torch.Tensor, keepdim: bool = False) -> torch.Tensor: + return (x * y).sum(-1, keepdim) + + +def normalized_orig(v: torch.Tensor, eps: float = 1e-6) -> torch.Tensor: + return v / batched_dot_product(v, v, keepdim=True).add(eps).sqrt() + + +def normalized_fnorm(v: torch.Tensor, eps: float = 1e-6) -> torch.Tensor: + return F.normalize(v, p=2, dim=-1, eps=eps) + + +# -- Reference normalization using explicit loops (earlier implementation) -- +def normalized_ref(v: torch.Tensor, eps: float = 1e-6) -> torch.Tensor: + # clone input and iterate over flattened vectors to preserve batch dims + out = v.clone() + flat = out.view(-1, v.size(-1)) + for i, vec in enumerate(flat): + norm = torch.sqrt((vec * vec).sum() + eps) + flat[i] = vec / norm + return out + + +@pytest.fixture(params=[(2, 3), (4, 5, 6), (10, 128)]) +def random_batch(request): + shape = request.param + torch.manual_seed(0) + return torch.randn(*shape, dtype=torch.float32) + + +def test_batched_dot_product_shape(random_batch): + x = random_batch + y = random_batch.clone() + out1 = batched_dot_product(x, y) + out2 = batched_dot_product(x, y, keepdim=True) + assert out1.shape == x.shape[:-1] + assert out2.shape == x.shape[:-1] + (1,) + + +@pytest.mark.parametrize("fn", [normalized_orig, normalized_fnorm]) +def test_unit_norm(random_batch, fn): + v = random_batch + out = fn(v) + assert out.shape == v.shape + norms = torch.linalg.norm(out.flatten(0, -2), dim=-1) + assert torch.allclose(norms, torch.ones_like(norms), atol=1e-6) + + +@pytest.mark.parametrize("fn", [normalized_orig, normalized_fnorm]) +def test_normalized_against_ref(fn, random_batch): + v = random_batch + out = fn(v) + ref = normalized_ref(v) + assert torch.allclose(out, ref, atol=1e-6), ( + f"{fn.__name__} differs from reference by {(out - ref).abs().max().item():.3e}" + ) + + +@pytest.mark.parametrize("fn", [normalized_fnorm]) +def test_optimized_matches_orig(fn, random_batch): + v = random_batch + out_opt = fn(v) + out_orig = normalized_orig(v) + assert torch.allclose(out_opt, out_orig, atol=1e-6), ( + f"{fn.__name__} differs from normalized_orig by {(out_opt - out_orig).abs().max().item():.3e}" + ) + + +def test_orig_matches_fnormalize(random_batch): + v = random_batch + out_orig = normalized_orig(v) + out_fnorm = F.normalize(v, p=2, dim=-1, eps=1e-6) + assert torch.allclose(out_orig, out_fnorm, atol=1e-6), ( + f"normalized_orig differs from F.normalize by {(out_orig - out_fnorm).abs().max().item():.3e}" + ) diff --git a/tests/geometry/test_plane.py b/tests/geometry/test_plane.py new file mode 100644 index 0000000..604c7c6 --- /dev/null +++ b/tests/geometry/test_plane.py @@ -0,0 +1,150 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.geometry.plane import Hyperplane, fit_plane +from kornia.geometry.vector import Vector3 + +from testing.base import BaseTester + + +# TODO: implement the rest of methods +class TestFitPlane(BaseTester): + @pytest.mark.parametrize("N", (4, 10)) + @pytest.mark.parametrize("D", (3,)) + # @pytest.mark.parametrize("D", (2, 3, 4)) + def test_smoke(self, device, dtype, N, D): + points = torch.ones(N, D, device=device, dtype=dtype) + plane = fit_plane(points) + assert isinstance(plane, Hyperplane) + assert plane.offset.shape == () + assert plane.normal.shape == (D,) + + @pytest.mark.skip(reason="not implemented yet") + def test_cardinality(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_jit(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_exception(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_module(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_gradcheck(self, device): + pass + + +# TODO: implement the rest of methods +class TestHyperplane(BaseTester): + @pytest.mark.parametrize("shape", (None, (1,), (2, 1))) + def test_smoke(self, device, dtype, shape): + p0 = Vector3.random(shape, device, dtype) + n0 = Vector3.random(shape, device, dtype).normalized() + pl0 = Hyperplane.from_vector(n0, p0) + assert pl0.normal.shape == shape or (3,) + assert pl0.offset.shape == ((*shape,) if shape is not None else ()) + + def test_serialization(self, device, dtype, tmp_path): + p = Vector3.random((), device, dtype) + n = Vector3.random((), device, dtype).normalized() + plane = Hyperplane.from_vector(n, p) + + file_path = tmp_path / "plane.pt" + torch.save(plane, file_path) + assert file_path.is_file() + + loaded_plane = torch.load(file_path, weights_only=False) + self.assert_close(plane.normal.unwrap(), loaded_plane.normal.unwrap()) + + # TODO: implement `Vector2` + # @pytest.mark.parametrize("batch_size", [1, 2]) + # def test_through_two(self, device, dtype, batch_size): + # v0 = _VectorType.random((batch_size, 2), device, dtype) + # v1 = _VectorType.random((batch_size, 2), device, dtype) + # # TODO: improve api so that we can accept Vector too + # p0 = Hyperplane.through(v0.data, v1.data) + # assert p0.offset.shape == (batch_size,) + # assert p0.normal.shape == (batch_size, 2) + + @pytest.mark.parametrize("shape", (None, (1,), (2, 1))) + def test_through_three(self, device, dtype, shape): + v0 = Vector3.random(shape, device, dtype) + v1 = Vector3.random(shape, device, dtype) + v2 = Vector3.random(shape, device, dtype) + # TODO: improve api so that we can accept Vector too + p0 = Hyperplane.through(v0, v1, v2) + assert p0.normal.shape == shape or (3,) + assert p0.offset.shape == ((*shape,) if shape is not None else ()) + + @pytest.mark.parametrize("shape", (None, (1,), (2, 1))) + def test_abs_signed_distance(self, device, dtype, shape): + p0 = Vector3.random(shape, device, dtype) + p1 = Vector3.random(shape, device, dtype) + + n0 = Vector3.random(shape, device, dtype).normalized() + n1 = Vector3.random(shape, device, dtype).normalized() + + s0 = torch.rand(shape or (), device=device, dtype=dtype) + s1 = torch.rand(shape or (), device=device, dtype=dtype) + + pl0 = Hyperplane.from_vector(n0, p0) + pl1 = Hyperplane.from_vector(n1, p1) + + expected = torch.ones(shape or (), device=device, dtype=dtype) + self.assert_close(pl1.signed_distance(p1 + n1 * s0[..., None]), s0) + assert (pl0.abs_distance(p0) < expected).all() + assert (pl1.signed_distance(pl1.projection(p0)) < expected).all() + assert (pl1.abs_distance(p1 + pl1.normal * s1) < expected).all() + + def test_projection(self, device, dtype): + v0 = Vector3.from_coords(0.0, 0.0, 0.0, device=device, dtype=dtype) + v1 = Vector3.from_coords(0.0, 1.0, 0.0, device=device, dtype=dtype) + v2 = Vector3.from_coords(0.0, 0.0, 1.0, device=device, dtype=dtype) + plane_in_world = Hyperplane.through(v0, v1, v2) + p_in_world = Vector3.from_coords(0.0, 0.0, 1.0, device=device, dtype=dtype) + p_in_plane = plane_in_world.projection(p_in_world) + p_in_plane_expected = torch.tensor([0.0, 0.0, 1.0], device=device, dtype=dtype) + self.assert_close(p_in_plane, p_in_plane_expected) + + @pytest.mark.skip(reason="not implemented yet") + def test_cardinality(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_jit(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_exception(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_module(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_gradcheck(self, device): + pass diff --git a/tests/geometry/test_pointcloud_io.py b/tests/geometry/test_pointcloud_io.py new file mode 100644 index 0000000..e2c69d8 --- /dev/null +++ b/tests/geometry/test_pointcloud_io.py @@ -0,0 +1,99 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +import numpy as np +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestSaveLoadPointCloud(BaseTester): + def test_save_pointcloud(self): + height, width = 10, 8 + xyz_save = torch.rand(height, width, 3) + + filename = "pointcloud.ply" + kornia.geometry.save_pointcloud_ply(filename, xyz_save) + + xyz_load = kornia.geometry.load_pointcloud_ply(filename) + self.assert_close(xyz_save.reshape(-1, 3), xyz_load) + + if os.path.exists(filename): + os.remove(filename) + + def test_inf_coordinates_save_pointcloud(self): + height, width = 10, 8 + xyz_save = torch.rand(height, width, 3) + + xyz_save[0, 0, :] = float("inf") # all inf → skipped + xyz_save[0, 1, 0] = float("inf") # partial inf → kept + xyz_save[1, 0, :-1] = float("inf") # partial inf → kept + + filename = "pointcloud.ply" + kornia.geometry.save_pointcloud_ply(filename, xyz_save) + + xyz_correct = xyz_save.reshape(-1, 3)[1:, :] + + xyz_load = kornia.geometry.load_pointcloud_ply(filename) + self.assert_close(xyz_correct, xyz_load) + + if os.path.exists(filename): + os.remove(filename) + + def test_invalid_filename_type(self): + xyz_save = torch.rand(10, 3) + with pytest.raises(TypeError): + kornia.geometry.save_pointcloud_ply(1234, xyz_save) + + def test_invalid_filename_extension(self): + xyz_save = torch.rand(10, 3) + with pytest.raises(TypeError): + kornia.geometry.save_pointcloud_ply("pointcloud.txt", xyz_save) + + def test_invalid_pointcloud_type(self): + with pytest.raises(TypeError): + kornia.geometry.save_pointcloud_ply("pointcloud.ply", [[1, 2, 3]]) + + def test_invalid_pointcloud_shape(self): + xyz_save = torch.rand(10, 4) + with pytest.raises(TypeError): + kornia.geometry.save_pointcloud_ply("pointcloud.ply", xyz_save) + + def test_save_pointcloud_with_nan(self): + xyz_save = torch.rand(5, 3) + xyz_save[0, :] = float("nan") + xyz_save[1, 0] = float("nan") + filename = "pointcloud_nan.ply" + kornia.geometry.save_pointcloud_ply(filename, xyz_save) + xyz_load = kornia.geometry.load_pointcloud_ply(filename) + expected = xyz_save[torch.isfinite(xyz_save).any(dim=1)] + + # Use numpy to compare with NaNs considered equal + np.testing.assert_allclose( + expected.detach().cpu().numpy(), + xyz_load.detach().cpu().numpy(), + atol=1e-9, + equal_nan=True, + ) + + if os.path.exists(filename): + os.remove(filename) diff --git a/tests/geometry/test_pose.py b/tests/geometry/test_pose.py new file mode 100644 index 0000000..6cc4e9e --- /dev/null +++ b/tests/geometry/test_pose.py @@ -0,0 +1,131 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.geometry import NamedPose +from kornia.geometry.liegroup import Se2, Se3, So2, So3 + +from testing.base import BaseTester + + +class TestNamedPose(BaseTester): + def test_smoke(self, device, dtype): + b_from_a = Se3.identity(device=device, dtype=dtype) + pose = NamedPose(b_from_a, frame_src="frame_a", frame_dst="frame_b") + assert isinstance(pose, NamedPose) + assert isinstance(pose.pose, Se3) + + @pytest.mark.skip(reason="not implemented yet") + def test_cardinality(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_jit(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_exception(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_module(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_gradcheck(self, device): + pass + + def test_mul(self, device, dtype): + b_from_a = NamedPose( + Se3.trans_x(torch.tensor([1.0], device=device, dtype=dtype)), frame_src="frame_a", frame_dst="frame_b" + ) + c_from_b = NamedPose( + Se3.trans_y(torch.tensor([1.0], device=device, dtype=dtype)), frame_src="frame_b", frame_dst="frame_c" + ) + c_from_a = c_from_b * b_from_a + assert isinstance(c_from_a, NamedPose) + assert isinstance(c_from_a.pose, Se3) + assert c_from_a.frame_src == "frame_a" + assert c_from_a.frame_dst == "frame_c" + + def test_from_rt(self, device, dtype): + b_from_a_rotation = So3.random(device=device, dtype=dtype) + b_from_a_translation = torch.tensor([1.0, 2.0, 3.0], device=device, dtype=dtype) + b_from_a = NamedPose.from_rt(b_from_a_rotation, b_from_a_translation, frame_src="frame_a", frame_dst="frame_b") + assert isinstance(b_from_a, NamedPose) + assert isinstance(b_from_a.pose, Se3) + + b_from_a_rotation = So2.random(device=device, dtype=dtype) + b_from_a_translation = torch.tensor([1.0, 2.0], device=device, dtype=dtype) + b_from_a = NamedPose.from_rt(b_from_a_rotation, b_from_a_translation, frame_src="frame_a", frame_dst="frame_b") + assert isinstance(b_from_a, NamedPose) + assert isinstance(b_from_a.pose, Se2) + + b_from_a_rotation = torch.eye(3, device=device, dtype=dtype) + b_from_a_translation = torch.tensor([1.0, 2.0, 3.0], device=device, dtype=dtype) + b_from_a = NamedPose.from_rt(b_from_a_rotation, b_from_a_translation, frame_src="frame_a", frame_dst="frame_b") + assert isinstance(b_from_a, NamedPose) + assert isinstance(b_from_a.pose, Se3) + + b_from_a_rotation = torch.eye(2, device=device, dtype=dtype) + b_from_a_translation = torch.tensor([1.0, 2.0], device=device, dtype=dtype) + b_from_a = NamedPose.from_rt(b_from_a_rotation, b_from_a_translation, frame_src="frame_a", frame_dst="frame_b") + assert isinstance(b_from_a, NamedPose) + assert isinstance(b_from_a.pose, Se2) + + def test_from_matrix(self, device, dtype): + b_from_a_matrix = Se3.identity(device=device, dtype=dtype).matrix() + b_from_a = NamedPose.from_matrix(b_from_a_matrix, frame_src="frame_a", frame_dst="frame_b") + point_in_a = torch.tensor([1.0, 2.0, 3.0], device=device, dtype=dtype) + point_in_b = b_from_a.transform_points(point_in_a) + self.assert_close(point_in_b, point_in_a) + assert isinstance(b_from_a, NamedPose) + assert isinstance(b_from_a.pose, Se3) + + b_from_a_matrix = torch.eye(3, device=device, dtype=dtype) + b_from_a = NamedPose.from_matrix(b_from_a_matrix, frame_src="frame_a", frame_dst="frame_b") + point_in_a = torch.tensor([1.0, 2.0], device=device, dtype=dtype) + point_in_b = b_from_a.transform_points(point_in_a) + self.assert_close(point_in_b, point_in_a) + assert isinstance(b_from_a, NamedPose) + assert isinstance(b_from_a.pose, Se2) + + def test_inverse(self, device, dtype): + b_from_a = NamedPose( + Se3.trans_x(torch.tensor([1.0], device=device, dtype=dtype)), frame_src="frame_a", frame_dst="frame_b" + ) + a_from_b = b_from_a.inverse() + assert isinstance(a_from_b, NamedPose) + assert isinstance(a_from_b.pose, Se3) + assert a_from_b.frame_src == "frame_b" + assert a_from_b.frame_dst == "frame_a" + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def transform_points(self, device, dtype, batch_size): + if batch_size is None: + points_in_a = torch.randn(3, device=device, dtype=dtype) + b_from_a_se3 = Se3.trans_x(torch.tensor(1.0, device=device, dtype=dtype)) + else: + points_in_a = torch.randn(batch_size, 3, device=device, dtype=dtype) + b_from_a_se3 = Se3.trans_x(torch.tensor([1.0], device=device, dtype=dtype)) + b_from_a = NamedPose(b_from_a_se3, frame_src="frame_a", frame_dst="frame_b") + a_from_b = b_from_a.inverse() + points_in_b = b_from_a.transform_points(points_in_a) + assert points_in_b.shape == points_in_a.shape + self.assert_close(a_from_b.transform_points(points_in_b), points_in_a) diff --git a/tests/geometry/test_quaternion.py b/tests/geometry/test_quaternion.py new file mode 100644 index 0000000..c136094 --- /dev/null +++ b/tests/geometry/test_quaternion.py @@ -0,0 +1,403 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Union + +import pytest +import torch +from torch.nn import Parameter + +from kornia.geometry.quaternion import Quaternion, average_quaternions + +from testing.base import BaseTester + + +class TestQuaternion(BaseTester): + def _make_rand_data(self, device, dtype, batch_size): + shape = [] if batch_size is None else [batch_size] + return torch.rand([*shape, 4], device=device, dtype=dtype) + + def test_smoke(self, device, dtype): + q = Quaternion.from_coeffs(1.0, 0.0, 0.0, 0.0) + q = q.to(device, dtype) + q_data = torch.tensor([1.0, 0.0, 0.0, 0.0], device=device, dtype=dtype) + assert isinstance(q, Quaternion) + assert q.shape == (4,) + self.assert_close(q.data, q_data) + self.assert_close(q.q, q_data) + self.assert_close(q.real, q_data[..., 0]) + self.assert_close(q.scalar, q_data[..., 0]) + self.assert_close(q.vec, q_data[..., 1:]) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_init(self, device, dtype, batch_size): + q1 = Quaternion.identity(batch_size, device, dtype) + q2 = Quaternion(q1.data) + assert isinstance(q2, Quaternion) + self.assert_close(q1, q2) + + def test_init_fail(self, device, dtype): + with pytest.raises(Exception): + _ = Quaternion("q") + + with pytest.raises(Exception): + _ = Quaternion([1, 0, 0, 0]) + + with pytest.raises(Exception): + _ = Quaternion(1, [0, 0, 0]) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_random(self, device, dtype, batch_size): + q = Quaternion.random(batch_size, device, dtype) + q_n = q.normalize().norm() + self.assert_close(q_n, q_n.new_ones(q_n.shape)) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_minus(self, device, dtype, batch_size): + data = self._make_rand_data(device, dtype, batch_size) + q = Quaternion(data) + q = q.to(device, dtype) + self.assert_close(-q, -data) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_add(self, device, dtype, batch_size): + d1 = self._make_rand_data(device, dtype, batch_size) + d2 = self._make_rand_data(device, dtype, batch_size) + q1 = Quaternion(d1) + q2 = Quaternion(d2) + q3 = q1 + q2 + assert isinstance(q3, Quaternion) + self.assert_close(q3, d1 + d2) + q1 += q2 + self.assert_close(q1, q3) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_subtract(self, device, dtype, batch_size): + d1 = self._make_rand_data(device, dtype, batch_size) + d2 = self._make_rand_data(device, dtype, batch_size) + q1 = Quaternion(d1) + q2 = Quaternion(d2) + q3 = q1 - q2 + assert isinstance(q3, Quaternion) + self.assert_close(q3, d1 - d2) + q1 -= q2 + self.assert_close(q1, q3) + + def test_multiplication_of_bases(self, device, dtype): + one = Quaternion.from_coeffs(1.0, 0.0, 0.0, 0.0).to(device, dtype) + i = Quaternion.from_coeffs(0.0, 1.0, 0.0, 0.0).to(device, dtype) + j = Quaternion.from_coeffs(0.0, 0.0, 1.0, 0.0).to(device, dtype) + k = Quaternion.from_coeffs(0.0, 0.0, 0.0, 1.0).to(device, dtype) + + self.assert_close(i * i, j * j) + self.assert_close(j * j, k * k) + self.assert_close(k * k, i * j * k) + self.assert_close(i * j * k, -one) + + self.assert_close(i * j, k) + self.assert_close(i * i, -one) + self.assert_close(i * k, -j) + self.assert_close(j * i, -k) + self.assert_close(j * j, -one) + self.assert_close(j * k, i) + self.assert_close(k * i, j) + self.assert_close(k * j, -i) + self.assert_close(k * k, -one) + self.assert_close(i * j * k, -one) + + def test_division_of_bases(self, device, dtype): + one = Quaternion.from_coeffs(1.0, 0.0, 0.0, 0.0).to(device, dtype) + i = Quaternion.from_coeffs(0.0, 1.0, 0.0, 0.0).to(device, dtype) + j = Quaternion.from_coeffs(0.0, 0.0, 1.0, 0.0).to(device, dtype) + k = Quaternion.from_coeffs(0.0, 0.0, 0.0, 1.0).to(device, dtype) + + self.assert_close(i / i, j / j) + self.assert_close(j / j, k / k) + self.assert_close(k / k, one) + self.assert_close(k / -k, -one) + + self.assert_close(i / j, -k) + self.assert_close(i / i, one) + self.assert_close(i / k, j) + self.assert_close(j / i, k) + self.assert_close(j / j, one) + self.assert_close(j / k, -i) + self.assert_close(k / i, -j) + self.assert_close(k / j, i) + self.assert_close(k / k, one) + self.assert_close(i / -j, k) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_pow(self, device, dtype, batch_size): + q = Quaternion.random(batch_size, device, dtype) + q1 = Quaternion.identity(batch_size, device, dtype) + self.assert_close(q**0, q1) + self.assert_close(q**1, q) + self.assert_close(q**2, q * q) + self.assert_close(q**-1, q.inv()) + self.assert_close((q**0.5) * (q**0.5), q) + self.assert_close((q1**1), q1) + self.assert_close((q1**2), q1) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_quaternion_scalar_multiplication(self, device, dtype, batch_size): + """Test scalar multiplication for issue #3101.""" + # Create a quaternion with parameters to test gradient flow + q_data = torch.tensor([1.0, 0.0, 0.0, 0.0], device=device, dtype=dtype, requires_grad=True) + q = Quaternion(Parameter(q_data)) + + # This should not raise a TypeError + result = q * q * 5 + + # Verify the result has proper gradient tracking + assert result.data.requires_grad + + # Backward pass should work + loss = result.data.sum() + loss.backward() + + # The quaternion's parameter should have gradients + assert q.data.grad is not None + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_inverse(self, device, dtype, batch_size): + q1 = Quaternion.random(batch_size, device, dtype) + q2 = Quaternion.identity(batch_size, device, dtype) + self.assert_close(q1 * q1.inv(), q2, rtol=1e-4, atol=1e-4) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_conjugate(self, device, dtype, batch_size): + q1 = Quaternion.random(batch_size, device, dtype) + q2 = Quaternion.random(batch_size, device, dtype) + self.assert_close((q1 * q2).conj(), q2.conj() * q1.conj()) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_double_conjugate(self, device, dtype, batch_size): + q1 = Quaternion.random(batch_size, device, dtype) + self.assert_close(q1, q1.conj().conj()) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_norm(self, device, dtype, batch_size): + q1 = Quaternion.random(batch_size, device, dtype) + q2 = Quaternion.random(batch_size, device, dtype) + self.assert_close((q1 * q2).norm(), q1.norm() * q2.norm()) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_norm_shape(self, device, dtype, batch_size): + q = Quaternion.random(batch_size, device, dtype) + expected_shape = () if batch_size is None else (batch_size,) + self.assert_close(tuple(q.norm().shape), expected_shape) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_normalize(self, device, dtype, batch_size): + q1 = Quaternion.random(batch_size, device, dtype) + q1_n = q1.normalize().norm() + self.assert_close(q1_n, q1_n.new_ones(q1_n.shape)) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_matrix(self, device, dtype, batch_size): + q1 = Quaternion.random(batch_size, device, dtype) + m1 = q1.matrix() + q2 = Quaternion.from_matrix(m1) + for qq1, qq2 in zip(q1.data, q2.data): + try: + self.assert_close(qq1, qq2) + except Exception: + self.assert_close(qq1, -qq2) + + @pytest.mark.parametrize("batch_size", (1, 2, 5)) + def test_getitem(self, device, dtype, batch_size): + q = Quaternion.random(batch_size, device, dtype) + for i in range(batch_size): + q1 = q[i] + self.assert_close(q1.data, q.data[i]) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_axis_angle(self, device, dtype, batch_size): + q1 = Quaternion.random(batch_size, device, dtype) + angle = 2 * q1.scalar.arccos()[..., None] + axis = q1.vec / (angle / 2).sin() + axis_angle = axis * angle + q2 = Quaternion.from_axis_angle(axis_angle) + q2 = q2.to(device, dtype) + self.assert_close(q1, q2) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_to_axis_angle(self, device, dtype, batch_size): + # batch_s = 5 + # random_coefs = Quaternion.random(batch_s).data + random_coefs = torch.tensor( + [ + [2.5398e-04, -2.2677e-01, -8.3897e-01, 4.9467e-01], + [-1.7005e-01, -1.0974e-01, 3.7635e-01, -9.0410e-01], + [9.1273e-01, 4.8935e-02, -6.2994e-03, 4.0558e-01], + [-9.8316e-01, 5.4078e-03, 1.4471e-01, 1.1145e-01], + [4.5794e-02, -7.0831e-01, 6.7577e-01, 1.9883e-01], + ], + device=device, + dtype=dtype, + ) + + q = Quaternion(random_coefs) + axis_angle_actual = q.to_axis_angle() + + axis_angle_expected = torch.tensor( + [ + [-0.7123, -2.6353, 1.5538], + [0.3118, -1.0693, 2.5687], + [0.1008, -0.0130, 0.8356], + [-0.0109, -0.2911, -0.2242], + [-2.1626, 2.0632, 0.6071], + ], + device=device, + dtype=dtype, + ) + + self.assert_close(axis_angle_expected, axis_angle_actual, 1e-4, 1e-4) + + @pytest.mark.parametrize("batch_size", (None, 1, 2, 5)) + def test_slerp(self, device, dtype, batch_size): + for axis in torch.tensor([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]): + axis = axis.to(device, dtype) + if batch_size is not None: + axis = axis.repeat(batch_size, 1) + q1 = Quaternion.from_axis_angle(axis * 0) + q1.to(device, dtype) + q2 = Quaternion.from_axis_angle(axis * 3.14159) + q2.to(device, dtype) + for t in torch.linspace(0.1, 1, 10): + q3 = q1.slerp(q2, t) + q4 = Quaternion.from_axis_angle(axis * t * 3.14159) + self.assert_close(q3, q4) + + def test_from_to_euler_values(self, device, dtype): + # num_samples = 5 + # data = 2 * torch.rand(3, num_samples, device=device, dtype=dtype) - 1 + # roll, pitch, yaw = torch.pi * data + roll = torch.tensor( + [2.6518599987, 0.0612506270, 1.2417907715, 2.8829660416, -1.9961174726, 0], device=device, dtype=dtype + ) + + pitch = torch.tensor( + [2.3267219067, -2.7309591770, -1.4011553526, -2.1962766647, 2.1454355717, 0], device=device, dtype=dtype + ) + + yaw = torch.tensor( + [-0.8856627345, 0.2605336905, 0.4579202533, -1.3095731735, 0.6096843481, 0], device=device, dtype=dtype + ) + + euler_expected = torch.tensor( + [ + [-0.4897327125, 0.8148705959, 2.2559301853], + [-3.0803420544, -0.4106334746, -2.8810589314], + [1.2417914867, -1.4011553526, 0.4579201937], + [-0.2586266696, -0.9453159571, 1.8320195675], + [1.1454752684, 0.9961569905, -2.5319085121], + [0, 0, 0], + ], + device=device, + dtype=dtype, + ) + + q = Quaternion.from_euler(roll, pitch, yaw) + euler = q.to_euler() + euler = torch.stack(euler, -1) + + self.assert_close(euler, euler_expected, 1e-4, 1e-4) + + +def _to_tensor(x: Union[torch.Tensor, Quaternion]) -> torch.Tensor: + # Unwrap Quaternion/Parameter to a plain Tensor for comparisons + if isinstance(x, Quaternion): + x = x.data + if isinstance(x, torch.nn.Parameter): + x = x.data + if x.ndim == 2 and x.shape[0] == 1: + x = x.squeeze(0) + return x + + +def _align_sign(q: torch.Tensor, ref: torch.Tensor) -> torch.Tensor: + # Flip sign of q so it points roughly in the same direction as ref + if torch.dot(q, ref) < 0: + return -q + return q + + +class TestQuaternionAverage(BaseTester): + @pytest.mark.parametrize("M", [1, 2, 5, 10]) + def test_average_identity(self, device, dtype, M): + """All identity quaternions → should return identity""" + Q = Quaternion.identity(M, device=device, dtype=dtype) + out = average_quaternions(Q) + q = _to_tensor(out) + + expected = torch.tensor([1.0, 0.0, 0.0, 0.0], device=device, dtype=dtype) + q = _align_sign(q, expected) + + self.assert_close(q, expected, rtol=1e-6, atol=1e-6) + + def test_output_is_normalized(self, device, dtype): + """Averaged quaternion should always have unit norm""" + Q = Quaternion.random(6, device=device, dtype=dtype).normalize() + out = average_quaternions(Q) + q = _to_tensor(out) + + self.assert_close(q.norm(), torch.tensor(1.0, device=device, dtype=dtype), rtol=1e-6, atol=1e-6) + + def test_weighted_bias(self, device, dtype): + """Heavier weights should bias the average toward the corresponding quaternion""" + q1 = Quaternion(torch.tensor([[1.0, 0.0, 0.0, 0.0]], device=device, dtype=dtype)) + q2 = Quaternion(torch.tensor([[0.0, 1.0, 0.0, 0.0]], device=device, dtype=dtype)) + Q = Quaternion(torch.cat([q1.data, q2.data], dim=0)) + + w = torch.tensor([0.9, 0.1], device=device, dtype=dtype) + out = average_quaternions(Q, w=w) + q = _to_tensor(out) + + dot1 = torch.dot(q, q1.data.squeeze()) + dot2 = torch.dot(q, q2.data.squeeze()) + assert dot1 > dot2 # should align closer to q1 + + def test_single_quaternion_returns_itself(self, device, dtype): + """Averaging a single quaternion should return it""" + q = Quaternion.random(1, device=device, dtype=dtype).normalize() + out = average_quaternions(q) + out_t = _to_tensor(out) + q_t = _to_tensor(q) + + out_t = _align_sign(out_t, q_t) + self.assert_close(out_t, q_t, rtol=1e-6, atol=1e-6) + + def test_opposite_quaternions(self, device, dtype): + """Opposite quaternions should average to something consistent (sign ambiguity)""" + q1 = Quaternion.identity(1, device=device, dtype=dtype) + q2 = Quaternion(-q1.data.clone()) + Q = Quaternion(torch.cat([q1.data, q2.data], dim=0)) + + out = average_quaternions(Q) + q = _to_tensor(out) + + # Should still be a valid unit quaternion + self.assert_close(q.norm(), torch.tensor(1.0, device=device, dtype=dtype), rtol=1e-6, atol=1e-6) + + def test_invalid_weights_raise(self, device, dtype): + """Mismatched number of weights should raise""" + Q = Quaternion.random(3, device=device, dtype=dtype) + w = torch.tensor([0.5, 0.5], device=device, dtype=dtype) # wrong length + with pytest.raises(ValueError): + average_quaternions(Q, w=w) diff --git a/tests/geometry/test_ransac.py b/tests/geometry/test_ransac.py new file mode 100644 index 0000000..5451b2e --- /dev/null +++ b/tests/geometry/test_ransac.py @@ -0,0 +1,370 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import pytest +import torch + +from kornia.geometry import RANSAC, transform_points +from kornia.geometry.epipolar import sampson_epipolar_distance + +from testing.base import BaseTester +from testing.casts import dict_to + + +class TestRANSACHomography(BaseTester): + def test_smoke(self, device, dtype): + torch.random.manual_seed(0) + points1 = torch.rand(4, 2, device=device, dtype=dtype) + points2 = torch.rand(4, 2, device=device, dtype=dtype) + ransac = RANSAC("homography").to(device=device, dtype=dtype) + torch.random.manual_seed(0) + H, _ = ransac(points1, points2) + assert H.shape == (3, 3) + + @pytest.mark.xfail(reason="might slightly and randomly imprecise due to RANSAC randomness") + def test_dirty_points(self, device, dtype): + # generate input data + torch.random.manual_seed(0) + + H = torch.eye(3, dtype=dtype, device=device) + H[:2] = H[:2] + 0.1 * torch.rand_like(H[:2]) + H[2:, :2] = H[2:, :2] + 0.001 * torch.rand_like(H[2:, :2]) + + points_src = 100.0 * torch.rand(1, 20, 2, device=device, dtype=dtype) + points_dst = transform_points(H[None], points_src) + + # making last point an outlier + points_dst[:, -1, :] += 800 + ransac = RANSAC("homography", inl_th=0.5, max_iter=20).to(device=device, dtype=dtype) + # compute transform from source to target + dst_homo_src, _ = ransac(points_src[0], points_dst[0]) + + self.assert_close( + transform_points(dst_homo_src[None], points_src[:, :-1]), points_dst[:, :-1], rtol=1e-3, atol=1e-3 + ) + + @pytest.mark.slow + @pytest.mark.parametrize("data", ["loftr_homo"], indirect=True) + def test_real_clean(self, device, dtype, data): + # generate input data + torch.random.manual_seed(0) + data_dev = dict_to(data, device, dtype) + homography_gt = torch.inverse(data_dev["H_gt"]) + homography_gt = homography_gt / homography_gt[2, 2] + pts_src = data_dev["pts0"] + pts_dst = data_dev["pts1"] + ransac = RANSAC("homography", inl_th=0.5, max_iter=20).to(device=device, dtype=dtype) + # compute transform from source to target + dst_homo_src, _ = ransac(pts_src, pts_dst) + + self.assert_close(transform_points(dst_homo_src[None], pts_src[None]), pts_dst[None], rtol=1e-2, atol=1.0) + + @pytest.mark.slow + @pytest.mark.xfail(reason="might slightly and randomly imprecise due to RANSAC randomness") + @pytest.mark.parametrize("data", ["loftr_homo"], indirect=True) + def test_real_dirty(self, device, dtype, data): + # generate input data + torch.random.manual_seed(0) + data_dev = dict_to(data, device, dtype) + homography_gt = torch.inverse(data_dev["H_gt"]) + homography_gt = homography_gt / homography_gt[2, 2] + pts_src = data_dev["pts0"] + pts_dst = data_dev["pts1"] + + kp1 = data_dev["loftr_outdoor_tentatives0"] + kp2 = data_dev["loftr_outdoor_tentatives1"] + + ransac = RANSAC("homography", inl_th=3.0, max_iter=30, max_lo_iters=10).to(device=device, dtype=dtype) + # compute transform from source to target + dst_homo_src, _ = ransac(kp1, kp2) + + # Reprojection error of 5px is OK + self.assert_close(transform_points(dst_homo_src[None], pts_src[None]), pts_dst[None], rtol=0.15, atol=5) + + @pytest.mark.skip(reason="find_homography_dlt is using try/except block") + def test_jit(self, device, dtype): + torch.random.manual_seed(0) + points1 = torch.rand(4, 2, device=device, dtype=dtype) + points2 = torch.rand(4, 2, device=device, dtype=dtype) + model = RANSAC("homography").to(device=device, dtype=dtype) + model_jit = torch.jit.script(RANSAC("homography").to(device=device, dtype=dtype)) + self.assert_close(model(points1, points2)[0], model_jit(points1, points2)[0], rtol=1e-4, atol=1e-4) + + +class TestRANSACHomographyLineSegments(BaseTester): + def test_smoke(self, device, dtype): + torch.random.manual_seed(0) + points1 = torch.rand(4, 2, 2, device=device, dtype=dtype) + points2 = torch.rand(4, 2, 2, device=device, dtype=dtype) + ransac = RANSAC("homography_from_linesegments").to(device=device, dtype=dtype) + torch.random.manual_seed(0) + H, _ = ransac(points1, points2) + assert H.shape == (3, 3) + + @pytest.mark.xfail(reason="might slightly and randomly imprecise due to RANSAC randomness") + def test_dirty_points(self, device, dtype): + # generate input data + torch.random.manual_seed(0) + + H = torch.eye(3, dtype=dtype, device=device) + H[:2] = H[:2] + 0.1 * torch.rand_like(H[:2]) + H[2:, :2] = H[2:, :2] + 0.001 * torch.rand_like(H[2:, :2]) + + points_src_st = 100.0 * torch.rand(1, 20, 2, device=device, dtype=dtype) + points_src_end = 100.0 * torch.rand(1, 20, 2, device=device, dtype=dtype) + + points_dst_st = transform_points(H[None], points_src_st) + points_dst_end = transform_points(H[None], points_src_end) + + # making last point an outlier + points_dst_st[:, -1, :] += 800 + ls1 = torch.stack([points_src_st, points_src_end], dim=2) + ls2 = torch.stack([points_dst_st, points_dst_end], dim=2) + + ransac = RANSAC("homography_from_linesegments", inl_th=0.5, max_iter=20).to(device=device, dtype=dtype) + # compute transform from source to target + dst_homo_src, _ = ransac(ls1[0], ls2[0]) + + self.assert_close( + transform_points(dst_homo_src[None], points_src_st[:, :-1]), points_dst_st[:, :-1], rtol=1e-3, atol=1e-3 + ) + + @pytest.mark.skip(reason="find_homography_dlt is using try/except block") + def test_jit(self, device, dtype): + torch.random.manual_seed(0) + points1 = torch.rand(4, 2, 2, device=device, dtype=dtype) + points2 = torch.rand(4, 2, 2, device=device, dtype=dtype) + model = RANSAC("homography_from_linesegments").to(device=device, dtype=dtype) + model_jit = torch.jit.script(RANSAC("homography_from_linesegments").to(device=device, dtype=dtype)) + self.assert_close(model(points1, points2)[0], model_jit(points1, points2)[0], rtol=1e-4, atol=1e-4) + + +class TestRANSACFundamental(BaseTester): + def test_smoke(self, device, dtype): + torch.random.manual_seed(0) + points1 = torch.rand(8, 2, device=device, dtype=dtype) + points2 = torch.rand(8, 2, device=device, dtype=dtype) + ransac = RANSAC("fundamental").to(device=device, dtype=dtype) + Fm, _ = ransac(points1, points2) + assert Fm.shape == (3, 3) + + @pytest.mark.slow + @pytest.mark.xfail(reason="might slightly and randomly imprecise due to RANSAC randomness") + @pytest.mark.parametrize("data", ["loftr_fund"], indirect=True) + def test_real_clean_8pt(self, device, dtype, data): + torch.random.manual_seed(0) + # generate input data + data_dev = dict_to(data, device, dtype) + pts_src = data_dev["pts0"] + pts_dst = data_dev["pts1"] + # compute transform from source to target + ransac = RANSAC("fundamental", inl_th=0.5, max_iter=20, max_lo_iters=10).to(device=device, dtype=dtype) + fundamental_matrix, _ = ransac(pts_src, pts_dst) + gross_errors = ( + sampson_epipolar_distance(pts_src[None], pts_dst[None], fundamental_matrix[None], squared=False) > 1.0 + ) + assert gross_errors.sum().item() == 0 + + @pytest.mark.slow + @pytest.mark.xfail(reason="might fail, because out F-RANSAC is not yet 7pt") + @pytest.mark.parametrize("data", ["loftr_fund"], indirect=True) + def test_real_clean_7pt(self, device, dtype, data): + torch.random.manual_seed(0) + # generate input data + data_dev = dict_to(data, device, dtype) + pts_src = data_dev["pts0"] + pts_dst = data_dev["pts1"] + # compute transform from source to target + ransac = RANSAC("fundamental_7pt", inl_th=1.0, max_iter=100, max_lo_iters=10).to(device=device, dtype=dtype) + fundamental_matrix, _ = ransac(pts_src, pts_dst) + gross_errors = ( + sampson_epipolar_distance(pts_src[None], pts_dst[None], fundamental_matrix[None], squared=False) > 1.0 + ) + assert gross_errors.sum().item() == 0 + + @pytest.mark.slow + @pytest.mark.xfail(reason="might slightly and randomly imprecise due to RANSAC randomness") + @pytest.mark.parametrize("data", ["loftr_fund"], indirect=True) + def test_real_dirty_8pt(self, device, dtype, data): + torch.random.manual_seed(0) + # generate input data + data_dev = dict_to(data, device, dtype) + pts_src = data_dev["pts0"] + pts_dst = data_dev["pts1"] + + kp1 = data_dev["loftr_indoor_tentatives0"] + kp2 = data_dev["loftr_indoor_tentatives1"] + + ransac = RANSAC("fundamental", inl_th=1.0, max_iter=20, max_lo_iters=10).to(device=device, dtype=dtype) + # compute transform from source to target + fundamental_matrix, _ = ransac(kp1, kp2) + gross_errors = ( + sampson_epipolar_distance(pts_src[None], pts_dst[None], fundamental_matrix[None], squared=False) > 10.0 + ) + assert gross_errors.sum().item() < 2 + + @pytest.mark.slow + @pytest.mark.xfail(reason="might fail, because this F-RANSAC is not 7pt") + @pytest.mark.parametrize("data", ["loftr_fund"], indirect=True) + def test_real_dirty_7pt(self, device, dtype, data): + torch.random.manual_seed(0) + # generate input data + data_dev = dict_to(data, device, dtype) + pts_src = data_dev["pts0"] + pts_dst = data_dev["pts1"] + + kp1 = data_dev["loftr_indoor_tentatives0"] + kp2 = data_dev["loftr_indoor_tentatives1"] + + ransac = RANSAC("fundamental_7pt", inl_th=1.0, max_iter=20, max_lo_iters=10).to(device=device, dtype=dtype) + # compute transform from source to target + fundamental_matrix, _ = ransac(kp1, kp2) + gross_errors = ( + sampson_epipolar_distance(pts_src[None], pts_dst[None], fundamental_matrix[None], squared=False) > 10.0 + ) + assert gross_errors.sum().item() < 2 + + @pytest.mark.skip(reason="try except block in python version") + def test_jit(self, device, dtype): + torch.random.manual_seed(0) + points1 = torch.rand(8, 2, device=device, dtype=dtype) + points2 = torch.rand(8, 2, device=device, dtype=dtype) + model = RANSAC("fundamental").to(device=device, dtype=dtype) + model_jit = torch.jit.script(model) + self.assert_close(model(points1, points2)[0], model_jit(points1, points2)[0], rtol=1e-3, atol=1e-3) + + @pytest.mark.skip(reason="RANSAC is random algorithm, so Jacobian is not defined") + def test_gradcheck(self, device): + torch.random.manual_seed(0) + points1 = torch.rand(8, 2, device=device, dtype=torch.float64, requires_grad=True) + points2 = torch.rand(8, 2, device=device, dtype=torch.float64) + model = RANSAC("fundamental").to(device=device, dtype=torch.float64) + + def gradfun(p1, p2): + return model(p1, p2)[0] + + self.gradcheck(gradfun, (points1, points2), fast_mode=False, requires_grad=(True, False, False)) + + +class TestRansacMethods: + def test_max_samples_by_conf(self): + """Test max_samples_by_conf with realistic scenarios.""" + conf = 0.99 + + # Test 1: Very few inliers (1 out of 1000) with sample_size=7 + # Returns 1 because n_inl <= sample_size (early exit condition) + x = RANSAC.max_samples_by_conf(n_inl=1, num_tc=1000, sample_size=7, conf=conf) + assert x == 1 # Early exit when n_inl <= sample_size + + # Test 2: Low inlier ratio (10 out of 1000, 1%) with sample_size=7 + # Should require many iterations + x = RANSAC.max_samples_by_conf(n_inl=10, num_tc=1000, sample_size=7, conf=conf) + assert x > 0 + # With 1% inliers, probability of all 7 samples being inliers is very low + # So we need a huge number of samples + assert x > 1_000_000 # Should be a very large number + + # Test 3: Medium inlier ratio (500 out of 1000, 50%) with sample_size=4 + # Should require moderate number of iterations + x = RANSAC.max_samples_by_conf(n_inl=500, num_tc=1000, sample_size=4, conf=conf) + assert x > 0 + # With 50% inliers, probability of all 4 samples being inliers is ~0.0625 + # So we need log(1-0.99)/log(1-0.0625) ≈ 70 samples + assert 50 < x < 100 + + # Test 4: High inlier ratio (900 out of 1000, 90%) with sample_size=4 + # Should require very few iterations + x = RANSAC.max_samples_by_conf(n_inl=900, num_tc=1000, sample_size=4, conf=conf) + assert x > 0 + # With 90% inliers, probability of all 4 samples being inliers is ~0.66 + # So we need log(1-0.99)/log(1-0.66) ≈ 4 samples + assert x < 10 + + # Test 5: Edge case - all points are inliers + x = RANSAC.max_samples_by_conf(n_inl=100, num_tc=100, sample_size=4, conf=conf) + assert x == 1 # Only need 1 sample if all are inliers + + # Test 6: Edge case - not enough points for sample + x = RANSAC.max_samples_by_conf(n_inl=10, num_tc=10, sample_size=15, conf=conf) + assert x == 1 # Returns 1 when num_tc <= sample_size + + # Test 7: Edge case - too few inliers (n_inl <= sample_size) + x = RANSAC.max_samples_by_conf(n_inl=2, num_tc=1000, sample_size=4, conf=conf) + assert x == 1 # Returns 1 when n_inl <= sample_size + + # Test 8: Edge case - confidence at boundaries + x = RANSAC.max_samples_by_conf(n_inl=50, num_tc=100, sample_size=4, conf=1.0) + assert x == 1 # Returns 1 when conf >= 1.0 + + x = RANSAC.max_samples_by_conf(n_inl=50, num_tc=100, sample_size=4, conf=0.0) + assert x == 1 # Returns 1 when conf <= 0.0 + + # Test 9: Verify monotonicity - more inliers should require fewer samples + x1 = RANSAC.max_samples_by_conf(n_inl=200, num_tc=1000, sample_size=4, conf=conf) + x2 = RANSAC.max_samples_by_conf(n_inl=500, num_tc=1000, sample_size=4, conf=conf) + x3 = RANSAC.max_samples_by_conf(n_inl=800, num_tc=1000, sample_size=4, conf=conf) + assert x1 > x2 > x3 # More inliers = fewer samples needed + + +class TestRANSACSeed: + def test_same_seed_reproducible(self, device, dtype): + """Same seed should produce identical results across two calls.""" + torch.manual_seed(42) + points1 = torch.rand(20, 2, device=device, dtype=dtype) + points2 = torch.rand(20, 2, device=device, dtype=dtype) + + ransac = RANSAC("homography", inl_th=2.0, max_iter=5, seed=123).to(device=device, dtype=dtype) + H1, inliers1 = ransac(points1, points2) + H2, inliers2 = ransac(points1, points2) + + assert torch.allclose(H1, H2) + assert torch.equal(inliers1, inliers2) + + def test_different_seeds_differ(self, device, dtype): + """Different seeds should (very likely) produce different results.""" + torch.manual_seed(42) + points1 = torch.rand(20, 2, device=device, dtype=dtype) + points2 = torch.rand(20, 2, device=device, dtype=dtype) + + ransac_a = RANSAC("homography", inl_th=2.0, max_iter=5, seed=1).to(device=device, dtype=dtype) + ransac_b = RANSAC("homography", inl_th=2.0, max_iter=5, seed=2).to(device=device, dtype=dtype) + + H_a, _ = ransac_a(points1, points2) + H_b, _ = ransac_b(points1, points2) + + assert not torch.allclose(H_a, H_b) + + def test_no_seed_differs_from_seeded(self, device, dtype): + """Without a seed, repeated calls should not be forced to be identical to a seeded call.""" + torch.manual_seed(0) + points1 = torch.rand(20, 2, device=device, dtype=dtype) + points2 = torch.rand(20, 2, device=device, dtype=dtype) + + ransac_seeded = RANSAC("homography", inl_th=2.0, max_iter=5, seed=42).to(device=device, dtype=dtype) + H_s1, _ = ransac_seeded(points1, points2) + H_s2, _ = ransac_seeded(points1, points2) + # Seeded should be reproducible + assert torch.allclose(H_s1, H_s2) + + def test_seed_stored_as_attribute(self, device, dtype): + ransac = RANSAC("homography", seed=7) + assert ransac.seed == 7 + + def test_none_seed_stored(self, device, dtype): + ransac = RANSAC("homography") + assert ransac.seed is None diff --git a/tests/geometry/test_ray.py b/tests/geometry/test_ray.py new file mode 100644 index 0000000..a776217 --- /dev/null +++ b/tests/geometry/test_ray.py @@ -0,0 +1,41 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch + +from kornia.geometry.line import ParametrizedLine +from kornia.geometry.ray import Ray + + +def test_ray_is_parametrized_line(): + assert Ray is ParametrizedLine + + +def test_ray_smoke(): + origin = torch.zeros(3) + direction = torch.tensor([0.0, 0.0, 1.0]) + ray = Ray(origin, direction) + assert isinstance(ray, ParametrizedLine) + + +def test_ray_point_at(): + origin = torch.zeros(3) + direction = torch.tensor([1.0, 0.0, 0.0]) + ray = Ray(origin, direction) + pt = ray.point_at(2.0) + expected = torch.tensor([2.0, 0.0, 0.0]) + assert torch.allclose(pt, expected) diff --git a/tests/geometry/test_vector.py b/tests/geometry/test_vector.py new file mode 100644 index 0000000..79c3f61 --- /dev/null +++ b/tests/geometry/test_vector.py @@ -0,0 +1,163 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.geometry.vector import Scalar, Vector2, Vector3 + +from testing.base import BaseTester + + +class TestVector3(BaseTester): + def test_smoke(self, device, dtype): + vec = Vector3.random(device=device, dtype=dtype) + assert vec.shape == (3,) + assert vec.x is not None + assert vec.y is not None + assert vec.z is not None + + @pytest.mark.parametrize("batch_size", (1, 2, 5)) + def test_getitem(self, device, dtype, batch_size): + xyz = torch.rand((batch_size, 3), device=device, dtype=dtype) + vec = Vector3(xyz) + for i in range(batch_size): + v = vec[i] + self.assert_close(v.data, xyz[i, ...]) + + @pytest.mark.parametrize("shape", ((), (1,), (2, 4))) + def test_cardinality(self, device, dtype, shape): + vec = Vector3.random(shape, device, dtype) + assert vec.shape[:-1] == shape + assert vec.x.shape == shape + assert vec.y.shape == shape + assert vec.z.shape == shape + + def test_from_coords(self): + vec = Vector3.from_coords(0.0, 1.0, 0.0) + assert vec.shape == (3,) + assert vec.x == 0.0 + assert vec.y == 1.0 + assert vec.z == 0.0 + + @pytest.mark.parametrize("shape", ((), (1,), (2, 4))) + def test_from_coords_tensor(self, device, dtype, shape): + xyz = torch.rand((*shape, 3), device=device, dtype=dtype) + vec = Vector3.from_coords(xyz[..., 0], xyz[..., 1], xyz[..., 2]) + assert vec.shape[:-1] == shape + assert vec.x.shape == shape + assert vec.y.shape == shape + assert vec.z.shape == shape + + @pytest.mark.parametrize("shape", (None, (1,), (2, 1))) + def test_dot(self, device, dtype, shape): + p0 = Vector3.random(shape, device, dtype) + n0 = Vector3.random(shape, device, dtype).normalized() + res: Scalar = p0.dot(n0) + assert res.shape == () if shape is None else shape + expected = torch.ones(shape or (), device=device, dtype=dtype) + self.assert_close(n0.dot(n0), expected) + + @pytest.mark.parametrize("shape", (None, (1,), (2, 1))) + def test_squared_norm(self, device, dtype, shape): + p0 = Vector3.random(shape, device, dtype) + res: Scalar = p0.squared_norm() + assert res.shape == () if shape is None else shape + + @pytest.mark.skip(reason="not implemented yet") + def test_jit(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_exception(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_module(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_gradcheck(self, device): + pass + + +class TestVector2(BaseTester): + def test_smoke(self, device, dtype): + vec = Vector2.random(device=device, dtype=dtype) + assert vec.shape == (2,) + assert vec.x is not None + assert vec.y is not None + + @pytest.mark.parametrize("batch_size", (1, 2, 5)) + def test_getitem(self, device, dtype, batch_size): + xy = torch.rand((batch_size, 2), device=device, dtype=dtype) + vec = Vector2(xy) + for i in range(batch_size): + v = vec[i] + self.assert_close(v.data, xy[i, ...]) + + @pytest.mark.parametrize("shape", ((), (1,), (2, 4))) + def test_cardinality(self, device, dtype, shape): + vec = Vector2.random(shape, device, dtype) + assert vec.shape[:-1] == shape + assert vec.x.shape == shape + assert vec.y.shape == shape + + def test_from_coords(self): + vec = Vector2.from_coords(0.0, 1.0) + assert vec.shape == (2,) + assert vec.x == 0.0 + assert vec.y == 1.0 + + @pytest.mark.parametrize("shape", ((), (1,), (2, 4))) + def test_from_coords_tensor(self, device, dtype, shape): + xy = torch.rand((*shape, 2), device=device, dtype=dtype) + vec = Vector2.from_coords(xy[..., 0], xy[..., 1]) + assert vec.shape[:-1] == shape + assert vec.x.shape == shape + assert vec.y.shape == shape + + @pytest.mark.parametrize("shape", (None, (1,), (2, 1))) + def test_dot(self, device, dtype, shape): + p0 = Vector2.random(shape, device, dtype) + n0 = Vector2.random(shape, device, dtype).normalized() + res: Scalar = p0.dot(n0) + assert res.shape == () if shape is None else shape + expected = torch.ones(shape or (), device=device, dtype=dtype) + self.assert_close(n0.dot(n0), expected) + + @pytest.mark.parametrize("shape", (None, (1,), (2, 1))) + def test_squared_norm(self, device, dtype, shape): + p0 = Vector2.random(shape, device, dtype) + res: Scalar = p0.squared_norm() + assert res.shape == () if shape is None else shape + + @pytest.mark.skip(reason="not implemented yet") + def test_jit(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_exception(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_module(self, device, dtype): + pass + + @pytest.mark.skip(reason="not implemented yet") + def test_gradcheck(self, device): + pass diff --git a/tests/geometry/transform/test_affine.py b/tests/geometry/transform/test_affine.py new file mode 100644 index 0000000..3ba4e47 --- /dev/null +++ b/tests/geometry/transform/test_affine.py @@ -0,0 +1,803 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestResize(BaseTester): + def test_smoke(self, device, dtype): + inp = torch.rand(1, 3, 3, 4, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, (3, 4), align_corners=False) + self.assert_close(inp, out, atol=1e-4, rtol=1e-4) + + # 2D + inp = torch.rand(3, 4, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, (3, 4), align_corners=False) + self.assert_close(inp, out, atol=1e-4, rtol=1e-4) + + # 3D + inp = torch.rand(3, 3, 4, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, (3, 4), align_corners=False) + self.assert_close(inp, out, atol=1e-4, rtol=1e-4) + + # arbitrary dim + inp = torch.rand(1, 2, 3, 2, 1, 3, 3, 4, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, (3, 4), align_corners=False) + self.assert_close(inp, out, atol=1e-4, rtol=1e-4) + + def test_upsize(self, device, dtype): + inp = torch.rand(1, 3, 3, 4, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, (6, 8), align_corners=False) + assert out.shape == (1, 3, 6, 8) + + # 2D + inp = torch.rand(3, 4, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, (6, 8), align_corners=False) + assert out.shape == (6, 8) + + # 3D + inp = torch.rand(3, 3, 4, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, (6, 8), align_corners=False) + assert out.shape == (3, 6, 8) + + # arbitrary dim + inp = torch.rand(1, 2, 3, 2, 1, 3, 3, 4, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, (6, 8), align_corners=False) + assert out.shape == (1, 2, 3, 2, 1, 3, 6, 8) + + def test_downsize(self, device, dtype): + inp = torch.rand(1, 3, 5, 2, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, (3, 1), align_corners=False) + assert out.shape == (1, 3, 3, 1) + + # 2D + inp = torch.rand(5, 2, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, (3, 1), align_corners=False) + assert out.shape == (3, 1) + + # 3D + inp = torch.rand(3, 5, 2, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, (3, 1), align_corners=False) + assert out.shape == (3, 3, 1) + + # arbitrary dim + inp = torch.rand(1, 2, 3, 2, 1, 3, 5, 2, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, (3, 1), align_corners=False) + assert out.shape == (1, 2, 3, 2, 1, 3, 3, 1) + + def test_downsizeAA(self, device, dtype): + inp = torch.rand(1, 3, 10, 8, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, (5, 3), align_corners=False, antialias=True) + assert out.shape == (1, 3, 5, 3) + + inp = torch.rand(1, 1, 20, 10, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, (15, 8), align_corners=False, antialias=True) + assert out.shape == (1, 1, 15, 8) + + # 2D + inp = torch.rand(10, 8, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, (5, 3), align_corners=False, antialias=True) + assert out.shape == (5, 3) + + # 3D + inp = torch.rand(3, 10, 8, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, (5, 3), align_corners=False, antialias=True) + assert out.shape == (3, 5, 3) + + # arbitrary dim + inp = torch.rand(1, 2, 3, 2, 1, 3, 10, 8, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, (5, 3), align_corners=False, antialias=True) + assert out.shape == (1, 2, 3, 2, 1, 3, 5, 3) + + def test_one_param(self, device, dtype): + inp = torch.rand(1, 3, 5, 2, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, 10, align_corners=False) + assert out.shape == (1, 3, 25, 10) + + # 2D + inp = torch.rand(5, 2, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, 10, align_corners=False) + assert out.shape == (25, 10) + + # 3D + inp = torch.rand(3, 5, 2, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, 10, align_corners=False) + assert out.shape == (3, 25, 10) + + # arbitrary dim + inp = torch.rand(1, 2, 3, 2, 1, 3, 5, 2, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, 10, align_corners=False) + assert out.shape == (1, 2, 3, 2, 1, 3, 25, 10) + + def test_one_param_long(self, device, dtype): + inp = torch.rand(1, 3, 5, 2, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, 10, align_corners=False, side="long") + assert out.shape == (1, 3, 10, 4) + + # 2D + inp = torch.rand(5, 2, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, 10, align_corners=False, side="long") + assert out.shape == (10, 4) + + # 3D + inp = torch.rand(3, 5, 2, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, 10, align_corners=False, side="long") + assert out.shape == (3, 10, 4) + + # arbitrary dim + inp = torch.rand(1, 2, 3, 2, 1, 3, 5, 2, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, 10, align_corners=False, side="long") + assert out.shape == (1, 2, 3, 2, 1, 3, 10, 4) + + def test_one_param_vert(self, device, dtype): + inp = torch.rand(1, 3, 5, 2, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, 10, align_corners=False, side="vert") + assert out.shape == (1, 3, 10, 4) + + # 2D + inp = torch.rand(5, 2, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, 10, align_corners=False, side="vert") + assert out.shape == (10, 4) + + # 3D + inp = torch.rand(3, 5, 2, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, 10, align_corners=False, side="vert") + assert out.shape == (3, 10, 4) + + # arbitrary dim + inp = torch.rand(1, 2, 3, 2, 1, 3, 5, 2, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, 10, align_corners=False, side="vert") + assert out.shape == (1, 2, 3, 2, 1, 3, 10, 4) + + def test_one_param_horz(self, device, dtype): + inp = torch.rand(1, 3, 2, 5, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, 10, align_corners=False, side="horz") + assert out.shape == (1, 3, 4, 10) + + # 2D + inp = torch.rand(1, 3, 2, 5, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, 10, align_corners=False, side="horz") + assert out.shape == (1, 3, 4, 10) + + # 3D + inp = torch.rand(1, 3, 2, 5, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, 10, align_corners=False, side="horz") + assert out.shape == (1, 3, 4, 10) + + # arbitrary dim + inp = torch.rand(1, 3, 2, 5, device=device, dtype=dtype) + out = kornia.geometry.transform.resize(inp, 10, align_corners=False, side="horz") + assert out.shape == (1, 3, 4, 10) + + def test_gradcheck(self, device): + # test parameters + new_size = 4 + inp = torch.rand(1, 2, 3, 4, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.transform.Resize(new_size, align_corners=False), (inp,)) + + @pytest.mark.parametrize("anti_alias", [True, False]) + def test_dynamo(self, device, dtype, anti_alias, torch_optimizer): + new_size = (5, 6) + inp = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + op = torch_optimizer(kornia.geometry.transform.resize) + out = op(inp, new_size, align_corners=False, antialias=anti_alias) + assert out.shape == (1, 2, 5, 6) + expected = op(inp, new_size, align_corners=False, antialias=anti_alias) + self.assert_close(out, expected) + + +class TestRescale(BaseTester): + def test_smoke(self, device, dtype): + input = torch.rand(1, 3, 3, 4, device=device, dtype=dtype) + output = kornia.geometry.transform.rescale(input, (1.0, 1.0), align_corners=False) + self.assert_close(input, output, atol=1e-4, rtol=1e-4) + + def test_upsize(self, device, dtype): + input = torch.rand(1, 3, 3, 4, device=device, dtype=dtype) + output = kornia.geometry.transform.rescale(input, (3.0, 2.0), align_corners=False) + assert output.shape == (1, 3, 9, 8) + + def test_downsize(self, device, dtype): + input = torch.rand(1, 3, 9, 8, device=device, dtype=dtype) + output = kornia.geometry.transform.rescale(input, (1.0 / 3.0, 1.0 / 2.0), align_corners=False) + assert output.shape == (1, 3, 3, 4) + + def test_downscale_values(self, device, dtype): + inp_x = torch.arange(20, device=device, dtype=dtype) / 20.0 + inp = inp_x[None].T @ inp_x[None] + inp = inp[None, None] + out = kornia.geometry.transform.rescale(inp, (0.25, 0.25), antialias=False, align_corners=False) + expected = torch.tensor( + [ + [ + [ + [0.0056, 0.0206, 0.0356, 0.0506, 0.0656], + [0.0206, 0.0756, 0.1306, 0.1856, 0.2406], + [0.0356, 0.1306, 0.2256, 0.3206, 0.4156], + [0.0506, 0.1856, 0.3206, 0.4556, 0.5906], + [0.0656, 0.2406, 0.4156, 0.5906, 0.7656], + ] + ] + ], + device=device, + dtype=dtype, + ) + self.assert_close(out, expected, atol=1e-3, rtol=1e-3) + + def test_downscale_values_AA(self, device, dtype): + inp_x = torch.arange(20, device=device, dtype=dtype) / 20.0 + inp = inp_x[None].T @ inp_x[None] + inp = inp[None, None] + out = kornia.geometry.transform.rescale(inp, (0.25, 0.25), antialias=True, align_corners=False) + expected = torch.tensor( + [ + [ + [ + [0.0074, 0.0237, 0.0409, 0.0581, 0.0743], + [0.0237, 0.0756, 0.1306, 0.1856, 0.2376], + [0.0409, 0.1306, 0.2256, 0.3206, 0.4104], + [0.0581, 0.1856, 0.3206, 0.4556, 0.5832], + [0.0743, 0.2376, 0.4104, 0.5832, 0.7464], + ] + ] + ], + device=device, + dtype=dtype, + ) + self.assert_close(out, expected, atol=1e-3, rtol=1e-3) + + def test_one_param(self, device, dtype): + input = torch.rand(1, 3, 3, 4, device=device, dtype=dtype) + output = kornia.geometry.transform.rescale(input, 2.0, align_corners=False) + assert output.shape == (1, 3, 6, 8) + + def test_gradcheck(self, device): + input = torch.rand(1, 2, 3, 4, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.transform.Rescale(2.0, align_corners=False), (input,), nondet_tol=1e-8) + + +class TestRotate(BaseTester): + def test_angle90(self, device, dtype): + # prepare input data + inp = torch.tensor([[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]]], device=device, dtype=dtype) + expected = torch.tensor([[[0.0, 0.0], [4.0, 6.0], [3.0, 5.0], [0.0, 0.0]]], device=device, dtype=dtype) + # prepare transformation + angle = torch.tensor([90.0], device=device, dtype=dtype) + transform = kornia.geometry.transform.Rotate(angle, align_corners=True) + self.assert_close(transform(inp), expected, atol=1e-4, rtol=1e-4) + + def test_angle90_batch2(self, device, dtype): + # prepare input data + inp = torch.tensor([[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]]], device=device, dtype=dtype).repeat( + 2, 1, 1, 1 + ) + expected = torch.tensor( + [[[[0.0, 0.0], [4.0, 6.0], [3.0, 5.0], [0.0, 0.0]]], [[[0.0, 0.0], [5.0, 3.0], [6.0, 4.0], [0.0, 0.0]]]], + device=device, + dtype=dtype, + ) + # prepare transformation + angle = torch.tensor([90.0, -90.0], device=device, dtype=dtype) + transform = kornia.geometry.transform.Rotate(angle, align_corners=True) + self.assert_close(transform(inp), expected, atol=1e-4, rtol=1e-4) + + def test_angle90_batch2_broadcast(self, device, dtype): + # prepare input data + inp = torch.tensor([[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]]], device=device, dtype=dtype).repeat( + 2, 1, 1, 1 + ) + expected = torch.tensor( + [[[[0.0, 0.0], [4.0, 6.0], [3.0, 5.0], [0.0, 0.0]]], [[[0.0, 0.0], [4.0, 6.0], [3.0, 5.0], [0.0, 0.0]]]], + device=device, + dtype=dtype, + ) + # prepare transformation + angle = torch.tensor([90.0], device=device, dtype=dtype) + transform = kornia.geometry.transform.Rotate(angle, align_corners=True) + self.assert_close(transform(inp), expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + # test parameters + angle = torch.tensor([90.0], device=device, dtype=torch.float64) + + # evaluate function gradient + input = torch.rand(1, 2, 3, 4, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.transform.rotate, (input, angle)) + + @pytest.mark.skip("Need deep look into it since crashes everywhere.") + @pytest.mark.skip(reason="turn off all jit for a while") + def test_jit(self, device, dtype): + angle = torch.tensor([90.0], device=device, dtype=dtype) + batch_size, channels, height, width = 2, 3, 64, 64 + img = torch.ones(batch_size, channels, height, width, device=device, dtype=dtype) + rot = kornia.geometry.transform.Rotate(angle) + rot_traced = torch.jit.trace(kornia.geometry.transform.Rotate(angle), img) + self.assert_close(rot(img), rot_traced(img)) + + +class TestTranslate(BaseTester): + def test_dxdy(self, device, dtype): + # prepare input data + inp = torch.tensor([[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]]], device=device, dtype=dtype) + expected = torch.tensor([[[0.0, 1.0], [0.0, 3.0], [0.0, 5.0], [0.0, 7.0]]], device=device, dtype=dtype) + # prepare transformation + translation = torch.tensor([[1.0, 0.0]], device=device, dtype=dtype) + transform = kornia.geometry.transform.Translate(translation, align_corners=True) + self.assert_close(transform(inp), expected, atol=1e-4, rtol=1e-4) + + def test_dxdy_batch(self, device, dtype): + # prepare input data + inp = torch.tensor([[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]]], device=device, dtype=dtype).repeat( + 2, 1, 1, 1 + ) + expected = torch.tensor( + [[[[0.0, 1.0], [0.0, 3.0], [0.0, 5.0], [0.0, 7.0]]], [[[0.0, 0.0], [0.0, 1.0], [0.0, 3.0], [0.0, 5.0]]]], + device=device, + dtype=dtype, + ) + # prepare transformation + translation = torch.tensor([[1.0, 0.0], [1.0, 1.0]], device=device, dtype=dtype) + transform = kornia.geometry.transform.Translate(translation, align_corners=True) + self.assert_close(transform(inp), expected, atol=1e-4, rtol=1e-4) + + def test_dxdy_batch_broadcast(self, device, dtype): + # prepare input data + inp = torch.tensor([[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]]], device=device, dtype=dtype).repeat( + 2, 1, 1, 1 + ) + expected = torch.tensor( + [[[[0.0, 1.0], [0.0, 3.0], [0.0, 5.0], [0.0, 7.0]]], [[[0.0, 1.0], [0.0, 3.0], [0.0, 5.0], [0.0, 7.0]]]], + device=device, + dtype=dtype, + ) + # prepare transformation + translation = torch.tensor([[1.0, 0.0]], device=device, dtype=dtype) + transform = kornia.geometry.transform.Translate(translation, align_corners=True) + self.assert_close(transform(inp), expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + # test parameters + translation = torch.tensor([[1.0, 0.0]], device=device, dtype=torch.float64) + + # evaluate function gradient + input = torch.rand(1, 2, 3, 4, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.transform.translate, (input, translation), requires_grad=(True, False)) + + @pytest.mark.skip("Need deep look into it since crashes everywhere.") + @pytest.mark.skip(reason="turn off all jit for a while") + def test_jit(self, device, dtype): + translation = torch.tensor([[1.0, 0.0]], device=device, dtype=dtype) + batch_size, channels, height, width = 2, 3, 64, 64 + img = torch.ones(batch_size, channels, height, width, device=device, dtype=dtype) + trans = kornia.geometry.transform.Translate(translation) + trans_traced = torch.jit.trace(kornia.geometry.transform.Translate(translation), img) + self.assert_close(trans(img), trans_traced(img), atol=1e-4, rtol=1e-4) + + +class TestScale(BaseTester): + def test_scale_factor_2(self, device, dtype): + # prepare input data + inp = torch.tensor( + [[[0.0, 0.0, 0.0, 0.0], [0.0, 1.0, 1.0, 0.0], [0.0, 1.0, 1.0, 0.0], [0.0, 0.0, 0.0, 0.0]]], + device=device, + dtype=dtype, + ) + # prepare transformation + scale_factor = torch.tensor([[2.0, 2.0]], device=device, dtype=dtype) + transform = kornia.geometry.transform.Scale(scale_factor) + self.assert_close(transform(inp).sum().item(), 12.25, atol=1e-4, rtol=1e-4) + + def test_scale_factor_05(self, device, dtype): + # prepare input data + inp = torch.tensor( + [[[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]], + device=device, + dtype=dtype, + ) + expected = torch.tensor( + [[[0.0, 0.0, 0.0, 0.0], [0.0, 1.0, 1.0, 0.0], [0.0, 1.0, 1.0, 0.0], [0.0, 0.0, 0.0, 0.0]]], + device=device, + dtype=dtype, + ) + # prepare transformation + scale_factor = torch.tensor([[0.5, 0.5]], device=device, dtype=dtype) + transform = kornia.geometry.transform.Scale(scale_factor) + self.assert_close(transform(inp), expected, atol=1e-4, rtol=1e-4) + + def test_scale_factor_05_batch2(self, device, dtype): + # prepare input data + inp = torch.tensor( + [[[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]], + device=device, + dtype=dtype, + ).repeat(2, 1, 1, 1) + expected = torch.tensor( + [[[0.0, 0.0, 0.0, 0.0], [0.0, 1.0, 1.0, 0.0], [0.0, 1.0, 1.0, 0.0], [0.0, 0.0, 0.0, 0.0]]], + device=device, + dtype=dtype, + ).repeat(2, 1, 1, 1) + # prepare transformation + scale_factor = torch.tensor([[0.5, 0.5]], device=device, dtype=dtype) + transform = kornia.geometry.transform.Scale(scale_factor) + self.assert_close(transform(inp), expected, atol=1e-4, rtol=1e-4) + + def test_scale_factor_05_batch2_broadcast(self, device, dtype): + # prepare input data + inp = torch.tensor( + [[[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]], + device=device, + dtype=dtype, + ).repeat(2, 1, 1, 1) + expected = torch.tensor( + [[[0.0, 0.0, 0.0, 0.0], [0.0, 1.0, 1.0, 0.0], [0.0, 1.0, 1.0, 0.0], [0.0, 0.0, 0.0, 0.0]]], + device=device, + dtype=dtype, + ).repeat(2, 1, 1, 1) + # prepare transformation + scale_factor = torch.tensor([[0.5, 0.5]], device=device, dtype=dtype) + transform = kornia.geometry.transform.Scale(scale_factor) + self.assert_close(transform(inp), expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + # test parameters + scale_factor = torch.tensor([[0.5, 0.5]], device=device, dtype=torch.float64) + + # evaluate function gradient + input = torch.rand(1, 2, 3, 4, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.transform.scale, (input, scale_factor), requires_grad=(True, False)) + + @pytest.mark.skip("Need deep look into it since crashes everywhere.") + @pytest.mark.skip(reason="turn off all jit for a while") + def test_jit(self, device, dtype): + scale_factor = torch.tensor([[0.5, 0.5]], device=device, dtype=dtype) + batch_size, channels, height, width = 2, 3, 64, 64 + img = torch.ones(batch_size, channels, height, width, device=device, dtype=dtype) + trans = kornia.geometry.transform.Scale(scale_factor) + trans_traced = torch.jit.trace(kornia.Scale(scale_factor), img) + self.assert_close(trans(img), trans_traced(img), atol=1e-4, rtol=1e-4) + + +class TestShear(BaseTester): + def test_shear_x(self, device, dtype): + # prepare input data + inp = torch.tensor( + [[[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]], + device=device, + dtype=dtype, + ) + expected = torch.tensor( + [[[0.75, 1.0, 1.0, 1.0], [0.25, 1.0, 1.0, 1.0], [0.0, 0.75, 1.0, 1.0], [0.0, 0.25, 1.0, 1.0]]], + device=device, + dtype=dtype, + ) + + # prepare transformation + shear = torch.tensor([[0.5, 0.0]], device=device, dtype=dtype) + transform = kornia.geometry.transform.Shear(shear, align_corners=False) + self.assert_close(transform(inp), expected, atol=1e-4, rtol=1e-4) + + def test_shear_y(self, device, dtype): + # prepare input data + inp = torch.tensor( + [[[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]], + device=device, + dtype=dtype, + ) + expected = torch.tensor( + [[[0.75, 0.25, 0.0, 0.0], [1.0, 1.0, 0.75, 0.25], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]], + device=device, + dtype=dtype, + ) + + # prepare transformation + shear = torch.tensor([[0.0, 0.5]], device=device, dtype=dtype) + transform = kornia.geometry.transform.Shear(shear, align_corners=False) + self.assert_close(transform(inp), expected, atol=1e-4, rtol=1e-4) + + def test_shear_batch2(self, device, dtype): + # prepare input data + inp = torch.tensor( + [[[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]], + device=device, + dtype=dtype, + ).repeat(2, 1, 1, 1) + + expected = torch.tensor( + [ + [[[0.75, 1.0, 1.0, 1.0], [0.25, 1.0, 1.0, 1.0], [0.0, 0.75, 1.0, 1.0], [0.0, 0.25, 1.0, 1.0]]], + [[[0.75, 0.25, 0.0, 0.0], [1.0, 1.0, 0.75, 0.25], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]], + ], + device=device, + dtype=dtype, + ) + + # prepare transformation + shear = torch.tensor([[0.5, 0.0], [0.0, 0.5]], device=device, dtype=dtype) + transform = kornia.geometry.transform.Shear(shear, align_corners=False) + self.assert_close(transform(inp), expected, atol=1e-4, rtol=1e-4) + + def test_shear_batch2_broadcast(self, device, dtype): + # prepare input data + inp = torch.tensor( + [[[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]], + device=device, + dtype=dtype, + ).repeat(2, 1, 1, 1) + + expected = torch.tensor( + [[[[0.75, 1.0, 1.0, 1.0], [0.25, 1.0, 1.0, 1.0], [0.0, 0.75, 1.0, 1.0], [0.0, 0.25, 1.0, 1.0]]]], + device=device, + dtype=dtype, + ).repeat(2, 1, 1, 1) + + # prepare transformation + shear = torch.tensor([[0.5, 0.0]], device=device, dtype=dtype) + transform = kornia.geometry.transform.Shear(shear, align_corners=False) + self.assert_close(transform(inp), expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + # test parameters + shear = torch.tensor([[0.5, 0.0]], device=device, dtype=torch.float64) + + # evaluate function gradient + input = torch.rand(1, 2, 3, 4, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.transform.shear, (input, shear), requires_grad=(True, False)) + + @pytest.mark.skip("Need deep look into it since crashes everywhere.") + @pytest.mark.skip(reason="turn off all jit for a while") + def test_jit(self, device, dtype): + shear = torch.tensor([[0.5, 0.0]], device=device, dtype=dtype) + batch_size, channels, height, width = 2, 3, 64, 64 + img = torch.ones(batch_size, channels, height, width, device=device, dtype=dtype) + trans = kornia.geometry.transform.Shear(shear, align_corners=False) + trans_traced = torch.jit.trace(kornia.geometry.transform.Shear(shear), img) + self.assert_close(trans(img), trans_traced(img), atol=1e-4, rtol=1e-4) + + +class TestAffine2d(BaseTester): + def test_affine_no_args(self): + with pytest.raises(RuntimeError): + kornia.geometry.transform.Affine() + + def test_affine_batch_size_mismatch(self, device, dtype): + with pytest.raises(RuntimeError): + angle = torch.rand(1, device=device, dtype=dtype) + translation = torch.rand(2, 2, device=device, dtype=dtype) + kornia.geometry.transform.Affine(angle, translation) + + def test_affine_rotate(self, device, dtype): + # TODO: Remove when #666 is implemented + if device.type == "cuda": + pytest.skip("Currently breaks in CUDA.See https://github.com/kornia/kornia/issues/666") + torch.manual_seed(0) + angle = torch.rand(1, device=device, dtype=dtype) * 90.0 + input = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + + transform = kornia.geometry.transform.Affine(angle=angle).to(device=device, dtype=dtype) + actual = transform(input) + expected = kornia.geometry.transform.rotate(input, angle) + self.assert_close(actual, expected, atol=1e-4, rtol=1e-4) + + def test_affine_translate(self, device, dtype): + # TODO: Remove when #666 is implemented + if device.type == "cuda": + pytest.skip("Currently breaks in CUDA.See https://github.com/kornia/kornia/issues/666") + torch.manual_seed(0) + translation = torch.rand(1, 2, device=device, dtype=dtype) * 2.0 + input = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + + transform = kornia.geometry.transform.Affine(translation=translation).to(device=device, dtype=dtype) + actual = transform(input) + expected = kornia.geometry.transform.translate(input, translation) + self.assert_close(actual, expected, atol=1e-4, rtol=1e-4) + + def test_affine_scale(self, device, dtype): + # TODO: Remove when #666 is implemented + if device.type == "cuda": + pytest.skip("Currently breaks in CUDA.See https://github.com/kornia/kornia/issues/666") + torch.manual_seed(0) + _scale_factor = torch.rand(1, device=device, dtype=dtype) * 2.0 + scale_factor = torch.stack([_scale_factor, _scale_factor], dim=1) + input = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + + transform = kornia.geometry.transform.Affine(scale_factor=scale_factor).to(device=device, dtype=dtype) + actual = transform(input) + expected = kornia.geometry.transform.scale(input, scale_factor) + self.assert_close(actual, expected, atol=1e-4, rtol=1e-4) + + @pytest.mark.skip( + "_compute_shear_matrix and get_affine_matrix2d yield different results. " + "See https://github.com/kornia/kornia/issues/629 for details." + ) + def test_affine_shear(self, device, dtype): + torch.manual_seed(0) + shear = torch.rand(1, 2, device=device, dtype=dtype) + input = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + + transform = kornia.geometry.transform.Affine(shear=shear).to(device, dtype) + actual = transform(input) + expected = kornia.geometry.transform.shear(input, shear) + self.assert_close(actual, expected, atol=1e-4, rtol=1e-4) + + def test_affine_rotate_translate(self, device, dtype): + # TODO: Remove when #666 is implemented + if device.type == "cuda": + pytest.skip("Currently breaks in CUDA.See https://github.com/kornia/kornia/issues/666") + batch_size = 2 + + input = torch.tensor( + [[[0.0, 0.0, 0.0, 1.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]]], + device=device, + dtype=dtype, + ).repeat(batch_size, 1, 1, 1) + + angle = torch.tensor(180.0, device=device, dtype=dtype).repeat(batch_size) + translation = torch.tensor([1.0, 0.0], device=device, dtype=dtype).repeat(batch_size, 1) + + expected = torch.tensor( + [[[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 1.0, 0.0, 0.0]]], + device=device, + dtype=dtype, + ).repeat(batch_size, 1, 1, 1) + + transform = kornia.geometry.transform.Affine(angle=angle, translation=translation, align_corners=True).to( + device=device, dtype=dtype + ) + actual = transform(input) + self.assert_close(actual, expected, atol=1e-4, rtol=1e-4) + + def test_compose_affine_matrix_3x3(self, device, dtype): + """To get parameters: + import torchvision as tv + from PIL import Image + from torch import Tensor as T + import math + import random + img_size = (96,96) + seed = 42 + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) # if you are using multi-GPU. + np.random.seed(seed) # Numpy module. + random.seed(seed) # Python random module. + torch.manual_seed(seed) + tfm = tv.transforms.RandomAffine(degrees=(-25.0,25.0), + scale=(0.6, 1.4) , + translate=(0, 0.1), + shear=(-25., 25., -20., 20.)) + angle, translations, scale, shear = tfm.get_params(tfm.degrees, tfm.translate, + tfm.scale, tfm.shear, img_size) + print (angle, translations, scale, shear) + output_size = img_size + center = (img.size[0] * 0.5 + 0.5, img.size[1] * 0.5 + 0.5) + + matrix = tv.transforms.functional._get_inverse_affine_matrix(center, angle, translations, scale, shear) + matrix = np.array(matrix).reshape(2,3) + print (matrix) + """ + import math + + from torch import Tensor as T + + batch_size, _, height, width = 1, 1, 96, 96 + angle, translations = 6.971339922894188, (0.0, -4.0) + scale, shear = [0.7785685905190581, 0.7785685905190581], [11.8235607082617, 7.06797949691645] + matrix_expected = T([[1.27536969, 4.26828945e-01, -3.2349e01], [2.18297196e-03, 1.29424165e00, -9.1996e00]]) + center = T([float(width), float(height)]).view(1, 2) / 2.0 + 0.5 + center = center.expand(batch_size, -1) + matrix_kornia = kornia.geometry.transform.get_affine_matrix2d( + T(translations).view(-1, 2), + center, + T([scale]).view(-1, 2), + T([angle]).view(-1), + T([math.radians(shear[0])]).view(-1, 1), + T([math.radians(shear[1])]).view(-1, 1), + ) + matrix_kornia = matrix_kornia.inverse()[0, :2].detach().cpu() + self.assert_close(matrix_kornia, matrix_expected, atol=1e-4, rtol=1e-4) + + def test_broadcasting_issue_3176(self, device, dtype): + # Issue 3176: RandomRotation with multi-channel masks caused crash + # Scenario: Tensor is (B=1, C, H, W) but Matrix is (B=8, 2, 3) + # This implies applying N different transformations to a single image + B_mat, B_ten = 8, 1 + C, H, W = 3, 64, 64 + + input_tensor = torch.rand(B_ten, C, H, W, device=device, dtype=dtype) + + matrix = torch.rand(B_mat, 2, 3, device=device, dtype=dtype) + + output = kornia.geometry.transform.affine(input_tensor, matrix, mode="bilinear") + + assert output.shape == (B_mat, C, H, W) + + def test_warp_affine_fill_value(self, device, dtype): + # Feature: Support padding_mode="fill" with custom fill_value + # Scenario: 1-channel mask, fill with 1.0 + B, C, H, W = 1, 1, 10, 10 + src = torch.zeros(B, C, H, W, device=device, dtype=dtype) + + M = torch.eye(2, 3, device=device, dtype=dtype).unsqueeze(0) + M[..., 0, 2] = 5.0 + + # Fill with 1.0 + fill_val = torch.tensor([1.0], device=device, dtype=dtype) + + out = kornia.geometry.transform.warp_affine(src, M, (H, W), padding_mode="fill", fill_value=fill_val) + + assert out[0, 0, 0, 0] == 1.0 + + assert out[0, 0, 0, 9] == 0.0 + + +class TestGetAffineMatrix(BaseTester): + def test_smoke(self, device, dtype): + H, W = 5, 5 + translation = torch.tensor([[0.0, 0.0]], device=device, dtype=dtype) + # NOTE: ideally the center should be [W * 0.5, H * 0.5] + center = torch.tensor([[W // 2, H // 2]], device=device, dtype=dtype) + zoom1 = torch.ones([1, 1], device=device, dtype=dtype) * 0.5 + zoom2 = torch.ones([1, 1], device=device, dtype=dtype) * 1.0 + zoom = torch.cat([zoom1, zoom2], -1) + angle = torch.zeros([1], device=device, dtype=dtype) + affine_mat = kornia.geometry.get_affine_matrix2d(translation, center, zoom, angle) + + img = torch.ones(1, 1, H, W, device=device, dtype=dtype) + expected = torch.zeros_like(img) + expected[..., 1:4] = 1.0 + + out = kornia.geometry.transform.warp_affine(img, affine_mat[:, :2], (H, W)) + self.assert_close(out, expected) + + +class TestGetShearMatrix(BaseTester): + def test_get_shear_matrix2d_with_controlled_values(self, device, dtype): + # Define controlled values for shear angles and center + sx = torch.tensor([0.5], device=device, dtype=dtype) + sy = torch.tensor([0.25], device=device, dtype=dtype) + center = torch.tensor([[0.0, 0.0]], device=device, dtype=dtype) + + # Calculate the shear matrix using your function + out = kornia.geometry.transform.get_shear_matrix2d(center, sx=sx, sy=sy) + + # Define the expected shear matrix with controlled numbers + expected = torch.tensor( + [[[1.0, -0.5463, 0.0], [-0.2553, 1.1395, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype + ) + + self.assert_close(out, expected, atol=1e-4, rtol=1e-4) + + def test_get_shear_matrix2d_default_params_device_dtype(self, device, dtype): + # When sx and sy are None, the defaults should inherit device and dtype from center. + center = torch.tensor([[128.0, 128.0]], device=device, dtype=dtype) + out = kornia.geometry.transform.get_shear_matrix2d(center, sx=None, sy=None) + assert out.device.type == center.device.type, "Output device must match center device" + assert out.dtype == center.dtype, "Output dtype must match center dtype" + + def test_get_shear_matrix3d_default_params_device_dtype(self, device, dtype): + # When shear params are None, the defaults should inherit device and dtype from center. + center = torch.tensor([[64.0, 64.0, 32.0]], device=device, dtype=dtype) + out = kornia.geometry.transform.get_shear_matrix3d( + center, sxy=None, sxz=None, syx=None, syz=None, szx=None, szy=None + ) + assert out.device.type == center.device.type, "Output device must match center device" + assert out.dtype == center.dtype, "Output dtype must match center dtype" diff --git a/tests/geometry/transform/test_crop2d.py b/tests/geometry/transform/test_crop2d.py new file mode 100644 index 0000000..0a27d82 --- /dev/null +++ b/tests/geometry/transform/test_crop2d.py @@ -0,0 +1,328 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestCropAndResize(BaseTester): + def test_align_corners_true(self, device, dtype): + inp = torch.tensor( + [[[[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [9.0, 10.0, 11.0, 12.0], [13.0, 14.0, 15.0, 16.0]]]], + device=device, + dtype=dtype, + ) + + height, width = 2, 3 + + expected = torch.tensor([[[[6.0000, 6.5000, 7.0000], [10.0000, 10.5000, 11.0000]]]], device=device, dtype=dtype) + + boxes = torch.tensor([[[1.0, 1.0], [2.0, 1.0], [2.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype) # 1x4x2 + + # default should use align_coners True + patches = kornia.geometry.transform.crop_and_resize(inp, boxes, (height, width)) + self.assert_close(patches, expected, rtol=1e-4, atol=1e-4) + + def test_align_corners_false(self, device, dtype): + inp = torch.tensor( + [[[[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [9.0, 10.0, 11.0, 12.0], [13.0, 14.0, 15.0, 16.0]]]], + device=device, + dtype=dtype, + ) + + height, width = 2, 3 + expected = torch.tensor([[[[6.7222, 7.1667, 7.6111], [9.3889, 9.8333, 10.2778]]]], device=device, dtype=dtype) + + boxes = torch.tensor([[[1.0, 1.0], [2.0, 1.0], [2.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype) # 1x4x2 + + patches = kornia.geometry.transform.crop_and_resize(inp, boxes, (height, width), align_corners=False) + self.assert_close(patches, expected, rtol=1e-4, atol=1e-4) + + def test_crop_batch(self, device, dtype): + inp = torch.tensor( + [ + [[[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [9.0, 10.0, 11.0, 12.0], [13.0, 14.0, 15.0, 16.0]]], + [[[1.0, 5.0, 9.0, 13.0], [2.0, 6.0, 10.0, 14.0], [3.0, 7.0, 11.0, 15.0], [4.0, 8.0, 12.0, 16.0]]], + ], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [[[[6.0, 7.0], [10.0, 11.0]]], [[[7.0, 15.0], [8.0, 16.0]]]], device=device, dtype=dtype + ) + + boxes = torch.tensor( + [[[1.0, 1.0], [2.0, 1.0], [2.0, 2.0], [1.0, 2.0]], [[1.0, 2.0], [3.0, 2.0], [3.0, 3.0], [1.0, 3.0]]], + device=device, + dtype=dtype, + ) # 2x4x2 + + patches = kornia.geometry.transform.crop_and_resize(inp, boxes, (2, 2)) + self.assert_close(patches, expected, rtol=1e-4, atol=1e-4) + + def test_crop_batch_broadcast(self, device, dtype): + inp = torch.tensor( + [ + [[[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [9.0, 10.0, 11.0, 12.0], [13.0, 14.0, 15.0, 16.0]]], + [[[1.0, 5.0, 9.0, 13.0], [2.0, 6.0, 10.0, 14.0], [3.0, 7.0, 11.0, 15.0], [4.0, 8.0, 12.0, 16.0]]], + ], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [[[[6.0, 7.0], [10.0, 11.0]]], [[[6.0, 10.0], [7.0, 11.0]]]], device=device, dtype=dtype + ) + + boxes = torch.tensor([[[1.0, 1.0], [2.0, 1.0], [2.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype) # 1x4x2 + + patches = kornia.geometry.transform.crop_and_resize(inp, boxes, (2, 2)) + self.assert_close(patches, expected, rtol=1e-4, atol=1e-4) + + def test_gradcheck(self, device): + img = torch.rand(1, 2, 5, 4, device=device, dtype=torch.float64) + + boxes = torch.tensor([[[1.0, 1.0], [2.0, 1.0], [2.0, 2.0], [1.0, 2.0]]], device=device, dtype=torch.float64) + + self.gradcheck( + kornia.geometry.transform.crop_and_resize, (img, boxes, (4, 2)), requires_grad=(True, False, False) + ) + + def test_dynamo(self, device, dtype, torch_optimizer): + # Define script + op = kornia.geometry.transform.crop_and_resize + op_optimized = torch_optimizer(op) + # Define input + img = torch.tensor( + [[[[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [9.0, 10.0, 11.0, 12.0], [13.0, 14.0, 15.0, 16.0]]]], + device=device, + dtype=dtype, + ) + boxes = torch.tensor([[[1.0, 1.0], [2.0, 1.0], [2.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype) # 1x4x2 + + crop_height, crop_width = 4, 2 + actual = op_optimized(img, boxes, (crop_height, crop_width)) + expected = op(img, boxes, (crop_height, crop_width)) + self.assert_close(actual, expected, rtol=1e-4, atol=1e-4) + + +class TestCenterCrop(BaseTester): + def test_center_crop_h2_w4(self, device, dtype): + inp = torch.tensor( + [[[[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [9.0, 10.0, 11.0, 12.0], [13.0, 14.0, 15.0, 16.0]]]], + device=device, + dtype=dtype, + ) + + expected = torch.tensor([[[[5.0, 6.0, 7.0, 8.0], [9.0, 10.0, 11.0, 12.0]]]], device=device, dtype=dtype) + + out_crop = kornia.geometry.transform.center_crop(inp, (2, 4)) + self.assert_close(out_crop, expected, rtol=1e-4, atol=1e-4) + self.assert_close(kornia.geometry.transform.CenterCrop2D((2, 4))(inp), expected, rtol=1e-4, atol=1e-4) + + def test_center_crop_h4_w2(self, device, dtype): + inp = torch.tensor( + [[[[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [9.0, 10.0, 11.0, 12.0], [13.0, 14.0, 15.0, 16.0]]]], + device=device, + dtype=dtype, + ) + + height, width = 4, 2 + expected = torch.tensor([[[[2.0, 3.0], [6.0, 7.0], [10.0, 11.0], [14.0, 15.0]]]], device=device, dtype=dtype) + + out_crop = kornia.geometry.transform.center_crop(inp, (height, width)) + self.assert_close(out_crop, expected, rtol=1e-4, atol=1e-4) + self.assert_close(kornia.geometry.transform.CenterCrop2D((height, width))(inp), expected, rtol=1e-4, atol=1e-4) + + def test_center_crop_h4_w2_batch(self, device, dtype): + inp = torch.tensor( + [ + [[[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [9.0, 10.0, 11.0, 12.0], [13.0, 14.0, 15.0, 16.0]]], + [[[1.0, 5.0, 9.0, 13.0], [2.0, 6.0, 10.0, 14.0], [3.0, 7.0, 11.0, 15.0], [4.0, 8.0, 12.0, 16.0]]], + ], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [ + [[[2.0, 3.0], [6.0, 7.0], [10.0, 11.0], [14.0, 15.0]]], + [[[5.0, 9.0], [6.0, 10.0], [7.0, 11.0], [8.0, 12.0]]], + ], + device=device, + dtype=dtype, + ) + + out_crop = kornia.geometry.transform.center_crop(inp, (4, 2)) + self.assert_close(out_crop, expected, rtol=1e-4, atol=1e-4) + self.assert_close(kornia.geometry.transform.CenterCrop2D((4, 2))(inp), expected, rtol=1e-4, atol=1e-4) + + def test_gradcheck(self, device): + img = torch.rand(1, 2, 5, 4, device=device, dtype=torch.float64) + + self.gradcheck(kornia.geometry.transform.center_crop, (img, (4, 2))) + self.gradcheck(kornia.geometry.transform.CenterCrop2D((4, 2)), (img)) + + def test_dynamo(self, device, dtype, torch_optimizer): + # Define script + op = kornia.geometry.transform.center_crop + op_script = torch_optimizer(op) + # Define input + img = torch.ones(1, 2, 5, 4, device=device, dtype=dtype) + + actual = op_script(img, (4, 2)) + expected = op(img, (4, 2)) + self.assert_close(actual, expected, rtol=1e-4, atol=1e-4) + + +class TestCropByBoxes(BaseTester): + def test_crop_by_boxes_no_resizing(self, device, dtype): + inp = torch.tensor( + [[[[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [9.0, 10.0, 11.0, 12.0], [13.0, 14.0, 15.0, 16.0]]]], + device=device, + dtype=dtype, + ) + + src = torch.tensor([[[1.0, 1.0], [2.0, 1.0], [2.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype) # 1x4x2 + + dst = torch.tensor([[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0]]], device=device, dtype=dtype) # 1x4x2 + + expected = torch.tensor([[[[6.0, 7.0], [10.0, 11.0]]]], device=device, dtype=dtype) + + patches = kornia.geometry.transform.crop_by_boxes(inp, src, dst) + self.assert_close(patches, expected) + + def test_crop_by_boxes_resizing(self, device, dtype): + inp = torch.tensor( + [[[[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [9.0, 10.0, 11.0, 12.0], [13.0, 14.0, 15.0, 16.0]]]], + device=device, + dtype=dtype, + ) + + src = torch.tensor([[[1.0, 1.0], [2.0, 1.0], [2.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype) # 1x4x2 + + dst = torch.tensor([[[0.0, 0.0], [2.0, 0.0], [2.0, 1.0], [0.0, 1.0]]], device=device, dtype=dtype) # 1x4x2 + + expected = torch.tensor([[[[6.0, 6.5, 7.0], [10.0, 10.5, 11.0]]]], device=device, dtype=dtype) + + patches = kornia.geometry.transform.crop_by_boxes(inp, src, dst) + self.assert_close(patches, expected, rtol=1e-4, atol=1e-4) + + def test_gradcheck(self, device): + dtype = torch.float64 + inp = torch.randn((1, 1, 3, 3), device=device, dtype=dtype) + src = torch.tensor([[[1.0, 0.0], [2.0, 0.0], [2.0, 1.0], [1.0, 1.0]]], device=device, dtype=dtype) + dst = torch.tensor([[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0]]], device=device, dtype=dtype) + + self.gradcheck(kornia.geometry.transform.crop_by_boxes, (inp, src, dst), requires_grad=(True, False, False)) + + +class TestCropByTransform(BaseTester): + def test_crop_by_transform_no_resizing(self, device, dtype): + inp = torch.tensor( + [[[[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [9.0, 10.0, 11.0, 12.0], [13.0, 14.0, 15.0, 16.0]]]], + device=device, + dtype=dtype, + ) + + transform = torch.tensor( + [[[1.0, 0.0, -1.0], [0.0, 1.0, -1.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype + ) # 1x3x3 + + expected = torch.tensor([[[[6.0, 7.0], [10.0, 11.0]]]], device=device, dtype=dtype) + + patches = kornia.geometry.transform.crop_by_transform_mat(inp, transform, (2, 2)) + self.assert_close(patches, expected) + + def test_crop_by_boxes_resizing(self, device, dtype): + inp = torch.tensor( + [[[[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [9.0, 10.0, 11.0, 12.0], [13.0, 14.0, 15.0, 16.0]]]], + device=device, + dtype=dtype, + ) + + transform = torch.tensor( + [[[2.0, 0.0, -2.0], [0.0, 1.0, -1.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype + ) # 1x3x3 + + expected = torch.tensor([[[[6.0, 6.5, 7.0], [10.0, 10.5, 11.0]]]], device=device, dtype=dtype) + + patches = kornia.geometry.transform.crop_by_transform_mat(inp, transform, (2, 3)) + self.assert_close(patches, expected, rtol=1e-4, atol=1e-4) + + def test_gradcheck(self, device): + inp = torch.randn((1, 1, 3, 3), device=device, dtype=torch.float64) + transform = torch.tensor( + [[[2.0, 0.0, -2.0], [0.0, 1.0, -1.0], [0.0, 0.0, 1.0]]], device=device, dtype=torch.float64 + ) # 1x3x3 + + self.gradcheck( + kornia.geometry.transform.crop_by_transform_mat, + (inp, transform, (2, 2)), + requires_grad=(True, False, False), + ) + + +class TestCropByIndices(BaseTester): + def test_crop_by_indices_no_resizing(self, device, dtype): + inp = torch.tensor([[[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7, 8, 9]]]], device=device, dtype=dtype) # 1x3x3 + + # provide the indices to crop as 4 points + indices = torch.tensor([[[0, 0], [1, 0], [1, 1], [0, 1]]], device=device, dtype=torch.int64) + expected = torch.tensor([[[[1.0, 2.0], [4.0, 5.0]]]], device=device, dtype=dtype) + + self.assert_close(kornia.geometry.transform.crop_by_indices(inp, indices), expected) + + def test_dynamo(self, device, dtype, torch_optimizer): + # Define script + op = kornia.geometry.transform.crop_by_indices + op_script = torch_optimizer(op) + # Define input + img = torch.ones(1, 2, 5, 4, device=device, dtype=dtype) + + actual = op_script(img, torch.tensor([[[0, 0], [1, 0], [1, 1], [0, 1]]])) + expected = op(img, torch.tensor([[[0, 0], [1, 0], [1, 1], [0, 1]]])) + self.assert_close(actual, expected, rtol=1e-4, atol=1e-4) + + +class TestCropSizeValidation: + """Tests that crop functions properly reject invalid size arguments.""" + + def test_crop_and_resize_rejects_wrong_length(self, device, dtype): + inp = torch.rand(1, 1, 4, 4, device=device, dtype=dtype) + boxes = torch.tensor([[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0]]], device=device, dtype=dtype) + with pytest.raises(ValueError, match="tuple/list of length 2"): + kornia.geometry.transform.crop_and_resize(inp, boxes, (2, 2, 2)) + + def test_crop_and_resize_rejects_non_tuple(self, device, dtype): + inp = torch.rand(1, 1, 4, 4, device=device, dtype=dtype) + boxes = torch.tensor([[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0]]], device=device, dtype=dtype) + # Passing an int instead of a tuple can raise either ValueError (from an + # explicit validation check) or TypeError (from calling len() on an int), + # depending on which code path runs first inside crop_and_resize. + with pytest.raises((ValueError, TypeError)): + kornia.geometry.transform.crop_and_resize(inp, boxes, 2) + + def test_center_crop_rejects_wrong_length(self, device, dtype): + inp = torch.rand(1, 1, 4, 4, device=device, dtype=dtype) + with pytest.raises(ValueError, match="tuple/list of length 2"): + kornia.geometry.transform.center_crop(inp, (2, 2, 2)) diff --git a/tests/geometry/transform/test_crop3d.py b/tests/geometry/transform/test_crop3d.py new file mode 100644 index 0000000..9a8698c --- /dev/null +++ b/tests/geometry/transform/test_crop3d.py @@ -0,0 +1,349 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia +from kornia.core._compat import torch_version + +from testing.base import BaseTester + + +class TestCropAndResize3D(BaseTester): + def test_crop(self, device, dtype): + inp = torch.arange(0.0, 64.0, device=device, dtype=dtype).view(1, 1, 4, 4, 4) + + depth, height, width = 2, 2, 2 + expected = torch.tensor( + [[[[[25.1667, 27.1667], [30.5000, 32.5000]], [[46.5000, 48.5000], [51.8333, 53.8333]]]]], + device=device, + dtype=dtype, + ) + + boxes = torch.tensor( + [[[0, 0, 1], [3, 0, 1], [3, 2, 1], [0, 2, 1], [0, 0, 3], [3, 0, 3], [3, 2, 3], [0, 2, 3]]], + device=device, + dtype=dtype, + ) # 1x8x3 + + patches = kornia.geometry.transform.crop_and_resize3d(inp, boxes, (depth, height, width)) + self.assert_close(patches, expected) + + def test_crop_batch(self, device, dtype): + inp = torch.cat( + [ + torch.arange(0.0, 64.0, device=device, dtype=dtype).view(1, 1, 4, 4, 4), + torch.arange(0.0, 128.0, step=2, device=device, dtype=dtype).view(1, 1, 4, 4, 4), + ], + dim=0, + ) + + depth, height, width = 2, 2, 2 + expected = torch.tensor( + [ + [[[[16.0000, 19.0000], [24.0000, 27.0000]], [[48.0000, 51.0000], [56.0000, 59.0000]]]], + [[[[0.0000, 6.0000], [16.0000, 22.0000]], [[64.0000, 70.0000], [80.0000, 86.0000]]]], + ], + device=device, + dtype=dtype, + ) + + boxes = torch.tensor( + [ + [[0, 0, 1], [3, 0, 1], [3, 2, 1], [0, 2, 1], [0, 0, 3], [3, 0, 3], [3, 2, 3], [0, 2, 3]], + [[0, 0, 0], [3, 0, 0], [3, 2, 0], [0, 2, 0], [0, 0, 2], [3, 0, 2], [3, 2, 2], [0, 2, 2]], + ], + device=device, + dtype=dtype, + ) # 2x8x3 + + patches = kornia.geometry.transform.crop_and_resize3d(inp, boxes, (depth, height, width), align_corners=True) + self.assert_close(patches, expected) + + def test_gradcheck(self, device): + img = torch.arange(0.0, 64.0, device=device, dtype=torch.float64).view(1, 1, 4, 4, 4) + + boxes = torch.tensor( + [[[0, 0, 1], [3, 0, 1], [3, 2, 1], [0, 2, 1], [0, 0, 3], [3, 0, 3], [3, 2, 3], [0, 2, 3]]], + device=device, + dtype=torch.float64, + ) # 1x8x3 + + self.gradcheck(kornia.geometry.transform.crop_and_resize3d, (img, boxes, (4, 3, 2))) + + def test_dynamo(self, device, dtype, torch_optimizer): + # Define script + op = kornia.geometry.transform.crop_and_resize3d + op_script = torch_optimizer(op) + + img = torch.arange(0.0, 64.0, device=device, dtype=dtype).view(1, 1, 4, 4, 4) + + boxes = torch.tensor( + [[[0, 0, 1], [3, 0, 1], [3, 2, 1], [0, 2, 1], [0, 0, 3], [3, 0, 3], [3, 2, 3], [0, 2, 3]]], + device=device, + dtype=dtype, + ) # 1x8x3 + + actual = op_script(img, boxes, (4, 3, 2)) + expected = op(img, boxes, (4, 3, 2)) + self.assert_close(actual, expected) + + +class TestCenterCrop3D(BaseTester): + @pytest.mark.parametrize("crop_size", [(3, 5, 7), (5, 3, 7), (7, 3, 5)]) + def test_center_crop_357(self, crop_size, device, dtype): + inp = torch.arange(0.0, 343.0, device=device, dtype=dtype).view(1, 1, 7, 7, 7) + expected = inp[ + :, + :, + (inp.size(2) // 2 - crop_size[0] // 2) : (inp.size(2) // 2 + crop_size[0] // 2 + 1), + (inp.size(3) // 2 - crop_size[1] // 2) : (inp.size(3) // 2 + crop_size[1] // 2 + 1), + (inp.size(4) // 2 - crop_size[2] // 2) : (inp.size(4) // 2 + crop_size[2] // 2 + 1), + ] + out_crop = kornia.geometry.transform.center_crop3d(inp, crop_size, align_corners=True) + self.assert_close(out_crop, expected, rtol=1e-4, atol=1e-4) + + @pytest.mark.parametrize("crop_size", [(3, 5, 7), (5, 3, 7), (7, 3, 5)]) + def test_center_crop_357_batch(self, crop_size, device, dtype): + inp = torch.cat( + [ + torch.arange(0.0, 343.0, device=device, dtype=dtype).view(1, 1, 7, 7, 7), + torch.arange(343.0, 686.0, device=device, dtype=dtype).view(1, 1, 7, 7, 7), + ] + ) + expected = inp[ + :, + :, + (inp.size(2) // 2 - crop_size[0] // 2) : (inp.size(2) // 2 + crop_size[0] // 2 + 1), + (inp.size(3) // 2 - crop_size[1] // 2) : (inp.size(3) // 2 + crop_size[1] // 2 + 1), + (inp.size(4) // 2 - crop_size[2] // 2) : (inp.size(4) // 2 + crop_size[2] // 2 + 1), + ] + out_crop = kornia.geometry.transform.center_crop3d(inp, crop_size, align_corners=True) + self.assert_close(out_crop, expected, rtol=1e-4, atol=1e-4) + + def test_gradcheck(self, device): + img = torch.arange(0.0, 343.0, device=device, dtype=torch.float64).view(1, 1, 7, 7, 7) + + self.gradcheck(kornia.geometry.transform.center_crop3d, (img, (3, 5, 7))) + + @pytest.mark.skipif( + torch_version() == "2.1.0", + reason=( + "https://github.com/pytorch/pytorch/issues/110680" + " - unsupported operand type(s) for @: 'FakeTensor' and 'FakeTensor' on `normalize_homography3d`" + ), + ) + def test_dynamo(self, device, dtype, torch_optimizer): + # Define script + op = kornia.geometry.transform.center_crop3d + op_script = torch_optimizer(op) + img = torch.ones(4, 3, 5, 6, 7, device=device, dtype=dtype) + + actual = op_script(img, (4, 3, 2)) + expected = kornia.geometry.transform.center_crop3d(img, (4, 3, 2)) + self.assert_close(actual, expected, rtol=1e-4, atol=1e-4) + + +class TestCropByBoxes3D(BaseTester): + def test_crop_by_boxes_no_resizing(self, device, dtype): + inp = torch.arange(0.0, 343.0, device=device, dtype=dtype).view(1, 1, 7, 7, 7) + src_box = torch.tensor( + [ + [ + [1.0, 1.0, 1.0], + [3.0, 1.0, 1.0], + [3.0, 3.0, 1.0], + [1.0, 3.0, 1.0], + [1.0, 1.0, 2.0], + [3.0, 1.0, 2.0], + [3.0, 3.0, 2.0], + [1.0, 3.0, 2.0], + ] + ], + device=device, + dtype=dtype, + ) # 1x8x3 + dst_box = torch.tensor( + [ + [ + [0.0, 0.0, 0.0], + [2.0, 0.0, 0.0], + [2.0, 2.0, 0.0], + [0.0, 2.0, 0.0], + [0.0, 0.0, 1.0], + [2.0, 0.0, 1.0], + [2.0, 2.0, 1.0], + [0.0, 2.0, 1.0], + ] + ], + device=device, + dtype=dtype, + ) # 1x8x3 + + expected = inp[:, :, 1:3, 1:4, 1:4] + + patches = kornia.geometry.transform.crop_by_boxes3d(inp, src_box, dst_box, align_corners=True) + self.assert_close(patches, expected, rtol=1e-4, atol=1e-4) + + def test_crop_by_boxes_resizing(self, device, dtype): + inp = torch.arange(0.0, 343.0, device=device, dtype=dtype).view(1, 1, 7, 7, 7) + src_box = torch.tensor( + [ + [ + [1.0, 1.0, 1.0], + [3.0, 1.0, 1.0], + [3.0, 3.0, 1.0], + [1.0, 3.0, 1.0], + [1.0, 1.0, 2.0], + [3.0, 1.0, 2.0], + [3.0, 3.0, 2.0], + [1.0, 3.0, 2.0], + ] + ], + device=device, + dtype=dtype, + ) # 1x8x3 + dst_box = torch.tensor( + [ + [ + [0.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [1.0, 1.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 0.0, 1.0], + [1.0, 0.0, 1.0], + [1.0, 1.0, 1.0], + [0.0, 1.0, 1.0], + ] + ], + device=device, + dtype=dtype, + ) # 1x8x3 + + expected = torch.tensor( + [[[[[57.0000, 59.0000], [71.0000, 73.0000]], [[106.0000, 108.0000], [120.0000, 122.0000]]]]], + device=device, + dtype=dtype, + ) + + patches = kornia.geometry.transform.crop_by_boxes3d(inp, src_box, dst_box, align_corners=True) + self.assert_close(patches, expected, rtol=1e-4, atol=1e-4) + + def test_dynamo(self, device, dtype, torch_optimizer): + # Define script + op = kornia.geometry.transform.crop_by_boxes3d + op_script = torch_optimizer(op) + # Define input + inp = torch.randn((1, 1, 7, 7, 7), device=device, dtype=dtype) + src_box = torch.tensor( + [ + [ + [1.0, 1.0, 1.0], + [3.0, 1.0, 1.0], + [3.0, 3.0, 1.0], + [1.0, 3.0, 1.0], + [1.0, 1.0, 2.0], + [3.0, 1.0, 2.0], + [3.0, 3.0, 2.0], + [1.0, 3.0, 2.0], + ] + ], + device=device, + dtype=dtype, + ) # 1x8x3 + dst_box = torch.tensor( + [ + [ + [0.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [1.0, 1.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 0.0, 1.0], + [1.0, 0.0, 1.0], + [1.0, 1.0, 1.0], + [0.0, 1.0, 1.0], + ] + ], + device=device, + dtype=dtype, + ) # 1x8x3 + + actual = op_script(inp, src_box, dst_box, align_corners=True) + expected = op(inp, src_box, dst_box, align_corners=True) + self.assert_close(actual, expected, rtol=1e-4, atol=1e-4) + + def test_gradcheck(self, device): + dtype = torch.float64 + inp = torch.randn((1, 1, 7, 7, 7), device=device, dtype=dtype) + src_box = torch.tensor( + [ + [ + [1.0, 1.0, 1.0], + [3.0, 1.0, 1.0], + [3.0, 3.0, 1.0], + [1.0, 3.0, 1.0], + [1.0, 1.0, 2.0], + [3.0, 1.0, 2.0], + [3.0, 3.0, 2.0], + [1.0, 3.0, 2.0], + ] + ], + device=device, + dtype=dtype, + ) # 1x8x3 + dst_box = torch.tensor( + [ + [ + [0.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [1.0, 1.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 0.0, 1.0], + [1.0, 0.0, 1.0], + [1.0, 1.0, 1.0], + [0.0, 1.0, 1.0], + ] + ], + device=device, + dtype=dtype, + ) # 1x8x3 + + self.gradcheck( + kornia.geometry.transform.crop_by_boxes3d, (inp, src_box, dst_box), requires_grad=(True, False, False) + ) + + +class TestCrop3DSizeValidation: + """Tests that 3D crop functions properly reject invalid size arguments.""" + + def test_crop_and_resize3d_rejects_wrong_length(self, device, dtype): + inp = torch.rand(1, 1, 4, 4, 4, device=device, dtype=dtype) + boxes = torch.rand(1, 8, 3, device=device, dtype=dtype) + with pytest.raises(ValueError, match="tuple/list of length 3"): + kornia.geometry.transform.crop_and_resize3d(inp, boxes, (2, 2)) + + def test_crop_and_resize3d_rejects_non_tuple(self, device, dtype): + inp = torch.rand(1, 1, 4, 4, 4, device=device, dtype=dtype) + boxes = torch.rand(1, 8, 3, device=device, dtype=dtype) + with pytest.raises((ValueError, TypeError)): + kornia.geometry.transform.crop_and_resize3d(inp, boxes, 2) + + def test_center_crop3d_rejects_wrong_length(self, device, dtype): + inp = torch.rand(1, 1, 4, 4, 4, device=device, dtype=dtype) + with pytest.raises(ValueError, match="tuple/list of length 3"): + kornia.geometry.transform.center_crop3d(inp, (2, 2)) diff --git a/tests/geometry/transform/test_elastic_transform.py b/tests/geometry/transform/test_elastic_transform.py new file mode 100644 index 0000000..f1d5522 --- /dev/null +++ b/tests/geometry/transform/test_elastic_transform.py @@ -0,0 +1,110 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.geometry.transform import elastic_transform2d + +from testing.base import BaseTester + + +class TestElasticTransform(BaseTester): + def test_smoke(self, device, dtype): + image = torch.rand(1, 4, 5, 5, device=device, dtype=dtype) + noise = torch.rand(1, 2, 5, 5, device=device, dtype=dtype) + assert elastic_transform2d(image, noise) is not None + + @pytest.mark.parametrize("batch, channels, height, width", [(1, 3, 3, 4), (2, 2, 2, 4), (1, 5, 4, 1)]) + def test_cardinality(self, device, dtype, batch, channels, height, width): + shape = batch, channels, height, width + img = torch.ones(shape, device=device, dtype=dtype) + noise = torch.ones((batch, 2, height, width), device=device, dtype=dtype) + assert elastic_transform2d(img, noise).shape == shape + + def test_exception(self, device, dtype): + from kornia.core.exceptions import ShapeError, TypeCheckError + + ex = torch.ones(1, device=device, dtype=dtype) + with pytest.raises(TypeCheckError) as errinfo: + elastic_transform2d([0.0], ex) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(TypeCheckError) as errinfo: + elastic_transform2d(ex, 1) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(ShapeError) as errinfo: + img = torch.ones(1, 1, 1, device=device, dtype=dtype) + noise = torch.ones(1, 2, 1, 1, device=device, dtype=dtype) + elastic_transform2d(img, noise) + assert "Shape dimension mismatch" in str(errinfo.value) + + with pytest.raises(ShapeError) as errinfo: + img = torch.ones(1, 1, 1, 1, device=device, dtype=dtype) + noise = torch.ones(2, 1, 1, device=device, dtype=dtype) + elastic_transform2d(img, noise) + assert "Shape dimension mismatch" in str(errinfo.value) + + with pytest.raises(RuntimeError) as errinfo: + img = torch.ones(1, 1, 1, 1, device=device, dtype=dtype) + noise = torch.ones(1, 3, 1, 1, device=device, dtype=dtype) + elastic_transform2d(img, noise) + assert "The size of tensor a (2) must match the size of tensor b (3)" in str(errinfo.value) + + @pytest.mark.parametrize( + "kernel_size, sigma, alpha", + [ + [(3, 3), (4.0, 4.0), (32.0, 32.0)], + [(5, 3), (4.0, 8.0), (16.0, 32.0)], + [(5, 5), torch.tensor([2.0, 8.0]), torch.tensor([16.0, 64.0])], + ], + ) + def test_valid_paramters(self, device, dtype, kernel_size, sigma, alpha): + image = torch.rand(1, 4, 5, 5, device=device, dtype=dtype) + noise = torch.rand(1, 2, 5, 5, device=device, dtype=dtype) + if isinstance(sigma, torch.Tensor): + sigma = sigma.to(device, dtype) + if isinstance(alpha, torch.Tensor): + alpha = alpha.to(device, dtype) + assert elastic_transform2d(image, noise, kernel_size, sigma, alpha) is not None + + def test_values(self, device, dtype): + image = torch.tensor( + [[[[0.0018, 0.7521, 0.7550], [0.2053, 0.4249, 0.1369], [0.1027, 0.3992, 0.8773]]]], + device=device, + dtype=dtype, + ) + + noise = torch.ones(1, 2, 3, 3, device=device, dtype=dtype) + + expected = torch.tensor( + [[[[0.0005, 0.3795, 0.1905], [0.1034, 0.4235, 0.0702], [0.0259, 0.2007, 0.2193]]]], + device=device, + dtype=dtype, + ) + + actual = elastic_transform2d(image, noise) + self.assert_close(actual, expected, atol=1e-3, rtol=1e-3) + + @pytest.mark.parametrize("requires_grad", [True, False]) + def test_gradcheck(self, device, dtype, requires_grad): + image = torch.rand(1, 1, 3, 3, device=device, dtype=torch.float64, requires_grad=requires_grad) + noise = torch.rand(1, 2, 3, 3, device=device, dtype=torch.float64, requires_grad=not requires_grad) + assert self.gradcheck( + elastic_transform2d, (image, noise), raise_exception=True, fast_mode=True, nondet_tol=1e-4 + ) diff --git a/tests/geometry/transform/test_flip.py b/tests/geometry/transform/test_flip.py new file mode 100644 index 0000000..d5641de --- /dev/null +++ b/tests/geometry/transform/test_flip.py @@ -0,0 +1,211 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestVflip(BaseTester): + def smoke_test(self, device, dtype): + f = kornia.geometry.transform.Vflip() + repr = "Vflip()" + assert str(f) == repr + + def test_vflip(self, device, dtype): + f = kornia.geometry.transform.Vflip() + input = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]], device=device, dtype=dtype) # 3 x 3 + + expected = torch.tensor( + [[0.0, 1.0, 1.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], device=device, dtype=dtype + ) # 3 x 3 + + self.assert_close(f(input), expected) + + def test_batch_vflip(self, device, dtype): + input = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]], device=device, dtype=dtype) # 3 x 3 + + input = input.repeat(2, 1, 1) # 2 x 3 x 3 + + f = kornia.geometry.transform.Vflip() + expected = torch.tensor( + [[[0.0, 1.0, 1.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]], device=device, dtype=dtype + ) # 1 x 3 x 3 + + expected = expected.repeat(2, 1, 1) # 2 x 3 x 3 + + self.assert_close(f(input), expected) + + @pytest.mark.skip(reason="turn off all jit for a while") + def test_jit(self, device, dtype): + @torch.jit.script + def op_script(data: torch.Tensor) -> torch.Tensor: + return kornia.geometry.transform.vflip(data) + + input = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]], device=device, dtype=dtype) # 3 x 3 + + # Build jit trace + op_trace = torch.jit.trace(op_script, (input,)) + + # Create new inputs + input = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [5.0, 5.0, 0.0]], device=device, dtype=dtype) # 3 x 3 + + input = input.repeat(2, 1, 1) # 2 x 3 x 3 + + expected = torch.tensor( + [[[5.0, 5.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]], device=device, dtype=dtype + ) # 3 x 3 + + expected = expected.repeat(2, 1, 1) + + actual = op_trace(input) + + self.assert_close(actual, expected) + + def test_gradcheck(self, device): + input = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]], device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.transform.Vflip(), (input,)) + + +class TestHflip(BaseTester): + def smoke_test(self, device, dtype): + f = kornia.geometry.transform.Hflip() + repr = "Hflip()" + assert str(f) == repr + + def test_hflip(self, device, dtype): + f = kornia.geometry.transform.Hflip() + input = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]], device=device, dtype=dtype) # 3 x 3 + + expected = torch.tensor( + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [1.0, 1.0, 0.0]], device=device, dtype=dtype + ) # 3 x 3 + + self.assert_close(f(input), expected) + + def test_batch_hflip(self, device, dtype): + input = torch.tensor( + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]], device=device, dtype=dtype + ) # 1 x 3 x 3 + + input = input.repeat(2, 1, 1) # 2 x 3 x 3 + + f = kornia.geometry.transform.Hflip() + expected = torch.tensor( + [[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [1.0, 1.0, 0.0]]], device=device, dtype=dtype + ) # 3 x 3 + + expected = expected.repeat(2, 1, 1) # 2 x 3 x 3 + + self.assert_close(f(input), expected) + + @pytest.mark.skip(reason="turn off all jit for a while") + def test_jit(self, device, dtype): + @torch.jit.script + def op_script(data: torch.Tensor) -> torch.Tensor: + return kornia.geometry.transform.hflip(data) + + input = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]], device=device, dtype=dtype) # 3 x 3 + + # Build jit trace + op_trace = torch.jit.trace(op_script, (input,)) + + # Create new inputs + input = torch.tensor([[0.0, 0.0, 0.0], [5.0, 5.0, 0.0], [0.0, 0.0, 0.0]], device=device, dtype=dtype) # 3 x 3 + + input = input.repeat(2, 1, 1) # 2 x 3 x 3 + + expected = torch.tensor( + [[[0.0, 0.0, 0.0], [0.0, 5.0, 5.0], [0.0, 0.0, 0.0]]], device=device, dtype=dtype + ) # 3 x 3 + + expected = expected.repeat(2, 1, 1) + + actual = op_trace(input) + + self.assert_close(actual, expected) + + def test_gradcheck(self, device): + input = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]], device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.transform.Hflip(), (input,)) + + +class TestRot180(BaseTester): + def smoke_test(self, device, dtype): + f = kornia.geometry.transform.Rot180() + repr = "Rot180()" + assert str(f) == repr + + def test_rot180(self, device, dtype): + f = kornia.geometry.transform.Rot180() + input = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]], device=device, dtype=dtype) # 3 x 3 + + expected = torch.tensor( + [[1.0, 1.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], device=device, dtype=dtype + ) # 3 x 3 + + self.assert_close(f(input), expected) + + def test_batch_rot180(self, device, dtype): + input = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]], device=device, dtype=dtype) # 3 x 3 + + input = input.repeat(2, 1, 1) # 2 x 3 x 3 + + f = kornia.geometry.transform.Rot180() + expected = torch.tensor( + [[1.0, 1.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], device=device, dtype=dtype + ) # 1 x 3 x 3 + + expected = expected.repeat(2, 1, 1) # 2 x 3 x 3 + + self.assert_close(f(input), expected) + + @pytest.mark.skip(reason="turn off all jit for a while") + def test_jit(self, device, dtype): + @torch.jit.script + def op_script(data: torch.Tensor) -> torch.Tensor: + return kornia.geometry.transform.rot180(data) + + input = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]], device=device, dtype=dtype) # 3 x 3 + + # Build jit trace + op_trace = torch.jit.trace(op_script, (input,)) + + # Create new inputs + input = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [5.0, 5.0, 0.0]], device=device, dtype=dtype) # 3 x 3 + + input = input.repeat(2, 1, 1) # 2 x 3 x 3 + + expected = torch.tensor( + [[[0.0, 5.0, 5.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]], device=device, dtype=dtype + ) # 3 x 3 + + expected = expected.repeat(2, 1, 1) + + actual = op_trace(input) + + self.assert_close(actual, expected) + + def test_gradcheck(self, device): + input = torch.tensor( + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 1.0]], device=device, dtype=torch.float64 + ) # 3 x 3 + + self.gradcheck(kornia.geometry.transform.Rot180(), (input,)) diff --git a/tests/geometry/transform/test_homography_warper.py b/tests/geometry/transform/test_homography_warper.py new file mode 100644 index 0000000..79fc266 --- /dev/null +++ b/tests/geometry/transform/test_homography_warper.py @@ -0,0 +1,463 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia +from kornia.core.ops import eye_like +from kornia.core.utils import _torch_inverse_cast + +from testing.base import BaseTester +from testing.error import compute_patch_abs_error + + +class TestHomographyWarper(BaseTester): + num_tests = 10 + threshold = 0.1 + + def test_identity(self, device, dtype): + # create input data + height, width = 2, 5 + patch_src = torch.rand(1, 1, height, width, device=device, dtype=dtype) + dst_homo_src = eye_like(3, patch_src) + + # instantiate warper + warper = kornia.geometry.transform.HomographyWarper(height, width, align_corners=True) + + # warp from source to destination + patch_dst = warper(patch_src, dst_homo_src) + self.assert_close(patch_src, patch_dst) + + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_normalize_homography_identity(self, batch_size, device, dtype): + # create input data + height, width = 2, 5 + res = torch.tensor([[[0.5, 0.0, -1.0], [0.0, 2.0, -1.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + dst_homo_src = eye_like(3, res.expand(batch_size, -1, -1, -1)) + + self.assert_close( + kornia.geometry.conversions.normal_transform_pixel(height, width, device=device, dtype=dtype), res + ) + + norm_homo = kornia.geometry.conversions.normalize_homography(dst_homo_src, (height, width), (height, width)) + self.assert_close(norm_homo, dst_homo_src) + + # change output scale + norm_homo = kornia.geometry.conversions.normalize_homography( + dst_homo_src, (height, width), (height * 2, width // 2) + ) + res = torch.tensor( + [[[4.0, 0.0, 3.0], [0.0, 1 / 3, -2 / 3], [0.0, 0.0, 1.0]]], device=device, dtype=dtype + ).repeat(batch_size, 1, 1) + self.assert_close(norm_homo, res, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_denormalize_homography_identity(self, batch_size, device, dtype): + # create input data + height, width = 2, 5 + + res = torch.tensor([[[0.5, 0.0, -1.0], [0.0, 2.0, -1.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype) + dst_homo_src = eye_like(3, res.expand(batch_size, -1, -1, -1)) + + self.assert_close( + kornia.geometry.conversions.normal_transform_pixel(height, width, device=device, dtype=dtype), res + ) + + denorm_homo = kornia.geometry.conversions.denormalize_homography(dst_homo_src, (height, width), (height, width)) + self.assert_close(denorm_homo, dst_homo_src) + + # change output scale + denorm_homo = kornia.geometry.conversions.denormalize_homography( + dst_homo_src, (height, width), (height * 2, width // 2) + ) + res = torch.tensor([[[0.25, 0.0, 0.0], [0.0, 3.0, 0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype).repeat( + batch_size, 1, 1 + ) + self.assert_close(denorm_homo, res, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_normalize_homography_general(self, batch_size, device, dtype): + # create input data + height, width = 2, 5 + dst_homo_src = torch.eye(3, device=device, dtype=dtype) + dst_homo_src[..., 0, 0] = 0.5 + dst_homo_src[..., 1, 1] = 2.0 + dst_homo_src[..., 0, 2] = 1.0 + dst_homo_src[..., 1, 2] = 2.0 + dst_homo_src = dst_homo_src.expand(batch_size, -1, -1) + + norm_homo = kornia.geometry.conversions.normalize_homography(dst_homo_src, (height, width), (height, width)) + res = torch.tensor([[[0.5, 0.0, 0.0], [0.0, 2.0, 5.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype).expand( + batch_size, -1, -1 + ) + self.assert_close(norm_homo, res) + + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_denormalize_homography_general(self, batch_size, device, dtype): + # create input data + height, width = 2, 5 + dst_homo_src = torch.eye(3, device=device, dtype=dtype) + dst_homo_src[..., 0, 0] = 0.5 + dst_homo_src[..., 1, 1] = 2.0 + dst_homo_src[..., 0, 2] = 1.0 + dst_homo_src[..., 1, 2] = 2.0 + dst_homo_src = dst_homo_src.expand(batch_size, -1, -1) + + denorm_homo = kornia.geometry.conversions.denormalize_homography(dst_homo_src, (height, width), (height, width)) + res = torch.tensor([[[0.5, 0.0, 3.0], [0.0, 2.0, 0.5], [0.0, 0.0, 1.0]]], device=device, dtype=dtype).expand( + batch_size, -1, -1 + ) + self.assert_close(denorm_homo, res) + + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_consistency(self, batch_size, device, dtype): + # create input data + height, width = 2, 5 + dst_homo_src = torch.eye(3, device=device, dtype=dtype) + dst_homo_src[..., 0, 0] = 0.5 + dst_homo_src[..., 1, 1] = 2.0 + dst_homo_src[..., 0, 2] = 1.0 + dst_homo_src[..., 1, 2] = 2.0 + dst_homo_src = dst_homo_src.expand(batch_size, -1, -1) + + denorm_homo = kornia.geometry.conversions.denormalize_homography(dst_homo_src, (height, width), (height, width)) + norm_denorm_homo = kornia.geometry.conversions.normalize_homography( + denorm_homo, (height, width), (height, width) + ) + self.assert_close(dst_homo_src, norm_denorm_homo) + norm_homo = kornia.geometry.conversions.normalize_homography(dst_homo_src, (height, width), (height, width)) + denorm_norm_homo = kornia.geometry.conversions.denormalize_homography( + norm_homo, (height, width), (height, width) + ) + self.assert_close(dst_homo_src, denorm_norm_homo) + + @pytest.mark.parametrize("offset", [1, 3, 7]) + @pytest.mark.parametrize("shape", [(4, 5), (2, 6), (4, 3), (5, 7)]) + def test_warp_grid_translation(self, shape, offset, device, dtype): + # create input data + height, width = shape + grid = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=False, device=device, dtype=dtype) + dst_homo_src = eye_like(3, grid) + dst_homo_src[..., 0, 2] = offset # apply offset in x + flow = kornia.geometry.transform.warp_grid(grid, dst_homo_src) + + # the grid the src plus the offset should be equal to the flow + # on the x-axis, y-axis remains the same. + self.assert_close(grid[..., 0].to(device=device, dtype=dtype) + offset, flow[..., 0]) + self.assert_close(grid[..., 1].to(device=device, dtype=dtype), flow[..., 1]) + + @pytest.mark.parametrize("batch_shape", [(1, 1, 4, 5), (2, 2, 4, 6), (3, 1, 5, 7)]) + def test_identity_resize(self, batch_shape, device, dtype): + # create input data + batch_size, channels, height, width = batch_shape + patch_src = torch.rand(batch_size, channels, height, width, device=device, dtype=dtype) + dst_homo_src = eye_like(3, patch_src) + + # instantiate warper warp from source to destination + warper = kornia.geometry.transform.HomographyWarper(height // 2, width // 2, align_corners=True) + patch_dst = warper(patch_src, dst_homo_src) + + # check the corners + self.assert_close(patch_src[..., 0, 0], patch_dst[..., 0, 0], atol=1e-4, rtol=1e-4) + self.assert_close(patch_src[..., 0, -1], patch_dst[..., 0, -1], atol=1e-4, rtol=1e-4) + self.assert_close(patch_src[..., -1, 0], patch_dst[..., -1, 0], atol=1e-4, rtol=1e-4) + self.assert_close(patch_src[..., -1, -1], patch_dst[..., -1, -1], atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("shape", [(4, 5), (2, 6), (4, 3), (5, 7)]) + def test_translation(self, shape, device, dtype): + # create input data + offset = 2.0 # in pixel + height, width = shape + patch_src = torch.rand(1, 1, height, width, device=device, dtype=dtype) + dst_homo_src = eye_like(3, patch_src) + + dst_homo_src[..., 0, 2] = offset / (width - 1) # apply offset in x + + # instantiate warper and from source to destination + warper = kornia.geometry.transform.HomographyWarper(height, width, align_corners=True) + patch_dst = warper(patch_src, dst_homo_src) + self.assert_close(patch_src[..., 1:], patch_dst[..., :-1], atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_shape", [(1, 1, 3, 5), (2, 2, 4, 3), (3, 1, 2, 3)]) + def test_rotation(self, batch_shape, device, dtype): + # create input data + batch_size, channels, height, width = batch_shape + patch_src = torch.rand(batch_size, channels, height, width, device=device, dtype=dtype) + # rotation of 90deg + dst_homo_src = torch.eye(3, device=device, dtype=dtype) + dst_homo_src[..., 0, 0] = 0.0 + dst_homo_src[..., 0, 1] = 1.0 + dst_homo_src[..., 1, 0] = -1.0 + dst_homo_src[..., 1, 1] = 0.0 + dst_homo_src = dst_homo_src.expand(batch_size, -1, -1) + + # instantiate warper and warp from source to destination + warper = kornia.geometry.transform.HomographyWarper(height, width, align_corners=True) + patch_dst = warper(patch_src, dst_homo_src) + + # check the corners + self.assert_close(patch_src[..., 0, 0], patch_dst[..., 0, -1], atol=1e-4, rtol=1e-4) + self.assert_close(patch_src[..., 0, -1], patch_dst[..., -1, -1], atol=1e-4, rtol=1e-4) + self.assert_close(patch_src[..., -1, 0], patch_dst[..., 0, 0], atol=1e-4, rtol=1e-4) + self.assert_close(patch_src[..., -1, -1], patch_dst[..., -1, 0], atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_size", [1, 2, 3]) + def test_homography_warper(self, batch_size, device, dtype): + # generate input data + height, width = 128, 64 + eye_size = 3 # identity 3x3 + + patch_src = torch.ones(batch_size, 1, height, width, device=device, dtype=dtype) + + # create base homography + dst_homo_src = eye_like(eye_size, patch_src) + + # instantiate warper + warper = kornia.geometry.transform.HomographyWarper(height, width, align_corners=True) + + for _ in range(self.num_tests): + # generate homography noise + homo_delta = torch.rand_like(dst_homo_src) * 0.3 + + dst_homo_src_i = dst_homo_src + homo_delta + + # transform the points from dst to ref + patch_dst = warper(patch_src, dst_homo_src_i) + patch_dst_to_src = warper(patch_dst, _torch_inverse_cast(dst_homo_src_i)) + + # same transform precomputing the grid + warper.precompute_warp_grid(_torch_inverse_cast(dst_homo_src_i)) + patch_dst_to_src_precomputed = warper(patch_dst) + self.assert_close(patch_dst_to_src_precomputed, patch_dst_to_src, atol=1e-4, rtol=1e-4) + + # projected should be equal as initial + error = compute_patch_abs_error(patch_src, patch_dst_to_src, height, width) + + assert error.item() < self.threshold + + # check functional api + patch_dst_to_src_functional = kornia.geometry.transform.homography_warp( + patch_dst, _torch_inverse_cast(dst_homo_src_i), (height, width), align_corners=True + ) + + self.assert_close(patch_dst_to_src, patch_dst_to_src_functional, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_shape", [(1, 1, 7, 5), (2, 3, 8, 5), (1, 1, 7, 16)]) + def test_gradcheck(self, batch_shape, device): + dtype = torch.float64 + # generate input data + eye_size = 3 # identity 3x3 + + # create checkerboard + patch_src = torch.rand(batch_shape, device=device, dtype=dtype) + + # create base homography + _batch_size, _, height, width = patch_src.shape + dst_homo_src = eye_like(eye_size, patch_src) + + # instantiate warper + warper = kornia.geometry.transform.HomographyWarper(height, width, align_corners=True) + # evaluate function gradient + self.gradcheck(warper, (patch_src, dst_homo_src), requires_grad=(True, False), nondet_tol=1e-8) + + @pytest.mark.parametrize("batch_size", [1, 2, 3]) + @pytest.mark.parametrize("align_corners", [True, False]) + @pytest.mark.parametrize("normalized_coordinates", [True, False]) + def test_dynamo(self, batch_size, align_corners, normalized_coordinates, device, dtype, torch_optimizer): + if ( + device == torch.device("cpu") + and batch_size != 1 + and kornia.core._compat.torch_version() in {"2.3.0", "2.3.1"} + ): + pytest.skip("Failing to compile batched inputs see pytorch/pytorch#126617") + + # generate input data + height, width = 128, 64 + eye_size = 3 # identity 3x3 + + patch_src = torch.rand(batch_size, 1, height, width, device=device, dtype=dtype) + + # create base homography + dst_homo_src = eye_like(eye_size, patch_src) + + # generate homography noise + homo_delta = torch.rand_like(dst_homo_src) * 0.3 + + dst_homo_src_i = dst_homo_src + homo_delta + + # transform the points with and without jit + patch_dst = kornia.geometry.transform.homography_warp( + patch_src, + dst_homo_src_i, + (height, width), + align_corners=align_corners, + normalized_coordinates=normalized_coordinates, + ) + patch_dst_optimized = torch_optimizer(kornia.geometry.transform.homography_warp)( + patch_src, + dst_homo_src_i, + (height, width), + align_corners=align_corners, + normalized_coordinates=normalized_coordinates, + ) + + self.assert_close(patch_dst, patch_dst_optimized, atol=1e-4, rtol=1e-4) + + +class TestHomographyNormalTransform(BaseTester): + expected_2d_0 = torch.tensor([[[0.5, 0.0, -1.0], [0.0, 2.0, -1.0], [0.0, 0.0, 1.0]]]) + + expected_2d_1 = torch.tensor([[[0.5, 0.0, -1.0], [0.0, 2e14, -1.0], [0.0, 0.0, 1.0]]]) + + expected_3d_0 = expected = torch.tensor( + [[[0.4, 0.0, 0.0, -1.0], [0.0, 2.0, 0.0, -1.0], [0.0, 0.0, 0.6667, -1.0], [0.0, 0.0, 0.0, 1.0]]] + ) + + expected_3d_1 = torch.tensor( + [[[0.4, 0.0, 0.0, -1.0], [0.0, 2e14, 0.0, -1.0], [0.0, 0.0, 0.6667, -1.0], [0.0, 0.0, 0.0, 1.0]]] + ) + + @pytest.mark.parametrize("height,width,expected", [(2, 5, expected_2d_0), (1, 5, expected_2d_1)]) + def test_transform2d(self, height, width, expected, device, dtype): + output = kornia.geometry.conversions.normal_transform_pixel(height, width, device=device, dtype=dtype) + + self.assert_close(output, expected.to(device=device, dtype=dtype), atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("height", [1, 2, 5]) + @pytest.mark.parametrize("width", [1, 2, 5]) + def test_divide_by_zero2d(self, height, width, device, dtype): + output = kornia.geometry.conversions.normal_transform_pixel(height, width, device=device, dtype=dtype) + assert torch.isinf(output).sum().item() == 0 + + def test_transform2d_apply(self, device, dtype): + height, width = 2, 5 + input = torch.tensor([[0.0, 0.0], [width - 1, height - 1]], device=device, dtype=dtype) + expected = torch.tensor([[-1.0, -1.0], [1.0, 1.0]], device=device, dtype=dtype) + transform = kornia.geometry.conversions.normal_transform_pixel(height, width, device=device, dtype=dtype) + output = kornia.geometry.linalg.transform_points(transform, input) + self.assert_close(output, expected.to(device=device, dtype=dtype), atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("height,width,depth,expected", [(2, 6, 4, expected_3d_0), (1, 6, 4, expected_3d_1)]) + def test_transform3d(self, height, width, depth, expected, device, dtype): + output = kornia.geometry.conversions.normal_transform_pixel3d(depth, height, width, device=device, dtype=dtype) + self.assert_close(output, expected.to(device=device, dtype=dtype), atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("height", [1, 2, 5]) + @pytest.mark.parametrize("width", [1, 2, 5]) + @pytest.mark.parametrize("depth", [1, 2, 5]) + def test_divide_by_zero3d(self, height, width, depth, device, dtype): + output = kornia.geometry.conversions.normal_transform_pixel3d(depth, height, width, device=device, dtype=dtype) + assert torch.isinf(output).sum().item() == 0 + + def test_transform3d_apply(self, device, dtype): + depth, height, width = 3, 2, 5 + input = torch.tensor([[0.0, 0.0, 0.0], [width - 1, height - 1, depth - 1]], device=device, dtype=dtype) + expected = torch.tensor([[-1.0, -1.0, -1.0], [1.0, 1.0, 1.0]], device=device, dtype=dtype) + transform = kornia.geometry.conversions.normal_transform_pixel3d( + depth, height, width, device=device, dtype=dtype + ) + output = kornia.geometry.linalg.transform_points(transform, input) + self.assert_close(output, expected.to(device=device, dtype=dtype), atol=1e-4, rtol=1e-4) + + +class TestHomographyWarper3D(BaseTester): + num_tests = 10 + threshold = 0.1 + + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_normalize_homography_identity(self, batch_size, device, dtype): + # create input data + input_shape = (4, 8, 5) + + res = torch.tensor( + [[[0.5000, 0.0, 0.0, -1.0], [0.0, 0.2857, 0.0, -1.0], [0.0, 0.0, 0.6667, -1.0], [0.0, 0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ) + norm = kornia.geometry.conversions.normal_transform_pixel3d( + input_shape[0], + input_shape[1], + input_shape[2], + ).to(device=device, dtype=dtype) + + dst_homo_src = eye_like(4, norm.expand(batch_size, -1, -1, -1)) + + _atol = 1e-3 if (device.type == "cuda" and dtype == torch.float32) else 1e-4 + self.assert_close(norm, res, rtol=1e-4, atol=_atol) + + norm_homo = kornia.geometry.conversions.normalize_homography3d(dst_homo_src, input_shape, input_shape).to( + device=device, dtype=dtype + ) + self.assert_close(norm_homo, dst_homo_src, rtol=1e-4, atol=_atol) + + norm_homo = kornia.geometry.conversions.normalize_homography3d(dst_homo_src, input_shape, input_shape).to( + device=device, dtype=dtype + ) + self.assert_close(norm_homo, dst_homo_src, rtol=1e-4, atol=_atol) + + # change output scale + norm_homo = kornia.geometry.conversions.normalize_homography3d( + dst_homo_src, input_shape, (input_shape[0] // 2, input_shape[1] * 2, input_shape[2] // 2) + ).to(device=device, dtype=dtype) + res = torch.tensor( + [[[4.0, 0.0, 0.0, 3.0], [0.0, 0.4667, 0.0, -0.5333], [0.0, 0.0, 3.0, 2.0], [0.0, 0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ).repeat(batch_size, 1, 1) + + atol = 1e-3 if (device.type == "cuda" and dtype == torch.float32) else 1e-4 + self.assert_close(norm_homo, res, rtol=1e-4, atol=atol) + + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_normalize_homography_general(self, batch_size, device, dtype): + # create input data + dst_homo_src = torch.eye(4, device=device, dtype=dtype) + dst_homo_src[..., 0, 0] = 0.5 + dst_homo_src[..., 1, 1] = 0.5 + dst_homo_src[..., 2, 2] = 2.0 + dst_homo_src[..., 0, 3] = 1.0 + dst_homo_src[..., 1, 3] = 2.0 + dst_homo_src[..., 2, 3] = 3.0 + dst_homo_src = dst_homo_src.expand(batch_size, -1, -1) + + norm_homo = kornia.geometry.conversions.normalize_homography3d(dst_homo_src, (2, 2, 5), (2, 2, 5)) + res = torch.tensor( + [[[0.5, 0.0, 0.0, 0.0], [0.0, 0.5, 0.0, 3.5], [0.0, 0.0, 2.0, 7.0], [0.0, 0.0, 0.0, 1.0]]], + device=device, + dtype=dtype, + ).expand(batch_size, -1, -1) + self.assert_close(norm_homo, res) + + @pytest.mark.parametrize("offset", [1, 3, 7]) + @pytest.mark.parametrize("shape", [(4, 5, 6), (2, 4, 6), (4, 3, 9), (5, 7, 8)]) + def test_warp_grid_translation(self, shape, offset, device, dtype): + # create input data + depth, height, width = shape + grid = kornia.geometry.create_meshgrid3d( + depth, height, width, normalized_coordinates=False, dtype=dtype, device=device + ) + dst_homo_src = eye_like(4, grid) + dst_homo_src[..., 0, 3] = offset # apply offset in x + + flow = kornia.geometry.transform.warp_grid3d(grid, dst_homo_src) + + # the grid the src plus the offset should be equal to the flow + # on the x-axis, y-axis remains the same. + self.assert_close(grid[..., 0].to(device=device, dtype=dtype) + offset, flow[..., 0], atol=1e-4, rtol=1e-4) + self.assert_close(grid[..., 1].to(device=device, dtype=dtype), flow[..., 1], atol=1e-4, rtol=1e-4) + self.assert_close(grid[..., 2].to(device=device, dtype=dtype), flow[..., 2], atol=1e-4, rtol=1e-4) diff --git a/tests/geometry/transform/test_image_registrator.py b/tests/geometry/transform/test_image_registrator.py new file mode 100644 index 0000000..3fe5bf3 --- /dev/null +++ b/tests/geometry/transform/test_image_registrator.py @@ -0,0 +1,143 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys + +import pytest +import torch + +import kornia +from kornia.core._compat import torch_version +from kornia.geometry import transform_points +from kornia.geometry.conversions import denormalize_homography +from kornia.geometry.transform import ImageRegistrator + +from testing.base import BaseTester +from testing.casts import dict_to + + +class TestSimilarity(BaseTester): + def test_smoke(self, device, dtype): + expected = torch.eye(3, device=device, dtype=dtype)[None] + for r, sc, sh in zip([True, False], [True, False], [True, False]): + sim = kornia.geometry.transform.Similarity(r, sc, sh).to(device, dtype) + self.assert_close(sim(), expected, atol=1e-4, rtol=1e-4) + + def test_smoke_inverse(self, device, dtype): + expected = torch.eye(3, device=device, dtype=dtype)[None] + for r, sc, sh in zip([True, False], [True, False], [True, False]): + s = kornia.geometry.transform.Similarity(r, sc, sh).to(device, dtype) + self.assert_close(s.forward_inverse(), expected, atol=1e-4, rtol=1e-4) + + def test_scale(self, device, dtype): + sc = 0.5 + sim = kornia.geometry.transform.Similarity(True, True, True).to(device, dtype) + sim.scale.data *= sc + expected = torch.tensor([[0.5, 0, 0.0], [0, 0.5, 0], [0, 0, 1]], device=device, dtype=dtype)[None] + inv_expected = torch.tensor([[2.0, 0, 0.0], [0, 2.0, 0], [0, 0, 1]], device=device, dtype=dtype)[None] + self.assert_close(sim.forward_inverse(), inv_expected, atol=1e-4, rtol=1e-4) + self.assert_close(sim(), expected, atol=1e-4, rtol=1e-4) + + def test_repr(self, device, dtype): + for r, sc, sh in zip([True, False], [True, False], [True, False]): + s = kornia.geometry.transform.Similarity(r, sc, sh).to(device, dtype) + assert s is not None + + +class TestHomography(BaseTester): + def test_smoke(self, device, dtype): + expected = torch.eye(3, device=device, dtype=dtype)[None] + h = kornia.geometry.transform.Homography().to(device, dtype) + self.assert_close(h(), expected, atol=1e-4, rtol=1e-4) + + def test_smoke_inverse(self, device, dtype): + expected = torch.eye(3, device=device, dtype=dtype)[None] + h = kornia.geometry.transform.Homography().to(device, dtype) + self.assert_close(h.forward_inverse(), expected, atol=1e-4, rtol=1e-4) + + def test_repr(self, device, dtype): + h = kornia.geometry.transform.Homography().to(device, dtype) + assert h is not None + + +class TestImageRegistrator(BaseTester): + @pytest.mark.parametrize("model_type", ["homography", "similarity", "translation", "scale", "rotation"]) + def test_smoke(self, device, dtype, model_type): + ir = kornia.geometry.transform.ImageRegistrator(model_type).to(device, dtype) + assert ir is not None + + @pytest.mark.slow + @pytest.mark.xfail( + torch_version() in {"2.0.0", "2.0.1", "2.1.2", "2.2.2", "2.4.0"}, reason="failing at some 2.x torch" + ) + def test_registration_toy(self, device, dtype): + ch, height, width = 3, 16, 18 + homography = torch.eye(3, device=device, dtype=dtype)[None] + homography[..., 0, 0] = 1.05 + homography[..., 1, 1] = 1.05 + homography[..., 0, 2] = 0.01 + img_src = torch.rand(1, ch, height, width, device=device, dtype=dtype) + img_dst = kornia.geometry.homography_warp(img_src, homography, (height, width), align_corners=False) + IR = ImageRegistrator("Similarity", num_iterations=500, lr=3e-4, pyramid_levels=2).to(device, dtype) + model = IR.register(img_src, img_dst) + self.assert_close(model, homography, atol=1e-3, rtol=1e-3) + model, intermediate = IR.register(img_src, img_dst, output_intermediate_models=True) + assert len(intermediate) == 2 + + @pytest.mark.slow + @pytest.mark.parametrize("data", ["loftr_homo"], indirect=True) + @pytest.mark.skipif( + torch_version() == "2.0.0" and "win" in sys.platform, reason="Tensor not matching on win with torch 2.0" + ) + def test_registration_real(self, device, dtype, data): + data_dev = dict_to(data, device, dtype) + IR = ImageRegistrator("homography", num_iterations=1200, lr=2e-2, pyramid_levels=5).to(device, dtype) + model = IR.register(data_dev["image0"], data_dev["image1"]) + homography_gt = torch.inverse(data_dev["H_gt"]) + homography_gt = homography_gt / homography_gt[2, 2] + h0, w0 = data["image0"].shape[2], data["image0"].shape[3] + h1, w1 = data["image1"].shape[2], data["image1"].shape[3] + + model_denormalized = denormalize_homography(model, (h0, w0), (h1, w1)) + model_denormalized = model_denormalized / model_denormalized[0, 2, 2] + + bbox = torch.tensor([[[0, 0], [w0, 0], [w0, h0], [0, h0]]], device=device, dtype=dtype) + bbox_in_2_gt = transform_points(homography_gt[None], bbox) + bbox_in_2_gt_est = transform_points(model_denormalized, bbox) + # The tolerance is huge, because the error is in pixels + # and transformation is quite significant, so + # 15 px reprojection error is not super huge + self.assert_close(bbox_in_2_gt, bbox_in_2_gt_est, atol=15, rtol=0.1) + + def test_register_with_shape_mismatch(self, device): + img1 = torch.rand(1, 1, 64, 64, device=device) + img2 = torch.rand(1, 1, 32, 32, device=device) + + registrator = ImageRegistrator("similarity", allow_shape_mismatch=True) + + out = registrator.register(img1, img2) + + assert out is not None + + def test_register_shape_mismatch_raises(self, device): + img1 = torch.rand(1, 1, 64, 64, device=device) + img2 = torch.rand(1, 1, 32, 32, device=device) + + registrator = ImageRegistrator("similarity", allow_shape_mismatch=False) + + with pytest.raises(ValueError): + registrator.register(img1, img2) diff --git a/tests/geometry/transform/test_imgwarp.py b/tests/geometry/transform/test_imgwarp.py new file mode 100644 index 0000000..6a59312 --- /dev/null +++ b/tests/geometry/transform/test_imgwarp.py @@ -0,0 +1,690 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys + +import pytest +import torch + +import kornia +from kornia.core._compat import torch_version, torch_version_lt +from kornia.core.utils import _torch_inverse_cast + +from testing.base import BaseTester + + +class DummyNNModule(torch.nn.Module): + def __init__(self, h: int, w: int, align_corners: bool, padding_mode: str): + super().__init__() + self.h = h + self.w = w + + def forward(self, x, y): + return kornia.geometry.transform.warp_affine(x, y, dsize=(self.h, self.w), align_corners=False) + + +class TestGetPerspectiveTransform(BaseTester): + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_smoke(self, device, dtype, batch_size): + points_src = torch.rand(batch_size, 4, 2, device=device, dtype=dtype) + points_dst = torch.rand(batch_size, 4, 2, device=device, dtype=dtype) + + dst_trans_src = kornia.geometry.get_perspective_transform(points_src, points_dst) + + assert dst_trans_src.shape == (batch_size, 3, 3) + + @pytest.mark.parametrize("batch_size", [1, 5]) + def test_crop_src_dst_type_mismatch(self, device, dtype, batch_size): + # generate input data + src_h, src_w = 3, 3 + dst_h, dst_w = 3, 3 + + # [x, y] origin + # top-left, top-right, bottom-right, bottom-left + points_src = torch.tensor( + [[[0, 0], [0, src_w - 1], [src_h - 1, src_w - 1], [src_h - 1, 0]]], device=device, dtype=torch.int64 + ) + + # [x, y] destination + # top-left, top-right, bottom-right, bottom-left + points_dst = torch.tensor( + [[[0, 0], [0, dst_w - 1], [dst_h - 1, dst_w - 1], [dst_h - 1, 0]]], device=device, dtype=dtype + ) + + # compute transformation between points + with pytest.raises(Exception): + _ = kornia.geometry.get_perspective_transform(points_src, points_dst) + + def test_back_and_forth(self, device, dtype): + # generate input data + h_max, w_max = 64, 32 # height, width + h = h_max * torch.rand(1, device=device, dtype=dtype) + w = w_max * torch.rand(1, device=device, dtype=dtype) + + norm = torch.rand(1, 4, 2, device=device, dtype=dtype) + points_src = torch.zeros_like(norm, device=device, dtype=dtype) + points_src[:, 1, 0] = h + points_src[:, 2, 1] = w + points_src[:, 3, 0] = h + points_src[:, 3, 1] = w + points_dst = points_src + norm + + # compute transform from source to target + dst_trans_src = kornia.geometry.get_perspective_transform(points_src, points_dst) + points_dst_hat = kornia.geometry.transform_points(dst_trans_src, points_src) + self.assert_close(points_dst, points_dst_hat) + + def test_hflip(self, device, dtype): + points_src = torch.tensor([[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0]]], device=device, dtype=dtype) + + points_dst = torch.tensor([[[1.0, 0.0], [0.0, 0.0], [0.0, 1.0], [1.0, 1.0]]], device=device, dtype=dtype) + + dst_trans_src = kornia.geometry.get_perspective_transform(points_src, points_dst) + + point_left = torch.tensor([[[0.0, 0.0]]], device=device, dtype=dtype) + point_right = torch.tensor([[[1.0, 0.0]]], device=device, dtype=dtype) + + self.assert_close(kornia.geometry.transform_points(dst_trans_src, point_left), point_right) + + def test_dynamo(self, device, dtype, torch_optimizer): + points_src = torch.rand(1, 4, 2, device=device, dtype=dtype) + points_dst = torch.rand(1, 4, 2, device=device, dtype=dtype) + + op = kornia.geometry.get_perspective_transform + op_optimized = torch_optimizer(op) + + self.assert_close(op(points_src, points_dst), op_optimized(points_src, points_dst)) + + @pytest.mark.skipif(torch_version_lt(1, 11, 0), reason="backward for LSTSQ not supported in pytorch < 1.11.0") + def test_gradcheck(self, device): + # compute gradient check + points_src = torch.rand(1, 4, 2, device=device, dtype=torch.float64, requires_grad=True) + points_dst = torch.rand(1, 4, 2, device=device, dtype=torch.float64, requires_grad=True) + self.gradcheck(kornia.geometry.get_perspective_transform, (points_src, points_dst)) + + +class TestRotationMatrix2d(BaseTester): + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_90deg_rotation(self, batch_size, device, dtype): + # generate input data + center_base = torch.zeros(batch_size, 2, device=device, dtype=dtype) + angle_base = torch.ones(batch_size, device=device, dtype=dtype) + scale_base = torch.ones(batch_size, 2, device=device, dtype=dtype) + + # 90 deg rotation + center = center_base + angle = 90.0 * angle_base + scale = scale_base + M = kornia.geometry.get_rotation_matrix2d(center, angle, scale) + + for i in range(batch_size): + self.assert_close(M[i, 0, 0].item(), 0.0, rtol=1e-4, atol=1e-4) + self.assert_close(M[i, 0, 1].item(), 1.0, rtol=1e-4, atol=1e-4) + self.assert_close(M[i, 1, 0].item(), -1.0, rtol=1e-4, atol=1e-4) + self.assert_close(M[i, 1, 1].item(), 0.0, rtol=1e-4, atol=1e-4) + + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_rotation_90deg_and_scale(self, batch_size, device, dtype): + # generate input data + center_base = torch.zeros(batch_size, 2, device=device, dtype=dtype) + angle_base = torch.ones(batch_size, device=device, dtype=dtype) + scale_base = torch.ones(batch_size, 2, device=device, dtype=dtype) + + # 90 deg rotation + 2x scale + center = center_base + angle = 90.0 * angle_base + scale = 2.0 * scale_base + M = kornia.geometry.get_rotation_matrix2d(center, angle, scale) + + for i in range(batch_size): + self.assert_close(M[i, 0, 0].item(), 0.0, rtol=1e-4, atol=1e-4) + self.assert_close(M[i, 0, 1].item(), 2.0, rtol=1e-4, atol=1e-4) + self.assert_close(M[i, 1, 0].item(), -2.0, rtol=1e-4, atol=1e-4) + self.assert_close(M[i, 1, 1].item(), 0.0, rtol=1e-4, atol=1e-4) + + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_rotation_45deg(self, batch_size, device, dtype): + # generate input data + center_base = torch.zeros(batch_size, 2, device=device, dtype=dtype) + angle_base = torch.ones(batch_size, device=device, dtype=dtype) + scale_base = torch.ones(batch_size, 2, device=device, dtype=dtype) + + # 45 deg rotation + center = center_base + angle = 45.0 * angle_base + scale = scale_base + + M = kornia.geometry.get_rotation_matrix2d(center, angle, scale) + + for i in range(batch_size): + self.assert_close(M[i, 0, 0].item(), 0.7071) + self.assert_close(M[i, 0, 1].item(), 0.7071) + self.assert_close(M[i, 1, 0].item(), -0.7071) + self.assert_close(M[i, 1, 1].item(), 0.7071) + + @pytest.mark.parametrize("batch_size", [1]) + def test_gradcheck(self, batch_size, device): + dtype = torch.float64 + # generate input data + center_base = torch.zeros(batch_size, 2, device=device, dtype=dtype) + angle_base = torch.ones(batch_size, device=device, dtype=dtype) + scale_base = torch.ones(batch_size, 2, device=device, dtype=dtype) + + # 45 deg rotation + center = center_base + angle = 45.0 * angle_base + scale = scale_base + + # evaluate function gradient + self.gradcheck(kornia.geometry.get_rotation_matrix2d, (center, angle, scale)) + + +class TestWarpAffine(BaseTester): + def test_smoke(self, device, dtype): + batch_size, channels, height, width = 1, 2, 3, 4 + aff_ab = torch.eye(2, 3, device=device, dtype=dtype)[None] # 1x2x3 + img_b = torch.rand(batch_size, channels, height, width, device=device, dtype=dtype) + img_a = kornia.geometry.warp_affine(img_b, aff_ab, (height, width)) + self.assert_close(img_b, img_a) + + @pytest.mark.parametrize("batch_shape", ([1, 3, 2, 5], [2, 4, 3, 4], [3, 5, 6, 2])) + @pytest.mark.parametrize("out_shape", ([2, 5], [3, 4], [6, 2])) + def test_cardinality(self, device, dtype, batch_shape, out_shape): + batch_size, channels, height, width = batch_shape + h_out, w_out = out_shape + aff_ab = torch.eye(2, 3, device=device, dtype=dtype).repeat(batch_size, 1, 1) # Bx2x3 + img_b = torch.rand(batch_size, channels, height, width, device=device, dtype=dtype) + img_a = kornia.geometry.warp_affine(img_b, aff_ab, (h_out, w_out)) + assert img_a.shape == (batch_size, channels, h_out, w_out) + + def test_exception(self, device, dtype): + img = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + aff = torch.eye(2, 3, device=device, dtype=dtype)[None] + size = (4, 5) + + with pytest.raises(TypeError): + assert kornia.geometry.warp_affine(0.0, aff, size) + + with pytest.raises(TypeError): + assert kornia.geometry.warp_affine(img, 0.0, size) + + with pytest.raises(ValueError): + img = torch.rand(2, 3, 4, device=device, dtype=dtype) + assert kornia.geometry.warp_affine(img, aff, size) + + with pytest.raises(ValueError): + aff = torch.eye(2, 2, device=device, dtype=dtype)[None] + assert kornia.geometry.warp_affine(img, aff, size) + + def test_translation(self, device, dtype): + offset = 1.0 + h, w = 3, 4 + aff_ab = torch.eye(2, 3, device=device, dtype=dtype)[None] + aff_ab[..., -1] += offset + + img_b = torch.arange(float(h * w), device=device, dtype=dtype).view(1, 1, h, w) + + expected = torch.zeros_like(img_b) + expected[..., 1:, 1:] = img_b[..., :2, :3] + + # Same as opencv: cv2.warpAffine(kornia.tensor_to_image(img_b), aff_ab[0].numpy(), (w, h)) + img_a = kornia.geometry.warp_affine(img_b, aff_ab, (h, w)) + self.assert_close(img_a, expected) + + def test_rotation_inverse(self, device, dtype): + h, w = 4, 4 + img_b = torch.rand(1, 1, h, w, device=device, dtype=dtype) + + # create rotation matrix of 90deg (anti-clockwise) + center = torch.tensor([[w - 1, h - 1]], device=device, dtype=dtype) / 2 + scale = torch.ones((1, 2), device=device, dtype=dtype) + angle = 90.0 * torch.ones(1, device=device, dtype=dtype) + aff_ab_2x3 = kornia.geometry.get_rotation_matrix2d(center, angle, scale) + # Same as opencv: cv2.getRotationMatrix2D(((w-1)/2,(h-1)/2), 90., 1.) + + # warp the tensor + # Same as opencv: cv2.warpAffine(kornia.tensor_to_image(img_b), aff_ab[0].numpy(), (w, h)) + img_a = kornia.geometry.warp_affine(img_b, aff_ab_2x3, (h, w)) + + # invert the transform + aff_ab_3x3 = kornia.geometry.conversions.convert_affinematrix_to_homography(aff_ab_2x3) + aff_ba_2x3 = _torch_inverse_cast(aff_ab_3x3)[..., :2, :] + img_b_hat = kornia.geometry.warp_affine(img_a, aff_ba_2x3, (h, w)) + self.assert_close(img_b_hat, img_b, atol=1e-3, rtol=1e-3) + + def test_dynamo(self, device, dtype, torch_optimizer): + aff_ab = torch.eye(2, 3, device=device, dtype=dtype)[None] + img = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + args = (img, aff_ab, (4, 5)) + op = kornia.geometry.warp_affine + op_optimized = torch_optimizer(op) + self.assert_close(op(*args), op_optimized(*args)) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 2, 3, 4 + aff_ab = torch.eye(2, 3, device=device, dtype=torch.float64)[None] + 1e-6 # 1x2x3 + img_b = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.warp_affine, (img_b, aff_ab, (height, width))) + + def test_fill_padding_translation(self, device, dtype): + offset = 1.0 + h, w = 3, 4 + aff_ab = torch.eye(2, 3, device=device, dtype=dtype)[None] + aff_ab[..., -1] += offset + + img_b = torch.arange(float(3 * h * w), device=device, dtype=dtype).view(1, 3, h, w) + + # normally fill_value will also be converted to the right device and type in warp_affine + fill_value = torch.tensor([0.5, 0.2, 0.1], device=device, dtype=dtype) + + img_a = kornia.geometry.warp_affine(img_b, aff_ab, (h, w), padding_mode="fill", fill_value=fill_value) + top_row_mean = img_a[..., :1, :].mean(dim=[0, 2, 3]) + first_col_mean = img_a[..., :1].mean(dim=[0, 2, 3]) + self.assert_close(top_row_mean, fill_value) + self.assert_close(first_col_mean, fill_value) + + @pytest.mark.parametrize("num_channels", [1, 3, 5]) + def test_fill_padding_channels(self, device, dtype, num_channels): + offset = 1.0 + h, w = 3, 4 + aff_ab = torch.eye(2, 3, device=device, dtype=dtype)[None] + aff_ab[..., -1] += offset + + img_b = torch.arange(float(num_channels * h * w), device=device, dtype=dtype).view(1, num_channels, h, w) + + fill_value = torch.zeros(num_channels, device=device, dtype=dtype) + + img_a = kornia.geometry.warp_affine(img_b, aff_ab, (h, w), padding_mode="fill", fill_value=fill_value) + + self.assert_close(img_a[:, :, :1, :1].squeeze(), fill_value.squeeze()) + + @pytest.mark.parametrize("align_corners", (True, False)) + @pytest.mark.parametrize("padding_mode", ("zeros", "fill")) + def test_jit_script(self, device, dtype, align_corners, padding_mode): + offset = 1.0 + h, w = 3, 4 + aff_ab = torch.eye(2, 3, device=device, dtype=dtype)[None] + aff_ab[..., -1] += offset + + img_b = torch.arange(float(3 * h * w), device=device, dtype=dtype).view(1, 3, h, w) + net = DummyNNModule(3, 4, align_corners, padding_mode) + script_net = torch.jit.script(net) + + assert isinstance(script_net, torch.jit.ScriptModule) + self.assert_close(script_net(img_b, aff_ab), net(img_b, aff_ab)) + + +class TestWarpPerspective(BaseTester): + def test_smoke(self, device, dtype): + batch_size, channels, height, width = 1, 2, 3, 4 + img_b = torch.rand(batch_size, channels, height, width, device=device, dtype=dtype) + H_ab = kornia.core.ops.eye_like(3, img_b) + img_a = kornia.geometry.warp_perspective(img_b, H_ab, (height, width)) + self.assert_close(img_b, img_a) + + @pytest.mark.parametrize("batch_shape", ([1, 3, 2, 5], [2, 4, 3, 4], [3, 5, 6, 2])) + @pytest.mark.parametrize("out_shape", ([2, 5], [3, 4], [6, 2])) + def test_cardinality(self, device, dtype, batch_shape, out_shape): + batch_size, channels, height, width = batch_shape + h_out, w_out = out_shape + img_b = torch.rand(batch_size, channels, height, width, device=device, dtype=dtype) + H_ab = kornia.core.ops.eye_like(3, img_b) + img_a = kornia.geometry.warp_perspective(img_b, H_ab, (h_out, w_out)) + assert img_a.shape == (batch_size, channels, h_out, w_out) + + def test_exception(self, device, dtype): + img = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + homo = torch.eye(3, device=device, dtype=dtype)[None] + size = (4, 5) + + with pytest.raises(TypeError): + assert kornia.geometry.warp_perspective(0.0, homo, size) + + with pytest.raises(TypeError): + assert kornia.geometry.warp_perspective(img, 0.0, size) + + with pytest.raises(ValueError): + img = torch.rand(2, 3, 4, device=device, dtype=dtype) + assert kornia.geometry.warp_perspective(img, homo, size) + + with pytest.raises(ValueError): + homo = torch.eye(2, 2, device=device, dtype=dtype)[None] + assert kornia.geometry.warp_perspective(img, homo, size) + + def test_translation(self, device, dtype): + offset = 1.0 + h, w = 3, 4 + + img_b = torch.arange(float(h * w), device=device, dtype=dtype).view(1, 1, h, w) + homo_ab = kornia.core.ops.eye_like(3, img_b) + homo_ab[..., :2, -1] += offset + + expected = torch.zeros_like(img_b) + expected[..., 1:, 1:] = img_b[..., :2, :3] + + # Same as opencv: cv2.warpPerspective(kornia.tensor_to_image(img_b), homo_ab[0].numpy(), (w, h)) + img_a = kornia.geometry.warp_perspective(img_b, homo_ab, (h, w)) + self.assert_close(img_a, expected, atol=1e-4, rtol=1e-4) + + def test_translation_normalized(self, device, dtype): + offset = 1.0 + h, w = 3, 4 + + img_b = torch.arange(float(h * w), device=device, dtype=dtype).view(1, 1, h, w) + homo_ab = kornia.core.ops.eye_like(3, img_b) + homo_ab[..., :2, -1] += offset + + expected = torch.zeros_like(img_b) + expected[..., 1:, 1:] = img_b[..., :2, :3] + + # Same as opencv: cv2.warpPerspective(kornia.tensor_to_image(img_b), homo_ab[0].numpy(), (w, h)) + img_a = kornia.geometry.transform.homography_warp(img_b, homo_ab, (h, w), normalized_homography=False) + self.assert_close(img_a, expected, atol=1e-4, rtol=1e-4) + + def test_rotation_inverse(self, device, dtype): + h, w = 4, 4 + img_b = torch.rand(1, 1, h, w, device=device, dtype=dtype) + + # create rotation matrix of 90deg (anti-clockwise) + center = torch.tensor([[w - 1, h - 1]], device=device, dtype=dtype) / 2 + scale = torch.ones((1, 2), device=device, dtype=dtype) + angle = 90.0 * torch.ones(1, device=device, dtype=dtype) + aff_ab = kornia.geometry.get_rotation_matrix2d(center, angle, scale) + # Same as opencv: cv2.getRotationMatrix2D(((w-1)/2,(h-1)/2), 90., 1.) + + H_ab = kornia.geometry.convert_affinematrix_to_homography(aff_ab) # Bx3x3 + + # warp the tensor + # Same as opencv: cv2.warpPerspecive(kornia.tensor_to_image(img_b), H_ab[0].numpy(), (w, h)) + img_a = kornia.geometry.warp_perspective(img_b, H_ab, (h, w)) + + # invert the transform + H_ba = _torch_inverse_cast(H_ab) + img_b_hat = kornia.geometry.warp_perspective(img_a, H_ba, (h, w)) + self.assert_close(img_b_hat, img_b, rtol=1e-4, atol=1e-4) + + @pytest.mark.parametrize("batch_size", [1, 5]) + @pytest.mark.parametrize("channels", [1, 5]) + def test_crop(self, batch_size, channels, device, dtype): + # generate input data + src_h, src_w = 3, 3 + dst_h, dst_w = 3, 3 + + # [x, y] origin + # top-left, top-right, bottom-right, bottom-left + points_src = torch.tensor( + [[[0, 0], [0, src_w - 1], [src_h - 1, src_w - 1], [src_h - 1, 0]]], device=device, dtype=dtype + ) + + # [x, y] destination + # top-left, top-right, bottom-right, bottom-left + points_dst = torch.tensor( + [[[0, 0], [0, dst_w - 1], [dst_h - 1, dst_w - 1], [dst_h - 1, 0]]], device=device, dtype=dtype + ) + + # compute transformation between points + dst_trans_src = kornia.geometry.get_perspective_transform(points_src, points_dst).expand(batch_size, -1, -1) + + # warp tensor + patch = torch.tensor( + [[[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]]]], device=device, dtype=dtype + ).expand(batch_size, channels, -1, -1) + + expected = patch[..., :3, :3] + + # warp and assert + patch_warped = kornia.geometry.warp_perspective(patch, dst_trans_src, (dst_h, dst_w)) + self.assert_close(patch_warped, expected) + + def test_crop_center_resize(self, device, dtype): + # generate input data + dst_h, dst_w = 4, 4 + + # [x, y] origin + # top-left, top-right, bottom-right, bottom-left + points_src = torch.tensor([[[1, 1], [1, 2], [2, 2], [2, 1]]], device=device, dtype=dtype) + + # [x, y] destination + # top-left, top-right, bottom-right, bottom-left + points_dst = torch.tensor( + [[[0, 0], [0, dst_w - 1], [dst_h - 1, dst_w - 1], [dst_h - 1, 0]]], device=device, dtype=dtype + ) + + # compute transformation between points + dst_trans_src = kornia.geometry.get_perspective_transform(points_src, points_dst) + + # warp tensor + patch = torch.tensor( + [[[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]]]], device=device, dtype=dtype + ) + + expected = torch.tensor( + [ + [ + [ + [6.0000, 6.3333, 6.6667, 7.0000], + [7.3333, 7.6667, 8.0000, 8.3333], + [8.6667, 9.0000, 9.3333, 9.6667], + [10.0000, 10.3333, 10.6667, 11.0000], + ] + ] + ], + device=device, + dtype=dtype, + ) + + # warp and assert + patch_warped = kornia.geometry.warp_perspective(patch, dst_trans_src, (dst_h, dst_w)) + self.assert_close(patch_warped, expected) + + def test_dynamo(self, device, dtype, torch_optimizer): + if dtype == torch.float64 and torch_version() in {"2.0.0", "2.0.1"} and sys.platform == "linux": + pytest.xfail("Failing on CI on ubuntu with torch 2.0.0 for float64") + img = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + H_ab = kornia.core.ops.eye_like(3, img) + args = (img, H_ab, (4, 5)) + op = kornia.geometry.warp_perspective + op_optimized = torch_optimizer(op) + self.assert_close(op(*args), op_optimized(*args)) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 2, 3, 4 + img_b = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + H_ab = kornia.core.ops.eye_like(3, img_b) + # TODO(dmytro/edgar): firgure out why gradient don't propagate for the tranaform + self.gradcheck( + kornia.geometry.warp_perspective, (img_b, H_ab, (height, width)), requires_grad=(True, False, False) + ) + + def test_fill_padding_translation(self, device, dtype): + offset = 1.0 + h, w = 3, 4 + + img_b = torch.arange(float(3 * h * w), device=device, dtype=dtype).view(1, 3, h, w) + homo_ab = kornia.core.ops.eye_like(3, img_b) + homo_ab[..., :2, -1] += offset + + # normally fill_value will also be converted to the right device and type in warp_perspective + fill_value = torch.tensor([0.5, 0.2, 0.1], device=device, dtype=dtype) + + img_a = kornia.geometry.warp_perspective(img_b, homo_ab, (h, w), padding_mode="fill", fill_value=fill_value) + top_row_mean = img_a[..., :1, :].mean(dim=[0, 2, 3]) + first_col_mean = img_a[..., :1].mean(dim=[0, 2, 3]) + self.assert_close(top_row_mean, fill_value) + self.assert_close(first_col_mean, fill_value) + + +class TestRemap(BaseTester): + def test_smoke(self, device, dtype): + height, width = 3, 4 + input_org = torch.ones(1, 1, height, width, device=device, dtype=dtype) + grid = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=False, device=device, dtype=dtype) + input_warped = kornia.geometry.remap( + input_org, grid[..., 0], grid[..., 1], normalized_coordinates=False, align_corners=True + ) + self.assert_close(input_org, input_warped, rtol=1e-4, atol=1e-4) + + def test_different_size(self, device, dtype): + height, width = 3, 4 + grid = kornia.geometry.create_meshgrid(height, width, device=device, dtype=dtype) + + img = torch.rand(1, 2, 6, 5, device=device, dtype=dtype) + img_warped = kornia.geometry.remap(img, grid[..., 0], grid[..., 1]) + assert img_warped.shape == (1, 2, height, width) + + def test_shift(self, device, dtype): + height, width = 3, 4 + inp = torch.tensor( + [[[[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]]], device=device, dtype=dtype + ) + expected = torch.tensor( + [[[[1.0, 1.0, 1.0, 0.0], [1.0, 1.0, 1.0, 0.0], [0.0, 0.0, 0.0, 0.0]]]], device=device, dtype=dtype + ) + + grid = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=False, device=device).to(dtype) + grid += 1.0 # apply shift in both x/y direction + + input_warped = kornia.geometry.remap(inp, grid[..., 0], grid[..., 1], align_corners=True) + self.assert_close(input_warped, expected, rtol=1e-4, atol=1e-4) + + def test_shift_batch(self, device, dtype): + height, width = 3, 4 + inp = torch.tensor( + [[[[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]]], device=device, dtype=dtype + ).repeat(2, 1, 1, 1) + + expected = torch.tensor( + [ + [[[1.0, 1.0, 1.0, 0.0], [1.0, 1.0, 1.0, 0.0], [1.0, 1.0, 1.0, 0.0]]], + [[[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [0.0, 0.0, 0.0, 0.0]]], + ], + device=device, + dtype=dtype, + ) + + # generate a batch of grids + grid = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=False, device=device).to(dtype) + grid = grid.repeat(2, 1, 1, 1) + grid[0, ..., 0] += 1.0 # apply shift in the x direction + grid[1, ..., 1] += 1.0 # apply shift in the y direction + + input_warped = kornia.geometry.remap(inp, grid[..., 0], grid[..., 1], align_corners=True) + self.assert_close(input_warped, expected, rtol=1e-4, atol=1e-4) + + def test_shift_batch_broadcast(self, device, dtype): + height, width = 3, 4 + inp = torch.tensor( + [[[[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]]], device=device, dtype=dtype + ).repeat(2, 1, 1, 1) + expected = torch.tensor( + [[[[1.0, 1.0, 1.0, 0.0], [1.0, 1.0, 1.0, 0.0], [0.0, 0.0, 0.0, 0.0]]]], device=device, dtype=dtype + ).repeat(2, 1, 1, 1) + + grid = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=False, device=device).to(dtype) + grid += 1.0 # apply shift in both x/y direction + + input_warped = kornia.geometry.remap(inp, grid[..., 0], grid[..., 1], align_corners=True) + self.assert_close(input_warped, expected, rtol=1e-4, atol=1e-4) + + def test_normalized_coordinates(self, device, dtype): + height, width = 3, 4 + normalized_coordinates = True + inp = torch.tensor( + [[[[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]]], device=device, dtype=dtype + ).repeat(2, 1, 1, 1) + expected = torch.tensor( + [[[[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]]], device=device, dtype=dtype + ).repeat(2, 1, 1, 1) + + grid = kornia.geometry.create_meshgrid( + height, width, normalized_coordinates=normalized_coordinates, device=device + ).to(dtype) + + # Normalized input coordinates + input_warped = kornia.geometry.remap( + inp, grid[..., 0], grid[..., 1], align_corners=True, normalized_coordinates=normalized_coordinates + ) + self.assert_close(input_warped, expected, rtol=1e-4, atol=1e-4) + + def test_gradcheck(self, device): + batch_size, channels, height, width = 1, 2, 3, 4 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + + grid = kornia.geometry.create_meshgrid( + height, width, normalized_coordinates=False, device=device, dtype=torch.float64 + ) + + self.gradcheck( + kornia.geometry.remap, + (img, grid[..., 0], grid[..., 1], "bilinear", "zeros", True), + requires_grad=(True, False, False, False, False, False), + ) + + @pytest.mark.skip(reason="Not fully support dynamo") + def test_dynamo(self, device, dtype, torch_optimizer): + # TODO: add dynamo support to create_meshgrid + batch_size, channels, height, width = 1, 1, 3, 4 + img = torch.ones(batch_size, channels, height, width, device=device, dtype=dtype) + + grid = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=False, device=device).to(dtype) + grid += 1.0 # apply some shift + + op = kornia.geometry.remap + op_script = torch_optimizer(op) + + inputs = (img, grid[..., 0], grid[..., 1], "bilinear", "zeros", True) + actual = op_script(*inputs) + expected = op(*inputs) + self.assert_close(actual, expected, rtol=1e-4, atol=1e-4) + + +class TestInvertAffineTransform(BaseTester): + def test_smoke(self, device, dtype): + matrix = torch.eye(2, 3, device=device, dtype=dtype)[None] + matrix_inv = kornia.geometry.invert_affine_transform(matrix) + self.assert_close(matrix, matrix_inv, rtol=1e-4, atol=1e-4) + + def test_rot90(self, device, dtype): + angle = torch.tensor([90.0], device=device, dtype=dtype) + scale = torch.tensor([[1.0, 1.0]], device=device, dtype=dtype) + center = torch.tensor([[0.0, 0.0]], device=device, dtype=dtype) + expected = torch.tensor([[[0.0, -1.0, 0.0], [1.0, 0.0, 0.0]]], device=device, dtype=dtype) + matrix = kornia.geometry.get_rotation_matrix2d(center, angle, scale) + matrix_inv = kornia.geometry.invert_affine_transform(matrix) + self.assert_close(matrix_inv, expected, rtol=1e-4, atol=1e-4) + + def test_rot90_batch(self, device, dtype): + angle = torch.tensor([90.0], device=device, dtype=dtype) + scale = torch.tensor([[1.0, 1.0]], device=device, dtype=dtype) + center = torch.tensor([[0.0, 0.0]], device=device, dtype=dtype) + expected = torch.tensor([[[0.0, -1.0, 0.0], [1.0, 0.0, 0.0]]], device=device, dtype=dtype).repeat(2, 1, 1) + matrix = kornia.geometry.get_rotation_matrix2d(center, angle, scale).repeat(2, 1, 1) + matrix_inv = kornia.geometry.invert_affine_transform(matrix) + self.assert_close(matrix_inv, expected, rtol=1e-4, atol=1e-4) + + def test_gradcheck(self, device): + matrix = torch.eye(2, 3, device=device, dtype=torch.float64)[None] + self.gradcheck(kornia.geometry.invert_affine_transform, (matrix,)) + + def test_dynamo(self, device, dtype, torch_optimizer): + op = kornia.geometry.invert_affine_transform + op_script = torch_optimizer(op) + + matrix = torch.eye(2, 3, device=device, dtype=dtype)[None] + actual = op_script(matrix) + expected = op(matrix) + self.assert_close(actual, expected, rtol=1e-4, atol=1e-4) diff --git a/tests/geometry/transform/test_imgwarp3d.py b/tests/geometry/transform/test_imgwarp3d.py new file mode 100644 index 0000000..c8521fb --- /dev/null +++ b/tests/geometry/transform/test_imgwarp3d.py @@ -0,0 +1,375 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia +import kornia.geometry.transform as proj +from kornia.core.utils import _torch_inverse_cast + +from testing.base import BaseTester + + +class TestWarpAffine3d(BaseTester): + def test_smoke(self, device, dtype): + sample = torch.rand(1, 3, 3, 4, 5, device=device, dtype=dtype) + P = torch.rand(1, 3, 4, device=device, dtype=dtype) + output = proj.warp_affine3d(sample, P, (3, 4, 5)) + assert output.shape == (1, 3, 3, 4, 5) + + @pytest.mark.parametrize("batch_size", [1, 3]) + @pytest.mark.parametrize("num_channels", [1, 3, 5]) + @pytest.mark.parametrize("out_shape", [(3, 3, 3), (4, 5, 6)]) + def test_batch(self, batch_size, num_channels, out_shape, device, dtype): + B, C = batch_size, num_channels + sample = torch.rand(B, C, 3, 4, 5, device=device, dtype=dtype) + P = torch.rand(B, 3, 4, device=device, dtype=dtype) + output = proj.warp_affine3d(sample, P, out_shape) + assert list(output.shape) == [B, C, *list(out_shape)] + + def test_gradcheck(self, device): + # generate input data + sample = torch.rand(1, 3, 3, 4, 5, device=device, dtype=torch.float64, requires_grad=True) + P = torch.rand(1, 3, 4, device=device, dtype=torch.float64) + self.gradcheck(proj.warp_affine3d, (sample, P, (3, 3, 3))) + + def test_forth_back(self, device, dtype): + out_shape = (3, 4, 5) + sample = torch.rand(2, 5, 3, 4, 5, device=device, dtype=dtype) + P = torch.rand(2, 3, 4, device=device, dtype=dtype) + P = kornia.geometry.convert_affinematrix_to_homography3d(P) + P_hat = (_torch_inverse_cast(P) @ P)[:, :3] + output = proj.warp_affine3d(sample, P_hat, out_shape, flags="nearest") + self.assert_close(output, sample, rtol=1e-4, atol=1e-4) + + def test_rotate_x(self, device, dtype): + sample = torch.tensor( + [ + [ + [ + [[0.0, 0.0, 0.0], [0.0, 2.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [ + [ + [ + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 2.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + ] + ] + ], + device=device, + dtype=dtype, + ) + + _, _, D, H, W = sample.shape + center = torch.tensor([[(W - 1) / 2, (H - 1) / 2, (D - 1) / 2]], device=device, dtype=dtype) + + angles = torch.tensor([[90.0, 0.0, 0.0]], device=device, dtype=dtype) + + scales: torch.Tensor = torch.ones_like(angles, device=device, dtype=dtype) + P = proj.get_projective_transform(center, angles, scales) + output = proj.warp_affine3d(sample, P, (3, 3, 3)) + self.assert_close(output, expected, rtol=1e-4, atol=1e-4) + + def test_rotate_y(self, device, dtype): + sample = torch.tensor( + [ + [ + [ + [[0.0, 0.0, 0.0], [0.0, 2.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [ + [ + [ + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [2.0, 1.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + ] + ] + ], + device=device, + dtype=dtype, + ) + + _, _, D, H, W = sample.shape + center = torch.tensor([[(W - 1) / 2, (H - 1) / 2, (D - 1) / 2]], device=device, dtype=dtype) + + angles = torch.tensor([[0.0, 90.0, 0.0]], device=device, dtype=dtype) + + scales: torch.Tensor = torch.ones_like(angles, device=device, dtype=dtype) + P = proj.get_projective_transform(center, angles, scales) + output = proj.warp_affine3d(sample, P, (3, 3, 3)) + self.assert_close(output, expected, rtol=1e-4, atol=1e-4) + + def test_rotate_z(self, device, dtype): + sample = torch.tensor( + [ + [ + [ + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 2.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + ] + ] + ], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [ + [ + [ + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 1.0, 2.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + ] + ] + ], + device=device, + dtype=dtype, + ) + + _, _, D, H, W = sample.shape + center = torch.tensor([[(W - 1) / 2, (H - 1) / 2, (D - 1) / 2]], device=device, dtype=dtype) + + angles = torch.tensor([[0.0, 0.0, 90.0]], device=device, dtype=dtype) + + scales: torch.Tensor = torch.ones_like(angles, device=device, dtype=dtype) + P = proj.get_projective_transform(center, angles, scales) + output = proj.warp_affine3d(sample, P, (3, 3, 3)) + self.assert_close(output, expected, rtol=1e-4, atol=1e-4) + + def test_rotate_y_large(self, device, dtype): + """Rotates 90deg anti-clockwise.""" + sample = torch.tensor( + [ + [ + [ + [[0.0, 4.0, 0.0], [0.0, 3.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 2.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + ], + [ + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 9.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 6.0, 7.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 8.0, 0.0], [0.0, 0.0, 0.0]], + ], + ] + ], + device=device, + dtype=dtype, + ) + + expected = torch.tensor( + [ + [ + [ + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + [[4.0, 2.0, 0.0], [3.0, 1.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + ], + [ + [[0.0, 0.0, 0.0], [0.0, 7.0, 0.0], [0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 6.0, 8.0], [9.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], + ], + ] + ], + device=device, + dtype=dtype, + ) + + _, _, D, H, W = sample.shape + center = torch.tensor([[(W - 1) / 2, (H - 1) / 2, (D - 1) / 2]], device=device, dtype=dtype) + + angles = torch.tensor([[0.0, 90.0, 0.0]], device=device, dtype=dtype) + + scales: torch.Tensor = torch.ones_like(angles, device=device, dtype=dtype) + P = proj.get_projective_transform(center, angles, scales) + output = proj.warp_affine3d(sample, P, (3, 3, 3)) + self.assert_close(output, expected, rtol=1e-4, atol=1e-4) + + +class TestGetRotationMatrix3d(BaseTester): + def test_smoke(self, device, dtype): + center = torch.rand(1, 3, device=device, dtype=dtype) + angle = torch.rand(1, 3, device=device, dtype=dtype) + scales: torch.Tensor = torch.ones_like(angle, device=device, dtype=dtype) + P = proj.get_projective_transform(center, angle, scales) + assert P.shape == (1, 3, 4) + + @pytest.mark.parametrize("batch_size", [1, 3, 6]) + def test_batch(self, batch_size, device, dtype): + B: int = batch_size + center = torch.rand(B, 3, device=device, dtype=dtype) + angle = torch.rand(B, 3, device=device, dtype=dtype) + scales: torch.Tensor = torch.ones_like(angle, device=device, dtype=dtype) + P = proj.get_projective_transform(center, angle, scales) + assert P.shape == (B, 3, 4) + + def test_identity(self, device, dtype): + center = torch.zeros(1, 3, device=device, dtype=dtype) + angle = torch.zeros(1, 3, device=device, dtype=dtype) + scales: torch.Tensor = torch.ones_like(angle, device=device, dtype=dtype) + P = proj.get_projective_transform(center, angle, scales) + P_expected = torch.tensor( + [[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0]], device=device, dtype=dtype + ).unsqueeze(0) + self.assert_close(P, P_expected, atol=1e-4, rtol=1e-4) + + def test_rot90x(self, device, dtype): + center = torch.zeros(1, 3, device=device, dtype=dtype) + angle = torch.tensor([[90.0, 0.0, 0.0]], device=device, dtype=dtype) + scales: torch.Tensor = torch.ones_like(angle, device=device, dtype=dtype) + P = proj.get_projective_transform(center, angle, scales) + P_expected = torch.tensor( + [[1.0, 0.0, 0.0, 0.0], [0.0, 0.0, -1.0, 0.0], [0.0, 1.0, 0.0, 0.0]], device=device, dtype=dtype + ).unsqueeze(0) + self.assert_close(P, P_expected, atol=1e-4, rtol=1e-4) + + def test_rot90y(self, device, dtype): + center = torch.zeros(1, 3, device=device, dtype=dtype) + angle = torch.tensor([[0.0, 90.0, 0.0]], device=device, dtype=dtype) + scales: torch.Tensor = torch.ones_like(angle, device=device, dtype=dtype) + P = proj.get_projective_transform(center, angle, scales) + P_expected = torch.tensor( + [[0.0, 0.0, 1.0, 0.0], [0.0, 1.0, 0.0, 0.0], [-1.0, 0.0, 0.0, 0.0]], device=device, dtype=dtype + ).unsqueeze(0) + self.assert_close(P, P_expected, atol=1e-4, rtol=1e-4) + + def test_rot90z(self, device, dtype): + center = torch.zeros(1, 3, device=device, dtype=dtype) + angle = torch.tensor([[0.0, 0.0, 90.0]], device=device, dtype=dtype) + scales: torch.Tensor = torch.ones_like(angle, device=device, dtype=dtype) + P = proj.get_projective_transform(center, angle, scales) + P_expected = torch.tensor( + [[0.0, -1.0, 0.0, 0.0], [1.0, 0.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0]], device=device, dtype=dtype + ).unsqueeze(0) + self.assert_close(P, P_expected, atol=1e-4, rtol=1e-4) + + def test_gradcheck(self, device): + # generate input data + center = torch.rand(1, 3, device=device, dtype=torch.float64, requires_grad=True) + angle = torch.rand(1, 3, device=device, dtype=torch.float64) + scales: torch.Tensor = torch.ones_like(angle, device=device, dtype=torch.float64) + self.gradcheck(proj.get_projective_transform, (center, angle, scales)) + + +class TestPerspectiveTransform3D(BaseTester): + @pytest.mark.skip("Not working") + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_get_perspective_transform3d(self, batch_size, device, dtype): + # generate input data + # d_max, h_max, w_max = 16, 64, 32 # height, width + # d = torch.ceil(d_max * torch.rand(batch_size, device=device, dtype=dtype)) + # h = torch.ceil(h_max * torch.rand(batch_size, device=device, dtype=dtype)) + # w = torch.ceil(w_max * torch.rand(batch_size, device=device, dtype=dtype)) + + norm = torch.rand(batch_size, 8, 3, device=device, dtype=dtype) + points_src = torch.rand_like(norm, device=device, dtype=dtype) + points_dst = points_src + norm + + # compute transform from source to target + dst_homo_src = kornia.geometry.transform.get_perspective_transform3d(points_src, points_dst) + + # TODO: get_perspective_transform3d seems to be correct since it would result in the + # expected output for cropping volumes. Not sure what is going on here. + self.assert_close( + kornia.geometry.linalg.transform_points(dst_homo_src, points_src), points_dst, rtol=1e-4, atol=1e-4 + ) + + # compute gradient check + self.gradcheck(kornia.geometry.transform.get_perspective_transform3d, (points_src, points_dst), fast_mode=False) + + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_get_perspective_transform3d_2(self, batch_size, device, dtype): + torch.manual_seed(0) + src = kornia.geometry.bbox.bbox_generator3d( + torch.randint_like(torch.ones(batch_size), 0, 50, dtype=dtype), + torch.randint_like(torch.ones(batch_size), 0, 50, dtype=dtype), + torch.randint_like(torch.ones(batch_size), 0, 50, dtype=dtype), + torch.randint(0, 50, (1,), dtype=dtype).repeat(batch_size), + torch.randint(0, 50, (1,), dtype=dtype).repeat(batch_size), + torch.randint(0, 50, (1,), dtype=dtype).repeat(batch_size), + ).to(device=device, dtype=dtype) + dst = kornia.geometry.bbox.bbox_generator3d( + torch.randint_like(torch.ones(batch_size), 0, 50, dtype=dtype), + torch.randint_like(torch.ones(batch_size), 0, 50, dtype=dtype), + torch.randint_like(torch.ones(batch_size), 0, 50, dtype=dtype), + torch.randint(0, 50, (1,), dtype=dtype).repeat(batch_size), + torch.randint(0, 50, (1,), dtype=dtype).repeat(batch_size), + torch.randint(0, 50, (1,), dtype=dtype).repeat(batch_size), + ).to(device=device, dtype=dtype) + out = kornia.geometry.transform.get_perspective_transform3d(src, dst) + if batch_size == 1: + expected = torch.tensor( + [ + [ + [3.3000, 0.0000, 0.0000, -118.2000], + [0.0000, 0.0769, 0.0000, 0.0000], + [0.0000, 0.0000, 0.5517, 28.7930], + [0.0000, 0.0000, 0.0000, 1.0000], + ] + ], + device=device, + dtype=dtype, + ) + if batch_size == 2: + expected = torch.tensor( + [ + [ + [0.9630, 0.0000, 0.0000, -9.3702], + [0.0000, 2.0000, 0.0000, -49.9999], + [0.0000, 0.0000, 0.3830, 44.0213], + [0.0000, 0.0000, 0.0000, 1.0000], + ], + [ + [0.9630, 0.0000, 0.0000, -36.5555], + [0.0000, 2.0000, 0.0000, -14.0000], + [0.0000, 0.0000, 0.3830, 16.8940], + [0.0000, 0.0000, 0.0000, 1.0000], + ], + ], + device=device, + dtype=dtype, + ) + + self.assert_close(out, expected, rtol=1e-4, atol=1e-4) + + # compute gradient check + self.gradcheck(kornia.geometry.transform.get_perspective_transform3d, (src, dst)) diff --git a/tests/geometry/transform/test_pyramid.py b/tests/geometry/transform/test_pyramid.py new file mode 100644 index 0000000..8f0e17e --- /dev/null +++ b/tests/geometry/transform/test_pyramid.py @@ -0,0 +1,185 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestPyrUp(BaseTester): + def test_shape(self, device, dtype): + inp = torch.zeros(1, 2, 4, 4, device=device, dtype=dtype) + pyr = kornia.geometry.PyrUp() + assert pyr(inp).shape == (1, 2, 8, 8) + + def test_shape_batch(self, device, dtype): + inp = torch.zeros(2, 2, 4, 4, device=device, dtype=dtype) + pyr = kornia.geometry.PyrUp() + assert pyr(inp).shape == (2, 2, 8, 8) + + def test_gradcheck(self, device): + img = torch.rand(1, 2, 5, 4, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.pyrup, (img,), nondet_tol=1e-8) + + +class TestPyrDown(BaseTester): + def test_shape(self, device, dtype): + inp = torch.zeros(1, 2, 4, 4, device=device, dtype=dtype) + pyr = kornia.geometry.PyrDown() + assert pyr(inp).shape == (1, 2, 2, 2) + + def test_shape_custom_factor(self, device, dtype): + inp = torch.zeros(1, 2, 9, 9, device=device, dtype=dtype) + pyr = kornia.geometry.PyrDown(factor=3.0) + assert pyr(inp).shape == (1, 2, 3, 3) + + def test_shape_batch(self, device, dtype): + inp = torch.zeros(2, 2, 4, 4, device=device, dtype=dtype) + pyr = kornia.geometry.PyrDown() + assert pyr(inp).shape == (2, 2, 2, 2) + + def test_symmetry_preserving(self, device, dtype): + inp = torch.zeros(1, 1, 6, 6, device=device, dtype=dtype) + inp[:, :, 2:4, 2:4] = 1.0 + pyr_out = kornia.geometry.PyrDown()(inp).squeeze() + self.assert_close(pyr_out, pyr_out.flip(0)) + self.assert_close(pyr_out, pyr_out.flip(1)) + + def test_gradcheck(self, device): + img = torch.rand(1, 2, 5, 4, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.pyrdown, (img,), nondet_tol=1e-8) + + +class TestScalePyramid(BaseTester): + def test_shape_tuple(self, device, dtype): + inp = torch.zeros(3, 2, 41, 41, device=device, dtype=dtype) + SP = kornia.geometry.ScalePyramid(n_levels=1, min_size=30) + out = SP(inp) + assert len(out) == 3 + assert len(out[0]) == 1 + assert len(out[1]) == 1 + assert len(out[2]) == 1 + + def test_shape_batch(self, device, dtype): + inp = torch.zeros(3, 2, 31, 31, device=device, dtype=dtype) + SP = kornia.geometry.ScalePyramid(n_levels=1) + sp, _, _ = SP(inp) + assert sp[0].shape == (3, 2, 3 + 1, 31, 31) + + def test_shape_batch_double(self, device, dtype): + inp = torch.zeros(3, 2, 31, 31, device=device, dtype=dtype) + SP = kornia.geometry.ScalePyramid(n_levels=1, double_image=True) + sp, _, _ = SP(inp) + assert sp[0].shape == (3, 2, 1 + 3, 62, 62) + + def test_n_levels_shape(self, device, dtype): + inp = torch.zeros(1, 1, 32, 32, device=device, dtype=dtype) + SP = kornia.geometry.ScalePyramid(n_levels=3) + sp, _, _ = SP(inp) + assert sp[0].shape == (1, 1, 3 + 3, 32, 32) + + def test_blur_order(self, device, dtype): + inp = torch.rand(1, 1, 31, 31, device=device, dtype=dtype) + SP = kornia.geometry.ScalePyramid(n_levels=3) + sp, _, _ = SP(inp) + for _, pyr_level in enumerate(sp): + for _, img in enumerate(pyr_level): + img = img.squeeze().view(3, -1) + max_per_blur_level_val, _ = img.max(dim=1) + assert torch.argmax(max_per_blur_level_val).item() == 0 + + def test_symmetry_preserving(self, device, dtype): + PS = 16 + R = 2 + inp = torch.zeros(1, 1, PS, PS, device=device, dtype=dtype) + inp[..., PS // 2 - R : PS // 2 + R, PS // 2 - R : PS // 2 + R] = 1.0 + SP = kornia.geometry.ScalePyramid(n_levels=3) + sp, _, _ = SP(inp) + for _, pyr_level in enumerate(sp): + for _, img in enumerate(pyr_level): + img = img.squeeze() + self.assert_close(img, img.flip(1)) + self.assert_close(img, img.flip(2)) + + def test_gradcheck(self, device): + img = torch.rand(1, 2, 7, 9, device=device, dtype=torch.float64) + from kornia.geometry import ScalePyramid as SP + + def sp_tuple(img): + sp, _, _ = SP()(img) + return tuple(sp) + + self.gradcheck(sp_tuple, (img,), nondet_tol=1e-4) + + +class TestBuildPyramid(BaseTester): + def test_smoke(self, device, dtype): + sample = torch.ones(1, 2, 4, 5, device=device, dtype=dtype) + pyramid = kornia.geometry.transform.build_pyramid(sample, max_level=1) + assert len(pyramid) == 1 + assert pyramid[0].shape == (1, 2, 4, 5) + + @pytest.mark.parametrize("batch_size", (1, 2, 3)) + @pytest.mark.parametrize("channels", (1, 3)) + @pytest.mark.parametrize("max_level", (2, 3, 4)) + def test_num_levels(self, batch_size, channels, max_level, device, dtype): + height, width = 16, 20 + sample = torch.rand(batch_size, channels, height, width, device=device, dtype=dtype) + pyramid = kornia.geometry.transform.build_pyramid(sample, max_level) + assert len(pyramid) == max_level + for i in range(1, max_level): + img = pyramid[i] + denom = 2**i + expected_shape = (batch_size, channels, height // denom, width // denom) + assert img.shape == expected_shape + + def test_gradcheck(self, device): + max_level = 1 + batch_size, channels, height, width = 1, 2, 7, 9 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.transform.build_pyramid, (img, max_level)) + + +class TestBuildLaplacianPyramid(BaseTester): + def test_smoke(self, device, dtype): + sample = torch.ones(1, 2, 4, 5, device=device, dtype=dtype) + pyramid = kornia.geometry.transform.build_laplacian_pyramid(sample, max_level=1) + assert len(pyramid) == 1 + assert pyramid[0].shape == (1, 2, 4, 5) + + @pytest.mark.parametrize("batch_size", (1, 2, 3)) + @pytest.mark.parametrize("channels", (1, 3)) + @pytest.mark.parametrize("max_level", (2, 3, 4)) + def test_num_levels(self, batch_size, channels, max_level, device, dtype): + height, width = 16, 32 + sample = torch.rand(batch_size, channels, height, width, device=device, dtype=dtype) + pyramid = kornia.geometry.transform.build_laplacian_pyramid(sample, max_level) + assert len(pyramid) == max_level + for i in range(1, max_level): + img = pyramid[i] + denom = 2**i + expected_shape = (batch_size, channels, height // denom, width // denom) + assert img.shape == expected_shape + + def test_gradcheck(self, device): + max_level = 1 + batch_size, channels, height, width = 1, 2, 7, 9 + img = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64) + self.gradcheck(kornia.geometry.transform.build_laplacian_pyramid, (img, max_level), nondet_tol=1e-8) diff --git a/tests/geometry/transform/test_thin_plate_spline.py b/tests/geometry/transform/test_thin_plate_spline.py new file mode 100644 index 0000000..cf0a3ef --- /dev/null +++ b/tests/geometry/transform/test_thin_plate_spline.py @@ -0,0 +1,276 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +def _sample_points(batch_size, device, dtype=torch.float32): + src = torch.tensor([[[0.0, 0.0], [0.0, 10.0], [10.0, 0.0], [10.0, 10.0], [5.0, 5.0]]], device=device, dtype=dtype) + src = src.repeat(batch_size, 1, 1) + dst = src + torch.rand_like(src) * 2.5 + return src, dst + + +class TestTransformParameters(BaseTester): + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_smoke(self, batch_size, device, dtype): + src = torch.rand(batch_size, 4, 2, device=device) + out = kornia.geometry.transform.get_tps_transform(src, src) + assert len(out) == 2 + + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_no_warp(self, batch_size, device, dtype): + src = torch.rand(batch_size, 5, 2, device=device) + kernel, affine = kornia.geometry.transform.get_tps_transform(src, src) + target_kernel = torch.zeros(batch_size, 5, 2, device=device) + target_affine = torch.zeros(batch_size, 3, 2, device=device) + target_affine[:, [1, 2], [0, 1]] = 1.0 + self.assert_close(kernel, target_kernel, atol=1e-4, rtol=1e-4) + self.assert_close(affine, target_affine, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_affine_only(self, batch_size, device, dtype): + src = torch.tensor([[[0.0, 0.0], [0.0, 1.0], [1.0, 0.0], [1.0, 1.0], [0.5, 0.5]]], device=device).repeat( + batch_size, 1, 1 + ) + dst = src.clone() * 2.0 + kernel, _ = kornia.geometry.transform.get_tps_transform(src, dst) + self.assert_close(kernel, torch.zeros_like(kernel), atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_exception(self, batch_size, device, dtype): + with pytest.raises(TypeError): + src = torch.rand(batch_size, 5, 2).numpy() + assert kornia.geometry.transform.get_tps_transform(src, src) + + with pytest.raises(ValueError): + src = torch.rand(batch_size, 5) + assert kornia.geometry.transform.get_tps_transform(src, src) + + @pytest.mark.grad() + @pytest.mark.parametrize("batch_size", [1, 3]) + @pytest.mark.parametrize("requires_grad", [True, False]) + def test_gradcheck(self, batch_size, device, dtype, requires_grad): + opts = {"device": device, "dtype": torch.float64} + src, dst = _sample_points(batch_size, **opts) + src.requires_grad_(requires_grad) + dst.requires_grad_(not requires_grad) + assert self.gradcheck( + kornia.geometry.transform.get_tps_transform, (src, dst), raise_exception=True, fast_mode=True + ) + + @pytest.mark.jit() + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_jit(self, batch_size, device, dtype): + src, dst = _sample_points(batch_size, device) + op = kornia.geometry.transform.get_tps_transform + op_jit = torch.jit.script(op) + op_output = op(src, dst) + jit_output = op_jit(src, dst) + self.assert_close(op_output[0], jit_output[0]) + self.assert_close(op_output[1], jit_output[1]) + + +class TestWarpPoints(BaseTester): + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_smoke(self, batch_size, device, dtype): + src, dst = _sample_points(batch_size, device) + kernel, affine = kornia.geometry.transform.get_tps_transform(src, dst) + warp = kornia.geometry.transform.warp_points_tps(src, dst, kernel, affine) + assert warp.shape == src.shape + + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_warp(self, batch_size, device, dtype): + src, dst = _sample_points(batch_size, device) + kernel, affine = kornia.geometry.transform.get_tps_transform(src, dst) + warp = kornia.geometry.transform.warp_points_tps(src, dst, kernel, affine) + self.assert_close(warp, dst, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_exception(self, batch_size, device, dtype): + src = torch.rand(batch_size, 5, 2) + kernel = torch.zeros_like(src) + affine = torch.zeros(batch_size, 3, 2) + + with pytest.raises(TypeError): + assert kornia.geometry.transform.warp_points_tps(src.numpy(), src, kernel, affine) + + with pytest.raises(TypeError): + assert kornia.geometry.transform.warp_points_tps(src, src.numpy(), kernel, affine) + + with pytest.raises(TypeError): + assert kornia.geometry.transform.warp_points_tps(src, src, kernel.numpy(), affine) + + with pytest.raises(TypeError): + assert kornia.geometry.transform.warp_points_tps(src, src, kernel, affine.numpy()) + + with pytest.raises(ValueError): + src_bad = torch.rand(batch_size, 5) + assert kornia.geometry.transform.warp_points_tps(src_bad, src, kernel, affine) + + with pytest.raises(ValueError): + src_bad = torch.rand(batch_size, 5) + assert kornia.geometry.transform.warp_points_tps(src, src_bad, kernel, affine) + + with pytest.raises(ValueError): + kernel_bad = torch.rand(batch_size, 5) + assert kornia.geometry.transform.warp_points_tps(src, src, kernel_bad, affine) + + with pytest.raises(ValueError): + affine_bad = torch.rand(batch_size, 3) + assert kornia.geometry.transform.warp_points_tps(src, src, kernel, affine_bad) + + @pytest.mark.grad() + @pytest.mark.parametrize("batch_size", [1, 3]) + @pytest.mark.parametrize("requires_grad", [True, False]) + def test_gradcheck(self, batch_size, device, dtype, requires_grad): + opts = {"device": device, "dtype": torch.float64} + src, dst = _sample_points(batch_size, **opts) + kernel, affine = kornia.geometry.transform.get_tps_transform(src, dst) + kernel.requires_grad_(requires_grad) + affine.requires_grad_(not requires_grad) + assert self.gradcheck( + kornia.geometry.transform.warp_points_tps, (src, dst, kernel, affine), raise_exception=True, fast_mode=True + ) + + @pytest.mark.jit() + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_jit(self, batch_size, device, dtype): + src, dst = _sample_points(batch_size, device) + kernel, affine = kornia.geometry.transform.get_tps_transform(src, dst) + op = kornia.geometry.transform.warp_points_tps + op_jit = torch.jit.script(op) + self.assert_close(op(src, dst, kernel, affine), op_jit(src, dst, kernel, affine)) + + +class TestWarpImage(BaseTester): + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_smoke(self, batch_size, device, dtype): + src, dst = _sample_points(batch_size, device) + tensor = torch.rand(batch_size, 3, 32, 32, device=device) + kernel, affine = kornia.geometry.transform.get_tps_transform(src, dst) + warp = kornia.geometry.transform.warp_image_tps(tensor, dst, kernel, affine) + assert warp.shape == tensor.shape + + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_warp(self, batch_size, device, dtype): + src = torch.tensor([[[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0], [0.0, 0.0]]], device=device).repeat( + batch_size, 1, 1 + ) + # zoom in by a factor of 2 + dst = src.clone() * 2.0 + tensor = torch.zeros(batch_size, 3, 8, 8, device=device) + tensor[:, :, 2:6, 2:6] = 1.0 + + expected = torch.ones_like(tensor) + # nn.grid_sample interpolates the at the edges it seems, so the boundaries have values < 1 + expected[:, :, [0, -1], :] *= 0.5 + expected[:, :, :, [0, -1]] *= 0.5 + + kernel, affine = kornia.geometry.transform.get_tps_transform(dst, src) + warp = kornia.geometry.transform.warp_image_tps(tensor, src, kernel, affine) + self.assert_close(warp, expected, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_exception(self, batch_size, device, dtype): + image = torch.rand(batch_size, 3, 32, 32) + dst = torch.rand(batch_size, 5, 2) + kernel = torch.zeros_like(dst) + affine = torch.zeros(batch_size, 3, 2) + + with pytest.raises(TypeError): + assert kornia.geometry.transform.warp_image_tps(image.numpy(), dst, kernel, affine) + + with pytest.raises(TypeError): + assert kornia.geometry.transform.warp_image_tps(image, dst.numpy(), kernel, affine) + + with pytest.raises(TypeError): + assert kornia.geometry.transform.warp_image_tps(image, dst, kernel.numpy(), affine) + + with pytest.raises(TypeError): + assert kornia.geometry.transform.warp_image_tps(image, dst, kernel, affine.numpy()) + + with pytest.raises(ValueError): + image_bad = torch.rand(batch_size, 32, 32) + assert kornia.geometry.transform.warp_image_tps(image_bad, dst, kernel, affine) + + with pytest.raises(ValueError): + dst_bad = torch.rand(batch_size, 5) + assert kornia.geometry.transform.warp_image_tps(image, dst_bad, kernel, affine) + + with pytest.raises(ValueError): + kernel_bad = torch.rand(batch_size, 5) + assert kornia.geometry.transform.warp_image_tps(image, dst, kernel_bad, affine) + + with pytest.raises(ValueError): + affine_bad = torch.rand(batch_size, 3) + assert kornia.geometry.transform.warp_image_tps(image, dst, kernel, affine_bad) + + @pytest.mark.grad() + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_gradcheck(self, batch_size, device, dtype): + if device.type != "cpu": + pytest.skip("gradcheck is unstable for warp_image_tps on CUDA") + if dtype != torch.float64: + pytest.skip("gradcheck requires float64") + + opts = {"device": device, "dtype": torch.float64} + src, dst = _sample_points(batch_size, **opts) + + # Compute TPS params without tracking gradients + with torch.no_grad(): + kernel, affine = kornia.geometry.transform.get_tps_transform(src, dst) + + image = torch.rand(batch_size, 3, 32, 32, requires_grad=True, **opts) + + assert self.gradcheck( + kornia.geometry.transform.warp_image_tps, + (image, dst, kernel, affine), + requires_grad=[True, False, False, False], + raise_exception=True, + atol=1e-4, + rtol=1e-4, + nondet_tol=1e-8, + fast_mode=True, + ) + + @pytest.mark.jit() + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_jit(self, batch_size, device, dtype): + src, dst = _sample_points(batch_size, device) + kernel, affine = kornia.geometry.transform.get_tps_transform(src, dst) + image = torch.rand(batch_size, 3, 32, 32, device=device) + op = kornia.geometry.transform.warp_image_tps + op_jit = torch.jit.script(op) + self.assert_close(op(image, dst, kernel, affine), op_jit(image, dst, kernel, affine), rtol=1e-4, atol=1e-4) + + @pytest.mark.parametrize("batch_size", [1]) + def test_identity_warp_align_corners(self, batch_size, device, dtype): + image = torch.arange(9.0, device=device, dtype=dtype).reshape(1, 1, 3, 3) + dst = torch.tensor( + [[[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0], [0.0, 0.0]]], + device=device, + dtype=dtype, + ).repeat(batch_size, 1, 1) + kernel, affine = kornia.geometry.transform.get_tps_transform(dst, dst) + warped = kornia.geometry.transform.warp_image_tps(image, dst, kernel, affine, align_corners=True) + self.assert_close(warped, image, atol=1e-4, rtol=1e-4) diff --git a/tests/image/test_draw.py b/tests/image/test_draw.py new file mode 100644 index 0000000..2afd38d --- /dev/null +++ b/tests/image/test_draw.py @@ -0,0 +1,550 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math + +import pytest +import torch + +from kornia.geometry import create_meshgrid +from kornia.image import draw_convex_polygon, draw_rectangle +from kornia.image.draw import draw_line, draw_point2d + +from testing.base import BaseTester + + +class TestDrawPoint(BaseTester): + """Test drawing individual pixels.""" + + def test_draw_point2d_rgb(self, dtype, device): + """Test plotting multiple [x, y] points.""" + points = torch.tensor([(1, 3), (2, 4)], device=device) + color = torch.tensor([5, 10, 15], dtype=dtype, device=device) + img = torch.zeros(3, 8, 8, dtype=dtype, device=device) + img = draw_point2d(img, points, color) + for x, y in points: + self.assert_close(img[:, y, x], color.to(img.dtype)) + + def test_draw_point2d_grayscale_third_order(self, dtype, device): + """Test plotting multiple [x, y] points on a (1, m, n) image.""" + points = torch.tensor([(1, 3), (2, 4)], device=device) + color = torch.tensor([100], dtype=dtype, device=device) + img = torch.zeros(1, 8, 8, dtype=dtype, device=device) + img = draw_point2d(img, points, color) + for x, y in points: + self.assert_close(img[:, y, x], color.to(img.dtype)) + + def test_draw_point2d_grayscale_second_order(self, dtype, device): + """Test plotting multiple [x, y] points on a (m, n) image.""" + points = torch.tensor([(1, 3), (2, 4)], device=device) + color = torch.tensor([100], dtype=dtype, device=device) + img = torch.zeros(8, 8, dtype=dtype, device=device) + img = draw_point2d(img, points, color) + for x, y in points: + self.assert_close(torch.unsqueeze(img[y, x], dim=0), color.to(img.dtype)) + + def test_draw_point2d_with_mismatched_dims(self, dtype, device): + """Test that we raise if the len of the color tensor != the # of image channels.""" + points = torch.tensor([(1, 3), (2, 4)], device=device) + color = torch.tensor([100], dtype=dtype, device=device) + img = torch.zeros(3, 8, 8, dtype=dtype, device=device) + with pytest.raises(Exception): + draw_point2d(img, points, color) + + def test_draw_point2d_with_mismatched_dtype(self, device): + """Test that the color is correctly cast to the image dtype when drawing points.""" + points = torch.tensor([(1, 3), (2, 4)], device=device) + color = torch.tensor([5, 10, 15], dtype=torch.float32, device=device) + img = torch.zeros(3, 8, 8, dtype=torch.uint8, device=device) + img = draw_point2d(img, points, color) + assert img.dtype is torch.uint8 + for x, y in points: + self.assert_close(img[:, y, x], color.to(torch.uint8)) + + def test_draw_point2d_with_singleton_color_dims(self, dtype, device): + """Ensure that plotting behavior is consistent if we have a singleton dim for the color.""" + points = torch.tensor([(1, 3), (2, 4)], device=device) + # Plot given a color tensor of shape [3] + color_vec = torch.tensor([5, 10, 15], dtype=torch.float32, device=device) + vec_img = torch.zeros(3, 8, 8, dtype=torch.uint8, device=device) + drawn_vec_img = draw_point2d(vec_img, points, color_vec) + # Plot given a color tensor of shape [3, 1] + color_mat = torch.unsqueeze(color_vec, dim=1) + mat_img = vec_img.clone() + drawn_mat_img = draw_point2d(mat_img, points, color_mat) + # Ensure that we get the same underlying image back + self.assert_close(drawn_vec_img, drawn_mat_img) + + +class TestDrawLine(BaseTester): + def test_draw_line_vertical(self, dtype, device): + """Test drawing a vertical line.""" + img = torch.zeros(1, 8, 8, dtype=dtype, device=device) + img = draw_line(img, torch.tensor([6, 2]), torch.tensor([6, 0]), torch.tensor([255])) + img_mask = torch.tensor( + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 255.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 255.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 255.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ] + ], + device=device, + dtype=dtype, + ) + self.assert_close(img, img_mask) + + def test_draw_line_horizontal(self, dtype, device): + """Test drawing a horizontal line.""" + img = torch.zeros(1, 8, 8, dtype=dtype, device=device) + img = draw_line(img, torch.tensor([6, 4]), torch.tensor([0, 4]), torch.tensor([255])) + img_mask = torch.tensor( + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [255.0, 255.0, 255.0, 255.0, 255.0, 255.0, 255.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ] + ], + device=device, + dtype=dtype, + ) + self.assert_close(img, img_mask) + + def test_draw_line_with_big_coordinates(self, dtype, device): + """Test drawing a line with big coordinates.""" + img = torch.zeros(1, 500, 500, dtype=dtype, device=device) + img = draw_line(img, torch.tensor([200, 200]), torch.tensor([400, 200]), torch.tensor([255])) + img_mask = torch.zeros(1, 500, 500, dtype=dtype, device=device) + img_mask[:, 200, 200:401] = 255 + self.assert_close(img, img_mask) + + def test_draw_line_m_lte_neg1(self, dtype, device): + """Test drawing a line with m <= -1.""" + img = torch.zeros(1, 8, 8, dtype=dtype, device=device) + img = draw_line(img, torch.tensor([0, 7]), torch.tensor([6, 0]), torch.tensor([255])) + img_mask = torch.tensor( + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 255.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 255.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 255.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 255.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 255.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 255.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 255.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [255.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ] + ], + device=device, + dtype=dtype, + ) + self.assert_close(img, img_mask) + + def test_draw_line_m_lt_0_gte_neg1(self, dtype, device): + """Test drawing a line with -1 < m < 0.""" + img = torch.zeros(1, 8, 8, dtype=dtype, device=device) + img = draw_line(img, torch.tensor([1, 5]), torch.tensor([7, 0]), torch.tensor([255])) + img_mask = torch.tensor( + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 255.0, 255.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 255.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 255.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 255.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 255.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 255.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ] + ], + device=device, + dtype=dtype, + ) + self.assert_close(img, img_mask) + + def test_draw_line_m_gt_0_lt_1(self, dtype, device): + """Test drawing a line with 0 < m < 1.""" + img = torch.zeros(1, 8, 8, dtype=dtype, device=device) + img = draw_line(img, torch.tensor([0, 0]), torch.tensor([6, 2]), torch.tensor([255])) + img_mask = torch.tensor( + [ + [ + [255.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 255.0, 255.0, 255.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 255.0, 255.0, 255.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ] + ], + device=device, + dtype=dtype, + ) + self.assert_close(img, img_mask) + + def test_draw_line_m_gte_1(self, dtype, device): + """Test drawing a line with m >= 1.""" + img = torch.zeros(1, 8, 8, dtype=dtype, device=device) + img = draw_line(img, torch.tensor([3, 7]), torch.tensor([1, 4]), torch.tensor([255])) + img_mask = torch.tensor( + [ + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 255.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 255.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 255.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 255.0, 0.0, 0.0, 0.0, 0.0], + ] + ], + device=device, + dtype=dtype, + ) + self.assert_close(img, img_mask) + + def test_draw_lines_batched(self, dtype, device): + """Test drawing a line with m <= -1.""" + img = torch.zeros(1, 8, 8, dtype=dtype, device=device) + img = draw_line( + img, torch.tensor([[0, 7], [0, 7], [0, 2]]), torch.tensor([[6, 0], [0, 0], [7, 7]]), torch.tensor([255]) + ) + img_mask = torch.tensor( + [ + [ + [255.0, 0.0, 0.0, 0.0, 0.0, 0.0, 255.0, 0.0], + [255.0, 0.0, 0.0, 0.0, 0.0, 0.0, 255.0, 0.0], + [255.0, 0.0, 0.0, 0.0, 0.0, 255.0, 0.0, 0.0], + [255.0, 255.0, 0.0, 0.0, 255.0, 0.0, 0.0, 0.0], + [255.0, 0.0, 255.0, 255.0, 0.0, 0.0, 0.0, 0.0], + [255.0, 0.0, 255.0, 255.0, 255.0, 0.0, 0.0, 0.0], + [255.0, 255.0, 0.0, 0.0, 0.0, 255.0, 0.0, 0.0], + [255.0, 0.0, 0.0, 0.0, 0.0, 0.0, 255.0, 255.0], + ] + ], + device=device, + dtype=dtype, + ) + self.assert_close(img, img_mask) + + @pytest.mark.parametrize( + "p1", [torch.tensor([-1, 0]), torch.tensor([0, -1]), torch.tensor([8, 0]), torch.tensor([0, 8])] + ) + def test_p1_out_of_bounds(self, p1, dtype, device): + """Tests that an exception is raised if p1 is out of bounds.""" + img = torch.zeros(1, 8, 8, dtype=dtype, device=device) + with pytest.raises(ValueError) as excinfo: + draw_line(img, p1, torch.tensor([0, 0]), torch.tensor([255])) + + assert "p1 is out of bounds." == str(excinfo.value) + + @pytest.mark.parametrize( + "p2", [torch.tensor([-1, 0]), torch.tensor([0, -1]), torch.tensor([8, 0]), torch.tensor([0, 8])] + ) + def test_p2_out_of_bounds(self, p2, dtype, device): + """Tests that an exception is raised if p2 is out of bounds.""" + img = torch.zeros(1, 8, 8, dtype=dtype, device=device) + with pytest.raises(ValueError) as excinfo: + draw_line(img, torch.tensor([0, 0]), p2, torch.tensor([255])) + + assert "p2 is out of bounds." == str(excinfo.value) + + @pytest.mark.parametrize( + "p1,p2", [(torch.tensor([0, 0]), torch.tensor([0, 0])), (torch.tensor([0, 0]), torch.tensor([7, 7]))] + ) + def test_p1_p2_bounds(self, p1, p2, dtype, device): + """Tests to verify that points on the bounds do not trigger errors.""" + img = torch.zeros(1, 8, 8, dtype=dtype, device=device) + img = draw_line(img, p1, p2, torch.tensor([255])) + assert img.shape == (1, 8, 8) + + @pytest.mark.parametrize("img_size", [(200, 100), (32, 3, 20, 20)]) + def test_image_size(self, img_size, dtype, device): + img = torch.zeros(*img_size, dtype=dtype, device=device) + with pytest.raises(ValueError) as excinfo: + draw_line(img, torch.tensor([0, 0]), torch.tensor([1, 1]), torch.tensor([255])) + + assert "image must have 3 dimensions (C,H,W)." == str(excinfo.value) + + @pytest.mark.parametrize("img_size,color", [((1, 8, 8), torch.tensor([23, 53])), ((3, 8, 8), torch.tensor([255]))]) + def test_color_image_channel_size(self, img_size, color, dtype, device): + img = torch.zeros(*img_size, dtype=dtype, device=device) + with pytest.raises(ValueError) as excinfo: + draw_line(img, torch.tensor([0, 0]), torch.tensor([1, 1]), color) + + assert "color must have the same number of channels as the image." == str(excinfo.value) + + @pytest.mark.parametrize("p1,p2", [(torch.rand([10, 2]), torch.rand([20, 2])), (torch.rand([2]), torch.rand([3]))]) + def test_point_size(self, p1, p2, dtype, device): + img = torch.zeros(1, 8, 8, dtype=dtype, device=device) + with pytest.raises(ValueError) as excinfo: + draw_line(img, p1, p2, torch.tensor([255])) + + assert "Input points must be 2D points with shape (2, ) or (B, 2) and must have the same batch sizes." == str( + excinfo.value + ) + + +class TestDrawRectangle(BaseTester): + @pytest.mark.parametrize("batch", (4, 17)) + @pytest.mark.parametrize("color", (torch.Tensor([1.0]), torch.Tensor([0.5]))) + def test_smoke(self, device, batch, color): + black_image = torch.zeros(batch, 1, 3, 3, device=device) # 1 channel 3x3 black_image + points = torch.tensor([1.0, 1.0, 1.0, 1.0]).to(device).expand(batch, 1, 4) # single pixel rectangle + + draw_rectangle(black_image, points, color=color) + + target = torch.zeros(batch, 1, 3, 3, device=device) + target[:, :, 1, 1] = color + + assert torch.all(black_image == target) + + @pytest.mark.parametrize("batch", (8, 11)) + @pytest.mark.parametrize("fill", (True, False)) + @pytest.mark.parametrize("height", (12, 106, 298)) + @pytest.mark.parametrize("width", (7, 123, 537)) + def test_fill_and_edges(self, device, batch, fill, height, width): + black_image = torch.zeros(batch, 3, height, width, device=device) + # we should pass height - 1 and width - 1 but rectangle should clip correctly + points = torch.tensor([0, 0, width, height]).to(device).expand(batch, 1, 4) + + image_w_rectangle = draw_rectangle(black_image, points, color=torch.tensor([1.0]), fill=fill) + + assert image_w_rectangle is black_image + if fill: + assert image_w_rectangle.sum() == batch * 3 * height * width + else: + # corners are double counted + assert image_w_rectangle.sum() == batch * 3 * (2 * height + 2 * width - 4) + + @pytest.mark.parametrize("batch", (4, 6)) + @pytest.mark.parametrize("N", (5, 12)) + @pytest.mark.parametrize("fill", (True, False)) + def test_n_rectangles(self, device, batch, N, fill): + points_list = [] + h, w = 20, 20 + for b in range(batch): + points_list.append([]) + for n in range(N): + points_list[b].append([]) + points_list[b][n].append(int(torch.randint(0, w - 1, (1,)))) + points_list[b][n].append(int(torch.randint(0, h - 1, (1,)))) + points_list[b][n].append(int(torch.randint(points_list[b][n][-2] + 1, w, (1,)))) + points_list[b][n].append(int(torch.randint(points_list[b][n][-2] + 1, h, (1,)))) + + points = torch.tensor(points_list, device=device) + + random_background = torch.rand(batch, 3, h, w, device=device) + random_w_rectangle = random_background.clone() + + draw_rectangle(random_w_rectangle, points, color=torch.tensor([1.0, 1.0, 1.0]), fill=fill) + + for b in range(batch): + for n in range(N): + if fill: + assert ( + random_w_rectangle[ + b, + :, + points_list[b][n][1] : points_list[b][n][3] + 1, + points_list[b][n][0] : points_list[b][n][2] + 1, + ].sum() + == (points_list[b][n][3] - points_list[b][n][1] + 1) + * (points_list[b][n][2] - points_list[b][n][0] + 1) + * 3 + ) + else: + assert ( + random_w_rectangle[ + b, :, points_list[b][n][1] : points_list[b][n][3] + 1, points_list[b][n][0] + ].sum() + == (points_list[b][n][3] - points_list[b][n][1] + 1) * 3 + ) + assert ( + random_w_rectangle[ + b, :, points_list[b][n][1] : points_list[b][n][3] + 1, points_list[b][n][2] + ].sum() + == (points_list[b][n][3] - points_list[b][n][1] + 1) * 3 + ) + assert ( + random_w_rectangle[ + b, :, points_list[b][n][1], points_list[b][n][0] : points_list[b][n][2] + 1 + ].sum() + == (points_list[b][n][2] - points_list[b][n][0] + 1) * 3 + ) + assert ( + random_w_rectangle[ + b, :, points_list[b][n][1], points_list[b][n][0] : points_list[b][n][2] + 1 + ].sum() + == (points_list[b][n][2] - points_list[b][n][0] + 1) * 3 + ) + + @pytest.mark.parametrize("color", (torch.tensor([0.5, 0.3, 0.15]), torch.tensor([0.23, 0.33, 0.8]))) + def test_color_background(self, device, color): + image = torch.zeros(1, 3, 40, 40, device=device) + image[:, 0, :, :] = color[0] + image[:, 1, :, :] = color[1] + image[:, 2, :, :] = color[2] + image_w_rectangle = image.clone() + p1 = (1, 5) + p2 = (30, 39) + points = torch.tensor([[[p1[1], p1[0], p2[1], p2[0]]]], device=device) + + draw_rectangle(image_w_rectangle, points, color=torch.tensor([1.0])) + assert ( + torch.abs( + (image_w_rectangle - image).sum() + - (1 - color[0]) * (2 * (p2[0] - p1[0] + 1) + 2 * (p2[1] - p1[1] + 1) - 4) + - (1 - color[1]) * (2 * (p2[0] - p1[0] + 1) + 2 * (p2[1] - p1[1] + 1) - 4) + - (1 - color[2]) * (2 * (p2[0] - p1[0] + 1) + 2 * (p2[1] - p1[1] + 1) - 4) + ) + <= 0.0001 + ) + + @pytest.mark.parametrize("color", (torch.tensor([0.34, 0.63, 0.16]), torch.tensor([0.29, 0.13, 0.48]))) + def test_color_foreground(self, device, color): + image = torch.zeros(1, 3, 50, 40, device=device) + image_w_rectangle = image.clone() + p1 = (10, 4) + p2 = (11, 40) + points = torch.tensor([[[p1[1], p1[0], p2[1], p2[0]]]], device=device) + + draw_rectangle(image_w_rectangle, points, color=color) + + # corners are double counted, no plus 1 for y since p2[1] of 40 already lies outside of the image + assert ( + torch.abs( + (image_w_rectangle - image).sum() + - (color[0]) * (2 * (p2[0] - p1[0] + 1) + 2 * (p2[1] - p1[1]) - 4) + - (color[1]) * (2 * (p2[0] - p1[0] + 1) + 2 * (p2[1] - p1[1]) - 4) + - (color[2]) * (2 * (p2[0] - p1[0] + 1) + 2 * (p2[1] - p1[1]) - 4) + ) + <= 0.0001 + ) + + +class TestFillConvexPolygon(BaseTester): + def test_circle(self, device, dtype): + b, c, h, w = 1, 3, 500, 500 + n = 5000 + im = torch.zeros(b, c, h, w, device=device, dtype=dtype) + t = torch.linspace(0, 1, steps=n, device=device, dtype=dtype)[None].expand(b, n) + color = torch.tensor([1, 1, 1], device=device, dtype=dtype)[None].expand(b, c) + x = (2 * math.pi * t).cos() + y = (2 * math.pi * t).sin() + ctr = 200 + radius = 200 + pts = ctr + radius * torch.stack((x, y), dim=-1) + poly_im = draw_convex_polygon(im, pts, color) + XY = create_meshgrid(h, w, normalized_coordinates=False, device=device, dtype=dtype) + inside = (((XY[..., 1] - ctr) ** 2 + (XY[..., 0] - ctr) ** 2).sqrt() <= radius)[:, None].expand(b, c, h, w) + circ_im = inside * color[..., None, None] + assert (circ_im - poly_im).abs().mean() <= 1e-4 + + def test_ellipse(self, device, dtype): + b, c, h, w = 1, 3, 500, 500 + n = 5000 + im = torch.zeros(b, c, h, w, device=device, dtype=dtype) + t = torch.linspace(0, 1, steps=n, device=device, dtype=dtype)[None].expand(b, n) + color = torch.tensor([1, 1, 1], device=device, dtype=dtype)[None].expand(b, c) + lam = 2 + x = lam * (2 * math.pi * t).cos() + y = (2 * math.pi * t).sin() + ctr = 200 + radius = 100 + pts = ctr + radius * torch.stack((x, y), dim=-1) + poly_im = draw_convex_polygon(im, pts, color) + XY = create_meshgrid(h, w, normalized_coordinates=False, device=device, dtype=dtype) + inside = (((XY[..., 1] - ctr) ** 2 + ((XY[..., 0] - ctr) / lam) ** 2).sqrt() <= radius)[:, None].expand( + b, c, h, w + ) + ellipse_im = inside * color[..., None, None] + assert (ellipse_im - poly_im).abs().mean() <= 1e-4 + + def test_rectangle(self, device, dtype): + b, c, h, w = 1, 3, 500, 500 + im = torch.zeros(b, c, h, w, device=device, dtype=dtype) + color = torch.tensor([1, 1, 1], device=device, dtype=dtype)[None].expand(b, c) + pts = torch.tensor([[[50, 50], [200, 50], [200, 250], [50, 250]]], device=device, dtype=dtype) + poly_im = draw_convex_polygon(im.clone(), pts, color) + rect = torch.cat((pts[..., 0, :], pts[..., 2, :]), dim=-1)[:, None] + rect_im = draw_rectangle(im.clone(), rect, color[:, None], fill=True) + self.assert_close(rect_im, poly_im) + + def test_batch(self, device, dtype): + im = torch.rand(2, 3, 12, 16, dtype=dtype, device=device) + pts = torch.tensor( + [[[4, 4], [12, 4], [12, 8], [4, 8]], [[0, 0], [4, 0], [4, 4], [0, 4]]], dtype=dtype, device=device + ) + color = torch.tensor([[0.5, 0.5, 0.5], [0.5, 0.5, 0.75]], dtype=dtype, device=device) + poly_im = draw_convex_polygon(im.clone(), pts, color) + rect = torch.tensor([[[4, 4, 12, 8]], [[0, 0, 4, 4]]], dtype=dtype, device=device) + rect_im = draw_rectangle(im.clone(), rect, color[:, None], fill=True) + self.assert_close(rect_im, poly_im) + + def test_batch_variable_size(self, device, dtype): + im = torch.rand(2, 3, 12, 16, dtype=dtype, device=device) + pts = [ + torch.tensor([[4, 4], [12, 4], [12, 8], [4, 8]], dtype=dtype, device=device), + torch.tensor([[0, 0], [2, 0], [4, 0], [4, 4], [0, 4]], dtype=dtype, device=device), + ] + color = torch.tensor([[0.5, 0.5, 0.5], [0.5, 0.5, 0.75]], dtype=dtype, device=device) + poly_im = draw_convex_polygon(im.clone(), pts, color) + rect = torch.tensor([[[4, 4, 12, 8]], [[0, 0, 4, 4]]], dtype=dtype, device=device) + rect_im = draw_rectangle(im.clone(), rect, color[:, None], fill=True) + self.assert_close(rect_im, poly_im) + + def test_batch_color_no_batch(self, device, dtype): + im = torch.rand(2, 3, 12, 16, dtype=dtype, device=device) + pts = [ + torch.tensor([[4, 4], [12, 4], [12, 8], [4, 8]], dtype=dtype, device=device), + torch.tensor([[0, 0], [2, 0], [4, 0], [4, 4], [0, 4]], dtype=dtype, device=device), + ] + color = torch.tensor([0.5, 0.5, 0.75], dtype=dtype, device=device) + poly_im = draw_convex_polygon(im.clone(), pts, color) + rect = torch.tensor([[[4, 4, 12, 8]], [[0, 0, 4, 4]]], dtype=dtype, device=device) + rect_im = draw_rectangle(im.clone(), rect, color, fill=True) + self.assert_close(rect_im, poly_im) + + def test_out_of_bounds_rectangle(self, device, dtype): + b, c, h, w = 1, 3, 500, 500 + im = torch.zeros(b, c, h, w, device=device, dtype=dtype) + color = torch.tensor([1, 1, 1], device=device, dtype=dtype)[None].expand(b, c) + pts = 350 + torch.tensor([[[50, 50], [200, 50], [200, 250], [50, 250]]], device=device, dtype=dtype) + poly_im = draw_convex_polygon(im.clone(), pts, color) + rect = torch.cat((pts[..., 0, :], pts[..., 2, :]), dim=-1)[:, None] + rect_im = draw_rectangle(im.clone(), rect, color[:, None], fill=True) + self.assert_close(rect_im, poly_im) + pts = -150 + torch.tensor([[[50, 50], [200, 50], [200, 250], [50, 250]]], device=device, dtype=dtype) + poly_im = draw_convex_polygon(im.clone(), pts, color) + rect = torch.cat((pts[..., 0, :], pts[..., 2, :]), dim=-1)[:, None] + rect_im = draw_rectangle(im.clone(), rect, color[:, None], fill=True) + self.assert_close(rect_im, poly_im) diff --git a/tests/image/test_image.py b/tests/image/test_image.py new file mode 100644 index 0000000..375c6cd --- /dev/null +++ b/tests/image/test_image.py @@ -0,0 +1,229 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from pathlib import Path + +import numpy as np +import pytest +import torch + +from kornia.core._compat import torch_version_le +from kornia.core.exceptions import ShapeError +from kornia.image.base import KORNIA_CHECK_IMAGE_LAYOUT, ChannelsOrder, ColorSpace, ImageLayout, ImageSize, PixelFormat +from kornia.image.image import Image + +from testing.base import BaseTester + + +class TestImage(BaseTester): + def test_smoke(self, device): + data = torch.randint(0, 255, (3, 4, 5), device=device, dtype=torch.uint8) + pixel_format = PixelFormat(color_space=ColorSpace.RGB, bit_depth=8) + layout = ImageLayout(image_size=ImageSize(4, 5), channels=3, channels_order=ChannelsOrder.CHANNELS_FIRST) + + img = Image(data, pixel_format, layout) + assert isinstance(img, Image) + assert img.channels == 3 + assert img.height == 4 + assert img.width == 5 + assert img.shape == (3, 4, 5) + assert img.device == device + assert img.dtype == torch.uint8 + assert img.layout == layout + assert img.pixel_format.color_space == ColorSpace.RGB + assert img.pixel_format.bit_depth == 8 + assert img.channels_order == ChannelsOrder.CHANNELS_FIRST + + def test_numpy(self, device): + # as it was from cv2.imread + data = np.ones((4, 5, 3), dtype=np.uint8) + img = Image.from_numpy(data, color_space=ColorSpace.RGB) + img = img.to(device) + assert isinstance(img, Image) + assert img.channels == 3 + assert img.height == 4 + assert img.width == 5 + assert img.pixel_format.color_space == ColorSpace.RGB + assert img.shape == (4, 5, 3) + assert img.device == device + assert img.dtype == torch.uint8 + np_img = np.asarray(img.to_numpy()) + np.testing.assert_array_equal(data, np_img) + + # check clone + img2 = img.clone() + assert isinstance(img2, Image) + img2 = img2.to(device) + assert img2.dtype == torch.uint8 + assert img2.device == device + img3 = img2.to(torch.uint8) + assert isinstance(img3, Image) + assert img3.dtype == torch.uint8 + assert img3.device == device + + def test_dlpack(self, device, dtype): + data = torch.rand((3, 4, 5), device=device, dtype=dtype) + pixel_format = PixelFormat(color_space=ColorSpace.RGB, bit_depth=data.element_size() * 8) + layout = ImageLayout(image_size=ImageSize(4, 5), channels=3, channels_order=ChannelsOrder.CHANNELS_FIRST) + img = Image(data, pixel_format=pixel_format, layout=layout) + self.assert_close(data, Image.from_dlpack(img.to_dlpack()).data) + + # Channel first + def test_rgb_gray_rgb_channels_first(self): + rgb_val = torch.tensor([0.5, 0.2, 0.1], dtype=torch.float32) + rgb_data = rgb_val.view(3, 1, 1) + + img_rgb = make_image(rgb_data, ColorSpace.RGB, ChannelsOrder.CHANNELS_FIRST) + + gray = img_rgb.to_gray() + rgb_back = gray.to_rgb() + + expected_gray = img_rgb.to_gray().data.squeeze() + self.assert_close(gray.data.squeeze(), expected_gray) + + # RGB reconstructed from gray should repeat luminance across channels + expected_rgb = expected_gray.repeat(3) + self.assert_close(rgb_back.data.squeeze(), expected_rgb) + + def test_bgr_gray_bgr_channels_first(self): + rgb_val = torch.tensor([0.5, 0.2, 0.1], dtype=torch.float32) + bgr_val = rgb_val.flip(0) + bgr_data = bgr_val.view(3, 1, 1) + + img_bgr = make_image(bgr_data, ColorSpace.BGR, ChannelsOrder.CHANNELS_FIRST) + + gray = img_bgr.to_gray() + bgr_back = gray.to_bgr() + + expected_gray = img_bgr.to_gray().data.squeeze() + self.assert_close(gray.data.squeeze(), expected_gray) + + expected_bgr = expected_gray.repeat(3).flip(0) + self.assert_close(bgr_back.data.squeeze(), expected_bgr) + + def test_rgb_bgr_swap_channels_first(self): + rgb_val = torch.tensor([0.5, 0.2, 0.1], dtype=torch.float32) + bgr_val = rgb_val.flip(0) + + rgb_data = rgb_val.view(3, 1, 1) + bgr_data = bgr_val.view(3, 1, 1) + + img_rgb = make_image(rgb_data, ColorSpace.RGB, ChannelsOrder.CHANNELS_FIRST) + img_bgr = make_image(bgr_data, ColorSpace.BGR, ChannelsOrder.CHANNELS_FIRST) + + self.assert_close(img_rgb.to_bgr().data.squeeze(), bgr_val) + self.assert_close(img_bgr.to_rgb().data.squeeze(), rgb_val) + + # Channel last + def test_rgb_gray_rgb_channels_last(self): + rgb_val = torch.tensor([0.5, 0.2, 0.1], dtype=torch.float32) + rgb_data = rgb_val.view(1, 1, 3) + + img_rgb = make_image(rgb_data, ColorSpace.RGB, ChannelsOrder.CHANNELS_LAST) + + gray = img_rgb.to_gray() + rgb_back = gray.to_rgb() + + expected_gray = img_rgb.to_gray().data.squeeze() + self.assert_close(gray.data.squeeze(), expected_gray) + + # RGB reconstructed from gray should repeat luminance across channels + expected_rgb = expected_gray.repeat(3) + self.assert_close(rgb_back.data.squeeze(), expected_rgb) + + def test_bgr_gray_bgr_channels_last(self): + rgb_val = torch.tensor([0.5, 0.2, 0.1], dtype=torch.float32) + bgr_val = rgb_val.flip(0) + bgr_data = bgr_val.view(1, 1, 3) + + img_bgr = make_image(bgr_data, ColorSpace.BGR, ChannelsOrder.CHANNELS_LAST) + + gray = img_bgr.to_gray() + bgr_back = gray.to_bgr() + + expected_gray = img_bgr.to_gray().data.squeeze() + self.assert_close(gray.data.squeeze(), expected_gray) + + expected_bgr = expected_gray.repeat(3).flip(0) + self.assert_close(bgr_back.data.squeeze(), expected_bgr) + + def test_rgb_bgr_swap_channels_last(self): + rgb_val = torch.tensor([0.5, 0.2, 0.1], dtype=torch.float32) + bgr_val = rgb_val.flip(0) + + rgb_data = rgb_val.view(1, 1, 3) + bgr_data = bgr_val.view(1, 1, 3) + + img_rgb = make_image(rgb_data, ColorSpace.RGB, ChannelsOrder.CHANNELS_LAST) + img_bgr = make_image(bgr_data, ColorSpace.BGR, ChannelsOrder.CHANNELS_LAST) + + self.assert_close(img_rgb.to_bgr().data.squeeze(), bgr_val) + self.assert_close(img_bgr.to_rgb().data.squeeze(), rgb_val) + + @pytest.mark.skipif(torch_version_le(1, 9, 1), reason="dlpack is broken in torch<=1.9.1") + @pytest.mark.xfail(reason="This may fail some time due to jpeg compression assertion") + def test_load_write(self, tmp_path: Path) -> None: + data = torch.randint(0, 255, (3, 4, 5), dtype=torch.uint8) + img = Image.from_numpy(data.numpy(), channels_order=ChannelsOrder.CHANNELS_FIRST) + + file_name = tmp_path / "image.jpg" + + img.write(file_name) + img2 = Image.from_file(file_name) + + # NOTE: the tolerance is high due to the jpeg compression + assert (img.float().data - img2.float().data).pow(2).mean() <= 0.75 + + def test_write_first_channel(self, tmp_path: Path) -> None: + data = np.ones((4, 5, 3), dtype=np.uint8) + img = Image.from_numpy(data, color_space=ColorSpace.RGB, channels_order=ChannelsOrder.CHANNELS_LAST) + img.write(tmp_path / "image.jpg") + + +def make_image(data: torch.Tensor, cs: ColorSpace, order: ChannelsOrder) -> Image: + if order not in [ChannelsOrder.CHANNELS_FIRST, ChannelsOrder.CHANNELS_LAST]: + pytest.skip(f"Skipping unsupported channels_order: {order}") + if order == ChannelsOrder.CHANNELS_FIRST: + C, H, W = data.shape + else: + H, W, C = data.shape + pf = PixelFormat(color_space=cs, bit_depth=data.element_size() * 8) + layout = ImageLayout(image_size=ImageSize(H, W), channels=C, channels_order=order) + return Image(data.clone(), pf, layout) + + +class TestCheckImageLayout(BaseTester): + def test_channels_first_valid(self, device): + data = torch.rand(3, 4, 5, device=device) + layout = ImageLayout(ImageSize(4, 5), 3, ChannelsOrder.CHANNELS_FIRST) + assert KORNIA_CHECK_IMAGE_LAYOUT(data, layout) + + def test_channels_last_valid(self, device): + data = torch.rand(4, 5, 3, device=device) + layout = ImageLayout(ImageSize(4, 5), 3, ChannelsOrder.CHANNELS_LAST) + assert KORNIA_CHECK_IMAGE_LAYOUT(data, layout) + + def test_invalid_shape_raises(self, device): + data = torch.rand(3, 4, 5, device=device) + layout = ImageLayout(ImageSize(10, 10), 3, ChannelsOrder.CHANNELS_FIRST) + with pytest.raises(ShapeError): + KORNIA_CHECK_IMAGE_LAYOUT(data, layout) + + def test_invalid_shape_no_raise(self, device): + data = torch.rand(3, 4, 5, device=device) + layout = ImageLayout(ImageSize(10, 10), 3, ChannelsOrder.CHANNELS_FIRST) + assert not KORNIA_CHECK_IMAGE_LAYOUT(data, layout, raises=False) diff --git a/tests/image/test_image_utils.py b/tests/image/test_image_utils.py new file mode 100644 index 0000000..f61b719 --- /dev/null +++ b/tests/image/test_image_utils.py @@ -0,0 +1,137 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import List + +import numpy as np +import pytest +import torch + +import kornia + +from testing.base import assert_close + + +@pytest.mark.parametrize( + "input_dtype, expected_dtype", [(np.uint8, torch.uint8), (np.float32, torch.float32), (np.float64, torch.float64)] +) +def test_image_to_tensor_keep_dtype(input_dtype, expected_dtype): + image = np.ones((1, 3, 4, 5), dtype=input_dtype) + tensor = kornia.image.image_to_tensor(image) + assert tensor.dtype == expected_dtype + + +@pytest.mark.parametrize("num_of_images, image_shape", [(2, (4, 3, 1)), (0, (1, 2, 3)), (5, (2, 3, 2, 5))]) +def test_list_of_images_to_tensor(num_of_images, image_shape): + images: List[np.array] = [] + if num_of_images == 0: + with pytest.raises(ValueError): + kornia.image.image_list_to_tensor([]) + return + for _ in range(num_of_images): + images.append(np.ones(shape=image_shape)) + if len(image_shape) != 3: + with pytest.raises(ValueError): + kornia.image.image_list_to_tensor(images) + return + tensor = kornia.image.image_list_to_tensor(images) + assert tensor.shape == (num_of_images, image_shape[-1], image_shape[-3], image_shape[-2]) + + +@pytest.mark.parametrize( + "input_shape, expected", + [ + ((4, 4), (4, 4)), + ((1, 4, 4), (4, 4)), + ((1, 1, 4, 4), (4, 4)), + ((3, 4, 4), (4, 4, 3)), + ((2, 3, 4, 4), (2, 4, 4, 3)), + ((1, 3, 4, 4), (4, 4, 3)), + ], +) +def test_tensor_to_image(device, input_shape, expected): + tensor = torch.ones(input_shape).to(device) + image = kornia.image.tensor_to_image(tensor) + assert image.shape == expected + assert isinstance(image, np.ndarray) + + +@pytest.mark.parametrize( + "input_shape, expected", + [ + ((4, 4), (4, 4)), + ((1, 4, 4), (4, 4)), + ((1, 1, 4, 4), (1, 4, 4)), + ((3, 4, 4), (4, 4, 3)), + ((2, 3, 4, 4), (2, 4, 4, 3)), + ((1, 3, 4, 4), (1, 4, 4, 3)), + ], +) +def test_tensor_to_image_keepdim(device, input_shape, expected): + tensor = torch.ones(input_shape).to(device) + image = kornia.image.tensor_to_image(tensor, keepdim=True) + assert image.shape == expected + assert isinstance(image, np.ndarray) + + +@pytest.mark.parametrize( + "input_shape, expected", + [ + ((4, 4), (1, 1, 4, 4)), + ((1, 4, 4), (1, 4, 1, 4)), + ((2, 3, 4), (1, 4, 2, 3)), + ((4, 4, 3), (1, 3, 4, 4)), + ((2, 4, 4, 3), (2, 3, 4, 4)), + ((1, 4, 4, 3), (1, 3, 4, 4)), + ], +) +def test_image_to_tensor(input_shape, expected): + image = np.ones(input_shape) + tensor = kornia.image.image_to_tensor(image, keepdim=False) + assert tensor.shape == expected + assert isinstance(tensor, torch.Tensor) + + to_tensor = kornia.image.ImageToTensor(keepdim=False) + assert_close(tensor, to_tensor(image)) + + +@pytest.mark.parametrize( + "input_shape, expected", + [ + ((4, 4), (1, 4, 4)), + ((1, 4, 4), (4, 1, 4)), + ((2, 3, 4), (4, 2, 3)), + ((4, 4, 3), (3, 4, 4)), + ((2, 4, 4, 3), (2, 3, 4, 4)), + ((1, 4, 4, 3), (1, 3, 4, 4)), + ], +) +def test_image_to_tensor_keepdim(input_shape, expected): + image = np.ones(input_shape) + tensor = kornia.image.image_to_tensor(image, keepdim=True) + assert tensor.shape == expected + assert isinstance(tensor, torch.Tensor) + + +def test_tensor_to_image_contiguous(device, dtype): + tensor = torch.rand(2, 3, 4, 4, device=device, dtype=dtype) + + image = kornia.image.tensor_to_image(tensor) + assert not image.flags["C_CONTIGUOUS"] + + image = kornia.image.tensor_to_image(tensor, force_contiguous=True) + assert image.flags["C_CONTIGUOUS"] diff --git a/tests/image/test_print.py b/tests/image/test_print.py new file mode 100644 index 0000000..8edefed --- /dev/null +++ b/tests/image/test_print.py @@ -0,0 +1,63 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.image import image_to_string, print_image + + +class TestImageToString: + def test_value(self): + image = torch.arange(16).reshape(1, 4, 4).repeat(3, 1, 1).long() * 16 + out = image_to_string(image) + + expected = ( + "\033[48;5;16m \033[48;5;16m \033[48;5;16m \033[48;5;59m \033[0m\n" + "\033[48;5;59m \033[48;5;59m \033[48;5;59m \033[48;5;59m \033[0m\n" + "\033[48;5;102m \033[48;5;102m \033[48;5;145m \033[48;5;145m \033[0m\n" + "\033[48;5;145m \033[48;5;188m \033[48;5;188m \033[48;5;231m \033[0m\n" + ) + assert out == expected + + def test_exception(self): + img = torch.rand(3, 15, 15) + image_to_string(img) + + img = torch.rand(3, 15, 15) + image_to_string(img, max_width=12) + + from kornia.core.exceptions import ShapeError + + with pytest.raises(ShapeError) as errinfo: + img = torch.rand(1, 3, 15, 15) + image_to_string(img) + assert "Shape dimension mismatch" in str(errinfo.value) or "Expected shape" in str(errinfo.value) + + from kornia.core.exceptions import ValueCheckError + + with pytest.raises(ValueCheckError) as errinfo: + img = torch.rand(3, 15, 15) * 10 + image_to_string(img) + assert "Value range mismatch" in str(errinfo.value) or "Invalid image value range" in str(errinfo.value) + + with pytest.raises(RuntimeError): + print_image([img]) # Do not accept list + + def test_print_smoke(self): + img = torch.rand(3, 15, 15) + print_image(img) diff --git a/tests/integration/models/__init__.py b/tests/integration/models/__init__.py new file mode 100644 index 0000000..4d273d5 --- /dev/null +++ b/tests/integration/models/__init__.py @@ -0,0 +1,16 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/integration/models/test_kimi_vl.py b/tests/integration/models/test_kimi_vl.py new file mode 100644 index 0000000..0fc62a2 --- /dev/null +++ b/tests/integration/models/test_kimi_vl.py @@ -0,0 +1,150 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Integration tests for KimiVL model. + +These tests verify compatibility with official model weights and require +external resources. They are separate from unit tests to keep the main +test suite fast and focused. +""" + +import json +import os + +import pytest +import torch +import torch.nn.functional as F + +from kornia.models.kimi_vl import KimiVLConfig, KimiVLModel +from kornia.models.kimi_vl.config import KimiVLProjectorConfig, MoonViTConfig + + +def test_kimi_vl_official_weights(): + """Integration test for loading official KimiVL vision weights. + + To run this test, set the environment variable ``KIMI_VL_WEIGHTS_DIR`` to a directory + containing the file ``model.safetensors.index.json`` and the referenced shard files. + If this variable is not set or the files are missing, the test is skipped. + """ + + weights_dir = os.environ.get("KIMI_VL_WEIGHTS_DIR") + if not weights_dir: + pytest.skip("KIMI_VL_WEIGHTS_DIR is not set; skipping test_kimi_vl_official_weights") + + safetensors_torch = pytest.importorskip("safetensors.torch") + load_file = safetensors_torch.load_file + + index_path = os.path.join(weights_dir, "model.safetensors.index.json") + + if not os.path.exists(index_path): + pytest.skip(f"Weights index not found at {index_path}; skipping test_kimi_vl_official_weights") + + with open(index_path) as f: + index = json.load(f) + + vision_keys = [k for k in index["weight_map"].keys() if "vision_tower" in k or "multi_modal_projector" in k] + shards = {index["weight_map"][k] for k in vision_keys} + + state_dict = {} + for shard in shards: + shard_path = os.path.join(weights_dir, shard) + shard_weights = load_file(shard_path) + for k in vision_keys: + if k in shard_weights: + state_dict[k] = shard_weights[k] + + vision_config = MoonViTConfig( + image_size=336, + patch_size=14, + hidden_size=1152, + num_hidden_layers=27, + num_attention_heads=16, + intermediate_size=4304, + ) + + projector_config = KimiVLProjectorConfig(input_dim=1152, hidden_dim=4608, output_dim=2048) + + config = KimiVLConfig(vision_config=vision_config, projector_config=projector_config) + + model = KimiVLModel(config) + model.eval() + + new_state_dict = {} + + def get_w(key): + return state_dict[f"vision_tower.{key}"] + + new_state_dict["vision_encoder.patch_embed.weight"] = get_w("patch_embed.proj.weight") + new_state_dict["vision_encoder.patch_embed.bias"] = get_w("patch_embed.proj.bias") + + pos_embed = get_w("patch_embed.pos_emb.weight") + pos_embed_reshaped = pos_embed.permute(2, 0, 1).unsqueeze(0) + pos_embed_interp = F.interpolate(pos_embed_reshaped, size=(24, 24), mode="bicubic", align_corners=False) + pos_embed_final = pos_embed_interp.flatten(2).transpose(1, 2) + new_state_dict["vision_encoder.pos_embed"] = pos_embed_final + + for i in range(config.vision_config.num_hidden_layers): + prefix_official = f"encoder.blocks.{i}" + prefix_kornia = f"vision_encoder.encoder.layers.{i}" + + new_state_dict[f"{prefix_kornia}.norm1.weight"] = get_w(f"{prefix_official}.norm0.weight") + new_state_dict[f"{prefix_kornia}.norm1.bias"] = get_w(f"{prefix_official}.norm0.bias") + new_state_dict[f"{prefix_kornia}.norm2.weight"] = get_w(f"{prefix_official}.norm1.weight") + new_state_dict[f"{prefix_kornia}.norm2.bias"] = get_w(f"{prefix_official}.norm1.bias") + + wqkv = get_w(f"{prefix_official}.wqkv.weight") + bqkv = get_w(f"{prefix_official}.wqkv.bias") + + hidden_size = config.vision_config.hidden_size + wq, wk, wv = wqkv.split(hidden_size, dim=0) + bq, bk, bv = bqkv.split(hidden_size, dim=0) + + new_state_dict[f"{prefix_kornia}.attn.q_proj.weight"] = wq + new_state_dict[f"{prefix_kornia}.attn.q_proj.bias"] = bq + new_state_dict[f"{prefix_kornia}.attn.k_proj.weight"] = wk + new_state_dict[f"{prefix_kornia}.attn.k_proj.bias"] = bk + new_state_dict[f"{prefix_kornia}.attn.v_proj.weight"] = wv + new_state_dict[f"{prefix_kornia}.attn.v_proj.bias"] = bv + + new_state_dict[f"{prefix_kornia}.attn.out_proj.weight"] = get_w(f"{prefix_official}.wo.weight") + new_state_dict[f"{prefix_kornia}.attn.out_proj.bias"] = get_w(f"{prefix_official}.wo.bias") + + new_state_dict[f"{prefix_kornia}.mlp.fc1.weight"] = get_w(f"{prefix_official}.mlp.fc0.weight") + new_state_dict[f"{prefix_kornia}.mlp.fc1.bias"] = get_w(f"{prefix_official}.mlp.fc0.bias") + new_state_dict[f"{prefix_kornia}.mlp.fc2.weight"] = get_w(f"{prefix_official}.mlp.fc1.weight") + new_state_dict[f"{prefix_kornia}.mlp.fc2.bias"] = get_w(f"{prefix_official}.mlp.fc1.bias") + + new_state_dict["vision_encoder.norm.weight"] = get_w("encoder.final_layernorm.weight") + new_state_dict["vision_encoder.norm.bias"] = get_w("encoder.final_layernorm.bias") + + new_state_dict["projector.pre_norm.weight"] = state_dict["multi_modal_projector.pre_norm.weight"] + new_state_dict["projector.pre_norm.bias"] = state_dict["multi_modal_projector.pre_norm.bias"] + + new_state_dict["projector.mlp.0.weight"] = state_dict["multi_modal_projector.linear_1.weight"] + new_state_dict["projector.mlp.0.bias"] = state_dict["multi_modal_projector.linear_1.bias"] + new_state_dict["projector.mlp.2.weight"] = state_dict["multi_modal_projector.linear_2.weight"] + new_state_dict["projector.mlp.2.bias"] = state_dict["multi_modal_projector.linear_2.bias"] + + missing, unexpected = model.load_state_dict(new_state_dict, strict=True) + assert len(missing) == 0 + assert len(unexpected) == 0 + + x = torch.randn(1, 3, 336, 336) + with torch.no_grad(): + out = model(x) + + assert out.shape == (1, 144, 2048) diff --git a/tests/integration/test_conversions.py b/tests/integration/test_conversions.py new file mode 100644 index 0000000..fc4a8c9 --- /dev/null +++ b/tests/integration/test_conversions.py @@ -0,0 +1,327 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import numpy as np +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +@pytest.fixture() +def atol(device, dtype): + """Lower tolerance for cuda-float16 only.""" + if "cuda" in device.type and dtype == torch.float16: + return 1.0e-3 + return 1.0e-4 + + +@pytest.fixture() +def rtol(device, dtype): + """Lower tolerance for cuda-float16 only.""" + if "cuda" in device.type and dtype == torch.float16: + return 1.0e-3 + return 1.0e-4 + + +class TestAngleAxisToQuaternionToAngleAxis(BaseTester): + def test_zero_angle(self, device, dtype, atol, rtol): + axis_angle = torch.tensor((0.0, 0.0, 0.0), device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.axis_angle_to_quaternion(axis_angle) + axis_angle_hat = kornia.geometry.conversions.quaternion_to_axis_angle(quaternion) + self.assert_close(axis_angle_hat, axis_angle, atol=atol, rtol=rtol) + + @pytest.mark.parametrize("axis", (0, 1, 2)) + def test_small_angle(self, axis, device, dtype, atol, rtol): + theta = 1.0e-2 + array = [0.0, 0.0, 0.0] + array[axis] = theta + axis_angle = torch.tensor(array, device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.axis_angle_to_quaternion(axis_angle) + axis_angle_hat = kornia.geometry.conversions.quaternion_to_axis_angle(quaternion) + self.assert_close(axis_angle_hat, axis_angle, atol=atol, rtol=rtol) + + @pytest.mark.parametrize("axis", (0, 1, 2)) + def test_rotation(self, axis, device, dtype, atol, rtol): + # half_sqrt2 = 0.5 * np.sqrt(2) + array = [0.0, 0.0, 0.0] + array[axis] = kornia.pi / 2.0 + axis_angle = torch.tensor(array, device=device, dtype=dtype) + quaternion = kornia.geometry.conversions.axis_angle_to_quaternion(axis_angle) + axis_angle_hat = kornia.geometry.conversions.quaternion_to_axis_angle(quaternion) + self.assert_close(axis_angle_hat, axis_angle, atol=atol, rtol=rtol) + + +class TestQuaternionToAngleAxisToQuaternion(BaseTester): + def test_unit_quaternion(self, device, dtype, atol, rtol): + quaternion = torch.tensor((1.0, 0.0, 0.0, 0.0), device=device, dtype=dtype) + axis_angle = kornia.geometry.conversions.quaternion_to_axis_angle(quaternion) + quaternion_hat = kornia.geometry.conversions.axis_angle_to_quaternion(axis_angle) + self.assert_close(quaternion_hat, quaternion, atol=atol, rtol=rtol) + + @pytest.mark.parametrize("axis", (0, 1, 2)) + def test_rotation(self, axis, device, dtype, atol, rtol): + array = [0.0, 0.0, 0.0, 0.0] + array[1 + axis] = 1.0 + quaternion = torch.tensor(array, device=device, dtype=dtype) + axis_angle = kornia.geometry.conversions.quaternion_to_axis_angle(quaternion) + quaternion_hat = kornia.geometry.conversions.axis_angle_to_quaternion(axis_angle) + self.assert_close(quaternion_hat, quaternion, atol=atol, rtol=rtol) + + @pytest.mark.parametrize("axis", (0, 1, 2)) + def test_small_angle(self, axis, device, dtype, atol, rtol): + theta = 1.0e-2 + array = [np.cos(theta / 2), 0.0, 0.0, 0.0] + array[1 + axis] = np.sin(theta / 2.0) + quaternion = torch.tensor(array, device=device, dtype=dtype) + axis_angle = kornia.geometry.conversions.quaternion_to_axis_angle(quaternion) + quaternion_hat = kornia.geometry.conversions.axis_angle_to_quaternion(axis_angle) + self.assert_close(quaternion_hat, quaternion, atol=atol, rtol=rtol) + + +class TestQuaternionToRotationMatrixToAngleAxis(BaseTester): + @pytest.mark.parametrize("axis", (0, 1, 2)) + def test_triplet_qma(self, axis, device, dtype, atol, rtol): + array = [[0.0, 0.0, 0.0, 0.0]] + array[0][1 + axis] = 1.0 # `1 + axis` this should fail when XYZW + quaternion = torch.tensor(array, device=device, dtype=dtype) + assert quaternion.shape[-1] == 4 + + mm = kornia.geometry.conversions.quaternion_to_rotation_matrix(quaternion) + assert mm.shape[-1] == 3 + assert mm.shape[-2] == 3 + + axis_angle = kornia.geometry.conversions.rotation_matrix_to_axis_angle(mm) + assert axis_angle.shape[-1] == 3 + axis_angle_expected = [[0.0, 0.0, 0.0]] + axis_angle_expected[0][axis] = kornia.pi + axis_angle_expected = torch.tensor(axis_angle_expected, device=device, dtype=dtype) + self.assert_close(axis_angle, axis_angle_expected, atol=atol, rtol=rtol) + + quaternion_hat = kornia.geometry.conversions.axis_angle_to_quaternion(axis_angle) + self.assert_close(quaternion_hat, quaternion, atol=atol, rtol=rtol) + + @pytest.mark.parametrize("axis", (0, 1, 2)) + def test_triplet_qam(self, axis, device, dtype, atol, rtol): + array = [[0.0, 0.0, 0.0, 0.0]] + array[0][1 + axis] = 1.0 + quaternion = torch.tensor(array, device=device, dtype=dtype) + assert quaternion.shape[-1] == 4 + + axis_angle = kornia.geometry.conversions.quaternion_to_axis_angle(quaternion) + assert axis_angle.shape[-1] == 3 + + rot_m = kornia.geometry.conversions.axis_angle_to_rotation_matrix(axis_angle) + assert rot_m.shape[-1] == 3 + assert rot_m.shape[-2] == 3 + + quaternion_hat = kornia.geometry.conversions.rotation_matrix_to_quaternion(rot_m) + self.assert_close(quaternion_hat, quaternion, atol=atol, rtol=rtol) + + @pytest.mark.parametrize("axis", (0, 1, 2)) + def test_triplet_amq(self, axis, device, dtype, atol, rtol): + array = [[0.0, 0.0, 0.0]] + array[0][axis] = kornia.pi / 2.0 + axis_angle = torch.tensor(array, device=device, dtype=dtype) + assert axis_angle.shape[-1] == 3 + + rot_m = kornia.geometry.conversions.axis_angle_to_rotation_matrix(axis_angle) + assert rot_m.shape[-1] == 3 + assert rot_m.shape[-2] == 3 + + quaternion = kornia.geometry.conversions.rotation_matrix_to_quaternion(rot_m) + assert quaternion.shape[-1] == 4 + + axis_angle_hat = kornia.geometry.conversions.quaternion_to_axis_angle(quaternion) + self.assert_close(axis_angle_hat, axis_angle, atol=atol, rtol=rtol) + + @pytest.mark.parametrize("axis", (0, 1, 2)) + def test_triplet_aqm(self, axis, device, dtype, atol, rtol): + array = [[0.0, 0.0, 0.0]] + array[0][axis] = kornia.pi / 2.0 + axis_angle = torch.tensor(array, device=device, dtype=dtype) + assert axis_angle.shape[-1] == 3 + + quaternion = kornia.geometry.conversions.axis_angle_to_quaternion(axis_angle) + assert quaternion.shape[-1] == 4 + + rot_m = kornia.geometry.conversions.quaternion_to_rotation_matrix(quaternion) + assert rot_m.shape[-1] == 3 + assert rot_m.shape[-2] == 3 + + axis_angle_hat = kornia.geometry.conversions.rotation_matrix_to_axis_angle(rot_m) + self.assert_close(axis_angle_hat, axis_angle, atol=atol, rtol=rtol) + + +class TestAngleOfRotations(BaseTester): + """See: https://arxiv.org/pdf/1711.02508.pdf.""" + + @staticmethod + def matrix_angle_abs(mx: torch.Tensor): + """Unsigned rotation matrix angle.""" + trace = torch.diagonal(mx[..., :3, :3], dim1=-1, dim2=-2).sum(-1, keepdim=True) + return torch.acos((trace - 1.0) / 2.0) + + @staticmethod + def axis_and_angle_to_rotation_matrix(axis_name: str, angle: torch.Tensor, device, dtype): + """See also: https://en.wikipedia.org/wiki/Rotation_matrix#Basic_rotations.""" + axis_name = axis_name.lower() + assert axis_name in ("x", "y", "z") + sn = torch.sin(angle) + cs = torch.cos(angle) + ones = torch.ones_like(sn) + zeros = torch.zeros_like(sn) + if axis_name == "x": + axis = torch.tensor((1.0, 0.0, 0.0), device=device, dtype=dtype).repeat(angle.size()) + rot_m = torch.stack((ones, zeros, zeros, zeros, cs, -sn, zeros, sn, cs), dim=2).view(-1, 3, 3) + elif axis_name == "y": + axis = torch.tensor((0.0, 1.0, 0.0), device=device, dtype=dtype).repeat(angle.size()) + rot_m = torch.stack((cs, zeros, sn, zeros, ones, zeros, -sn, zeros, cs), dim=2).view(-1, 3, 3) + elif axis_name == "z": + axis = torch.tensor((0.0, 0.0, 1.0), device=device, dtype=dtype).repeat(angle.size()) + rot_m = torch.stack((cs, -sn, zeros, sn, cs, zeros, zeros, zeros, ones), dim=2).view(-1, 3, 3) + else: + raise NotImplementedError(f"Not prepared for axis with name {axis_name}") + + return rot_m, axis + + @pytest.mark.parametrize("axis_name", ("x", "y", "z")) + def test_axis_angle_to_rotation_matrix(self, axis_name, device, dtype, atol, rtol): + # Random angle in [-pi..pi] + angle = torch.tensor( + (np.random.default_rng().random(size=(2, 1)) * 2.0 * np.pi - np.pi), device=device, dtype=dtype + ) + rot_m, axis = TestAngleOfRotations.axis_and_angle_to_rotation_matrix( + axis_name=axis_name, angle=angle, device=device, dtype=dtype + ) + assert rot_m.dim() == 3 + assert rot_m.shape[-1] == 3 + assert rot_m.shape[-2] == 3 + assert rot_m.shape[-3] == angle.numel() + assert axis.shape[-1] == 3 + assert axis.shape[-2] == angle.numel() + + # Make sure the returned axis matches the named one, and the appropriate column + if axis_name == "x": + self.assert_close(axis, torch.tensor(((1.0, 0.0, 0.0),) * angle.numel(), device=device, dtype=dtype)) + self.assert_close(axis, rot_m[..., :3, 0]) + elif axis_name == "y": + self.assert_close(axis, torch.tensor(((0.0, 1.0, 0.0),) * angle.numel(), device=device, dtype=dtype)) + self.assert_close(axis, rot_m[..., :3, 1]) + elif axis_name == "z": + self.assert_close(axis, torch.tensor(((0.0, 0.0, 1.0),) * angle.numel(), device=device, dtype=dtype)) + self.assert_close(axis, rot_m[..., :3, 2]) + else: + raise NotImplementedError(f"Not prepared for axis_name {axis_name}") + + # Make sure axes are perpendicular + zero = torch.zeros_like(angle).unsqueeze(-1) + self.assert_close(rot_m[..., :3, 1:2].permute((0, 2, 1)) @ rot_m[..., :3, 0:1], zero, atol=atol, rtol=rtol) + self.assert_close(rot_m[..., :3, 2:3].permute((0, 2, 1)) @ rot_m[..., :3, 1:2], zero, atol=atol, rtol=rtol) + self.assert_close(rot_m[..., :3, 2:3].permute((0, 2, 1)) @ rot_m[..., :3, 0:1], zero, atol=atol, rtol=rtol) + + # Make sure axes are unit norm + one = torch.ones_like(angle) + self.assert_close(rot_m[..., :3, 0].norm(p=2, dim=-1, keepdim=True), one, atol=atol, rtol=rtol) + self.assert_close(rot_m[..., :3, 1].norm(p=2, dim=-1, keepdim=True), one, atol=atol, rtol=rtol) + self.assert_close(rot_m[..., :3, 2].norm(p=2, dim=-1, keepdim=True), one, atol=atol, rtol=rtol) + + @pytest.mark.parametrize("axis_name", ("x", "y", "z")) + @pytest.mark.parametrize("angle_deg", (-179.9, -135.0, -90.0, -45.0, 0.0, 45, 90, 135, 179.9)) + def test_matrix_angle(self, axis_name, angle_deg, device, dtype): + angle = (angle_deg * kornia.pi / 180.0).to(dtype).to(device).view(1, 1) + rot_m, _ = TestAngleOfRotations.axis_and_angle_to_rotation_matrix( + axis_name=axis_name, angle=angle, device=device, dtype=dtype + ) + matrix_angle_abs = TestAngleOfRotations.matrix_angle_abs(rot_m) + self.assert_close(torch.abs(angle), matrix_angle_abs) + + @pytest.mark.parametrize("axis_name", ("x", "y", "z")) + @pytest.mark.parametrize("angle_deg", (-179.9, -90.0, -45.0, 0.0, 45, 90, 179.9)) + def test_quaternion(self, axis_name, angle_deg, device, dtype, atol, rtol): + eps = torch.finfo(dtype).eps + angle = torch.tensor((angle_deg * kornia.pi / 180.0,), device=device, dtype=dtype).repeat(2, 1) + pi = torch.ones_like(angle) * kornia.pi + assert 2 <= len(angle.shape) + rot_m, axis = TestAngleOfRotations.axis_and_angle_to_rotation_matrix( + axis_name=axis_name, angle=angle, device=device, dtype=dtype + ) + quaternion = kornia.geometry.conversions.rotation_matrix_to_quaternion(rot_m, eps=eps) + # compute quaternion rotation angle + # See Section 2.4.4 Equation (105a) in https://arxiv.org/pdf/1711.02508.pdf + angle_hat = 2.0 * torch.atan2(quaternion[..., 1:4].norm(p=2, dim=-1, keepdim=True), quaternion[..., 0:1]) + # make sure it lands between [-pi..pi) + mask = pi < angle_hat + while torch.any(mask): + angle_hat = torch.where(mask, angle_hat - 2.0 * kornia.pi, angle_hat) + mask = pi < angle_hat + # invert angle, if quaternion axis points in the opposite direction of the original axis + dots = (quaternion[..., 1:4] * axis).sum(dim=-1, keepdim=True) + angle_hat = torch.where(dots < 0.0, angle_hat * -1.0, angle_hat) + # quaternion angle should match input angle + self.assert_close(angle_hat, angle, atol=atol, rtol=rtol) + # magnitude of angle should match matrix rotation angle + matrix_angle_abs = TestAngleOfRotations.matrix_angle_abs(rot_m) + self.assert_close(torch.abs(angle_hat), matrix_angle_abs, atol=atol, rtol=rtol) + + @pytest.mark.parametrize("axis_name", ("x", "y", "z")) + @pytest.mark.parametrize("angle_deg", (-179.9, -90.0, -45.0, 0, 45, 90, 179.9)) + def test_axis_angle(self, axis_name, angle_deg, device, dtype, atol, rtol): + angle = (angle_deg * kornia.pi / 180.0).to(dtype).to(device).repeat(2, 1) + rot_m, axis = TestAngleOfRotations.axis_and_angle_to_rotation_matrix( + axis_name=axis_name, angle=angle, device=device, dtype=dtype + ) + axis_angle = kornia.geometry.conversions.rotation_matrix_to_axis_angle(rot_m) + # compute axis_angle rotation angle + angle_hat = axis_angle.norm(p=2, dim=-1, keepdim=True) + # invert angle, if axis_angle axis points in the opposite direction of the original axis + dots = (axis_angle * axis).sum(dim=-1, keepdim=True) + angle_hat = torch.where(dots < 0.0, angle_hat * -1.0, angle_hat) + # axis_angle angle should match input angle + self.assert_close(angle_hat, angle, atol=atol, rtol=rtol) + # magnitude of angle should match matrix rotation angle + matrix_angle_abs = TestAngleOfRotations.matrix_angle_abs(rot_m) + self.assert_close(torch.abs(angle_hat), matrix_angle_abs, atol=atol, rtol=rtol) + + @pytest.mark.parametrize("axis_name", ("x", "y", "z")) + @pytest.mark.parametrize("angle_deg", (-179.9, -90.0, -45.0, 0, 45, 90, 179.9)) + def test_log_quaternion(self, axis_name, angle_deg, device, dtype, atol, rtol): + eps = torch.finfo(dtype).eps + angle = (angle_deg * kornia.pi / 180.0).to(dtype).to(device).repeat(2, 1) + pi = torch.ones_like(angle) * kornia.pi + rot_m, axis = TestAngleOfRotations.axis_and_angle_to_rotation_matrix( + axis_name=axis_name, angle=angle, device=device, dtype=dtype + ) + quaternion = kornia.geometry.conversions.rotation_matrix_to_quaternion(rot_m, eps=eps) + log_q = kornia.geometry.conversions.quaternion_exp_to_log(quaternion, eps=eps) + # compute axis_angle rotation angle + angle_hat = 2.0 * log_q.norm(p=2, dim=-1, keepdim=True) + # make sure it lands between [-pi..pi) + mask = pi < angle_hat + while torch.any(mask): + angle_hat = torch.where(mask, angle_hat - 2.0 * kornia.pi, angle_hat) + mask = pi < angle_hat + # invert angle, if axis_angle axis points in the opposite direction of the original axis + dots = (log_q * axis).sum(dim=-1, keepdim=True) + angle_hat = torch.where(dots < 0.0, angle_hat * -1.0, angle_hat) + # axis_angle angle should match input angle + self.assert_close(angle_hat, angle, atol=atol, rtol=rtol) + # magnitude of angle should match matrix rotation angle + matrix_angle_abs = TestAngleOfRotations.matrix_angle_abs(rot_m) + self.assert_close(torch.abs(angle_hat), matrix_angle_abs, atol=atol, rtol=rtol) diff --git a/tests/integration/test_focal.py b/tests/integration/test_focal.py new file mode 100644 index 0000000..e4d995b --- /dev/null +++ b/tests/integration/test_focal.py @@ -0,0 +1,92 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging + +import pytest +import torch +import torch.nn.functional as F +from torch import nn, optim + +import kornia + +logger = logging.getLogger(__name__) + + +class TestIntegrationFocalLoss: + # optimization + thresh = 1e-1 + lr = 1e-3 + num_iterations = 1000 + num_classes = 2 + + # focal loss + alpha = 0.5 + gamma = 2.0 + + def generate_sample(self, base_target, std_val=0.1): + target = base_target.float() / base_target.max() + noise = std_val * torch.rand(1, 1, 6, 5).to(base_target.device) + return target + noise + + @staticmethod + def init_weights(m): + if isinstance(m, nn.Conv2d): + torch.nn.init.xavier_uniform_(m.weight) + + @pytest.mark.slow + def test_conv2d_relu(self, device): + # we generate base sample + target = torch.LongTensor(1, 6, 5).fill_(0).to(device) + for i in range(1, self.num_classes): + target[..., i:-i, i:-i] = i + + m = nn.Sequential( + nn.Conv2d(1, self.num_classes // 2, kernel_size=3, padding=1), + nn.ReLU(True), + nn.Conv2d(self.num_classes // 2, self.num_classes, kernel_size=3, padding=1), + ).to(device) + m.apply(self.init_weights) + + optimizer = optim.Adam(m.parameters(), lr=self.lr) + + criterion = kornia.losses.FocalLoss(alpha=self.alpha, gamma=self.gamma, reduction="mean") + # NOTE: uncomment to compare against vanilla cross entropy + # criterion = nn.CrossEntropyLoss() + + for _ in range(self.num_iterations): + sample = self.generate_sample(target).to(device) + output = m(sample) + loss = criterion(output, target.to(device)) + logger.debug(f"Loss: {loss.item()}") + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + sample = self.generate_sample(target).to(device) + output_argmax = torch.argmax(m(sample), dim=1) + logger.debug(f"Output argmax: \n{output_argmax}") + + # TODO(edgar): replace by IoU or find a more stable solution + # for this test. The issue is that depending on + # the seed to initialize the weights affects the + # final results and slows down the convergence of + # the algorithm. + val = F.mse_loss(output_argmax.float(), target.float()) + if not val.item() < self.thresh: + pytest.xfail("Wrong seed or initial weight values.") diff --git a/tests/integration/test_soft_argmax2d.py b/tests/integration/test_soft_argmax2d.py new file mode 100644 index 0000000..6d635fe --- /dev/null +++ b/tests/integration/test_soft_argmax2d.py @@ -0,0 +1,82 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging + +import pytest +import torch +from torch import nn, optim + +import kornia + +from testing.base import BaseTester + +logger = logging.getLogger(__name__) + + +class TestIntegrationSoftArgmax2d(BaseTester): + # optimization + lr = 1e-3 + num_iterations = 500 + + # data params + height = 240 + width = 320 + + def generate_sample(self, base_target, std_val=1.0): + """Generate a random sample around the given point. + + The standard deviation is in pixel. + """ + noise = std_val * torch.rand_like(base_target) + return base_target + noise + + @pytest.mark.slow + def test_regression_2d(self, device): + # create the parameters to estimate: the heatmap + params = nn.Parameter(torch.rand(1, 1, self.height, self.width).to(device)) + + # generate base sample + target = torch.zeros(1, 1, 2).to(device) + target[..., 0] = self.width / 2 + target[..., 1] = self.height / 2 + + # create the optimizer and pass the heatmap + optimizer = optim.Adam([params], lr=self.lr) + + # loss criterion + criterion = nn.MSELoss() + + # spatial soft-argmax2d module + soft_argmax2d = kornia.geometry.SpatialSoftArgmax2d(normalized_coordinates=False) + + # NOTE: check where this comes from + temperature = (self.height * self.width) ** (0.5) + + for _ in range(self.num_iterations): + x = params + sample = self.generate_sample(target).to(device) + pred = soft_argmax2d(temperature * x) + loss = criterion(pred, sample) + logger.debug(f"Loss: {loss.item():.3f} Pred: {pred}") + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + self.assert_close(pred[..., 0], target[..., 0], rtol=1e-2, atol=1e-2) + self.assert_close(pred[..., 1], target[..., 1], rtol=1e-2, atol=1e-2) diff --git a/tests/integration/test_warp.py b/tests/integration/test_warp.py new file mode 100644 index 0000000..3f462c5 --- /dev/null +++ b/tests/integration/test_warp.py @@ -0,0 +1,66 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn, optim + +import kornia + + +class MyHomography(nn.Module): + def __init__(self, init_homo: torch.Tensor) -> None: + super().__init__() + self.homo = nn.Parameter(init_homo.clone().detach()) + + def forward(self) -> torch.Tensor: + return torch.unsqueeze(self.homo, dim=0) + + +class TestWarping: + # optimization + lr = 1e-3 + num_iterations = 100 + + def test_smoke(self, device): + img_src_t: torch.Tensor = torch.rand(1, 3, 120, 120).to(device) + img_dst_t: torch.Tensor = torch.rand(1, 3, 120, 120).to(device) + + init_homo: torch.Tensor = torch.from_numpy( + np.array([[0.0415, 1.2731, -1.1731], [-0.9094, 0.5072, 0.4272], [0.0762, 1.3981, 1.0646]]) + ).float() + + height, width = img_dst_t.shape[-2:] + warper = kornia.geometry.transform.HomographyWarper(height, width) + dst_homo_src = MyHomography(init_homo=init_homo).to(device) + + learning_rate = self.lr + optimizer = optim.Adam(dst_homo_src.parameters(), lr=learning_rate) + + for _ in range(self.num_iterations): + # warp the reference image to the destiny with current homography + img_src_to_dst = warper(img_src_t, dst_homo_src()) + + # compute the photometric loss + loss = F.l1_loss(img_src_to_dst, img_dst_t) + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + assert not bool(torch.isnan(dst_homo_src.homo.grad).any()) diff --git a/tests/io/test_io_image.py b/tests/io/test_io_image.py new file mode 100644 index 0000000..82b8dc2 --- /dev/null +++ b/tests/io/test_io_image.py @@ -0,0 +1,168 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import io +import sys +from pathlib import Path + +import numpy as np +import pytest +import requests +import torch + +from kornia.core._compat import torch_version_ge +from kornia.io import ImageLoadType, load_image, write_image + +try: + import kornia_rs +except ImportError: + kornia_rs = None + + +def available_package() -> bool: + return sys.version_info >= (3, 7, 0) and torch_version_ge(1, 10, 0) and kornia_rs is not None + + +def create_random_img8(height: int, width: int, channels: int) -> np.ndarray: + return (np.random.rand(height, width, channels) * 255).astype(np.uint8) # noqa: NPY002 + + +def create_random_img8_torch(height: int, width: int, channels: int, device=None) -> torch.Tensor: + return (torch.rand(channels, height, width, device=device) * 255).to(torch.uint8) + + +def _download_image(url: str, filename: str = "") -> Path: + # TODO: move this to testing + + filename = url.rsplit("/", maxsplit=1)[-1] if len(filename) == 0 else filename + # Download + bytesio = io.BytesIO(requests.get(url, timeout=60).content) + # Save file + with open(filename, "wb") as outfile: + outfile.write(bytesio.getbuffer()) + + return Path(filename) + + +@pytest.fixture(scope="session") +def png_image(tmp_path_factory): + url = "https://github.com/kornia/data/raw/main/simba.png" + filename = tmp_path_factory.mktemp("data") / "image.png" + filename = _download_image(url, str(filename)) + return filename + + +@pytest.fixture(scope="session") +def rgba_png_image(tmp_path_factory): + """Create an RGBA PNG image for testing.""" + filename = tmp_path_factory.mktemp("data") / "rgba_image.png" + img_rgba = np.random.randint(0, 255, (32, 32, 4), dtype=np.uint8) # noqa: NPY002 + kornia_rs.write_image_png_u8(str(filename), img_rgba, mode="rgba") + return filename + + +@pytest.fixture(scope="session") +def jpg_image(tmp_path_factory): + url = "https://github.com/kornia/data/raw/main/crowd.jpg" + filename = tmp_path_factory.mktemp("data") / "image.jpg" + filename = _download_image(url, str(filename)) + return filename + + +@pytest.fixture(scope="session") +def images_fn(png_image, jpg_image): + return {"png": png_image, "jpg": jpg_image} + + +@pytest.mark.skipif(not available_package(), reason="kornia_rs only supports python >=3.7 and pt >= 1.10.0") +class TestIoImage: + def test_smoke(self, tmp_path: Path) -> None: + height, width = 4, 5 + img_th: torch.Tensor = create_random_img8_torch(height, width, 3) + + file_path = tmp_path / "image.jpg" + write_image(str(file_path), img_th) + + assert file_path.is_file() + + img_load: torch.Tensor = load_image(str(file_path), ImageLoadType.UNCHANGED) + + assert img_th.shape == img_load.shape + assert img_th.shape[1:] == (height, width) + assert str(img_th.device) == "cpu" + + def test_device(self, device, png_image: Path) -> None: + file_path = Path(png_image) + + assert file_path.is_file() + + img_th: torch.Tensor = load_image(file_path, ImageLoadType.UNCHANGED, str(device)) + assert str(img_th.device) == str(device) + + @pytest.mark.parametrize("ext", ["png", "jpg"]) + @pytest.mark.parametrize( + "channels,load_type,expected_type,expected_channels", + [ + # NOTE: these tests which should write and load images with channel size != 3, didn't do it + # (1, ImageLoadType.GRAY8, torch.uint8, 1), + (3, ImageLoadType.GRAY8, torch.uint8, 1), + # (4, ImageLoadType.GRAY8, torch.uint8, 1), + # (1, ImageLoadType.GRAY32, torch.float32, 1), + (3, ImageLoadType.GRAY32, torch.float32, 1), + # (4, ImageLoadType.GRAY32, torch.float32, 1), + (3, ImageLoadType.RGB8, torch.uint8, 3), + # (1, ImageLoadType.RGB8, torch.uint8, 3), + (3, ImageLoadType.RGBA8, torch.uint8, 4), + # (1, ImageLoadType.RGB32, torch.float32, 3), + (3, ImageLoadType.RGB32, torch.float32, 3), + ], + ) + def test_load_image(self, images_fn, ext, channels, load_type, expected_type, expected_channels): + file_path = images_fn[ext] + + assert file_path.is_file() + + img = load_image(file_path, load_type) + assert img.shape[0] == expected_channels + assert img.dtype == expected_type + + @pytest.mark.parametrize( + "load_type,expected_type,expected_channels", + [ + (ImageLoadType.UNCHANGED, torch.uint8, 4), + (ImageLoadType.GRAY8, torch.uint8, 1), + (ImageLoadType.GRAY32, torch.float32, 1), + (ImageLoadType.RGB8, torch.uint8, 3), + (ImageLoadType.RGBA8, torch.uint8, 4), + (ImageLoadType.RGB32, torch.float32, 3), + ], + ) + def test_load_rgba_png(self, rgba_png_image, load_type, expected_type, expected_channels): + img = load_image(rgba_png_image, load_type) + assert img.shape[0] == expected_channels + assert img.dtype == expected_type + + @pytest.mark.parametrize("ext", ["jpg"]) + @pytest.mark.parametrize("channels", [3]) + def test_write_image(self, device, tmp_path, ext, channels): + height, width = 4, 5 + img_th: torch.Tensor = create_random_img8_torch(height, width, channels, device) + + file_path = tmp_path / f"image.{ext}" + write_image(file_path, img_th) + + assert file_path.is_file() diff --git a/tests/losses/test_cauchy.py b/tests/losses/test_cauchy.py new file mode 100644 index 0000000..6be0408 --- /dev/null +++ b/tests/losses/test_cauchy.py @@ -0,0 +1,122 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestCauchyLoss(BaseTester): + @pytest.mark.parametrize("reduction", ["mean", "sum", "none", None]) + @pytest.mark.parametrize("shape", [(1, 2, 9, 9), (2, 4, 3, 6)]) + def test_smoke(self, device, dtype, reduction, shape): + img1 = torch.rand(shape, device=device, dtype=dtype) + img2 = torch.rand(shape, device=device, dtype=dtype) + + assert kornia.losses.cauchy_loss(img1, img2, reduction) is not None + + def test_exception(self, device, dtype): + img = torch.rand(3, 3, 3, device=device, dtype=dtype) + + # wrong reduction + from kornia.core.exceptions import BaseError + + with pytest.raises(BaseError) as execinfo: + kornia.losses.cauchy_loss(img, img, reduction="test") + assert "Given type of reduction is not supported. Got: test" in str(execinfo.value) + + # Check if both are tensors + from kornia.core.exceptions import TypeCheckError + + with pytest.raises(TypeCheckError) as errinfo: + kornia.losses.cauchy_loss(1.0, img) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(TypeCheckError) as errinfo: + kornia.losses.cauchy_loss(img, 1.0) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + # Check if same shape + from kornia.core.exceptions import ShapeError + + img_b = torch.rand(1, 1, 3, 3, 4, device=device, dtype=dtype) + with pytest.raises(ShapeError) as errinfo: + kornia.losses.cauchy_loss(img, img_b) + assert "Shape mismatch" in str(errinfo.value) + + @pytest.mark.parametrize("shape", [(1, 3, 5, 5), (2, 5, 5)]) + def test_cardinality(self, shape, device, dtype): + img = torch.rand(shape, device=device, dtype=dtype) + + actual = kornia.losses.cauchy_loss(img, img, reduction="none") + assert actual.shape == shape + + actual = kornia.losses.cauchy_loss(img, img, reduction="sum") + assert actual.shape == () + + actual = kornia.losses.cauchy_loss(img, img, reduction="mean") + assert actual.shape == () + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + img1 = torch.rand(2, 3, 3, 3, device=device, dtype=torch.float64) + img2 = torch.rand(2, 3, 3, 3, device=device, dtype=torch.float64) + + self.gradcheck(kornia.losses.cauchy_loss, (img1, img2)) + + def test_dynamo(self, device, dtype, torch_optimizer): + img1 = torch.rand(2, 3, 3, 3, device=device, dtype=dtype) + img2 = torch.rand(2, 3, 3, 3, device=device, dtype=dtype) + + op = kornia.losses.cauchy_loss + op_optimized = torch_optimizer(op) + + self.assert_close(op(img1, img2), op_optimized(img1, img2)) + + def test_module(self, device, dtype): + img1 = torch.rand(2, 3, 3, 3, device=device, dtype=dtype) + img2 = torch.rand(2, 3, 3, 3, device=device, dtype=dtype) + + op = kornia.losses.cauchy_loss + op_module = kornia.losses.CauchyLoss() + + self.assert_close(op(img1, img2), op_module(img1, img2)) + + @pytest.mark.parametrize("reduction", ["mean", "sum"]) + @pytest.mark.parametrize("shape", [(1, 2, 9, 9), (2, 4, 3, 6)]) + def test_perfect_prediction(self, device, dtype, reduction, shape): + # Sanity test + img = torch.rand(shape, device=device, dtype=dtype) + actual = kornia.losses.cauchy_loss(img, img, reduction=reduction) + expected = torch.tensor(0.0, device=device, dtype=dtype) + self.assert_close(actual, expected) + + # Check loss computation + img1 = torch.ones(shape, device=device, dtype=dtype) + img2 = torch.zeros(shape, device=device, dtype=dtype) + + actual = kornia.losses.cauchy_loss(img1, img2, reduction=reduction) + + if reduction == "mean": + expected = torch.tensor(0.40546512603759766, device=device, dtype=dtype) + elif reduction == "sum": + expected = (torch.ones_like(img1, device=device, dtype=dtype) * 0.40546512603759766).sum() + + self.assert_close(actual, expected) diff --git a/tests/losses/test_charbonnier.py b/tests/losses/test_charbonnier.py new file mode 100644 index 0000000..b3dd2cb --- /dev/null +++ b/tests/losses/test_charbonnier.py @@ -0,0 +1,122 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestCharbonnierLoss(BaseTester): + @pytest.mark.parametrize("reduction", ["mean", "sum", "none", None]) + @pytest.mark.parametrize("shape", [(1, 2, 9, 9), (2, 4, 3, 6)]) + def test_smoke(self, device, dtype, reduction, shape): + img1 = torch.rand(shape, device=device, dtype=dtype) + img2 = torch.rand(shape, device=device, dtype=dtype) + + assert kornia.losses.charbonnier_loss(img1, img2, reduction) is not None + + def test_exception(self, device, dtype): + img = torch.rand(3, 3, 3, device=device, dtype=dtype) + + # wrong reduction + from kornia.core.exceptions import BaseError + + with pytest.raises(BaseError) as execinfo: + kornia.losses.charbonnier_loss(img, img, reduction="test") + assert "Given type of reduction is not supported. Got: test" in str(execinfo.value) + + # Check if both are tensors + from kornia.core.exceptions import TypeCheckError + + with pytest.raises(TypeCheckError) as errinfo: + kornia.losses.charbonnier_loss(1.0, img) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(TypeCheckError) as errinfo: + kornia.losses.charbonnier_loss(img, 1.0) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + # Check if same shape + from kornia.core.exceptions import ShapeError + + img_b = torch.rand(1, 1, 3, 3, 4, device=device, dtype=dtype) + with pytest.raises(ShapeError) as errinfo: + kornia.losses.charbonnier_loss(img, img_b) + assert "Shape mismatch" in str(errinfo.value) + + @pytest.mark.parametrize("shape", [(1, 3, 5, 5), (2, 5, 5)]) + def test_cardinality(self, shape, device, dtype): + img = torch.rand(shape, device=device, dtype=dtype) + + actual = kornia.losses.charbonnier_loss(img, img, reduction="none") + assert actual.shape == shape + + actual = kornia.losses.charbonnier_loss(img, img, reduction="sum") + assert actual.shape == () + + actual = kornia.losses.charbonnier_loss(img, img, reduction="mean") + assert actual.shape == () + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + img1 = torch.rand(2, 3, 3, 3, device=device, dtype=torch.float64) + img2 = torch.rand(2, 3, 3, 3, device=device, dtype=torch.float64) + + self.gradcheck(kornia.losses.charbonnier_loss, (img1, img2)) + + def test_dynamo(self, device, dtype, torch_optimizer): + img1 = torch.rand(2, 3, 3, 3, device=device, dtype=dtype) + img2 = torch.rand(2, 3, 3, 3, device=device, dtype=dtype) + + op = kornia.losses.charbonnier_loss + op_optimized = torch_optimizer(op) + + self.assert_close(op(img1, img2), op_optimized(img1, img2)) + + def test_module(self, device, dtype): + img1 = torch.rand(2, 3, 3, 3, device=device, dtype=dtype) + img2 = torch.rand(2, 3, 3, 3, device=device, dtype=dtype) + + op = kornia.losses.charbonnier_loss + op_module = kornia.losses.CharbonnierLoss() + + self.assert_close(op(img1, img2), op_module(img1, img2)) + + @pytest.mark.parametrize("reduction", ["mean", "sum"]) + @pytest.mark.parametrize("shape", [(1, 2, 9, 9), (2, 4, 3, 6)]) + def test_perfect_prediction(self, device, dtype, reduction, shape): + # Sanity test + img = torch.rand(shape, device=device, dtype=dtype) + actual = kornia.losses.charbonnier_loss(img, img, reduction=reduction) + expected = torch.tensor(0.0, device=device, dtype=dtype) + self.assert_close(actual, expected) + + # Check loss computation + img1 = torch.ones(shape, device=device, dtype=dtype) + img2 = torch.zeros(shape, device=device, dtype=dtype) + + actual = kornia.losses.charbonnier_loss(img1, img2, reduction=reduction) + + if reduction == "mean": + expected = torch.tensor(0.41421356237, device=device, dtype=dtype) + elif reduction == "sum": + expected = (torch.ones_like(img1, device=device, dtype=dtype) * 0.41421356237).sum() + + self.assert_close(actual, expected) diff --git a/tests/losses/test_depth_smoothness.py b/tests/losses/test_depth_smoothness.py new file mode 100644 index 0000000..44ed66b --- /dev/null +++ b/tests/losses/test_depth_smoothness.py @@ -0,0 +1,88 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestDepthSmoothnessLoss(BaseTester): + @pytest.mark.parametrize("data_shape", [(1, 1, 10, 16), (2, 4, 8, 15)]) + def test_smoke(self, device, dtype, data_shape): + image = torch.rand(data_shape, device=device, dtype=dtype) + depth = torch.rand(data_shape, device=device, dtype=dtype) + + criterion = kornia.losses.InverseDepthSmoothnessLoss() + assert criterion(depth, image) is not None + + def test_exception(self): + with pytest.raises(TypeError) as errinf: + kornia.losses.InverseDepthSmoothnessLoss()(1, 1) + assert "Input idepth type is not a torch.Tensor. Got" in str(errinf) + + with pytest.raises(TypeError) as errinf: + kornia.losses.InverseDepthSmoothnessLoss()(torch.rand(1), 1) + assert "Input image type is not a torch.Tensor. Got" in str(errinf) + + with pytest.raises(ValueError) as errinf: + kornia.losses.InverseDepthSmoothnessLoss()(torch.rand(1, 1), torch.rand(1, 1, 1, 1)) + assert "Invalid idepth shape, we expect BxCxHxW. Got" in str(errinf) + + with pytest.raises(ValueError) as errinf: + kornia.losses.InverseDepthSmoothnessLoss()(torch.rand(1, 1, 1, 1), torch.rand(1, 1, 1)) + assert "Invalid image shape, we expect BxCxHxW. Got:" in str(errinf) + + with pytest.raises(ValueError) as errinf: + kornia.losses.InverseDepthSmoothnessLoss()(torch.rand(1, 1, 1, 1), torch.rand(1, 1, 1, 2)) + assert "idepth and image shapes must be the same. Got" in str(errinf) + + with pytest.raises(ValueError) as errinf: + kornia.losses.InverseDepthSmoothnessLoss()(torch.rand(1, 1, 1, 1), torch.rand(1, 1, 1, 1, device="meta")) + assert "idepth and image must be in the same device. Got:" in str(errinf) + + with pytest.raises(ValueError) as errinf: + kornia.losses.InverseDepthSmoothnessLoss()( + torch.rand(1, 1, 1, 1, dtype=torch.float32), torch.rand(1, 1, 1, 1, dtype=torch.float64) + ) + assert "idepth and image must be in the same dtype. Got:" in str(errinf) + + def test_dynamo(self, device, dtype, torch_optimizer): + image = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + depth = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + + op = kornia.losses.inverse_depth_smoothness_loss + op_optimized = torch_optimizer(op) + + self.assert_close(op(image, depth), op_optimized(image, depth)) + + def test_module(self, device, dtype): + image = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + depth = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + + op = kornia.losses.inverse_depth_smoothness_loss + op_module = kornia.losses.InverseDepthSmoothnessLoss() + + self.assert_close(op(image, depth), op_module(image, depth)) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + image = torch.rand(1, 2, 3, 4, device=device, dtype=torch.float64) + depth = torch.rand(1, 2, 3, 4, device=device, dtype=torch.float64) + self.gradcheck(kornia.losses.inverse_depth_smoothness_loss, (depth, image)) diff --git a/tests/losses/test_dice.py b/tests/losses/test_dice.py new file mode 100644 index 0000000..50cd3eb --- /dev/null +++ b/tests/losses/test_dice.py @@ -0,0 +1,226 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestDiceLoss(BaseTester): + def test_smoke(self, device, dtype): + num_classes = 3 + logits = torch.rand(2, num_classes, 3, 2, device=device, dtype=dtype) + labels = torch.rand(2, 3, 2) * num_classes + labels = labels.to(device).long() + + criterion = kornia.losses.DiceLoss() + assert criterion(logits, labels) is not None + + @pytest.mark.parametrize("ignore_index", [-100, None]) + def test_all_zeros(self, device, dtype, ignore_index): + num_classes = 3 + logits = torch.zeros(2, num_classes, 1, 2, device=device, dtype=dtype) + logits[:, 0] = 10.0 + logits[:, 1] = 1.0 + logits[:, 2] = 1.0 + labels = torch.zeros(2, 1, 2, device=device, dtype=torch.int64) + + criterion = kornia.losses.DiceLoss(ignore_index=ignore_index) + loss = criterion(logits, labels) + self.assert_close(loss, torch.zeros_like(loss), rtol=1e-3, atol=1e-3) + + def test_exception(self): + with pytest.raises(ValueError) as errinf: + kornia.losses.DiceLoss()(torch.rand(1, 1, 1), torch.rand(1, 1, 1)) + assert "Invalid pred shape, we expect BxNxHxW. Got:" in str(errinf) + + with pytest.raises(ValueError) as errinf: + kornia.losses.DiceLoss()(torch.rand(1, 1, 1, 1), torch.rand(1, 1, 1, 2)) + assert "pred and target shapes must be the same. Got: " in str(errinf) + + with pytest.raises(ValueError) as errinf: + kornia.losses.DiceLoss()(torch.rand(1, 1, 1, 1), torch.rand(1, 1, 1, 1, device="meta")) + assert "pred and target must be in the same device. Got:" in str(errinf) + + def test_averaging_micro(self, device, dtype): + num_classes = 2 + eps = 1e-8 + + logits = torch.zeros(1, num_classes, 4, 1, device=device, dtype=dtype) + logits[:, 0, 0:3] = 10.0 + logits[:, 0, 3:4] = 1.0 + logits[:, 1, 0:3] = 1.0 + logits[:, 1, 3:4] = 10.0 + + labels = torch.zeros(2, 4, 1, device=device, dtype=torch.int64) + + exp_1_0 = torch.exp(torch.tensor([1.0], device=device, dtype=dtype)) + exp_10_0 = torch.exp(torch.tensor([10.0], device=device, dtype=dtype)) + + expected_intersection = (3.0 * exp_10_0 + 1.0 * exp_1_0) / (exp_1_0 + exp_10_0) + expected_cardinality = 8.0 # for micro averaging cardinality is equal 2 * H * W + expected_loss = 1.0 - 2.0 * expected_intersection / (expected_cardinality + eps) + expected_loss = expected_loss.squeeze() + + criterion = kornia.losses.DiceLoss(average="micro", eps=eps) + loss = criterion(logits, labels) + self.assert_close(loss, expected_loss, rtol=1e-3, atol=1e-3) + + @pytest.mark.parametrize("avg", ["micro", "macro"]) + def test_weight(self, device, dtype, avg): + num_classes = 3 + eps = 1e-8 + logits = torch.zeros(4, num_classes, 1, 4, device=device, dtype=dtype) + logits[:, 0, :, 0] = 100.0 + logits[:, 2, :, 1:] = 100.0 + labels = torch.tensor([0, 1, 2, 2], device=device, dtype=torch.int64).expand((4, 1, -1)) + + # class 0 is all correct + expected_loss = torch.tensor([0.0], device=device, dtype=dtype).squeeze() + weight = torch.tensor([1.0, 0.0, 0.0], device=device, dtype=dtype) + criterion = kornia.losses.DiceLoss(average=avg, eps=eps, weight=weight) + loss = criterion(logits, labels) + self.assert_close(loss, expected_loss, rtol=1e-3, atol=1e-3) + + # class 1 is all incorrect + expected_loss = torch.tensor([1.0], device=device, dtype=dtype).squeeze() + weight = torch.tensor([0.0, 1.0, 0.0], device=device, dtype=dtype) + criterion = kornia.losses.DiceLoss(average=avg, eps=eps, weight=weight) + loss = criterion(logits, labels) + self.assert_close(loss, expected_loss, rtol=1e-3, atol=1e-3) + + # class 2 is partially correct + expected_loss = torch.tensor([1.0 / 5.0], device=device, dtype=dtype).squeeze() + weight = torch.tensor([0.0, 0.0, 1.0], device=device, dtype=dtype) + criterion = kornia.losses.DiceLoss(average=avg, eps=eps, weight=weight) + loss = criterion(logits, labels) + self.assert_close(loss, expected_loss, rtol=1e-3, atol=1e-3) + + # ignore class 3 + expected_loss = kornia.losses.dice_loss(logits, labels, average=avg, eps=eps) + weight = torch.tensor([1.0, 1.0, 1.0, 0.0], device=device, dtype=dtype) + criterion = kornia.losses.DiceLoss(average=avg, eps=eps, weight=weight) + loss = criterion(torch.cat([logits, logits.new_zeros((4, 1, 1, 4))], dim=1), labels) + self.assert_close(loss, expected_loss, rtol=1e-3, atol=1e-3) + + # test non binary weights + w_cl_0, w_cl_1 = 0.3, 0.7 + if avg == "macro": + expected_loss = torch.tensor([0.7], device=device, dtype=dtype).squeeze() + else: + dims = (1, 2) + preds = logits.argmax(1) + tp_cl_0 = ((preds == 0) & (labels == 0)).sum(dims) + tp_cl_1 = ((preds == 1) & (labels == 1)).sum(dims) + + fnfp_cl_0 = ((preds == 0) ^ (labels == 0)).sum(dims) + fnfp_cl_1 = ((preds == 1) ^ (labels == 1)).sum(dims) + + expected_loss = ( + ( + 1 + - 2 + * (w_cl_0 * tp_cl_0 + w_cl_1 * tp_cl_1) ** 2 + / (w_cl_0 * (2 * tp_cl_0 + fnfp_cl_0) + w_cl_1 * (2 * tp_cl_1 + fnfp_cl_1) + eps) + ) + .mean() + .to(dtype) + ) + + weight = torch.tensor([w_cl_0, w_cl_1, 0.0], device=device, dtype=dtype) + criterion = kornia.losses.DiceLoss(average=avg, eps=eps, weight=weight) + loss = criterion(logits, labels) + self.assert_close(loss, expected_loss, rtol=1e-3, atol=1e-3) + + def test_averaging_macro(self, device, dtype): + num_classes = 2 + eps = 1e-8 + + logits = torch.zeros(1, num_classes, 1, 4, device=device, dtype=dtype) + logits[:, 0, :, 0:3] = 10.0 + logits[:, 0, :, 3:4] = 1.0 + logits[:, 1, :, 0:3] = 1.0 + logits[:, 1, :, 3:4] = 10.0 + + labels = torch.zeros(2, 1, 4, device=device, dtype=torch.int64) + + exp_1_0 = torch.exp(torch.tensor([1.0], device=device, dtype=dtype)) + exp_10_0 = torch.exp(torch.tensor([10.0], device=device, dtype=dtype)) + + expected_intersection_1 = (3.0 * exp_10_0 + exp_1_0) / (exp_1_0 + exp_10_0) + expected_intersection_2 = 0.0 # all labels are 0 so the intersection for the second class is empty + expected_cardinality_1 = 4.0 + (3.0 * exp_10_0 + 1.0 * exp_1_0) / (exp_1_0 + exp_10_0) + expected_cardinality_2 = 0.0 + (1.0 * exp_10_0 + 3.0 * exp_1_0) / (exp_1_0 + exp_10_0) + + expected_loss_1 = 1.0 - 2.0 * expected_intersection_1 / (expected_cardinality_1 + eps) + expected_loss_2 = 1.0 - 2.0 * expected_intersection_2 / (expected_cardinality_2 + eps) + expected_loss = (expected_loss_1 + expected_loss_2) / 2.0 + expected_loss = expected_loss.squeeze() + + criterion = kornia.losses.DiceLoss(average="macro", eps=eps) + loss = criterion(logits, labels) + self.assert_close(loss, expected_loss, rtol=1e-3, atol=1e-3) + + @pytest.mark.parametrize("ignore_index", [-100, 255]) + def test_ignore_index(self, device, dtype, ignore_index): + num_classes = 2 + eps = 1e-8 + + logits = torch.zeros(2, num_classes, 1, 4, device=device, dtype=dtype) + logits[:, 0, :, 0] = 100.0 + logits[:, 1, :, 1:] = 100.0 + labels = torch.zeros(2, 1, 4, device=device, dtype=torch.int64) + + labels[..., 2:] = ignore_index + expected_loss = torch.tensor([1.0 / 2.0], device=device, dtype=dtype).squeeze() + criterion = kornia.losses.DiceLoss(average="micro", eps=eps, ignore_index=ignore_index) + loss = criterion(logits, labels) + self.assert_close(loss, expected_loss, rtol=1e-3, atol=1e-3) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + num_classes = 3 + logits = torch.rand(2, num_classes, 3, 2, device=device, dtype=torch.float64) + labels = torch.randint(0, num_classes, (2, 3, 2), device=device) + ignore = torch.rand(2, 3, 2, device=device) > 0.8 + labels[ignore] = -100 + self.gradcheck(kornia.losses.dice_loss, (logits, labels), dtypes=[torch.float64, torch.int64]) + + def test_dynamo(self, device, dtype, torch_optimizer): + num_classes = 3 + logits = torch.rand(2, num_classes, 1, 2, device=device, dtype=dtype) + labels = torch.rand(2, 1, 2) * num_classes + labels = labels.to(device).long() + + op = kornia.losses.dice_loss + op_optimized = torch_optimizer(op) + + self.assert_close(op(logits, labels), op_optimized(logits, labels)) + + def test_module(self, device, dtype): + num_classes = 3 + logits = torch.rand(2, num_classes, 1, 2, device=device, dtype=dtype) + labels = torch.rand(2, 1, 2) * num_classes + labels = labels.to(device).long() + + op = kornia.losses.dice_loss + op_module = kornia.losses.DiceLoss() + + self.assert_close(op(logits, labels), op_module(logits, labels)) diff --git a/tests/losses/test_divergence.py b/tests/losses/test_divergence.py new file mode 100644 index 0000000..a99d055 --- /dev/null +++ b/tests/losses/test_divergence.py @@ -0,0 +1,134 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import math + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestDivergenceLoss(BaseTester): + @pytest.mark.parametrize( + "pred,target,expected", + [ + (torch.full((1, 1, 2, 4), 0.125), torch.full((1, 1, 2, 4), 0.125), 0.0), + (torch.full((1, 7, 2, 4), 0.125), torch.full((1, 7, 2, 4), 0.125), 0.0), + (torch.full((1, 7, 2, 4), 0.125), torch.zeros((1, 7, 2, 4)), 0.346574), + (torch.zeros((1, 7, 2, 4)), torch.full((1, 7, 2, 4), 0.125), 0.346574), + ], + ) + def test_js_div_loss_2d(self, device, dtype, pred, target, expected): + actual = kornia.losses.js_div_loss_2d(pred.to(device, dtype), target.to(device, dtype)) + expected = torch.tensor(expected).to(device, dtype) + self.assert_close(actual, expected) + + @pytest.mark.parametrize( + "pred,target,expected", + [ + (torch.full((1, 1, 2, 4), 0.125), torch.full((1, 1, 2, 4), 0.125), 0.0), + (torch.full((1, 7, 2, 4), 0.125), torch.full((1, 7, 2, 4), 0.125), 0.0), + (torch.full((1, 7, 2, 4), 0.125), torch.zeros((1, 7, 2, 4)), 0.0), + (torch.zeros((1, 7, 2, 4)), torch.full((1, 7, 2, 4), 0.125), math.inf), + ], + ) + def test_kl_div_loss_2d(self, device, dtype, pred, target, expected): + actual = kornia.losses.kl_div_loss_2d(pred.to(device, dtype), target.to(device, dtype)) + expected = torch.tensor(expected).to(device, dtype) + self.assert_close(actual, expected) + + @pytest.mark.parametrize( + "pred,target,expected", + [ + (torch.full((1, 1, 2, 4), 0.125), torch.full((1, 1, 2, 4), 0.125), torch.full((1, 1), 0.0)), + (torch.full((1, 7, 2, 4), 0.125), torch.full((1, 7, 2, 4), 0.125), torch.full((1, 7), 0.0)), + (torch.full((1, 7, 2, 4), 0.125), torch.zeros((1, 7, 2, 4)), torch.full((1, 7), 0.0)), + (torch.zeros((1, 7, 2, 4)), torch.full((1, 7, 2, 4), 0.125), torch.full((1, 7), math.inf)), + ], + ) + def test_kl_div_loss_2d_without_reduction(self, device, dtype, pred, target, expected): + actual = kornia.losses.kl_div_loss_2d(pred.to(device, dtype), target.to(device, dtype), reduction="none") + self.assert_close(actual, expected.to(device, dtype)) + + @pytest.mark.parametrize( + "pred,target,expected", + [ + (torch.full((1, 1, 2, 4), 0.125), torch.full((1, 1, 2, 4), 0.125), 0.0), + (torch.full((1, 7, 2, 4), 0.125), torch.full((1, 7, 2, 4), 0.125), 0.0), + (torch.full((1, 7, 2, 4), 0.125), torch.zeros((1, 7, 2, 4)), 0.0), + (torch.zeros((1, 7, 2, 4)), torch.full((1, 7, 2, 4), 0.125), math.inf), + ], + ) + def test_noncontiguous_kl(self, device, dtype, pred, target, expected): + pred = pred.to(device, dtype).view(pred.shape[::-1]).transpose(-2, -1) + target = target.to(device, dtype).view(target.shape[::-1]).transpose(-2, -1) + actual = kornia.losses.kl_div_loss_2d(pred, target) + expected = torch.tensor(expected).to(device, dtype) + self.assert_close(actual, expected) + + @pytest.mark.parametrize( + "pred,target,expected", + [ + (torch.full((1, 1, 2, 4), 0.125), torch.full((1, 1, 2, 4), 0.125), 0.0), + (torch.full((1, 7, 2, 4), 0.125), torch.full((1, 7, 2, 4), 0.125), 0.0), + (torch.full((1, 7, 2, 4), 0.125), torch.zeros((1, 7, 2, 4)), 0.303251), + (torch.zeros((1, 7, 2, 4)), torch.full((1, 7, 2, 4), 0.125), 0.303251), + ], + ) + def test_noncontiguous_js(self, device, dtype, pred, target, expected): + pred = pred.to(device, dtype).view(pred.shape[::-1]).transpose(-2, -1) + target = target.to(device, dtype).view(target.shape[::-1]).transpose(-2, -1) + actual = kornia.losses.js_div_loss_2d(pred, target) + expected = torch.tensor(expected).to(device, dtype) + self.assert_close(actual, expected) + + @pytest.mark.grad() + def test_gradcheck_kl(self, device, dtype): + dtype = torch.float64 + pred = torch.rand(1, 1, 10, 16, device=device, dtype=dtype) + target = torch.rand(1, 1, 10, 16, device=device, dtype=dtype) + + # evaluate function gradient + self.gradcheck(kornia.losses.kl_div_loss_2d, (pred, target)) + + @pytest.mark.grad() + def test_gradcheck_js(self, device, dtype): + dtype = torch.float64 + pred = torch.rand(1, 1, 10, 16, device=device, dtype=dtype) + target = torch.rand(1, 1, 10, 16, device=device, dtype=dtype) + + # evaluate function gradient + self.gradcheck(kornia.losses.js_div_loss_2d, (pred, target)) + + def test_dynamo_kl(self, device, dtype, torch_optimizer): + pred = torch.full((1, 1, 2, 4), 0.125, dtype=dtype, device=device) + target = torch.full((1, 1, 2, 4), 0.125, dtype=dtype, device=device) + args = (pred, target) + op = kornia.losses.kl_div_loss_2d + op_optimized = torch_optimizer(op) + self.assert_close(op(*args), op_optimized(*args), rtol=0, atol=1e-5) + + def test_dynamo_js(self, device, dtype, torch_optimizer): + pred = torch.full((1, 1, 2, 4), 0.125, dtype=dtype, device=device) + target = torch.full((1, 1, 2, 4), 0.125, dtype=dtype, device=device) + args = (pred, target) + op = kornia.losses.js_div_loss_2d + op_optimized = torch_optimizer(op) + self.assert_close(op(*args), op_optimized(*args), rtol=0, atol=1e-5) diff --git a/tests/losses/test_focal_loss.py b/tests/losses/test_focal_loss.py new file mode 100644 index 0000000..9ccf1ef --- /dev/null +++ b/tests/losses/test_focal_loss.py @@ -0,0 +1,255 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +import torch.nn.functional as F + +import kornia + +from testing.base import BaseTester + + +class TestBinaryFocalLossWithLogits(BaseTester): + @pytest.mark.parametrize("reduction", ["none", "mean", "sum"]) + @pytest.mark.parametrize("ignore_index", [-100, None]) + def test_value_same_as_torch_bce_loss(self, device, dtype, reduction, ignore_index): + logits = torch.rand(2, 3, 2, dtype=dtype, device=device) + labels = torch.rand(2, 3, 2, dtype=dtype, device=device) + + focal_equivalent_bce_loss = kornia.losses.binary_focal_loss_with_logits( + logits, labels, alpha=None, gamma=0, reduction=reduction, ignore_index=ignore_index + ) + torch_bce_loss = F.binary_cross_entropy_with_logits(logits, labels, reduction=reduction) + self.assert_close(focal_equivalent_bce_loss, torch_bce_loss) + + @pytest.mark.parametrize("reduction", ["none", "mean", "sum"]) + def test_value_same_as_torch_bce_loss_pos_weight_weight(self, device, dtype, reduction): + num_classes = 3 + logits = torch.rand(2, num_classes, 2, dtype=dtype, device=device) + labels = torch.rand(2, num_classes, 2, dtype=dtype, device=device) + + pos_weight = torch.rand(num_classes, 1, dtype=dtype, device=device) + weight = torch.rand(num_classes, 1, dtype=dtype, device=device) + + focal_equivalent_bce_loss = kornia.losses.binary_focal_loss_with_logits( + logits, labels, alpha=None, gamma=0, reduction=reduction, pos_weight=pos_weight, weight=weight + ) + torch_bce_loss = F.binary_cross_entropy_with_logits( + logits, labels, reduction=reduction, pos_weight=pos_weight, weight=weight + ) + self.assert_close(focal_equivalent_bce_loss, torch_bce_loss) + + @pytest.mark.parametrize("reduction,expected_shape", [("none", (2, 3, 2)), ("mean", ()), ("sum", ())]) + @pytest.mark.parametrize("alpha", [None, 0.2, 0.5]) + @pytest.mark.parametrize("gamma", [0.0, 1.0, 2.0]) + def test_shape_alpha_gamma(self, device, dtype, reduction, expected_shape, alpha, gamma): + logits = torch.rand(2, 3, 2, dtype=dtype, device=device) + labels = torch.rand(2, 3, 2, dtype=dtype, device=device) + + actual_shape = kornia.losses.binary_focal_loss_with_logits( + logits, labels, alpha=alpha, gamma=gamma, reduction=reduction + ).shape + assert actual_shape == expected_shape + + @pytest.mark.parametrize("reduction,expected_shape", [("none", (2, 3, 2)), ("mean", ()), ("sum", ())]) + @pytest.mark.parametrize("pos_weight", [None, (1, 2, 5)]) + @pytest.mark.parametrize("weight", [None, (0.2, 0.5, 0.8)]) + def test_shape_pos_weight_weight(self, device, dtype, reduction, expected_shape, pos_weight, weight): + logits = torch.rand(2, 3, 2, dtype=dtype, device=device) + labels = torch.rand(2, 3, 2, dtype=dtype, device=device) + + pos_weight = None if pos_weight is None else torch.tensor(pos_weight, dtype=dtype, device=device) + weight = None if weight is None else torch.tensor(weight, dtype=dtype, device=device) + + actual_shape = kornia.losses.binary_focal_loss_with_logits( + logits, labels, alpha=0.8, gamma=0.5, reduction=reduction, pos_weight=pos_weight, weight=weight + ).shape + assert actual_shape == expected_shape + + @pytest.mark.parametrize("reduction,expected_shape", [("none", (2, 3, 2)), ("mean", ()), ("sum", ())]) + @pytest.mark.parametrize("ignore_index", [-100, 255]) + def test_shape_ignore_index(self, device, dtype, reduction, expected_shape, ignore_index): + logits = torch.rand(2, 3, 2, dtype=dtype, device=device) + labels = torch.rand(2, 3, 2, dtype=dtype, device=device) + + ignore = torch.rand(2, 3, 2, device=device) > 0.6 + labels[ignore] = ignore_index + + actual_shape = kornia.losses.binary_focal_loss_with_logits( + logits, labels, alpha=0.8, gamma=0.5, reduction=reduction, ignore_index=ignore_index + ).shape + assert actual_shape == expected_shape + + def test_dynamo(self, device, dtype, torch_optimizer): + logits = torch.rand(2, 3, 2, dtype=dtype, device=device) + labels = torch.rand(2, 3, 2, dtype=dtype, device=device) + + op = kornia.losses.binary_focal_loss_with_logits + op_optimized = torch_optimizer(op) + + args = (0.25, 2.0) + actual = op_optimized(logits, labels, *args) + expected = op(logits, labels, *args) + self.assert_close(actual, expected) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + logits = torch.rand(2, 3, 2, device=device, dtype=torch.float64) + labels = torch.rand(2, 3, 2, device=device, dtype=torch.float64) + + args = (0.25, 2.0) + op = kornia.losses.binary_focal_loss_with_logits + self.gradcheck(op, (logits, labels, *args)) + + @pytest.mark.grad() + def test_gradcheck_ignore_index(self, device, dtype): + logits = torch.rand(2, 3, 2, device=device, dtype=torch.float64) + labels = torch.rand(2, 3, 2, device=device, dtype=torch.float64) + ignore = torch.rand(2, 3, 2, device=device) > 0.8 + labels[ignore] = -100 + + args = (0.25, 2.0) + op = kornia.losses.binary_focal_loss_with_logits + self.gradcheck(op, (logits, labels, *args), requires_grad=[True, False, False, False]) + + def test_module(self, device, dtype): + logits = torch.rand(2, 3, 2, dtype=dtype, device=device) + labels = torch.rand(2, 3, 2, dtype=dtype, device=device) + + args = (0.25, 2.0) + op = kornia.losses.binary_focal_loss_with_logits + op_module = kornia.losses.BinaryFocalLossWithLogits(*args) + self.assert_close(op_module(logits, labels), op(logits, labels, *args)) + + def test_numeric_stability(self, device, dtype): + logits = torch.tensor([[100.0, -100]], dtype=dtype, device=device) + labels = torch.tensor([[1.0, 0.0]], dtype=dtype, device=device) + + args = (0.25, 2.0) + actual = kornia.losses.binary_focal_loss_with_logits(logits, labels, *args) + expected = torch.tensor([[0.0, 0.0]], dtype=dtype, device=device) + self.assert_close(actual, expected) + + +class TestFocalLoss(BaseTester): + @pytest.mark.parametrize("reduction,expected_shape", [("none", (2, 3, 3, 2)), ("mean", ()), ("sum", ())]) + @pytest.mark.parametrize("alpha", [None, 0.2, 0.5]) + @pytest.mark.parametrize("gamma", [0.0, 1.0, 2.0]) + def test_shape_alpha_gamma(self, device, dtype, reduction, expected_shape, alpha, gamma): + num_classes = 3 + logits = torch.rand(2, num_classes, 3, 2, device=device, dtype=dtype) + labels = torch.randint(num_classes, (2, 3, 2), device=device) + + actual_shape = kornia.losses.focal_loss(logits, labels, alpha=alpha, gamma=gamma, reduction=reduction).shape + assert actual_shape == expected_shape + + @pytest.mark.parametrize("reduction,expected_shape", [("none", (2, 3)), ("mean", ()), ("sum", ())]) + def test_shape_target_with_only_one_dim(self, device, dtype, reduction, expected_shape): + num_classes = 3 + logits = torch.rand(2, num_classes, device=device, dtype=dtype) + labels = torch.randint(num_classes, (2,), device=device) + + actual_shape = kornia.losses.focal_loss(logits, labels, alpha=0.1, gamma=1.5, reduction=reduction).shape + assert actual_shape == expected_shape + + @pytest.mark.parametrize("reduction,expected_shape", [("none", (2, 3, 3, 2)), ("mean", ()), ("sum", ())]) + @pytest.mark.parametrize("weight", [None, (0.2, 0.5, 0.8)]) + def test_shape_weight(self, device, dtype, reduction, expected_shape, weight): + num_classes = 3 + logits = torch.rand(2, num_classes, 3, 2, device=device, dtype=dtype) + labels = torch.randint(num_classes, (2, 3, 2), device=device) + + weight = None if weight is None else torch.tensor(weight, dtype=dtype, device=device) + + actual_shape = kornia.losses.focal_loss( + logits, labels, alpha=0.8, gamma=0.5, reduction=reduction, weight=weight + ).shape + assert actual_shape == expected_shape + + @pytest.mark.parametrize("reduction,expected_shape", [("none", (2, 3, 3, 2)), ("mean", ()), ("sum", ())]) + @pytest.mark.parametrize("ignore_index", [-100, 255]) + def test_shape_ignore_index(self, device, dtype, reduction, expected_shape, ignore_index): + num_classes = 3 + logits = torch.rand(2, num_classes, 3, 2, device=device, dtype=dtype) + labels = torch.randint(num_classes, (2, 3, 2), device=device) + + ignore = torch.rand(2, 3, 2, device=device) > 0.6 + labels[ignore] = ignore_index + + actual_shape = kornia.losses.focal_loss( + logits, labels, alpha=0.8, gamma=0.5, reduction=reduction, ignore_index=ignore_index + ).shape + assert actual_shape == expected_shape + + @pytest.mark.parametrize("ignore_index", [-100, 255]) + def test_value_ignore_index(self, device, dtype, ignore_index): + num_classes = 3 + logits = torch.rand(2, num_classes, 3, 2, device=device, dtype=dtype) + labels = torch.randint(num_classes, (2, 3, 2), device=device) + + ignore = torch.rand(2, 3, 2, device=device) > 0.6 + labels[ignore] = ignore_index + + labels_extra_class = labels.clone() + labels_extra_class[ignore] = num_classes + logits_extra_class = torch.cat([logits, logits.new_full((2, 1, 3, 2), float("-inf"))], dim=1) + + expected_values = kornia.losses.focal_loss( + logits_extra_class, labels_extra_class, alpha=0.8, gamma=0.5, reduction="none" + )[:, :-1, ...] + + actual_values = kornia.losses.focal_loss( + logits, labels, alpha=0.8, gamma=0.5, reduction="none", ignore_index=ignore_index + ) + + self.assert_close(actual_values, expected_values) + + def test_dynamo(self, device, dtype, torch_optimizer): + num_classes = 3 + logits = torch.rand(2, num_classes, device=device, dtype=dtype) + labels = torch.randint(num_classes, (2,), device=device) + + op = kornia.losses.focal_loss + op_optimized = torch_optimizer(op) + + args = (0.25, 2.0) + actual = op_optimized(logits, labels, *args) + expected = op(logits, labels, *args) + self.assert_close(actual, expected) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + num_classes = 3 + logits = torch.rand(2, num_classes, 3, 2, device=device, dtype=torch.float64) + labels = torch.randint(num_classes, (2, 3, 2), device=device).long() + ignore = torch.rand(2, 3, 2, device=device) > 0.8 + labels[ignore] = -100 + + self.gradcheck( + kornia.losses.focal_loss, (logits, labels, 0.25, 2.0), dtypes=[torch.float64, torch.int64, None, None] + ) + + def test_module(self, device, dtype): + num_classes = 3 + logits = torch.rand(2, num_classes, device=device, dtype=dtype) + labels = torch.randint(num_classes, (2,), device=device) + + args = (0.25, 2.0) + op = kornia.losses.focal_loss + op_module = kornia.losses.FocalLoss(*args) + self.assert_close(op_module(logits, labels), op(logits, labels, *args)) diff --git a/tests/losses/test_geman_macclure.py b/tests/losses/test_geman_macclure.py new file mode 100644 index 0000000..9ee106e --- /dev/null +++ b/tests/losses/test_geman_macclure.py @@ -0,0 +1,122 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestGemanMcclureLossLoss(BaseTester): + @pytest.mark.parametrize("reduction", ["mean", "sum", "none", None]) + @pytest.mark.parametrize("shape", [(1, 2, 9, 9), (2, 4, 3, 6)]) + def test_smoke(self, device, dtype, reduction, shape): + img1 = torch.rand(shape, device=device, dtype=dtype) + img2 = torch.rand(shape, device=device, dtype=dtype) + + assert kornia.losses.geman_mcclure_loss(img1, img2, reduction) is not None + + def test_exception(self, device, dtype): + img = torch.rand(3, 3, 3, device=device, dtype=dtype) + + # wrong reduction + from kornia.core.exceptions import BaseError + + with pytest.raises(BaseError) as execinfo: + kornia.losses.geman_mcclure_loss(img, img, reduction="test") + assert "Given type of reduction is not supported. Got: test" in str(execinfo.value) + + # Check if both are tensors + from kornia.core.exceptions import TypeCheckError + + with pytest.raises(TypeCheckError) as errinfo: + kornia.losses.geman_mcclure_loss(1.0, img) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(TypeCheckError) as errinfo: + kornia.losses.geman_mcclure_loss(img, 1.0) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + # Check if same shape + from kornia.core.exceptions import ShapeError + + img_b = torch.rand(1, 1, 3, 3, 4, device=device, dtype=dtype) + with pytest.raises(ShapeError) as errinfo: + kornia.losses.geman_mcclure_loss(img, img_b) + assert "Shape mismatch" in str(errinfo.value) + + @pytest.mark.parametrize("shape", [(1, 3, 5, 5), (2, 5, 5)]) + def test_cardinality(self, shape, device, dtype): + img = torch.rand(shape, device=device, dtype=dtype) + + actual = kornia.losses.geman_mcclure_loss(img, img, reduction="none") + assert actual.shape == shape + + actual = kornia.losses.geman_mcclure_loss(img, img, reduction="sum") + assert actual.shape == () + + actual = kornia.losses.geman_mcclure_loss(img, img, reduction="mean") + assert actual.shape == () + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + img1 = torch.rand(2, 3, 3, 3, device=device, dtype=torch.float64) + img2 = torch.rand(2, 3, 3, 3, device=device, dtype=torch.float64) + + self.gradcheck(kornia.losses.geman_mcclure_loss, (img1, img2)) + + def test_dynamo(self, device, dtype, torch_optimizer): + img1 = torch.rand(2, 3, 3, 3, device=device, dtype=dtype) + img2 = torch.rand(2, 3, 3, 3, device=device, dtype=dtype) + + op = kornia.losses.geman_mcclure_loss + op_optimized = torch_optimizer(op) + + self.assert_close(op(img1, img2), op_optimized(img1, img2)) + + def test_module(self, device, dtype): + img1 = torch.rand(2, 3, 3, 3, device=device, dtype=dtype) + img2 = torch.rand(2, 3, 3, 3, device=device, dtype=dtype) + + op = kornia.losses.geman_mcclure_loss + op_module = kornia.losses.GemanMcclureLoss() + + self.assert_close(op(img1, img2), op_module(img1, img2)) + + @pytest.mark.parametrize("reduction", ["mean", "sum"]) + @pytest.mark.parametrize("shape", [(1, 2, 9, 9), (2, 4, 3, 6)]) + def test_perfect_prediction(self, device, dtype, reduction, shape): + # Sanity test + img = torch.rand(shape, device=device, dtype=dtype) + actual = kornia.losses.geman_mcclure_loss(img, img, reduction=reduction) + expected = torch.tensor(0.0, device=device, dtype=dtype) + self.assert_close(actual, expected) + + # Check loss computation + img1 = torch.ones(shape, device=device, dtype=dtype) + img2 = torch.zeros(shape, device=device, dtype=dtype) + + actual = kornia.losses.geman_mcclure_loss(img1, img2, reduction=reduction) + + if reduction == "mean": + expected = torch.tensor(0.4, device=device, dtype=dtype) + elif reduction == "sum": + expected = (torch.ones_like(img1, device=device, dtype=dtype) * 0.4).sum() + + self.assert_close(actual, expected) diff --git a/tests/losses/test_hd.py b/tests/losses/test_hd.py new file mode 100644 index 0000000..3feac6e --- /dev/null +++ b/tests/losses/test_hd.py @@ -0,0 +1,102 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestHausdorffLoss(BaseTester): + @pytest.mark.parametrize("reduction", ["mean", "none", "sum"]) + @pytest.mark.parametrize( + "hd,shape", [[kornia.losses.HausdorffERLoss, (10, 10)], [kornia.losses.HausdorffERLoss3D, (10, 10, 10)]] + ) + def test_smoke_none(self, hd, shape, reduction, device, dtype): + num_classes = 3 + logits = torch.rand(2, num_classes, *shape, dtype=dtype, device=device) + labels = (torch.rand(2, 1, *shape, dtype=dtype, device=device) * (num_classes - 1)).long() + loss = hd(reduction=reduction) + + loss(logits, labels) + + def test_exception_2d(self): + with pytest.raises(ValueError) as errinf: + kornia.losses.HausdorffERLoss()((torch.rand(1, 2, 1) > 0.5) * 1, (torch.rand(1, 1, 1, 2) > 0.5) * 1) + assert "Only 2D images supported. Got " in str(errinf) + + with pytest.raises(ValueError) as errinf: + kornia.losses.HausdorffERLoss()( + (torch.rand(1, 2, 1, 1) > 0.5) * 1, torch.tensor([[[[1]]]], dtype=torch.float32) + ) + assert "Expect long type target value in range" in str(errinf) + + with pytest.raises(ValueError) as errinf: + kornia.losses.HausdorffERLoss()((torch.rand(1, 2, 1, 1) > 0.5) * 1, (torch.rand(1, 1, 1, 2) > 0.5) * 1) + assert "Prediction and target need to be of same size, and target should not be one-hot." in str(errinf) + + def test_exception_3d(self): + with pytest.raises(ValueError) as errinf: + kornia.losses.HausdorffERLoss3D()((torch.rand(1, 2, 1) > 0.5) * 1, (torch.rand(1, 1, 1, 2) > 0.5) * 1) + assert "Only 3D images supported. Got " in str(errinf) + + with pytest.raises(ValueError) as errinf: + kornia.losses.HausdorffERLoss3D()( + (torch.rand(1, 2, 1, 1, 1) > 0.5) * 1, torch.tensor([[[[[5]]]]], dtype=torch.float32) + ) + assert "Invalid target value" in str(errinf) + + def test_numeric(self, device, dtype): + if dtype == torch.float64: + pytest.xfail("Sometimes failing on float64") + num_classes = 3 + shape = (50, 50) + hd = kornia.losses.HausdorffERLoss + logits = torch.rand(2, num_classes, *shape, dtype=dtype, device=device) + labels = (torch.rand(2, 1, *shape, dtype=dtype, device=device) * (num_classes - 1)).long() + loss = hd(k=10) + + expected = torch.tensor(0.025, device=device, dtype=dtype) + + actual = loss(logits, labels) + self.assert_close(actual, expected, rtol=0.005, atol=0.005) + + def test_numeric_3d(self, device, dtype): + num_classes = 3 + shape = (50, 50, 50) + hd = kornia.losses.HausdorffERLoss3D + logits = torch.rand(2, num_classes, *shape, dtype=dtype, device=device) + labels = (torch.rand(2, 1, *shape, dtype=dtype, device=device) * (num_classes - 1)).long() + loss = hd(k=10) + + expected = torch.tensor(0.011, device=device, dtype=dtype) + actual = loss(logits, labels) + self.assert_close(actual, expected, rtol=1e-2, atol=1e-2) + + @pytest.mark.parametrize( + "hd,shape", [[kornia.losses.HausdorffERLoss, (5, 5)], [kornia.losses.HausdorffERLoss3D, (5, 5, 5)]] + ) + @pytest.mark.grad() + def test_gradcheck(self, hd, shape, device, dtype): + num_classes = 3 + logits = torch.rand(2, num_classes, *shape, device=device) + labels = (torch.rand(2, 1, *shape, device=device) * (num_classes - 1)).long() + loss = hd(k=2) + + self.gradcheck(loss, (logits, labels), dtypes=[torch.float64, torch.int64]) diff --git a/tests/losses/test_lovaz_hinge.py b/tests/losses/test_lovaz_hinge.py new file mode 100644 index 0000000..90d71c3 --- /dev/null +++ b/tests/losses/test_lovaz_hinge.py @@ -0,0 +1,95 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestLovaszHingeLoss(BaseTester): + def test_smoke(self, device, dtype): + num_classes = 1 + logits = torch.rand(2, num_classes, 1, 1, device=device, dtype=dtype) + labels = torch.rand(2, 1, 1) * num_classes + labels = labels.to(device).long() + + criterion = kornia.losses.LovaszHingeLoss() + assert criterion(logits, labels) is not None + + def test_exception(self): + criterion = kornia.losses.LovaszHingeLoss() + + with pytest.raises(ValueError) as errinfo: + criterion(torch.rand(1, 1, 1, 2), torch.rand(1, 1, 1)) + assert "pred and target shapes must be the same. Got:" in str(errinfo) + + with pytest.raises(ValueError) as errinfo: + criterion(torch.rand(1, 1, 1, 1), torch.rand(1, 1, 1, device="meta")) + assert "pred and target must be in the same device. Got:" in str(errinfo) + + def test_multi_class(self, device, dtype): + num_classes = 5 + logits = torch.rand(2, num_classes, 3, 2, device=device, dtype=dtype) + labels = torch.rand(2, 3, 2) * num_classes + labels = labels.to(device).long() + + criterion = kornia.losses.LovaszHingeLoss() + with pytest.raises(Exception): + criterion(logits, labels) + + def test_perfect_prediction(self, device, dtype): + num_classes = 1 + prediction = torch.ones(2, num_classes, 1, 2, device=device, dtype=dtype) + labels = torch.ones(2, 1, 2, device=device, dtype=torch.int64) + + criterion = kornia.losses.LovaszHingeLoss() + loss = criterion(prediction, labels) + self.assert_close(loss, torch.zeros_like(loss), rtol=1e-3, atol=1e-3) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + dtype = torch.float64 + num_classes = 1 + logits = torch.rand(2, num_classes, 3, 2, device=device, dtype=dtype) + labels = torch.randint(0, num_classes, (2, 3, 2), device=device, dtype=dtype) + + self.gradcheck(kornia.losses.lovasz_hinge_loss, (logits, labels)) + + def test_dynamo(self, device, dtype, torch_optimizer): + num_classes = 1 + logits = torch.rand(2, num_classes, 1, 2, device=device, dtype=dtype) + labels = torch.rand(2, 1, 2) * num_classes + labels = labels.to(device).long() + + op = kornia.losses.lovasz_hinge_loss + op_optimized = torch_optimizer(op) + + self.assert_close(op(logits, labels), op_optimized(logits, labels)) + + def test_module(self, device, dtype): + num_classes = 1 + logits = torch.rand(2, num_classes, 1, 2, device=device, dtype=dtype) + labels = torch.rand(2, 1, 2) * num_classes + labels = labels.to(device).long() + + op = kornia.losses.lovasz_hinge_loss + op_module = kornia.losses.LovaszHingeLoss() + + self.assert_close(op(logits, labels), op_module(logits, labels)) diff --git a/tests/losses/test_lovaz_softmax.py b/tests/losses/test_lovaz_softmax.py new file mode 100644 index 0000000..9ce3689 --- /dev/null +++ b/tests/losses/test_lovaz_softmax.py @@ -0,0 +1,125 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestLovaszSoftmaxLoss(BaseTester): + def test_smoke(self, device, dtype): + num_classes = 3 + logits = torch.rand(2, num_classes, 1, 1, device=device, dtype=dtype) + labels = torch.rand(2, 1, 1) * num_classes + labels = labels.to(device).long() + + criterion = kornia.losses.LovaszSoftmaxLoss() + assert criterion(logits, labels) is not None + + def test_exception(self): + from kornia.core.exceptions import ShapeError + + criterion = kornia.losses.LovaszSoftmaxLoss() + + with pytest.raises(ShapeError) as errinfo: + criterion(torch.rand(1), torch.rand(1)) + assert "Shape dimension mismatch" in str(errinfo.value) or "Expected shape" in str(errinfo.value) + + with pytest.raises(ShapeError) as errinfo: + criterion(torch.rand(1, 1, 1, 1), torch.rand(1)) + assert "Shape dimension mismatch" in str(errinfo.value) or "Expected shape" in str(errinfo.value) + + with pytest.raises(ValueError) as errinfo: + criterion(torch.rand(1, 1, 1, 1), torch.rand(1, 1, 1)) + assert "Invalid pred shape, we expect BxNxHxW, with N > 1." in str(errinfo) + + with pytest.raises(ValueError) as errinfo: + criterion(torch.rand(1, 2, 1, 1), torch.rand(1, 1, 2)) + assert "pred and target shapes must be the same. Got:" in str(errinfo) + + with pytest.raises(ValueError) as errinfo: + criterion(torch.rand(1, 2, 1, 1), torch.rand(1, 1, 1, device="meta")) + assert "pred and target must be in the same device. Got:" in str(errinfo) + + def test_binary(self, device, dtype): + num_classes = 1 + logits = torch.rand(2, num_classes, 3, 2, device=device, dtype=dtype) + labels = torch.rand(2, 3, 2) * num_classes + labels = labels.to(device).long() + + criterion = kornia.losses.LovaszSoftmaxLoss() + with pytest.raises(Exception): + criterion(logits, labels) + + def test_all_ones(self, device, dtype): + num_classes = 2 + # make perfect prediction + # note that softmax(prediction[:, 1]) == 1. softmax(prediction[:, 0]) == 0. + prediction = torch.zeros(2, num_classes, 1, 2, device=device, dtype=dtype) + prediction[:, 1] = 100.0 + labels = torch.ones(2, 1, 2, device=device, dtype=torch.int64) + + criterion = kornia.losses.LovaszSoftmaxLoss() + loss = criterion(prediction, labels) + + self.assert_close(loss, torch.zeros_like(loss), rtol=1e-3, atol=1e-3) + + def test_weight(self, device, dtype): + num_classes = 2 + # make perfect prediction + # note that softmax(prediction[:, 1]) == 1. softmax(prediction[:, 0]) == 0. + prediction = torch.zeros(2, num_classes, 1, 2, device=device, dtype=dtype) + prediction[:, 0] = 100.0 + labels = torch.ones(2, 1, 2, device=device, dtype=torch.int64) + + criterion = kornia.losses.LovaszSoftmaxLoss(weight=torch.tensor([1.0, 0.0], device=device, dtype=dtype)) + loss = criterion(prediction, labels) + + self.assert_close(loss, 0.5 * torch.ones_like(loss), rtol=1e-3, atol=1e-3) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + num_classes = 4 + logits = torch.rand(2, num_classes, 3, 2, device=device, dtype=torch.float64) + labels = torch.randint(0, num_classes, (2, 3, 2), device=device) + self.gradcheck(kornia.losses.lovasz_softmax_loss, (logits, labels), dtypes=[torch.float64, torch.int64]) + + @pytest.mark.skip(reason="Not matching results") + def test_dynamo(self, device, dtype, torch_optimizer): + # TODO: investigate if we can fix it or report the issue + num_classes = 6 + logits = torch.rand(2, num_classes, 1, 2, device=device, dtype=dtype) + labels = torch.randint(0, num_classes, (2, 1, 2), device=device) + + op = kornia.losses.lovasz_softmax_loss + op_optimized = torch_optimizer(op) + + self.assert_close(op(logits, labels), op_optimized(logits, labels)) + + def test_module(self, device, dtype): + num_classes = 5 + logits = torch.rand(2, num_classes, 1, 2, device=device, dtype=dtype) + labels = torch.rand(2, 1, 2) * num_classes + labels = labels.to(device).long() + + op = kornia.losses.lovasz_softmax_loss + op_module = kornia.losses.LovaszSoftmaxLoss() + + self.assert_close(op(logits, labels), op_module(logits, labels)) diff --git a/tests/losses/test_mutual_information.py b/tests/losses/test_mutual_information.py new file mode 100644 index 0000000..10826f9 --- /dev/null +++ b/tests/losses/test_mutual_information.py @@ -0,0 +1,252 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import pytest +import torch + +from kornia.losses.mutual_information import ( + MIKernel, + MILossFromRef, + NMILossFromRef, + mutual_information_loss, + mutual_information_loss_2d, + normalized_mutual_information_loss, + normalized_mutual_information_loss_2d, +) + +from testing.base import BaseTester + + +class TestMutualInformationLoss(BaseTester): + @staticmethod + def relative_mi(img_1, img_2, window_radius): + """Should theoretically be 0 if img_1 and img_2 are independent and 1 if img_1 = f(img_2), f one to one.""" + numerator = mutual_information_loss(img_1, img_2, window_radius=window_radius) + denominator = mutual_information_loss(img_2, img_2, window_radius=window_radius) + return numerator / denominator + + @staticmethod + def sampling_function(n_samples, device, dtype): + data = torch.rand(n_samples, device=device, dtype=dtype) + return 400 * torch.sin(data * torch.pi) + + def value_ranges_check(self, device, dtype, n_samples=10000, num_bins=64): + img_1 = self.sampling_function(n_samples, device, dtype) + img_2 = 50 * img_1 + 1 + img_3 = self.sampling_function(n_samples, device, dtype) + + for radius in [1 / 2, 1, 2, 3]: + # relative MI, expect 1 + assert torch.allclose( + self.relative_mi(img_1, img_2, window_radius=radius), torch.ones(1, dtype=dtype, device=device) + ), "Wrong MI behaviour, correlated case." + # relative MI, expect 0 + # NOTE: mutual_information_loss is a finite-sample, histogram-based estimator applied to random data. + # For independent variables the theoretical value is 0, but sampling noise and binning effects across + # radii introduce noticeable variance, so we use a slightly looser atol here for test robustness. + assert torch.allclose( + self.relative_mi(img_1, img_3, window_radius=radius), + torch.zeros(1, dtype=dtype, device=device), + atol=0.2, + ), "Wrong MI behaviour, uncorrelated case." + + assert torch.allclose( + self.relative_mi(img_2, img_3, window_radius=radius), + torch.zeros(1, dtype=dtype, device=device), + atol=0.2, + ), "Wrong MI behaviour, uncorrelated case." + + # NMI, expect -2 + assert torch.allclose( + normalized_mutual_information_loss(img_1, img_2, window_radius=radius, num_bins=num_bins), + -2 * torch.ones(1, dtype=dtype, device=device), + atol=0.2 * radius + 0.15, + ), "Wrong NMI behaviour, correlated case." + + # NMI, expect -1 + assert torch.allclose( + normalized_mutual_information_loss(img_1, img_3, window_radius=radius, num_bins=num_bins), + -torch.ones(1, dtype=dtype, device=device), + atol=0.1, + ), "Wrong NMI behaviour, uncorrelated case." + assert torch.allclose( + normalized_mutual_information_loss(img_2, img_3, window_radius=radius, num_bins=num_bins), + -torch.ones(1, dtype=dtype, device=device), + atol=0.1, + ), "Wrong NMI behaviour, uncorrelated case." + + def test_smoke(self, device, dtype): + """Basic functionality test""" + img1 = torch.rand(100, device=device, dtype=dtype) + img2 = torch.rand(100, device=device, dtype=dtype) + + loss = mutual_information_loss(img1, img2, num_bins=64) + assert isinstance(loss, torch.Tensor) + assert loss.shape == torch.Size([]) + + normalized_loss = normalized_mutual_information_loss(img1, img2, num_bins=64) + assert isinstance(normalized_loss, torch.Tensor) + assert normalized_loss.shape == torch.Size([]) + + def test_exception(self, device, dtype): + """Test error conditions""" + # Test with mismatched shapes + img1 = torch.rand(10, device=device, dtype=dtype) + img2 = torch.rand(20, device=device, dtype=dtype) + + with pytest.raises(Exception): + mutual_information_loss(img1, img2) + + with pytest.raises(Exception): + normalized_mutual_information_loss(img1, img2) + + def test_gradcheck(self, device): + """Gradient checking""" + img1 = torch.rand(50, device=device, dtype=torch.float64, requires_grad=True) + img2 = torch.rand(50, device=device, dtype=torch.float64) + + self.gradcheck(mutual_information_loss, (img1, img2)) + self.gradcheck(normalized_mutual_information_loss, (img1, img2)) + + def test_differentiability(self, device, dtype): + for _ in range(10): + img_1 = self.sampling_function(10000, device, dtype) + img_2 = self.sampling_function(10000, device, dtype) + param = torch.tensor(1 / 2.0, requires_grad=True) + mi = mutual_information_loss(img_1 + param * img_2, img_2) + mi.backward() + # negative gradient, order of magnitude 1/2 + assert -1 < param.grad < -1 / 10, f"Differentiability issue for mi, {param.grad=}." + param = torch.tensor(1 / 2.0, requires_grad=True) + nmi = normalized_mutual_information_loss(img_1 + param * img_2, img_2) + nmi.backward() + # negative gradient, order of magnitude 1/20 + assert -1 / 10 < param.grad < -1 / 100, f"Differentiability issue for nmi, {param.grad=}." + + def test_value_ranges(self, device, dtype): + for _ in range(10): + self.value_ranges_check(device, dtype) + + @pytest.mark.parametrize("kernel", [MIKernel.xu, MIKernel.rectangular, MIKernel.truncated_gaussian]) + @pytest.mark.parametrize("dim_param", range(5)) + def test_batch_consistency(self, device, dtype, kernel, dim_param): + torch.manual_seed(0) # Fix seed for reproducibility + + # Create random dimensions + dims = torch.randint(low=1, high=8, size=(dim_param + 1,)) + dims = tuple(map(int, dims)) + + for _ in range(3): + img1 = torch.rand(dims, device=device, dtype=dtype) + img2 = torch.rand(dims, device=device, dtype=dtype) + + # flatten batch dims + unique_batch_dim_1 = img1.reshape((-1,) + img1.shape[-1:]) + unique_batch_dim_2 = img2.reshape((-1,) + img1.shape[-1:]) + + # Compute batch loss + loss_batch = mutual_information_loss(img1, img2, num_bins=64, kernel_function=kernel) + normalized_loss_batch = normalized_mutual_information_loss(img1, img2, num_bins=64, kernel_function=kernel) + + # Compute iterative loss for verification + losses = [] + normalized_losses = [] + for i in range(unique_batch_dim_1.shape[0]): + loss = mutual_information_loss( + unique_batch_dim_1[i], unique_batch_dim_2[i], num_bins=64, kernel_function=kernel + ) + normalized_loss = normalized_mutual_information_loss( + unique_batch_dim_1[i], unique_batch_dim_2[i], num_bins=64, kernel_function=kernel + ) + losses.append(loss) + normalized_losses.append(normalized_loss) + + loss_iterative = torch.stack(losses) + normalized_loss_iterative = torch.stack(normalized_losses) + + # Compare + assert loss_batch.shape == dims[:-1], ( + f"The shape of the batched losses for mi is wrong: {loss_batch.shape} vs {dims[:-1]}." + ) + assert normalized_loss_batch.shape == dims[:-1], ( + f"The shape of the batched losses for nmi is wrong: {normalized_loss_batch.shape} vs {dims[:-1]}." + ) + + assert torch.allclose(loss_batch.flatten(), loss_iterative, atol=1e-4), ( + f"Batch mismatch for mi! Batch: {loss_batch}, Iterative: {loss_iterative}" + ) + assert torch.allclose(normalized_loss_batch.flatten(), normalized_loss_iterative, atol=1e-4), ( + f"Batch mismatch for nmi! Batch: {normalized_loss_batch}, Iterative: {normalized_loss_iterative}" + ) + + def test_module(self, device, dtype): + pred = torch.rand(2, 3, 3, 2, device=device, dtype=dtype) + target = torch.rand(2, 3, 3, 2, device=device, dtype=dtype) + + args = (pred, target) + + op = normalized_mutual_information_loss + op_module = NMILossFromRef(target) + + self.assert_close(op(*args), op_module(pred)) + + op = mutual_information_loss + op_module = MILossFromRef(target) + + self.assert_close(op(*args), op_module(pred)) + + def test_masking(self, device, dtype): + """test masking works on a 2d signal.""" + pred = torch.rand(2, 3, 200, 200, device=device, dtype=dtype) + target = torch.rand(2, 3, 200, 200, device=device, dtype=dtype) + target_mask = torch.zeros(pred.shape[-2:], dtype=torch.bool, device=device) + pred_mask = target_mask.clone() + target_mask[:100] = True + pred_mask[:, :100] = True + # we tweak the values of target and pred for the normalization to be the same with or without the mask + target[..., 0, 0] = 0 + target[..., 0, 1] = 1 + pred[..., 0, 0] = 0 + pred[..., 0, 1] = 1 + restricted_pred = pred[..., :100, :100] + restricted_target = target[..., :100, :100] + + masked_kwargs = {"input": pred, "target": target, "input_mask": pred_mask, "target_mask": target_mask} + restricted_kwargs = { + "input": restricted_pred, + "target": restricted_target, + } + self.assert_close(mutual_information_loss_2d(**masked_kwargs), mutual_information_loss_2d(**restricted_kwargs)) + self.assert_close( + normalized_mutual_information_loss_2d(**masked_kwargs), + normalized_mutual_information_loss_2d(**restricted_kwargs), + ) + + def test_dynamo(self, device, dtype, torch_optimizer): + pred = torch.rand(2, 3, 3, 2, device=device, dtype=dtype) + target = torch.rand(2, 3, 3, 2, device=device, dtype=dtype) + + args = (pred, target) + + op = mutual_information_loss + op_optimized = torch_optimizer(op) + + self.assert_close(op(*args), op_optimized(*args), low_tolerance=True) + + op = normalized_mutual_information_loss + op_optimized = torch_optimizer(op) + + self.assert_close(op(*args), op_optimized(*args), low_tolerance=True) diff --git a/tests/losses/test_one_hot.py b/tests/losses/test_one_hot.py new file mode 100644 index 0000000..d58fd3e --- /dev/null +++ b/tests/losses/test_one_hot.py @@ -0,0 +1,40 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch + +from kornia.losses.one_hot import one_hot + +from testing.base import assert_close + + +class TestOneHot: + def test_smoke(self, device, dtype): + num_classes = 4 + labels = torch.zeros(2, 2, 1, dtype=torch.int64, device=device) + labels[0, 0, 0] = 0 + labels[0, 1, 0] = 1 + labels[1, 0, 0] = 2 + labels[1, 1, 0] = 3 + + # convert labels to one hot tensor + one_hot_tensor = one_hot(labels, num_classes, device, dtype) + + assert_close(one_hot_tensor[0, labels[0, 0, 0], 0, 0].item(), 1.0) + assert_close(one_hot_tensor[0, labels[0, 1, 0], 1, 0].item(), 1.0) + assert_close(one_hot_tensor[1, labels[1, 0, 0], 0, 0].item(), 1.0) + assert_close(one_hot_tensor[1, labels[1, 1, 0], 1, 0].item(), 1.0) diff --git a/tests/losses/test_psnr.py b/tests/losses/test_psnr.py new file mode 100644 index 0000000..5a75207 --- /dev/null +++ b/tests/losses/test_psnr.py @@ -0,0 +1,83 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestPSNRLoss(BaseTester): + def test_smoke(self, device, dtype): + pred = torch.rand(2, 3, 3, 2, device=device, dtype=dtype) + target = torch.rand(2, 3, 3, 2, device=device, dtype=dtype) + + criterion = kornia.losses.PSNRLoss(1.0) + loss = criterion(pred, target) + + assert loss is not None + + def test_type(self, device, dtype): + # Expecting an exception + # since we pass integers instead of torch tensors + criterion = kornia.losses.PSNRLoss(1.0).to(device, dtype) + with pytest.raises(Exception): + criterion(1, 2) + + def test_shape(self, device, dtype): + # Expecting an exception + # since we pass tensors of different shapes + criterion = kornia.losses.PSNRLoss(1.0).to(device, dtype) + with pytest.raises(Exception): + criterion(torch.rand(2, 3, 3, 2), torch.rand(2, 3, 3)) + + def test_loss(self, device, dtype): + pred = torch.ones(1, device=device, dtype=dtype) + expected = torch.tensor(-20.0, device=device, dtype=dtype) + actual = kornia.losses.psnr_loss(pred, 1.2 * pred, 2.0) + self.assert_close(actual, expected) + + def test_dynamo(self, device, dtype, torch_optimizer): + pred = torch.rand(2, 3, 3, 2, device=device, dtype=dtype) + target = torch.rand(2, 3, 3, 2, device=device, dtype=dtype) + + args = (pred, target, 1.0) + + op = kornia.losses.psnr_loss + op_optimized = torch_optimizer(op) + + self.assert_close(op(*args), op_optimized(*args)) + + def test_module(self, device, dtype): + pred = torch.rand(2, 3, 3, 2, device=device, dtype=dtype) + target = torch.rand(2, 3, 3, 2, device=device, dtype=dtype) + + args = (pred, target, 1.0) + + op = kornia.losses.psnr_loss + op_module = kornia.losses.PSNRLoss(1.0) + + self.assert_close(op(*args), op_module(pred, target)) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + dtype = torch.float64 + pred = torch.rand(2, 3, 3, 2, device=device, dtype=dtype) + target = torch.rand(2, 3, 3, 2, device=device, dtype=dtype) + self.gradcheck(kornia.losses.psnr_loss, (pred, target, 1.0)) diff --git a/tests/losses/test_ssim.py b/tests/losses/test_ssim.py new file mode 100644 index 0000000..f673f1d --- /dev/null +++ b/tests/losses/test_ssim.py @@ -0,0 +1,221 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestSSIMLoss(BaseTester): + def test_ssim_equal_none(self, device, dtype): + # input data + img1 = torch.rand(1, 1, 10, 16, device=device, dtype=dtype) + img2 = torch.rand(1, 1, 10, 16, device=device, dtype=dtype) + + ssim1 = kornia.losses.ssim_loss(img1, img1, window_size=5, reduction="none") + ssim2 = kornia.losses.ssim_loss(img2, img2, window_size=5, reduction="none") + + self.assert_close(ssim1, torch.zeros_like(img1), low_tolerance=True) # rtol=tol_val, atol=tol_val) + self.assert_close(ssim2, torch.zeros_like(img2), low_tolerance=True) # rtol=tol_val, atol=tol_val) + + @pytest.mark.parametrize("window_size", [5, 11]) + @pytest.mark.parametrize("reduction_type", ["mean", "sum", "none"]) + @pytest.mark.parametrize("batch_shape", [(1, 1, 10, 16), (2, 4, 8, 15)]) + def test_ssim(self, device, dtype, batch_shape, window_size, reduction_type): + if device.type == "xla": + pytest.skip("test highly unstable with tpu") + + # input data + img = torch.rand(batch_shape, device=device, dtype=dtype) + + loss = kornia.losses.ssim_loss(img, img, window_size, reduction=reduction_type) + + if reduction_type == "none": + expected = torch.zeros_like(img) + else: + expected = torch.tensor(0.0, device=device, dtype=dtype) + + self.assert_close(loss, expected) + + def test_module(self, device, dtype): + img1 = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + img2 = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + + args = (img1, img2, 5, 1.0, 1e-12, "mean") + + op = kornia.losses.ssim_loss + op_module = kornia.losses.SSIMLoss(*args[2:]) + + self.assert_close(op(*args), op_module(*args[:2])) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + # input data + window_size = 3 + img1 = torch.rand(1, 1, 5, 4, device=device, dtype=torch.float64) + img2 = torch.rand(1, 1, 5, 4, device=device, dtype=torch.float64) + + # evaluate function gradient + + # TODO: review method since it needs `nondet_tol` in cuda sometimes. + self.gradcheck(kornia.losses.ssim_loss, (img1, img2, window_size), nondet_tol=1e-8) + + +class TestMS_SSIMLoss(BaseTester): + def test_msssim_equal_none(self, device, dtype): + # input data + img1 = torch.rand(1, 3, 10, 16, device=device, dtype=dtype) + img2 = torch.rand(1, 3, 10, 16, device=device, dtype=dtype) + + msssim = kornia.losses.MS_SSIMLoss().to(device, dtype) + msssim1 = msssim(img1, img1) + msssim2 = msssim(img2, img2) + + self.assert_close(msssim1.item(), 0.0) + self.assert_close(msssim2.item(), 0.0) + + def test_exception(self): + criterion = kornia.losses.MS_SSIMLoss() + + with pytest.raises(TypeError) as errinfo: + criterion(1, 2) + assert "Input type is not a torch.Tensor. Got" in str(errinfo) + + with pytest.raises(TypeError) as errinfo: + criterion(torch.rand(1), 2) + assert "Output type is not a torch.Tensor. Got" in str(errinfo) + + with pytest.raises(ValueError) as errinfo: + criterion(torch.rand(1), torch.rand(1, 2)) + assert "Input shapes should be same. Got" in str(errinfo) + + # TODO: implement for single channel image + @pytest.mark.parametrize("reduction_type", ["mean", "sum", "none"]) + @pytest.mark.parametrize("batch_shape", [(2, 1, 2, 3), (1, 3, 10, 16)]) + def test_msssim(self, device, dtype, batch_shape, reduction_type): + img = torch.rand(batch_shape, device=device, dtype=dtype) + + msssiml1 = kornia.losses.MS_SSIMLoss(reduction=reduction_type).to(device, dtype) + loss = msssiml1(img, img) + + self.assert_close(loss.sum().item(), 0.0) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + # input data + dtype = torch.float64 + img1 = torch.rand(1, 1, 5, 5, device=device, dtype=dtype) + img2 = torch.rand(1, 1, 5, 5, device=device, dtype=dtype) + + # evaluate function gradient + loss = kornia.losses.MS_SSIMLoss().to(device, dtype) + + self.gradcheck(loss, (img1, img2), nondet_tol=1e-8) + + def test_jit(self, device, dtype): + img1 = torch.rand(1, 3, 10, 10, device=device, dtype=dtype) + img2 = torch.rand(1, 3, 10, 10, device=device, dtype=dtype) + + args = (img1, img2) + + op = kornia.losses.MS_SSIMLoss().to(device, dtype) + op_script = torch.jit.script(op) + + self.assert_close(op(*args), op_script(*args)) + + +class TestSSIM3DLoss(BaseTester): + def test_smoke(self, device, dtype): + # input data + img1 = torch.rand(1, 1, 2, 4, 3, device=device, dtype=dtype) + img2 = torch.rand(1, 1, 2, 4, 4, device=device, dtype=dtype) + + ssim1 = kornia.losses.ssim3d_loss(img1, img1, window_size=3, reduction="none") + ssim2 = kornia.losses.ssim3d_loss(img2, img2, window_size=3, reduction="none") + + self.assert_close(ssim1, torch.zeros_like(img1)) + self.assert_close(ssim2, torch.zeros_like(img2)) + + @pytest.mark.parametrize("window_size", [5, 11]) + @pytest.mark.parametrize("reduction_type", ["mean", "sum", "none"]) + @pytest.mark.parametrize("shape", [(1, 1, 2, 16, 16), (2, 4, 2, 15, 20)]) + def test_ssim(self, device, dtype, shape, window_size, reduction_type): + if device.type == "xla": + pytest.skip("test highly unstable with tpu") + + # Sanity test + img = torch.rand(shape, device=device, dtype=dtype) + actual = kornia.losses.ssim3d_loss(img, img, window_size, reduction=reduction_type) + if reduction_type == "none": + expected = torch.zeros_like(img) + else: + expected = torch.tensor(0.0, device=device, dtype=dtype) + + self.assert_close(actual, expected) + + # Check loss computation + img1 = torch.ones(shape, device=device, dtype=dtype) + img2 = torch.zeros(shape, device=device, dtype=dtype) + + actual = kornia.losses.ssim3d_loss(img1, img2, window_size, reduction=reduction_type) + + if reduction_type == "mean": + expected = torch.tensor(0.9999, device=device, dtype=dtype) + elif reduction_type == "sum": + expected = (torch.ones_like(img1, device=device, dtype=dtype) * 0.9999).sum() + elif reduction_type == "none": + expected = torch.ones_like(img1, device=device, dtype=dtype) * 0.9999 + + self.assert_close(actual, expected) + + def test_module(self, device, dtype): + img1 = torch.rand(1, 2, 3, 4, 5, device=device, dtype=dtype) + img2 = torch.rand(1, 2, 3, 4, 5, device=device, dtype=dtype) + + args = (img1, img2, 5, 1.0, 1e-12, "mean") + + op = kornia.losses.ssim3d_loss + op_module = kornia.losses.SSIM3DLoss(*args[2:]) + + self.assert_close(op(*args), op_module(*args[:2])) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + # input data + img = torch.rand(1, 1, 5, 4, 3, device=device) + + # evaluate function gradient + + # TODO: review method since it needs `nondet_tol` in cuda sometimes. + self.gradcheck(kornia.losses.ssim3d_loss, (img, img, 3), nondet_tol=1e-8) + + @pytest.mark.parametrize("shape", [(1, 2, 3, 5, 5), (2, 4, 2, 5, 5)]) + def test_cardinality(self, shape, device, dtype): + img = torch.rand(shape, device=device, dtype=dtype) + + actual = kornia.losses.SSIM3DLoss(5, reduction="none")(img, img) + assert actual.shape == shape + + actual = kornia.losses.SSIM3DLoss(5)(img, img) + assert actual.shape == () + + @pytest.mark.skip("loss have no exception case") + def test_exception(self): + pass diff --git a/tests/losses/test_total_variation.py b/tests/losses/test_total_variation.py new file mode 100644 index 0000000..5cc3200 --- /dev/null +++ b/tests/losses/test_total_variation.py @@ -0,0 +1,171 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestTotalVariation(BaseTester): + # Total variation of constant vectors is 0 + @pytest.mark.parametrize( + "pred, expected", + [ + (torch.ones(3, 4, 5), torch.tensor([0.0, 0.0, 0.0])), + (2 * torch.ones(2, 3, 4, 5), torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]])), + ], + ) + def test_tv_on_constant(self, device, dtype, pred, expected): + actual = kornia.losses.total_variation(pred.to(device, dtype)) + self.assert_close(actual, expected.to(device, dtype)) + + # Total variation of constant vectors is 0 + @pytest.mark.parametrize( + "pred, expected", + [ + (torch.ones(3, 4, 5), torch.tensor([0.0, 0.0, 0.0])), + (2 * torch.ones(2, 3, 4, 5), torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]])), + ], + ) + def test_tv_on_constant_int(self, device, pred, expected): + actual = kornia.losses.total_variation(pred.to(device, dtype=torch.int32), reduction="mean") + self.assert_close(actual, expected.to(device)) + + # Total variation for 3D tensors + @pytest.mark.parametrize( + "pred, expected", + [ + ( + torch.tensor( + [ + [ + [0.11747694, 0.5717714, 0.89223915, 0.2929412, 0.63556224], + [0.5371079, 0.13416398, 0.7782737, 0.21392655, 0.1757018], + [0.62360305, 0.8563448, 0.25304103, 0.68539226, 0.6956515], + [0.9350611, 0.01694632, 0.78724295, 0.4760313, 0.73099905], + ], + [ + [0.4788819, 0.45253807, 0.932798, 0.5721999, 0.7612051], + [0.5455887, 0.8836531, 0.79551977, 0.6677338, 0.74293613], + [0.4830376, 0.16420758, 0.15784949, 0.21445751, 0.34168917], + [0.8675162, 0.5468113, 0.6117004, 0.01305223, 0.17554593], + ], + [ + [0.6423703, 0.5561105, 0.54304767, 0.20339686, 0.8553698], + [0.98024786, 0.31562763, 0.10122144, 0.17686582, 0.26260805], + [0.20522952, 0.14523649, 0.8601968, 0.02593213, 0.7382898], + [0.71935296, 0.9625162, 0.42287344, 0.07979459, 0.9149871], + ], + ] + ), + torch.tensor([12.6647, 7.9527, 12.3838]), + ), + ( + torch.tensor([[[0.09094203, 0.32630223, 0.8066123], [0.10921168, 0.09534764, 0.48588026]]]), + torch.tensor([1.6900]), + ), + ], + ) + def test_tv_on_3d(self, device, dtype, pred, expected): + actual = kornia.losses.total_variation(pred.to(device, dtype)) + self.assert_close(actual, expected.to(device, dtype), rtol=1e-3, atol=1e-3) + + # Total variation for 4D tensors + @pytest.mark.parametrize( + "pred, expected", + [ + ( + torch.tensor( + [ + [ + [[0.8756, 0.0920], [0.8034, 0.3107]], + [[0.3069, 0.2981], [0.9399, 0.7944]], + [[0.6269, 0.1494], [0.2493, 0.8490]], + ], + [ + [[0.3256, 0.9923], [0.2856, 0.9104]], + [[0.4107, 0.4387], [0.2742, 0.0095]], + [[0.7064, 0.3674], [0.6139, 0.2487]], + ], + ] + ), + torch.tensor([[1.5672, 1.2836, 2.1544], [1.4134, 0.8584, 0.9154]]), + ), + ( + torch.tensor( + [ + [[[0.1104, 0.2284, 0.4371], [0.4569, 0.1906, 0.8035]]], + [[[0.0552, 0.6831, 0.8310], [0.3589, 0.5044, 0.0802]]], + [[[0.5078, 0.5703, 0.9110], [0.4765, 0.8401, 0.2754]]], + ] + ), + torch.tensor([[1.9566], [2.5787], [2.2682]]), + ), + ], + ) + def test_tv_on_4d(self, device, dtype, pred, expected): + actual = kornia.losses.total_variation(pred.to(device, dtype)) + self.assert_close(actual, expected.to(device, dtype), rtol=1e-3, atol=1e-3) + + @pytest.mark.parametrize("pred", [torch.rand(3, 5, 5), torch.rand(4, 3, 5, 5), torch.rand(4, 2, 3, 5, 5)]) + def test_tv_shapes(self, device, dtype, pred): + pred = pred.to(device, dtype) + actual_lesser_dims = [] + for slice in torch.unbind(pred, dim=0): + slice_tv = kornia.losses.total_variation(slice) + actual_lesser_dims.append(slice_tv) + actual_lesser_dims = torch.stack(actual_lesser_dims, dim=0) + actual_higher_dims = kornia.losses.total_variation(pred) + self.assert_close(actual_lesser_dims, actual_higher_dims.to(device, dtype), rtol=1e-3, atol=1e-3) + + @pytest.mark.parametrize("reduction, expected", [("sum", torch.tensor(20)), ("mean", torch.tensor(1))]) + def test_tv_reduction(self, device, dtype, reduction, expected): + pred, _ = torch.meshgrid([torch.arange(5), torch.arange(5)], indexing="ij") + pred = pred.to(device, dtype) + actual = kornia.losses.total_variation(pred, reduction=reduction) + self.assert_close(actual, expected.to(device, dtype), rtol=1e-3, atol=1e-3) + + # Expect TypeError to be raised when non-torch tensors are passed + @pytest.mark.parametrize("pred", [1, [1, 2]]) + def test_tv_on_invalid_types(self, device, dtype, pred): + with pytest.raises(TypeError): + kornia.losses.total_variation(pred) + + def test_dynamo(self, device, dtype, torch_optimizer): + image = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + + op = kornia.losses.total_variation + op_optimized = torch_optimizer(op) + + self.assert_close(op(image), op_optimized(image)) + + def test_module(self, device, dtype): + image = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + + op = kornia.losses.total_variation + op_module = kornia.losses.TotalVariation() + + self.assert_close(op(image), op_module(image)) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + dtype = torch.float64 + image = torch.rand(1, 2, 3, 4, device=device, dtype=dtype) + self.gradcheck(kornia.losses.total_variation, (image,)) diff --git a/tests/losses/test_tversky.py b/tests/losses/test_tversky.py new file mode 100644 index 0000000..242a825 --- /dev/null +++ b/tests/losses/test_tversky.py @@ -0,0 +1,122 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestTverskyLoss(BaseTester): + def test_smoke(self, device, dtype): + num_classes = 3 + logits = torch.rand(2, num_classes, 3, 2, device=device, dtype=dtype) + labels = torch.rand(2, 3, 2) * num_classes + labels = labels.to(device).long() + + criterion = kornia.losses.TverskyLoss(alpha=0.5, beta=0.5) + assert criterion(logits, labels) is not None + + def test_exception(self): + criterion = kornia.losses.TverskyLoss(alpha=0.5, beta=0.5) + + with pytest.raises(TypeError) as errinfo: + criterion("not a tensor", torch.rand(1)) + assert "pred type is not a torch.Tensor. Got" in str(errinfo) + + with pytest.raises(ValueError) as errinfo: + criterion(torch.rand(1), torch.rand(1)) + assert "Invalid pred shape, we expect BxNxHxW. Got:" in str(errinfo) + + with pytest.raises(ValueError) as errinfo: + criterion(torch.rand(1, 1, 1, 1), torch.rand(1, 1, 1, 2)) + assert "pred and target shapes must be the same. Got:" in str(errinfo) + + with pytest.raises(ValueError) as errinfo: + criterion(torch.rand(1, 1, 1, 1), torch.rand(1, 1, 1, 1, device="meta")) + assert "pred and target must be in the same device. Got:" in str(errinfo) + + @pytest.mark.parametrize("ignore_index", [-100, None]) + def test_all_zeros(self, device, dtype, ignore_index): + num_classes = 3 + logits = torch.zeros(2, num_classes, 1, 2, device=device, dtype=dtype) + logits[:, 0] = 10.0 + logits[:, 1] = 1.0 + logits[:, 2] = 1.0 + labels = torch.zeros(2, 1, 2, device=device, dtype=torch.int64) + + criterion = kornia.losses.TverskyLoss(alpha=0.5, beta=0.5, ignore_index=ignore_index) + loss = criterion(logits, labels) + self.assert_close(loss, torch.zeros_like(loss), atol=1e-3, rtol=1e-3) + + @pytest.mark.parametrize("ignore_index", [-100, 255]) + def test_ignore_index(self, device, dtype, ignore_index): + num_classes = 2 + + logits = torch.zeros(2, num_classes, 1, 4, device=device, dtype=dtype) + logits[:, 0, :, 0] = 100.0 + logits[:, 1, :, 1:] = 100.0 + labels = torch.zeros(2, 1, 4, device=device, dtype=torch.int64) + + labels[..., 2:] = ignore_index + expected_loss = torch.tensor([1.0 / 2.0], device=device, dtype=dtype).squeeze() + criterion = kornia.losses.TverskyLoss(alpha=0.5, beta=0.5, ignore_index=ignore_index) + loss = criterion(logits, labels) + self.assert_close(loss, expected_loss, rtol=1e-3, atol=1e-3) + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + num_classes = 3 + alpha, beta = 0.5, 0.5 # for tversky loss + logits = torch.rand(2, num_classes, 3, 2, device=device, dtype=torch.float64) + labels = torch.randint(0, num_classes, (2, 3, 2), device=device) + ignore = torch.rand(2, 3, 2, device=device) > 0.8 + labels[ignore] = -100 + + self.gradcheck( + kornia.losses.tversky_loss, (logits, labels, alpha, beta), dtypes=[torch.float64, torch.int64, None, None] + ) + + def test_dynamo(self, device, dtype, torch_optimizer): + num_classes = 3 + params = (0.5, 0.05) + logits = torch.rand(2, num_classes, 3, 2, device=device, dtype=dtype) + labels = torch.rand(2, 3, 2) * num_classes + labels = labels.to(device).long() + + op = kornia.losses.tversky_loss + op_optimized = torch_optimizer(op) + + actual = op_optimized(logits, labels, *params) + expected = op(logits, labels, *params) + self.assert_close(actual, expected) + + def test_module(self, device, dtype): + num_classes = 3 + params = (0.5, 0.5) + logits = torch.rand(2, num_classes, 3, 2, device=device, dtype=dtype) + labels = torch.rand(2, 3, 2) * num_classes + labels = labels.to(device).long() + + op = kornia.losses.tversky_loss + op_module = kornia.losses.TverskyLoss(*params) + + actual = op_module(logits, labels) + expected = op(logits, labels, *params) + self.assert_close(actual, expected) diff --git a/tests/losses/test_welcsh.py b/tests/losses/test_welcsh.py new file mode 100644 index 0000000..9b1953a --- /dev/null +++ b/tests/losses/test_welcsh.py @@ -0,0 +1,119 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestWelschLoss(BaseTester): + def test_smoke(self, device, dtype): + img1 = torch.rand(2, 3, 2, device=device, dtype=dtype) + img2 = torch.rand(2, 3, 2, device=device, dtype=dtype) + + criterion = kornia.losses.WelschLoss() + + assert criterion(img1, img2) is not None + + @pytest.mark.parametrize("shape", [(1, 3, 5, 5), (2, 5, 5)]) + def test_cardinality(self, shape, device, dtype): + img = torch.rand(shape, device=device, dtype=dtype) + + actual = kornia.losses.WelschLoss(reduction="none")(img, img) + assert actual.shape == shape + + actual = kornia.losses.WelschLoss(reduction="mean")(img, img) + assert actual.shape == () + + @pytest.mark.grad() + def test_gradcheck(self, device, dtype): + img1 = torch.rand(2, 3, 3, 3, device=device, dtype=torch.float64) + img2 = torch.rand(2, 3, 3, 3, device=device, dtype=torch.float64) + + self.gradcheck(kornia.losses.welsch_loss, (img1, img2)) + + def test_dynamo(self, device, dtype, torch_optimizer): + img1 = torch.rand(2, 3, 3, 3, device=device, dtype=dtype) + img2 = torch.rand(2, 3, 3, 3, device=device, dtype=dtype) + + op = kornia.losses.welsch_loss + op_optimized = torch_optimizer(op) + + self.assert_close(op(img1, img2), op_optimized(img1, img2)) + + def test_module(self, device, dtype): + img1 = torch.rand(2, 3, 3, 3, device=device, dtype=dtype) + img2 = torch.rand(2, 3, 3, 3, device=device, dtype=dtype) + + op = kornia.losses.welsch_loss + op_module = kornia.losses.WelschLoss() + + self.assert_close(op(img1, img2), op_module(img1, img2)) + + @pytest.mark.parametrize("reduction", ["mean", "sum"]) + @pytest.mark.parametrize("shape", [(1, 2, 9, 9), (2, 4, 3, 6)]) + def test_perfect_prediction(self, device, dtype, reduction, shape): + # Sanity test + img = torch.rand(shape, device=device, dtype=dtype) + actual = kornia.losses.welsch_loss(img, img, reduction=reduction) + expected = torch.tensor(0.0, device=device, dtype=dtype) + self.assert_close(actual, expected) + + # Check loss computation + img1 = torch.ones(shape, device=device, dtype=dtype) + img2 = torch.zeros(shape, device=device, dtype=dtype) + + actual = kornia.losses.welsch_loss(img1, img2, reduction=reduction) + + if reduction == "mean": + expected = torch.tensor(0.39346934028, device=device, dtype=dtype) + elif reduction == "sum": + expected = (torch.ones_like(img1, device=device, dtype=dtype) * 0.39346934028).sum() + + self.assert_close(actual, expected) + + def test_exception(self, device, dtype): + img = torch.rand(3, 3, 3, device=device, dtype=dtype) + + # wrong reduction + from kornia.core.exceptions import BaseError + + with pytest.raises(BaseError) as execinfo: + kornia.losses.welsch_loss(img, img, reduction="test") + assert "Given type of reduction is not supported. Got: test" in str(execinfo.value) + + # Check if both are tensors + from kornia.core.exceptions import TypeCheckError + + with pytest.raises(TypeCheckError) as errinfo: + kornia.losses.welsch_loss(1.0, img) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(TypeCheckError) as errinfo: + kornia.losses.welsch_loss(img, 1.0) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + # Check if same shape + from kornia.core.exceptions import ShapeError + + img_b = torch.rand(1, 1, 3, 3, 4, device=device, dtype=dtype) + with pytest.raises(ShapeError) as errinfo: + kornia.losses.welsch_loss(img, img_b) + assert "Shape mismatch" in str(errinfo.value) diff --git a/tests/metrics/test_accuracy.py b/tests/metrics/test_accuracy.py new file mode 100644 index 0000000..f2ccb89 --- /dev/null +++ b/tests/metrics/test_accuracy.py @@ -0,0 +1,57 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch + +from kornia.metrics import accuracy + + +class TestAccuracy: + def test_top1_perfect(self): + logits = torch.tensor([[0.0, 1.0, 0.0], [1.0, 0.0, 0.0]]) + target = torch.tensor([[1], [0]]) + result = accuracy(logits, target, topk=(1,)) + assert len(result) == 1 + assert result[0].item() == 100.0 + + def test_top1_zero(self): + logits = torch.tensor([[1.0, 0.0, 0.0]]) + target = torch.tensor([[2]]) + result = accuracy(logits, target, topk=(1,)) + assert result[0].item() == 0.0 + + def test_topk(self): + # 3 classes; true class is index 2 (lowest logit) — wrong for both top-1 and top-2 + logits = torch.tensor([[0.3, 0.5, 0.2]]) + target = torch.tensor([[2]]) # true class is index 2 (third highest) + top1, top2 = accuracy(logits, target, topk=(1, 2)) + assert top1.item() == 0.0 + assert top2.item() == 0.0 + + logits2 = torch.tensor([[0.3, 0.2, 0.5]]) + target2 = torch.tensor([[2]]) + top1b, _ = accuracy(logits2, target2, topk=(1, 2)) + assert top1b.item() == 100.0 + + def test_topk_exceeds_num_classes_is_clipped(self): + # topk=5 but only 3 classes — should not crash + logits = torch.tensor([[0.1, 0.8, 0.1]]) + target = torch.tensor([[1]]) + result = accuracy(logits, target, topk=(5,)) + assert result[0].item() == 100.0 diff --git a/tests/metrics/test_aepe.py b/tests/metrics/test_aepe.py new file mode 100644 index 0000000..1e1a207 --- /dev/null +++ b/tests/metrics/test_aepe.py @@ -0,0 +1,89 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestAepe(BaseTester): + def test_metric_mean_reduction(self, device, dtype): + sample = torch.ones(4, 4, 2, device=device, dtype=dtype) + expected = torch.tensor(0.565685424, device=device, dtype=dtype) + actual = kornia.metrics.aepe(sample, 1.4 * sample, reduction="mean") + self.assert_close(actual, expected) + + def test_metric_sum_reduction(self, device, dtype): + sample = torch.ones(4, 4, 2, device=device, dtype=dtype) + expected = torch.tensor(1.4142, device=device, dtype=dtype) * 4**2 + actual = kornia.metrics.aepe(sample, 2.0 * sample, reduction="sum") + self.assert_close(actual, expected) + + def test_metric_no_reduction(self, device, dtype): + sample = torch.ones(4, 4, 2, device=device, dtype=dtype) + expected = torch.zeros(4, 4, device=device, dtype=dtype) + 1.4142 + actual = kornia.metrics.aepe(sample, 2.0 * sample, reduction="none") + self.assert_close(actual, expected) + + def test_perfect_fit(self, device, dtype): + sample = torch.ones(4, 4, 2, device=device, dtype=dtype) + expected = torch.zeros(4, 4, device=device, dtype=dtype) + actual = kornia.metrics.aepe(sample, sample, reduction="none") + self.assert_close(actual, expected) + + def test_aepe_alias(self, device, dtype): + sample = torch.ones(4, 4, 2, device=device, dtype=dtype) + expected = torch.zeros(4, 4, device=device, dtype=dtype) + actual_aepe = kornia.metrics.aepe(sample, sample, reduction="none") + actual_alias = kornia.metrics.average_endpoint_error(sample, sample, reduction="none") + self.assert_close(actual_aepe, expected) + self.assert_close(actual_alias, expected) + self.assert_close(actual_aepe, actual_alias) + + def test_exception(self, device, dtype): + from kornia.core.exceptions import TypeCheckError + + with pytest.raises(TypeCheckError) as errinfo: + criterion = kornia.metrics.AEPE() + criterion(None, torch.ones(4, 4, 2, device=device, dtype=dtype)) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(NotImplementedError) as errinfo: + sample = torch.ones(4, 4, 2, device=device, dtype=dtype) + _ = kornia.metrics.aepe(sample, 2.0 * sample, reduction="foo") + assert "Invalid reduction option." in str(errinfo) + + from kornia.core.exceptions import ShapeError + + with pytest.raises(ShapeError) as errinfo: + sample = torch.ones(4, 4, 2, device=device, dtype=dtype) + _ = kornia.metrics.aepe(sample, 2.0 * sample[..., 0], reduction="mean") + assert ( + "Shape dimension mismatch" in str(errinfo.value) + or "Expected shape" in str(errinfo.value) + or "shape must be" in str(errinfo.value) + ) + + def test_smoke(self, device, dtype): + input = torch.rand(3, 3, 2, device=device, dtype=dtype) + target = torch.rand(3, 3, 2, device=device, dtype=dtype) + + criterion = kornia.metrics.AEPE() + assert criterion(input, target) is not None diff --git a/tests/metrics/test_average_meter.py b/tests/metrics/test_average_meter.py new file mode 100644 index 0000000..0b09d0f --- /dev/null +++ b/tests/metrics/test_average_meter.py @@ -0,0 +1,57 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch + +from kornia.metrics import AverageMeter + + +class TestAverageMeter: + def test_initial_state(self): + m = AverageMeter() + assert m.val == 0 + assert m.avg == 0 + assert m.count == 0 + + def test_update_scalar(self): + m = AverageMeter() + m.update(0.8, n=1) + m.update(0.4, n=1) + assert abs(m.avg - 0.6) < 1e-6 + + def test_update_weighted(self): + m = AverageMeter() + m.update(1.0, n=3) + m.update(0.0, n=1) + assert abs(m.avg - 0.75) < 1e-6 + + def test_update_tensor(self): + m = AverageMeter() + m.update(torch.tensor(0.9), n=1) + # avg property converts tensor to float + assert isinstance(m.avg, float) + assert abs(m.avg - 0.9) < 1e-6 + + def test_reset(self): + m = AverageMeter() + m.update(1.0, n=5) + m.reset() + assert m.val == 0 + assert m.avg == 0 + assert m.count == 0 diff --git a/tests/metrics/test_confusion.py b/tests/metrics/test_confusion.py new file mode 100644 index 0000000..ad2bbb3 --- /dev/null +++ b/tests/metrics/test_confusion.py @@ -0,0 +1,151 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestConfusionMatrix(BaseTester): + def test_two_classes(self, device, dtype): + num_classes = 2 + actual = torch.tensor([[1, 1, 1, 1, 0, 0, 0, 0]], device=device, dtype=torch.long) + predicted = torch.tensor([[1, 1, 1, 1, 0, 0, 0, 1]], device=device, dtype=torch.long) + + conf_mat = kornia.metrics.confusion_matrix(predicted, actual, num_classes) + conf_mat_real = torch.tensor([[[3, 1], [0, 4]]], device=device, dtype=torch.float32) + self.assert_close(conf_mat, conf_mat_real) + + def test_two_classes_batch2(self, device, dtype): + batch_size = 2 + num_classes = 2 + actual = torch.tensor([[1, 1, 1, 1, 0, 0, 0, 0]], device=device, dtype=torch.long).repeat(batch_size, 1) + predicted = torch.tensor([[1, 1, 1, 1, 0, 0, 0, 1]], device=device, dtype=torch.long).repeat(batch_size, 1) + + conf_mat = kornia.metrics.confusion_matrix(predicted, actual, num_classes) + conf_mat_real = torch.tensor([[[3, 1], [0, 4]], [[3, 1], [0, 4]]], device=device, dtype=torch.float32) + self.assert_close(conf_mat, conf_mat_real) + + def test_three_classes(self, device, dtype): + num_classes = 3 + actual = torch.tensor([[2, 2, 0, 0, 1, 0, 0, 2, 1, 1, 0, 0, 1, 2, 1, 0]], device=device, dtype=torch.long) + predicted = torch.tensor([[2, 1, 0, 0, 0, 0, 0, 1, 0, 2, 2, 1, 0, 0, 2, 2]], device=device, dtype=torch.long) + + conf_mat = kornia.metrics.confusion_matrix(predicted, actual, num_classes) + conf_mat_real = torch.tensor([[[4, 1, 2], [3, 0, 2], [1, 2, 1]]], device=device, dtype=torch.float32) + self.assert_close(conf_mat, conf_mat_real) + + def test_four_classes_one_missing(self, device, dtype): + num_classes = 4 + actual = torch.tensor([[3, 3, 1, 1, 2, 1, 1, 3, 2, 2, 1, 1, 2, 3, 2, 1]], device=device, dtype=torch.long) + predicted = torch.tensor([[3, 2, 1, 1, 1, 1, 1, 2, 1, 3, 3, 2, 1, 1, 3, 3]], device=device, dtype=torch.long) + + conf_mat = kornia.metrics.confusion_matrix(predicted, actual, num_classes) + conf_mat_real = torch.tensor( + [[[0, 0, 0, 0], [0, 4, 1, 2], [0, 3, 0, 2], [0, 1, 2, 1]]], device=device, dtype=torch.float32 + ) + self.assert_close(conf_mat, conf_mat_real) + + def test_three_classes_normalized(self, device, dtype): + num_classes = 3 + normalized = True + actual = torch.tensor([[2, 2, 0, 0, 1, 0, 0, 2, 1, 1, 0, 0, 1, 2, 1, 0]], device=device, dtype=torch.long) + predicted = torch.tensor([[2, 1, 0, 0, 0, 0, 0, 1, 0, 2, 2, 1, 0, 0, 2, 2]], device=device, dtype=torch.long) + + conf_mat = kornia.metrics.confusion_matrix(predicted, actual, num_classes, normalized) + + conf_mat_real = torch.tensor( + [[[0.5000, 0.3333, 0.4000], [0.3750, 0.0000, 0.4000], [0.1250, 0.6667, 0.2000]]], + device=device, + dtype=torch.float32, + ) + + self.assert_close(conf_mat, conf_mat_real) + + def test_four_classes_2d_perfect(self, device, dtype): + num_classes = 4 + actual = torch.tensor( + [[[0, 0, 1, 1], [0, 0, 1, 1], [2, 2, 3, 3], [2, 2, 3, 3]]], device=device, dtype=torch.long + ) + predicted = torch.tensor( + [[[0, 0, 1, 1], [0, 0, 1, 1], [2, 2, 3, 3], [2, 2, 3, 3]]], device=device, dtype=torch.long + ) + + conf_mat = kornia.metrics.confusion_matrix(predicted, actual, num_classes) + conf_mat_real = torch.tensor( + [[[4, 0, 0, 0], [0, 4, 0, 0], [0, 0, 4, 0], [0, 0, 0, 4]]], device=device, dtype=torch.float32 + ) + self.assert_close(conf_mat, conf_mat_real) + + def test_four_classes_2d_one_class_nonperfect(self, device, dtype): + num_classes = 4 + actual = torch.tensor( + [[[0, 0, 1, 1], [0, 0, 1, 1], [2, 2, 3, 3], [2, 2, 3, 3]]], device=device, dtype=torch.long + ) + predicted = torch.tensor( + [[[0, 0, 1, 1], [0, 3, 0, 1], [2, 2, 1, 3], [2, 2, 3, 3]]], device=device, dtype=torch.long + ) + + conf_mat = kornia.metrics.confusion_matrix(predicted, actual, num_classes) + conf_mat_real = torch.tensor( + [[[3, 0, 0, 1], [1, 3, 0, 0], [0, 0, 4, 0], [0, 1, 0, 3]]], device=device, dtype=torch.float32 + ) + self.assert_close(conf_mat, conf_mat_real) + + def test_four_classes_2d_one_class_missing(self, device, dtype): + num_classes = 4 + actual = torch.tensor( + [[[0, 0, 1, 1], [0, 0, 1, 1], [2, 2, 3, 3], [2, 2, 3, 3]]], device=device, dtype=torch.long + ) + predicted = torch.tensor( + [[[3, 3, 1, 1], [3, 3, 1, 1], [2, 2, 3, 3], [2, 2, 3, 3]]], device=device, dtype=torch.long + ) + + conf_mat = kornia.metrics.confusion_matrix(predicted, actual, num_classes) + conf_mat_real = torch.tensor( + [[[0, 0, 0, 4], [0, 4, 0, 0], [0, 0, 4, 0], [0, 0, 0, 4]]], device=device, dtype=torch.float32 + ) + self.assert_close(conf_mat, conf_mat_real) + + def test_four_classes_2d_one_class_no_predicted(self, device, dtype): + num_classes = 4 + actual = torch.tensor( + [[[0, 0, 0, 0], [0, 0, 0, 0], [2, 2, 3, 3], [2, 2, 3, 3]]], device=device, dtype=torch.long + ) + predicted = torch.tensor( + [[[3, 3, 2, 2], [3, 3, 2, 2], [2, 2, 3, 3], [2, 2, 3, 3]]], device=device, dtype=torch.long + ) + + conf_mat = kornia.metrics.confusion_matrix(predicted, actual, num_classes) + conf_mat_real = torch.tensor( + [[[0, 0, 4, 4], [0, 0, 0, 0], [0, 0, 4, 0], [0, 0, 0, 4]]], device=device, dtype=torch.float32 + ) + self.assert_close(conf_mat, conf_mat_real) + + def test_exception_shape_mismatch(self, device, dtype): + pred = torch.zeros(1, 4, dtype=torch.long, device=device) + target = torch.zeros(1, 5, dtype=torch.long, device=device) + with pytest.raises(ValueError, match="same shape"): + kornia.metrics.confusion_matrix(pred, target, num_classes=2) + + def test_exception_num_classes_too_small(self, device, dtype): + pred = torch.zeros(1, 4, dtype=torch.long, device=device) + with pytest.raises(ValueError, match="bigger than two"): + kornia.metrics.confusion_matrix(pred, pred, num_classes=1) diff --git a/tests/metrics/test_disparity.py b/tests/metrics/test_disparity.py new file mode 100644 index 0000000..2a74baa --- /dev/null +++ b/tests/metrics/test_disparity.py @@ -0,0 +1,248 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia +from kornia.core.exceptions import BaseError, ShapeError, TypeCheckError + +from testing.base import BaseTester + + +class TestMeanAbsoluteDisparityError(BaseTester): + def test_smoke(self, device, dtype): + input = torch.rand(2, 1, 4, 5, device=device, dtype=dtype) + target = torch.rand(2, 1, 4, 5, device=device, dtype=dtype) + actual = kornia.metrics.mean_absolute_disparity_error(input, target) + assert actual.shape == torch.Size([]) + + def test_metric_mean_reduction(self, device, dtype): + sample = torch.ones(4, 4, device=device, dtype=dtype) + expected = torch.tensor(0.5, device=device, dtype=dtype) + actual = kornia.metrics.mean_absolute_disparity_error(sample, 1.5 * sample, reduction="mean") + self.assert_close(actual, expected) + + def test_metric_sum_reduction(self, device, dtype): + sample = torch.ones(4, 4, device=device, dtype=dtype) + expected = torch.tensor(8.0, device=device, dtype=dtype) + actual = kornia.metrics.mean_absolute_disparity_error(sample, 1.5 * sample, reduction="sum") + self.assert_close(actual, expected) + + def test_metric_no_reduction(self, device, dtype): + sample = torch.ones(4, 4, device=device, dtype=dtype) + expected = torch.full((4, 4), 0.5, device=device, dtype=dtype) + actual = kornia.metrics.mean_absolute_disparity_error(sample, 1.5 * sample, reduction="none") + self.assert_close(actual, expected) + + def test_perfect_prediction(self, device, dtype): + sample = torch.rand(4, 4, device=device, dtype=dtype) + expected = torch.tensor(0.0, device=device, dtype=dtype) + actual = kornia.metrics.mean_absolute_disparity_error(sample, sample) + self.assert_close(actual, expected) + + def test_valid_mask(self, device, dtype): + input = torch.zeros(2, 2, device=device, dtype=dtype) + target = torch.tensor([[1.0, 2.0], [3.0, 4.0]], device=device, dtype=dtype) + mask = torch.tensor([[True, False], [True, True]], device=device) + + actual_mean = kornia.metrics.mean_absolute_disparity_error(input, target, mask, reduction="mean") + self.assert_close(actual_mean, torch.tensor(8.0 / 3.0, device=device, dtype=dtype)) + + actual_sum = kornia.metrics.mean_absolute_disparity_error(input, target, mask, reduction="sum") + self.assert_close(actual_sum, torch.tensor(8.0, device=device, dtype=dtype)) + + actual_none = kornia.metrics.mean_absolute_disparity_error(input, target, mask, reduction="none") + expected_none = torch.tensor([[1.0, 0.0], [3.0, 4.0]], device=device, dtype=dtype) + self.assert_close(actual_none, expected_none) + + def test_valid_mask_numeric(self, device, dtype): + input = torch.zeros(2, 2, device=device, dtype=dtype) + target = torch.tensor([[1.0, 2.0], [3.0, 4.0]], device=device, dtype=dtype) + mask = torch.tensor([[1.0, 0.0], [1.0, 1.0]], device=device, dtype=dtype) + actual = kornia.metrics.mean_absolute_disparity_error(input, target, mask) + self.assert_close(actual, torch.tensor(8.0 / 3.0, device=device, dtype=dtype)) + + def test_valid_mask_broadcast(self, device, dtype): + input = torch.zeros(2, 2, 2, device=device, dtype=dtype) + target = torch.ones(2, 2, 2, device=device, dtype=dtype) + mask = torch.tensor([[True, False], [False, False]], device=device) + actual = kornia.metrics.mean_absolute_disparity_error(input, target, mask, reduction="sum") + self.assert_close(actual, torch.tensor(2.0, device=device, dtype=dtype)) + + def test_empty_valid_mask(self, device, dtype): + input = torch.zeros(2, 2, device=device, dtype=dtype) + target = torch.ones(2, 2, device=device, dtype=dtype) + mask = torch.zeros(2, 2, device=device, dtype=torch.bool) + + actual_mean = kornia.metrics.mean_absolute_disparity_error(input, target, mask, reduction="mean") + assert torch.isnan(actual_mean) + + actual_sum = kornia.metrics.mean_absolute_disparity_error(input, target, mask, reduction="sum") + self.assert_close(actual_sum, torch.tensor(0.0, device=device, dtype=dtype)) + + def test_exception(self, device, dtype): + sample = torch.ones(4, 4, device=device, dtype=dtype) + + with pytest.raises(TypeCheckError) as errinfo: + kornia.metrics.mean_absolute_disparity_error(None, sample) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(ShapeError) as errinfo: + kornia.metrics.mean_absolute_disparity_error(sample, sample[..., :2]) + assert "Shape mismatch" in str(errinfo.value) + + with pytest.raises(BaseError) as errinfo: + mask = torch.ones(3, device=device, dtype=torch.bool) + kornia.metrics.mean_absolute_disparity_error(sample, sample, mask) + assert "broadcastable" in str(errinfo.value) + + with pytest.raises(NotImplementedError) as errinfo: + kornia.metrics.mean_absolute_disparity_error(sample, 2.0 * sample, reduction="foo") + assert "Invalid reduction option." in str(errinfo.value) + + +class TestRootMeanSquaredDisparityError(BaseTester): + def test_smoke(self, device, dtype): + input = torch.rand(2, 1, 4, 5, device=device, dtype=dtype) + target = torch.rand(2, 1, 4, 5, device=device, dtype=dtype) + actual = kornia.metrics.root_mean_squared_disparity_error(input, target) + assert actual.shape == torch.Size([]) + + def test_metric_mean_reduction(self, device, dtype): + sample = torch.ones(4, 4, device=device, dtype=dtype) + expected = torch.tensor(0.5, device=device, dtype=dtype) + actual = kornia.metrics.root_mean_squared_disparity_error(sample, sample + 0.5, reduction="mean") + self.assert_close(actual, expected) + + def test_metric_sum_reduction(self, device, dtype): + sample = torch.ones(4, 4, device=device, dtype=dtype) + expected = torch.tensor(2.0, device=device, dtype=dtype) + actual = kornia.metrics.root_mean_squared_disparity_error(sample, sample + 0.5, reduction="sum") + self.assert_close(actual, expected) + + def test_metric_no_reduction(self, device, dtype): + sample = torch.ones(4, 4, device=device, dtype=dtype) + expected = torch.full((4, 4), 0.5, device=device, dtype=dtype) + actual = kornia.metrics.root_mean_squared_disparity_error(sample, sample + 0.5, reduction="none") + self.assert_close(actual, expected) + + def test_perfect_prediction(self, device, dtype): + sample = torch.rand(4, 4, device=device, dtype=dtype) + expected = torch.tensor(0.0, device=device, dtype=dtype) + actual = kornia.metrics.root_mean_squared_disparity_error(sample, sample) + self.assert_close(actual, expected) + + def test_valid_mask(self, device, dtype): + input = torch.zeros(2, 2, device=device, dtype=dtype) + target = torch.tensor([[1.0, 2.0], [3.0, 4.0]], device=device, dtype=dtype) + mask = torch.tensor([[True, True], [True, False]], device=device) + + # sqrt((1 + 4 + 9) / 3) + actual_mean = kornia.metrics.root_mean_squared_disparity_error(input, target, mask, reduction="mean") + self.assert_close(actual_mean, torch.tensor(2.1602468994, device=device, dtype=dtype)) + + # sqrt(1 + 4 + 9) + actual_sum = kornia.metrics.root_mean_squared_disparity_error(input, target, mask, reduction="sum") + self.assert_close(actual_sum, torch.tensor(3.7416573867, device=device, dtype=dtype)) + + actual_none = kornia.metrics.root_mean_squared_disparity_error(input, target, mask, reduction="none") + expected_none = torch.tensor([[1.0, 2.0], [3.0, 0.0]], device=device, dtype=dtype) + self.assert_close(actual_none, expected_none) + + def test_exception(self, device, dtype): + sample = torch.ones(4, 4, device=device, dtype=dtype) + + with pytest.raises(TypeCheckError) as errinfo: + kornia.metrics.root_mean_squared_disparity_error(None, sample) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(NotImplementedError) as errinfo: + kornia.metrics.root_mean_squared_disparity_error(sample, 2.0 * sample, reduction="foo") + assert "Invalid reduction option." in str(errinfo.value) + + +class TestMeanBadPixelError(BaseTester): + def test_smoke(self, device, dtype): + input = torch.rand(2, 1, 4, 5, device=device, dtype=dtype) + target = torch.rand(2, 1, 4, 5, device=device, dtype=dtype) + actual = kornia.metrics.mean_bad_pixel_error(input, target) + assert actual.shape == torch.Size([]) + + def test_metric_mean_reduction(self, device, dtype): + input = torch.zeros(1, 6, device=device, dtype=dtype) + target = torch.tensor([[0.0, 1.0, 2.0, 3.0, 4.0, 5.0]], device=device, dtype=dtype) + expected = torch.tensor(2.0 / 6.0, device=device, dtype=dtype) + actual = kornia.metrics.mean_bad_pixel_error(input, target, reduction="mean") + self.assert_close(actual, expected) + + def test_metric_sum_reduction(self, device, dtype): + input = torch.zeros(1, 6, device=device, dtype=dtype) + target = torch.tensor([[0.0, 1.0, 2.0, 3.0, 4.0, 5.0]], device=device, dtype=dtype) + expected = torch.tensor(2.0, device=device, dtype=dtype) + actual = kornia.metrics.mean_bad_pixel_error(input, target, reduction="sum") + self.assert_close(actual, expected) + + def test_metric_no_reduction(self, device, dtype): + input = torch.zeros(1, 6, device=device, dtype=dtype) + target = torch.tensor([[0.0, 1.0, 2.0, 3.0, 4.0, 5.0]], device=device, dtype=dtype) + expected = torch.tensor([[0.0, 0.0, 0.0, 0.0, 1.0, 1.0]], device=device, dtype=dtype) + actual = kornia.metrics.mean_bad_pixel_error(input, target, reduction="none") + self.assert_close(actual, expected) + + def test_perfect_prediction(self, device, dtype): + sample = torch.rand(4, 4, device=device, dtype=dtype) + expected = torch.tensor(0.0, device=device, dtype=dtype) + actual = kornia.metrics.mean_bad_pixel_error(sample, sample) + self.assert_close(actual, expected) + + def test_threshold(self, device, dtype): + input = torch.zeros(1, 6, device=device, dtype=dtype) + target = torch.tensor([[0.0, 1.0, 2.0, 3.0, 4.0, 5.0]], device=device, dtype=dtype) + + actual = kornia.metrics.mean_bad_pixel_error(input, target, threshold=1.0) + self.assert_close(actual, torch.tensor(4.0 / 6.0, device=device, dtype=dtype)) + + actual = kornia.metrics.mean_bad_pixel_error(input, target, threshold=4.5) + self.assert_close(actual, torch.tensor(1.0 / 6.0, device=device, dtype=dtype)) + + # an error exactly equal to the threshold is not a bad pixel + actual = kornia.metrics.mean_bad_pixel_error(input, target, threshold=5.0) + self.assert_close(actual, torch.tensor(0.0, device=device, dtype=dtype)) + + def test_valid_mask(self, device, dtype): + input = torch.zeros(1, 6, device=device, dtype=dtype) + target = torch.tensor([[0.0, 1.0, 2.0, 3.0, 4.0, 5.0]], device=device, dtype=dtype) + mask = torch.tensor([[True, True, True, True, False, True]], device=device) + + actual_mean = kornia.metrics.mean_bad_pixel_error(input, target, valid_mask=mask, reduction="mean") + self.assert_close(actual_mean, torch.tensor(1.0 / 5.0, device=device, dtype=dtype)) + + actual_none = kornia.metrics.mean_bad_pixel_error(input, target, valid_mask=mask, reduction="none") + expected_none = torch.tensor([[0.0, 0.0, 0.0, 0.0, 0.0, 1.0]], device=device, dtype=dtype) + self.assert_close(actual_none, expected_none) + + def test_exception(self, device, dtype): + sample = torch.ones(4, 4, device=device, dtype=dtype) + + with pytest.raises(TypeCheckError) as errinfo: + kornia.metrics.mean_bad_pixel_error(None, sample) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(NotImplementedError) as errinfo: + kornia.metrics.mean_bad_pixel_error(sample, 2.0 * sample, reduction="foo") + assert "Invalid reduction option." in str(errinfo.value) diff --git a/tests/metrics/test_map.py b/tests/metrics/test_map.py new file mode 100644 index 0000000..df0cd72 --- /dev/null +++ b/tests/metrics/test_map.py @@ -0,0 +1,49 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestMeanAveragePrecision(BaseTester): + def test_smoke(self, device, dtype): + boxes = torch.tensor([[100, 50, 150, 100.0]], device=device, dtype=dtype) + labels = torch.tensor([1], device=device, dtype=torch.long) + scores = torch.tensor([0.7], device=device, dtype=dtype) + + gt_boxes = torch.tensor([[100, 50, 150, 100.0]], device=device, dtype=dtype) + gt_labels = torch.tensor([1], device=device, dtype=torch.long) + + mean_ap = kornia.metrics.mean_average_precision([boxes], [labels], [scores], [gt_boxes], [gt_labels], 2) + + self.assert_close(mean_ap[0], torch.tensor(1.0, device=device, dtype=dtype)) + self.assert_close(mean_ap[1][1], 1.0) + + def test_raise(self, device, dtype): + boxes = torch.tensor([[100, 50, 150, 100.0]], device=device, dtype=dtype) + labels = torch.tensor([1], device=device, dtype=torch.long) + scores = torch.tensor([0.7], device=device, dtype=dtype) + + gt_boxes = torch.tensor([[100, 50, 150, 100.0]], device=device, dtype=dtype) + gt_labels = torch.tensor([1], device=device, dtype=torch.long) + + with pytest.raises(AssertionError): + _ = kornia.metrics.mean_average_precision(boxes[0], [labels], [scores], [gt_boxes], [gt_labels], 2) diff --git a/tests/metrics/test_mean_iou.py b/tests/metrics/test_mean_iou.py new file mode 100644 index 0000000..d350098 --- /dev/null +++ b/tests/metrics/test_mean_iou.py @@ -0,0 +1,170 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestMeanIoU(BaseTester): + def test_two_classes_perfect(self, device, dtype): + batch_size = 1 + num_classes = 2 + actual = torch.tensor([[1, 1, 1, 1, 0, 0, 0, 0]], device=device, dtype=torch.long) + predicted = torch.tensor([[1, 1, 1, 1, 0, 0, 0, 0]], device=device, dtype=torch.long) + + mean_iou = kornia.metrics.mean_iou(predicted, actual, num_classes) + mean_iou_real = torch.tensor([[1.0, 1.0]], device=device, dtype=torch.float32) + assert mean_iou.shape == (batch_size, num_classes) + self.assert_close(mean_iou, mean_iou_real) + + def test_two_classes_perfect_batch2(self, device, dtype): + batch_size = 2 + num_classes = 2 + actual = torch.tensor([[1, 1, 1, 1, 0, 0, 0, 0]], device=device, dtype=torch.long).repeat(batch_size, 1) + predicted = torch.tensor([[1, 1, 1, 1, 0, 0, 0, 0]], device=device, dtype=torch.long).repeat(batch_size, 1) + + mean_iou = kornia.metrics.mean_iou(predicted, actual, num_classes) + mean_iou_real = torch.tensor([[1.0, 1.0], [1.0, 1.0]], device=device, dtype=torch.float32) + assert mean_iou.shape == (batch_size, num_classes) + self.assert_close(mean_iou, mean_iou_real) + + def test_two_classes(self, device, dtype): + batch_size = 1 + num_classes = 2 + actual = torch.tensor([[1, 1, 1, 1, 0, 0, 0, 0]], device=device, dtype=torch.long) + predicted = torch.tensor([[1, 1, 1, 1, 0, 0, 0, 1]], device=device, dtype=torch.long) + + mean_iou = kornia.metrics.mean_iou(predicted, actual, num_classes) + mean_iou = kornia.metrics.mean_iou(predicted, actual, num_classes) + mean_iou_real = torch.tensor([[0.75, 0.80]], device=device, dtype=torch.float32) + assert mean_iou.shape == (batch_size, num_classes) + self.assert_close(mean_iou, mean_iou_real) + + def test_four_classes_2d_perfect(self, device, dtype): + batch_size = 1 + num_classes = 4 + actual = torch.tensor( + [[[0, 0, 1, 1], [0, 0, 1, 1], [2, 2, 3, 3], [2, 2, 3, 3]]], device=device, dtype=torch.long + ) + predicted = torch.tensor( + [[[0, 0, 1, 1], [0, 0, 1, 1], [2, 2, 3, 3], [2, 2, 3, 3]]], device=device, dtype=torch.long + ) + + mean_iou = kornia.metrics.mean_iou(predicted, actual, num_classes) + mean_iou_real = torch.tensor([[1.0, 1.0, 1.0, 1.0]], device=device, dtype=torch.float32) + assert mean_iou.shape == (batch_size, num_classes) + self.assert_close(mean_iou, mean_iou_real) + + def test_four_classes_one_missing(self, device, dtype): + batch_size = 1 + num_classes = 4 + actual = torch.tensor( + [[[0, 0, 0, 0], [0, 0, 0, 0], [2, 2, 3, 3], [2, 2, 3, 3]]], device=device, dtype=torch.long + ) + predicted = torch.tensor( + [[[3, 3, 2, 2], [3, 3, 2, 2], [2, 2, 3, 3], [2, 2, 3, 3]]], device=device, dtype=torch.long + ) + + mean_iou = kornia.metrics.mean_iou(predicted, actual, num_classes) + mean_iou_real = torch.tensor([[0.0, 1.0, 0.5, 0.5]], device=device, dtype=torch.float32) + assert mean_iou.shape == (batch_size, num_classes) + self.assert_close(mean_iou, mean_iou_real) + + def test_exception_shape_mismatch(self, device, dtype): + pred = torch.zeros(1, 4, dtype=torch.long, device=device) + target = torch.zeros(1, 5, dtype=torch.long, device=device) + with pytest.raises(ValueError, match="same shape"): + kornia.metrics.mean_iou(pred, target, num_classes=2) + + def test_exception_num_classes_too_small(self, device, dtype): + pred = torch.zeros(1, 4, dtype=torch.long, device=device) + with pytest.raises(ValueError, match="bigger than two"): + kornia.metrics.mean_iou(pred, pred, num_classes=1) + + +class TestMeanIoUBBox(BaseTester): + """Tests for mean_iou_bbox with different box formats.""" + + def test_bbox_xyxy_format(self, device, dtype): + """Test XYXY format (original behavior).""" + boxes_1 = torch.tensor([[40, 40, 60, 60], [30, 40, 50, 60]], device=device, dtype=dtype) + boxes_2 = torch.tensor([[40, 50, 60, 70], [30, 40, 40, 50]], device=device, dtype=dtype) + + iou = kornia.metrics.mean_iou_bbox(boxes_1, boxes_2, box_format="xyxy") + expected = torch.tensor([[0.3333, 0.0000], [0.1429, 0.2500]], device=device, dtype=dtype) + + self.assert_close(iou, expected, rtol=1e-3, atol=1e-4) + + def test_bbox_xywh_format(self, device, dtype): + """Test XYWH format.""" + # Same boxes as xyxy test, but in xywh format + boxes_1_xywh = torch.tensor([[40, 40, 20, 20], [30, 40, 20, 20]], device=device, dtype=dtype) + boxes_2_xywh = torch.tensor([[40, 50, 20, 20], [30, 40, 10, 10]], device=device, dtype=dtype) + + iou = kornia.metrics.mean_iou_bbox(boxes_1_xywh, boxes_2_xywh, box_format="xywh") + expected = torch.tensor([[0.3333, 0.0000], [0.1429, 0.2500]], device=device, dtype=dtype) + + self.assert_close(iou, expected, rtol=1e-3, atol=1e-4) + + def test_bbox_cxcywh_format(self, device, dtype): + """Test CXCYWH format.""" + # Same boxes as xyxy test, but in cxcywh format + boxes_1_cxcywh = torch.tensor([[50, 50, 20, 20], [40, 50, 20, 20]], device=device, dtype=dtype) + boxes_2_cxcywh = torch.tensor([[50, 60, 20, 20], [35, 45, 10, 10]], device=device, dtype=dtype) + + iou = kornia.metrics.mean_iou_bbox(boxes_1_cxcywh, boxes_2_cxcywh, box_format="cxcywh") + expected = torch.tensor([[0.3333, 0.0000], [0.1429, 0.2500]], device=device, dtype=dtype) + + self.assert_close(iou, expected, rtol=1e-3, atol=1e-4) + + def test_bbox_format_consistency(self, device, dtype): + """Test that all formats produce same results for equivalent boxes.""" + # Define same boxes in three formats + boxes_xyxy = torch.tensor([[10, 10, 20, 20]], device=device, dtype=dtype) + boxes_xywh = torch.tensor([[10, 10, 10, 10]], device=device, dtype=dtype) + boxes_cxcywh = torch.tensor([[15, 15, 10, 10]], device=device, dtype=dtype) + + iou_xyxy = kornia.metrics.mean_iou_bbox(boxes_xyxy, boxes_xyxy, box_format="xyxy") + iou_xywh = kornia.metrics.mean_iou_bbox(boxes_xywh, boxes_xywh, box_format="xywh") + iou_cxcywh = kornia.metrics.mean_iou_bbox(boxes_cxcywh, boxes_cxcywh, box_format="cxcywh") + + # All should give perfect IoU (1.0) + expected = torch.tensor([[1.0]], device=device, dtype=dtype) + self.assert_close(iou_xyxy, expected) + self.assert_close(iou_xywh, expected) + self.assert_close(iou_cxcywh, expected) + + def test_bbox_invalid_format(self, device, dtype): + """Test that invalid format raises ValueError.""" + boxes = torch.tensor([[10, 10, 20, 20]], device=device, dtype=dtype) + + with pytest.raises(ValueError, match="Unsupported box format"): + kornia.metrics.mean_iou_bbox(boxes, boxes, box_format="invalid") + + def test_bbox_default_format(self, device, dtype): + """Test that default format is xyxy.""" + boxes_1 = torch.tensor([[40, 40, 60, 60]], device=device, dtype=dtype) + boxes_2 = torch.tensor([[40, 50, 60, 70]], device=device, dtype=dtype) + + iou_default = kornia.metrics.mean_iou_bbox(boxes_1, boxes_2) + iou_explicit = kornia.metrics.mean_iou_bbox(boxes_1, boxes_2, box_format="xyxy") + + self.assert_close(iou_default, iou_explicit) diff --git a/tests/metrics/test_psnr_metric.py b/tests/metrics/test_psnr_metric.py new file mode 100644 index 0000000..db00fde --- /dev/null +++ b/tests/metrics/test_psnr_metric.py @@ -0,0 +1,37 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestPsnr(BaseTester): + def test_metric(self, device, dtype): + sample = torch.ones(1, device=device, dtype=dtype) + expected = torch.tensor(20.0, device=device, dtype=dtype) + actual = kornia.metrics.psnr(sample, 1.2 * sample, 2.0) + self.assert_close(actual, expected) + + def test_exception_shape_mismatch(self, device, dtype): + a = torch.ones(4, device=device, dtype=dtype) + b = torch.ones(8, device=device, dtype=dtype) + with pytest.raises(TypeError, match="Expected tensors of equal shapes"): + kornia.metrics.psnr(a, b, max_val=1.0) diff --git a/tests/metrics/test_ssim3d.py b/tests/metrics/test_ssim3d.py new file mode 100644 index 0000000..eadf944 --- /dev/null +++ b/tests/metrics/test_ssim3d.py @@ -0,0 +1,158 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +import kornia + +from testing.base import BaseTester + + +class TestSSIM3d(BaseTester): + @pytest.mark.parametrize( + "shape,padding,window_size,max_value", + [ + ((1, 1, 3, 3, 3), "same", 5, 1.0), + ((1, 1, 3, 3, 3), "same", 3, 2.0), + ((1, 1, 3, 3, 3), "same", 3, 0.5), + ((1, 1, 3, 3, 3), "valid", 3, 1.0), + ((2, 4, 3, 3, 3), "same", 3, 1.0), + ], + ) + def test_smoke(self, shape, padding, window_size, max_value, device, dtype): + img_a = (torch.ones(shape, device=device, dtype=dtype) * max_value).clamp(0.0, max_value) + img_b = torch.zeros(shape, device=device, dtype=dtype) + + actual = kornia.metrics.ssim3d(img_a, img_b, window_size, max_value, padding=padding) + expected = torch.ones_like(actual, device=device, dtype=dtype) + + self.assert_close(actual, expected * 0.0001) + + actual = kornia.metrics.ssim3d(img_a, img_a, window_size, max_value, padding=padding) + self.assert_close(actual, expected) + + @pytest.mark.parametrize( + "shape,padding,window_size,expected", + [ + ((1, 1, 2, 2, 3), "same", 3, (1, 1, 2, 2, 3)), + ((1, 1, 3, 3, 3), "same", 5, (1, 1, 3, 3, 3)), + ((1, 1, 3, 3, 3), "valid", 3, (1, 1, 1, 1, 1)), + ((2, 4, 3, 3, 3), "same", 3, (2, 4, 3, 3, 3)), + ], + ) + def test_cardinality(self, shape, padding, window_size, expected, device, dtype): + img = torch.rand(shape, device=device, dtype=dtype) + + actual = kornia.metrics.ssim3d(img, img, window_size, padding=padding) + + assert actual.shape == expected + + def test_exception(self, device, dtype): + img = torch.rand(1, 1, 3, 3, 3, device=device, dtype=dtype) + + # Check if both are tensors + from kornia.core.exceptions import TypeCheckError + + with pytest.raises(TypeCheckError) as errinfo: + kornia.metrics.ssim3d(1.0, img, 3) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + with pytest.raises(TypeCheckError) as errinfo: + kornia.metrics.ssim3d(img, 1.0, 3) + assert "Type mismatch: expected Tensor" in str(errinfo.value) + + # Check both shapes + from kornia.core.exceptions import ShapeError + + img_wrong_shape = torch.rand(3, 3, device=device, dtype=dtype) + with pytest.raises(ShapeError) as errinfo: + kornia.metrics.ssim3d(img, img_wrong_shape, 3) + assert "Shape dimension mismatch" in str(errinfo.value) or "Expected shape" in str(errinfo.value) + + with pytest.raises(ShapeError) as errinfo: + kornia.metrics.ssim3d(img_wrong_shape, img, 3) + assert "Shape dimension mismatch" in str(errinfo.value) or "Expected shape" in str(errinfo.value) + + # Check if same shape + img_b = torch.rand(1, 1, 3, 3, 4, device=device, dtype=dtype) + with pytest.raises(Exception) as errinfo: + kornia.metrics.ssim3d(img, img_b, 3) + assert "img1 and img2 shapes must be the same. Got:" in str(errinfo) + + def test_unit(self, device, dtype): + img_a = torch.tensor( + [ + [ + [ + [[0.7, 1.0, 0.5], [1.0, 0.3, 1.0], [0.2, 1.0, 0.1]], + [[0.2, 1.0, 0.1], [1.0, 0.3, 1.0], [0.7, 1.0, 0.5]], + [[1.0, 0.3, 1.0], [0.7, 1.0, 0.5], [0.2, 1.0, 0.1]], + ] + ] + ], + device=device, + dtype=dtype, + ) + + img_b = torch.ones(1, 1, 3, 3, 3, device=device, dtype=dtype) * 0.5 + + actual = kornia.metrics.ssim3d(img_a, img_b, 3, padding="same") + + expected = torch.tensor( + [ + [ + [ + [[0.0093, 0.0080, 0.0075], [0.0075, 0.0068, 0.0063], [0.0067, 0.0060, 0.0056]], + [[0.0077, 0.0070, 0.0065], [0.0077, 0.0069, 0.0064], [0.0075, 0.0066, 0.0062]], + [[0.0075, 0.0069, 0.0064], [0.0078, 0.0070, 0.0065], [0.0077, 0.0067, 0.0064]], + ] + ] + ], + device=device, + dtype=dtype, + ) + + self.assert_close(actual, expected, atol=1e-4, rtol=1e-4) + + @pytest.mark.parametrize( + "shape,padding,window_size,max_value", + [ + ((1, 1, 3, 3, 3), "same", 5, 1.0), + ((1, 1, 3, 3, 3), "same", 3, 2.0), + ((1, 1, 3, 3, 3), "same", 3, 0.5), + ((1, 1, 3, 3, 3), "valid", 3, 1.0), + ], + ) + def test_module(self, shape, padding, window_size, max_value, device, dtype): + img_a = torch.rand(shape, device=device, dtype=dtype).clamp(0.0, max_value) + img_b = torch.rand(shape, device=device, dtype=dtype).clamp(0.0, max_value) + + ops = kornia.metrics.ssim3d + mod = kornia.metrics.SSIM3D(window_size, max_value, padding=padding) + + ops_out = ops(img_a, img_b, window_size, max_value, padding=padding) + mod_out = mod(img_a, img_b) + + self.assert_close(ops_out, mod_out) + + def test_gradcheck(self, device): + img = torch.rand(1, 1, 3, 3, 3, device=device, dtype=torch.float64) + + op = kornia.metrics.ssim3d + + self.gradcheck(op, (img, img, 3), nondet_tol=1e-8) diff --git a/tests/metrics/test_ssim_metric.py b/tests/metrics/test_ssim_metric.py new file mode 100644 index 0000000..4f77a08 --- /dev/null +++ b/tests/metrics/test_ssim_metric.py @@ -0,0 +1,83 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import pytest +import torch + +from kornia.metrics.ssim import SSIM, ssim + +from testing.base import BaseTester + + +class TestSsim(BaseTester): + def test_same_image_returns_ones(self, device, dtype): + img = torch.rand(1, 3, 16, 16, device=device, dtype=dtype) + out = ssim(img, img, window_size=5) + assert out.shape == img.shape + assert (out > 0.99).all() + + def test_padding_valid(self, device, dtype): + img1 = torch.rand(1, 1, 16, 16, device=device, dtype=dtype) + img2 = torch.rand(1, 1, 16, 16, device=device, dtype=dtype) + out_same = ssim(img1, img2, window_size=5, padding="same") + out_valid = ssim(img1, img2, window_size=5, padding="valid") + # valid crops the border — output is smaller than 'same' + assert out_valid.shape[2] < out_same.shape[2] + assert out_valid.shape[3] < out_same.shape[3] + + def test_exception_non_tensor_img1(self, device, dtype): + img2 = torch.rand(1, 1, 8, 8, device=device, dtype=dtype) + with pytest.raises(TypeError, match=r"Input img1 type is not a torch\.Tensor"): + ssim([1, 2, 3], img2, window_size=3) + + def test_exception_non_tensor_img2(self, device, dtype): + img1 = torch.rand(1, 1, 8, 8, device=device, dtype=dtype) + with pytest.raises(TypeError, match=r"Input img2 type is not a torch\.Tensor"): + ssim(img1, [1, 2, 3], window_size=3) + + def test_exception_non_float_max_val(self, device, dtype): + img = torch.rand(1, 1, 8, 8, device=device, dtype=dtype) + with pytest.raises(TypeError, match="Input max_val type is not a float"): + ssim(img, img, window_size=3, max_val=1) + + def test_exception_wrong_ndim_img1(self, device, dtype): + img1 = torch.rand(1, 8, 8, device=device, dtype=dtype) + img2 = torch.rand(1, 1, 8, 8, device=device, dtype=dtype) + with pytest.raises(ValueError, match="Invalid img1 shape"): + ssim(img1, img2, window_size=3) + + def test_exception_wrong_ndim_img2(self, device, dtype): + img1 = torch.rand(1, 1, 8, 8, device=device, dtype=dtype) + img2 = torch.rand(1, 8, 8, device=device, dtype=dtype) + with pytest.raises(ValueError, match="Invalid img2 shape"): + ssim(img1, img2, window_size=3) + + def test_exception_shape_mismatch(self, device, dtype): + img1 = torch.rand(1, 1, 8, 8, device=device, dtype=dtype) + img2 = torch.rand(1, 1, 8, 16, device=device, dtype=dtype) + with pytest.raises(ValueError, match="img1 and img2 shapes must be the same"): + ssim(img1, img2, window_size=3) + + def test_ssim_module(self, device, dtype): + img1 = torch.rand(2, 3, 16, 16, device=device, dtype=dtype) + img2 = torch.rand(2, 3, 16, 16, device=device, dtype=dtype) + module = SSIM(window_size=5) + out_module = module(img1, img2) + out_fn = ssim(img1, img2, window_size=5) + assert out_module.shape == out_fn.shape diff --git a/tests/models/__init__.py b/tests/models/__init__.py new file mode 100644 index 0000000..28c8d2f --- /dev/null +++ b/tests/models/__init__.py @@ -0,0 +1,20 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Tests for Kornia models.""" + +__all__ = [] diff --git a/tests/models/qwen25/test_qwen2_vl.py b/tests/models/qwen25/test_qwen2_vl.py new file mode 100644 index 0000000..38d62ed --- /dev/null +++ b/tests/models/qwen25/test_qwen2_vl.py @@ -0,0 +1,33 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.models.qwen25 import Qwen2VLVisionTransformer + + +class TestQwen2VL: + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_smoke(self, batch_size, device, dtype): + model = Qwen2VLVisionTransformer().to(device=device, dtype=dtype) + input = torch.randn(batch_size, 3, 224, 224, device=device, dtype=dtype) + + output = model(input) + + assert output.shape[0] == batch_size + assert output.shape[2] == 1280 diff --git a/tests/models/sam3/__init__.py b/tests/models/sam3/__init__.py new file mode 100644 index 0000000..4d273d5 --- /dev/null +++ b/tests/models/sam3/__init__.py @@ -0,0 +1,16 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/models/sam3/test_sam3_architecture.py b/tests/models/sam3/test_sam3_architecture.py new file mode 100644 index 0000000..7256a32 --- /dev/null +++ b/tests/models/sam3/test_sam3_architecture.py @@ -0,0 +1,143 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Tests for SAM-3 architecture components.""" + +from __future__ import annotations + +import pytest +import torch +from torch import nn + +from kornia.models.sam3.architecture import ImageEncoderHiera +from kornia.models.sam3.architecture.common import Attention, MLPBlock + +from testing.base import BaseTester + + +class TestSam3Common(BaseTester): + """Test common components used in SAM-3.""" + + @pytest.mark.parametrize("embed_dim", [64, 256, 512]) + def test_layer_norm_smoke(self, device: str, embed_dim: int) -> None: + """Test LayerNorm basic functionality with different dimensions.""" + B, N = 2, 64 + ln = nn.LayerNorm(embed_dim).to(device) + x = torch.randn(B, N, embed_dim, device=device) + out = ln(x) + assert out.shape == x.shape + assert out.dtype == x.dtype + + @pytest.mark.parametrize("embed_dim,mlp_ratio", [(64, 4.0), (256, 4.0), (512, 2.0)]) + def test_mlp_block_smoke(self, device: str, embed_dim: int, mlp_ratio: float) -> None: + """Test MLPBlock basic functionality with different configurations.""" + B, N = 2, 64 + mlp_dim = int(embed_dim * mlp_ratio) + mlp = MLPBlock(embed_dim, mlp_dim).to(device) + x = torch.randn(B, N, embed_dim, device=device) + out = mlp(x) + assert out.shape == x.shape + assert out.dtype == x.dtype + + @pytest.mark.parametrize("dim,heads", [(256, 8), (512, 16), (768, 12)]) + def test_attention_smoke(self, device: str, dim: int, heads: int) -> None: + """Test Attention basic functionality with different configurations.""" + B, N = 2, 64 + attn = Attention(dim, heads=heads).to(device) + x = torch.randn(B, N, dim, device=device) + out = attn(x) + assert out.shape == x.shape + assert out.dtype == x.dtype + + +class TestImageEncoderHiera(BaseTester): + """Test ImageEncoderHiera architecture.""" + + @pytest.mark.parametrize("img_size,embed_dim", [(512, 128), (1024, 256)]) + def test_image_encoder_hiera_smoke(self, device: str, img_size: int, embed_dim: int) -> None: + """Test ImageEncoderHiera basic functionality with different configs.""" + patch_size = 16 + encoder = ImageEncoderHiera( + img_size=img_size, + patch_size=patch_size, + embed_dim=embed_dim, + depth=2, + num_heads=4, + ).to(device) + + B = 1 + x = torch.randn(B, 3, img_size, img_size, device=device) + features = encoder(x) + + # Expected output shape + num_patches = (img_size // patch_size) ** 2 + assert features.shape == (B, num_patches, embed_dim) + + @pytest.mark.parametrize("batch_size", [1, 2, 4]) + def test_image_encoder_hiera_cardinality(self, device: str, batch_size: int) -> None: + """Test ImageEncoderHiera output cardinality with different batch sizes.""" + encoder = ImageEncoderHiera( + img_size=512, + patch_size=16, + embed_dim=256, + depth=2, + num_heads=8, + ).to(device) + + x = torch.randn(batch_size, 3, 512, 512, device=device) + features = encoder(x) + assert features.shape == (batch_size, (512 // 16) ** 2, 256) + + def test_image_encoder_hiera_output_shape(self) -> None: + """Test output shape computation method.""" + encoder = ImageEncoderHiera( + img_size=1024, + patch_size=16, + embed_dim=256, + ) + + output_shape = encoder.get_output_shape((1, 3, 1024, 1024)) + expected = (1, (1024 // 16) ** 2, 256) + assert output_shape == expected + + def test_mlp_block_gradcheck(self, device: str) -> None: + """Test MLPBlock gradient computation.""" + mlp = MLPBlock(64, 256).to(device).double() + x = torch.randn(2, 4, 64, device=device, dtype=torch.float64, requires_grad=True) + self.gradcheck(mlp, (x,), raise_exception=True) + + def test_attention_gradcheck(self, device: str) -> None: + """Test Attention gradient computation.""" + attn = Attention(256, heads=8).to(device).double() + x = torch.randn(1, 16, 256, device=device, dtype=torch.float64, requires_grad=True) + self.gradcheck(attn, (x,), raise_exception=True) + + def test_image_encoder_hiera_gradcheck(self, device: str) -> None: + """Test ImageEncoderHiera gradient computation.""" + encoder = ( + ImageEncoderHiera( + img_size=64, + patch_size=16, + embed_dim=64, + depth=1, + num_heads=2, + ) + .to(device) + .double() + ) + x = torch.randn(1, 3, 64, 64, device=device, dtype=torch.float64, requires_grad=True) + self.gradcheck(encoder, (x,), raise_exception=True) diff --git a/tests/models/sam3/test_sam3_prompt_and_mask_architecture.py b/tests/models/sam3/test_sam3_prompt_and_mask_architecture.py new file mode 100644 index 0000000..1806cab --- /dev/null +++ b/tests/models/sam3/test_sam3_prompt_and_mask_architecture.py @@ -0,0 +1,246 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Tests for SAM-3 Phase 2 architecture modules.""" + +from __future__ import annotations + +import torch + +from kornia.models.sam3.architecture.mask_decoder import MaskDecoder +from kornia.models.sam3.architecture.prompt_encoder import PromptEncoder + + +class TestPromptEncoderPoints: + """Test PromptEncoder with point prompts.""" + + def test_prompt_encoder_with_points(self) -> None: + """Test basic prompt encoding with point prompts.""" + embed_dim = 256 + batch_size = 2 + num_points = 5 + + encoder = PromptEncoder(embed_dim=embed_dim) + + # Create dummy point prompts + coords = torch.rand(batch_size, num_points, 2) # (B, N, 2) + labels = torch.randint(0, 2, (batch_size, num_points)) # (B, N) with 0 or 1 + + sparse_emb, dense_emb = encoder(points=(coords, labels)) + + # Check output shapes + assert sparse_emb.shape == (batch_size, num_points, embed_dim), f"Got {sparse_emb.shape}" + assert dense_emb.shape[0] == batch_size, f"Got {dense_emb.shape}" + assert dense_emb.shape[1] == embed_dim, f"Got {dense_emb.shape}" + + def test_prompt_encoder_without_prompts(self) -> None: + """Test prompt encoder with no prompts.""" + embed_dim = 256 + batch_size = 1 + + encoder = PromptEncoder(embed_dim=embed_dim) + + sparse_emb, dense_emb = encoder() + + # Check output shapes + assert sparse_emb.shape[0] == batch_size, f"Got {sparse_emb.shape}" + assert sparse_emb.shape[2] == embed_dim, f"Got {sparse_emb.shape}" + assert dense_emb.shape[0] == batch_size, f"Got {dense_emb.shape}" + assert dense_emb.shape[1] == embed_dim, f"Got {dense_emb.shape}" + + def test_prompt_encoder_with_boxes(self) -> None: + """Test prompt encoder with box prompts.""" + embed_dim = 256 + batch_size = 2 + num_boxes = 3 + + encoder = PromptEncoder(embed_dim=embed_dim) + + # Create dummy box prompts + boxes = torch.rand(batch_size, num_boxes, 4) + + sparse_emb, dense_emb = encoder(boxes=boxes) + + # Check output shapes + assert sparse_emb.shape == (batch_size, num_boxes, embed_dim), f"Got {sparse_emb.shape}" + assert dense_emb.shape[0] == batch_size, f"Got {dense_emb.shape}" + + def test_prompt_encoder_with_masks(self) -> None: + """Test prompt encoder with mask prompts.""" + embed_dim = 256 + batch_size = 2 + mask_in_chans = 16 + + encoder = PromptEncoder(embed_dim=embed_dim, input_image_size=256, mask_in_chans=mask_in_chans) + + # Create dummy mask prompts + masks = torch.rand(batch_size, 1, 256, 256) + + sparse_emb, dense_emb = encoder(masks=masks) + + # Check output shapes + assert sparse_emb.shape[0] == batch_size, f"Got {sparse_emb.shape}" + assert sparse_emb.shape[2] == embed_dim, f"Got {sparse_emb.shape}" + assert dense_emb.shape[0] == batch_size, f"Got {dense_emb.shape}" + assert dense_emb.ndim == 4, f"Dense embedding should be 4D, got {dense_emb.ndim}D" + assert dense_emb.shape[2] == 64, f"Got spatial size {dense_emb.shape[2]}" # 256 // 4 + + def test_prompt_encoder_with_combined_prompts(self) -> None: + """Test prompt encoder with combined point and box prompts.""" + embed_dim = 256 + batch_size = 2 + num_points = 3 + num_boxes = 2 + + encoder = PromptEncoder(embed_dim=embed_dim) + + # Create dummy prompts + coords = torch.rand(batch_size, num_points, 2) + labels = torch.randint(0, 2, (batch_size, num_points)) + boxes = torch.rand(batch_size, num_boxes, 4) + + sparse_emb, dense_emb = encoder(points=(coords, labels), boxes=boxes) + + # Check output shapes + expected_num_sparse = num_points + num_boxes + assert sparse_emb.shape == (batch_size, expected_num_sparse, embed_dim), f"Got {sparse_emb.shape}" + assert dense_emb.shape[0] == batch_size, f"Got {dense_emb.shape}" + + +class TestMaskDecoderSmoke: + """Smoke tests for MaskDecoder.""" + + def test_mask_decoder_forward(self) -> None: + """Test basic mask decoder forward pass.""" + embed_dim = 256 + batch_size = 2 + num_patches = 1024 # 32x32 + num_prompts = 5 + + decoder = MaskDecoder(embed_dim=embed_dim) + + # Create dummy embeddings + image_embeddings = torch.randn(batch_size, num_patches, embed_dim) + sparse_prompts = torch.randn(batch_size, num_prompts, embed_dim) + dense_prompts = torch.randn(batch_size, embed_dim, 32, 32) + + masks, iou_pred = decoder( + image_embeddings, + sparse_prompts, + dense_prompts, + multimask_output=True, + ) + + # Check output shapes + assert masks.ndim == 4, f"Masks should be 4D, got {masks.ndim}D" + assert masks.shape[0] == batch_size, f"Got {masks.shape}" + assert iou_pred.shape == (batch_size, decoder.num_multimask_outputs), f"Got {iou_pred.shape}" + + def test_mask_decoder_single_mask_output(self) -> None: + """Test mask decoder output (Phase 2 generates single mask only).""" + embed_dim = 256 + batch_size = 1 + num_patches = 1024 + num_prompts = 3 + + decoder = MaskDecoder(embed_dim=embed_dim) + + # Create dummy embeddings + image_embeddings = torch.randn(batch_size, num_patches, embed_dim) + sparse_prompts = torch.randn(batch_size, num_prompts, embed_dim) + dense_prompts = torch.randn(batch_size, embed_dim, 32, 32) + + masks, iou_pred = decoder( + image_embeddings, + sparse_prompts, + dense_prompts, + multimask_output=False, + ) + + # Check output shapes + # Phase 2 generates single mask regardless of multimask_output flag + assert masks.ndim == 4, f"Masks should be 4D, got {masks.ndim}D" + assert masks.shape[0] == batch_size, f"Got {masks.shape}" + assert iou_pred.shape == (batch_size, decoder.num_multimask_outputs), f"Got {iou_pred.shape}" + + def test_mask_decoder_no_sparse_prompts(self) -> None: + """Test mask decoder with no sparse prompts.""" + embed_dim = 256 + batch_size = 1 + num_patches = 1024 + + decoder = MaskDecoder(embed_dim=embed_dim) + + # Create dummy embeddings with empty sparse prompts + image_embeddings = torch.randn(batch_size, num_patches, embed_dim) + sparse_prompts = torch.randn(batch_size, 0, embed_dim) + dense_prompts = torch.randn(batch_size, embed_dim, 32, 32) + + masks, iou_pred = decoder( + image_embeddings, + sparse_prompts, + dense_prompts, + ) + + # Check output shapes + assert masks.ndim == 4, f"Masks should be 4D, got {masks.ndim}D" + assert iou_pred.shape[0] == batch_size, f"Got {iou_pred.shape}" + + def test_mask_decoder_no_dense_prompts(self) -> None: + """Test mask decoder with no dense prompts.""" + embed_dim = 256 + batch_size = 1 + num_patches = 1024 + num_prompts = 2 + + decoder = MaskDecoder(embed_dim=embed_dim) + + # Create dummy embeddings with zero dense prompts + image_embeddings = torch.randn(batch_size, num_patches, embed_dim) + sparse_prompts = torch.randn(batch_size, num_prompts, embed_dim) + dense_prompts = torch.zeros(batch_size, embed_dim, 32, 32) + + masks, iou_pred = decoder( + image_embeddings, + sparse_prompts, + dense_prompts, + ) + + # Check output shapes + assert masks.ndim == 4, f"Masks should be 4D, got {masks.ndim}D" + assert iou_pred.shape == (batch_size, decoder.num_multimask_outputs), f"Got {iou_pred.shape}" + + def test_mask_decoder_batch_processing(self) -> None: + """Test mask decoder with different batch sizes.""" + embed_dim = 256 + num_patches = 1024 + + decoder = MaskDecoder(embed_dim=embed_dim) + + for batch_size in [1, 2, 4]: + image_embeddings = torch.randn(batch_size, num_patches, embed_dim) + sparse_prompts = torch.randn(batch_size, 3, embed_dim) + dense_prompts = torch.randn(batch_size, embed_dim, 32, 32) + + masks, iou_pred = decoder( + image_embeddings, + sparse_prompts, + dense_prompts, + ) + + assert masks.shape[0] == batch_size, f"Batch size mismatch: {masks.shape[0]} vs {batch_size}" + assert iou_pred.shape[0] == batch_size, f"Batch size mismatch in IoU: {iou_pred.shape[0]} vs {batch_size}" diff --git a/tests/models/test_box_filtering.py b/tests/models/test_box_filtering.py new file mode 100644 index 0000000..809a289 --- /dev/null +++ b/tests/models/test_box_filtering.py @@ -0,0 +1,116 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +from numpy.testing import assert_almost_equal + +from kornia.contrib.object_detection import BoxFiltering + + +class TestBoxFiltering: + @pytest.fixture + def sample_boxes(self): + # Setup some sample boxes with the format [class_id, confidence_score, x, y, w, h] + return torch.tensor( + [ + [ + [1, 0.9, 10, 10, 20, 20], # High confidence, class 1 + [2, 0.7, 15, 15, 25, 25], # Medium confidence, class 2 + [3, 0.7, 15, 15, 25, 25], # Medium confidence, class 3 + [4, 0.3, 5, 5, 10, 10], + ], # Low confidence, class 4 + [ + [1, 0.95, 12, 12, 18, 18], # High confidence, class 1 + [2, 0.5, 13, 13, 20, 20], # Low confidence, class 2 + [3, 0.5, 13, 13, 20, 20], # Low confidence, class 3 + [4, 0.2, 7, 7, 14, 14], + ], # Very low confidence, class 4 + [ + [1, 0.1, 12, 12, 18, 18], # Very Low confidence, class 1 + [2, 0.1, 13, 13, 20, 20], # Very Low confidence, class 2 + [3, 0.1, 13, 13, 20, 20], # Very Low confidence, class 3 + [4, 0.1, 7, 7, 14, 14], + ], # Very Low confidence, class 4 + ] + ) # Shape: [3, 4, 6], i.e., [B, D, 6] + + def test_confidence_filtering(self, sample_boxes): + """Test filtering based on confidence threshold.""" + # Set a confidence threshold of 0.7 + filter = BoxFiltering(confidence_threshold=0.7) + filtered_boxes = filter(sample_boxes) + + # Expected output: only boxes with confidence > 0.7 should be kept + assert len(filtered_boxes[0]) == 1 # Only one box in the first batch + assert_almost_equal(filtered_boxes[0][0][1].item(), 0.9) # Box with confidence 0.9 + assert len(filtered_boxes[1]) == 1 # Only one box in the second batch + assert_almost_equal(filtered_boxes[1][0][1].item(), 0.95) # Box with confidence 0.95 + assert len(filtered_boxes[2]) == 0 # No boxes in the third batch + + def test_class_filtering(self, sample_boxes): + """Test filtering based on class IDs.""" + # Set classes_to_keep to [1, 2] + filter = BoxFiltering(classes_to_keep=torch.tensor([1, 2])) + filtered_boxes = filter(sample_boxes) + + # Expected output: only boxes with class_id 1 and 2 should be kept + assert len(filtered_boxes[0]) == 2 # Two boxes in the first batch + assert filtered_boxes[0][0][0].item() == 1 # Box with class_id 1 + assert filtered_boxes[0][1][0].item() == 2 # Box with class_id 2 + assert len(filtered_boxes[1]) == 2 # Two boxes in the second batch + assert filtered_boxes[1][0][0].item() == 1 # Box with class_id 1 + assert filtered_boxes[1][1][0].item() == 2 # Box with class_id 2 + assert len(filtered_boxes[2]) == 2 # Two boxes in the third batch + assert filtered_boxes[2][0][0].item() == 1 # Box with class_id 1 + assert filtered_boxes[2][1][0].item() == 2 # Box with class_id 2 + + def test_combined_confidence_and_class_filtering(self, sample_boxes): + """Test filtering based on both confidence and class IDs.""" + # Set confidence threshold to 0.6 and classes_to_keep to [1, 3] + filter = BoxFiltering(confidence_threshold=0.6, classes_to_keep=torch.tensor([1, 3])) + filtered_boxes = filter(sample_boxes) + + # Expected output: only boxes with confidence > 0.6 and class_id in [1, 3] should be kept + assert len(filtered_boxes[0]) == 2 # Two boxes in the first batch + assert filtered_boxes[0][0][0].item() == 1 # Class_id 1 + assert filtered_boxes[0][1][0].item() == 3 # Class_id 3 + assert filtered_boxes[1][0][0].item() == 1 # Class_id 1 + assert len(filtered_boxes[1]) == 1 # No boxes in the second batch + assert len(filtered_boxes[2]) == 0 # No boxes in the third batch + + def test_filter_as_zero(self, sample_boxes): + """Test filtering boxes as zero when filter_as_zero is True.""" + filter = BoxFiltering(confidence_threshold=0.8, filter_as_zero=True) + filtered_boxes = filter(sample_boxes) + + # Expected output: boxes with confidence <= 0.8 should be zeroed out + assert torch.all(filtered_boxes[0][0] != 0) # Box with confidence 0.9 should remain + assert torch.all(filtered_boxes[0][1:] == 0) # Remaining boxes should be zeroed + assert torch.all(filtered_boxes[1][0] != 0) # Box with confidence 0.95 should remain + assert torch.all(filtered_boxes[1][1:] == 0) # Remaining boxes should be zeroed + assert torch.all(filtered_boxes[2] == 0) # All boxes in the third batch should be zeroed + + def test_no_class_or_confidence_filtering(self, sample_boxes): + """Test when no class or confidence filtering is applied.""" + filter = BoxFiltering() # No thresholds set + filtered_boxes = filter(sample_boxes) + + # Expected output: all boxes should be returned as-is + assert len(filtered_boxes[0]) == 4 # All boxes in the first batch should be kept + assert len(filtered_boxes[1]) == 4 # All boxes in the second batch should be kept + assert len(filtered_boxes[2]) == 4 # All boxes in the third batch should be kept diff --git a/tests/models/test_dexined.py b/tests/models/test_dexined.py new file mode 100644 index 0000000..4ad08b1 --- /dev/null +++ b/tests/models/test_dexined.py @@ -0,0 +1,56 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.models.dexined import DexiNed + +from testing.base import BaseTester + + +class TestDexiNed(BaseTester): + def test_smoke(self, device, dtype): + img = torch.rand(2, 3, 32, 32, device=device, dtype=dtype) + net = DexiNed(pretrained=False).to(device, dtype) + feat = net.get_features(img) + assert len(feat) == 6 + out = net(img) + assert out.shape == (2, 1, 32, 32) + + @pytest.mark.slow + @pytest.mark.parametrize("data", ["dexined"], indirect=True) + def test_inference(self, device, dtype, data): + model = DexiNed(pretrained=False) + model.load_state_dict(data, strict=True) + model = model.to(device, dtype) + model.eval() + + img = torch.tensor([[[[0.0, 255.0, 0.0], [0.0, 255.0, 0.0], [0.0, 255.0, 0.0]]]], device=device, dtype=dtype) + img = img.repeat(1, 3, 1, 1) + + expect = torch.tensor( + [[[[-0.3709, 0.0519, -0.2839], [0.0627, 0.6587, -0.1276], [-0.1840, -0.3917, -0.8240]]]], + device=device, + dtype=dtype, + ) + + out = model(img) + self.assert_close(out, expect, atol=1e-3, rtol=1e-2) + + @pytest.mark.skip(reason="DexiNed do not compile with dynamo.") + def test_dynamo(self, device, dtype, torch_optimizer): ... diff --git a/tests/models/test_dexined_onnx.py b/tests/models/test_dexined_onnx.py new file mode 100644 index 0000000..0c7bb22 --- /dev/null +++ b/tests/models/test_dexined_onnx.py @@ -0,0 +1,35 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest + +from kornia.models.dexined import DexiNed + + +def test_dexined_to_onnx(tmp_path): + """Ensure `DexiNed.to_onnx` exports a valid ONNX model and saves the file.""" + onnx = pytest.importorskip("onnx") + pytest.importorskip("onnxscript") + + model = DexiNed(pretrained=False) + model.eval() + + onnx_path = tmp_path / "dexined.onnx" + op = model.to_onnx(save=True, onnx_name=str(onnx_path), pseudo_shape=[1, 3, 32, 32]) + + assert isinstance(op, onnx.ModelProto) + assert onnx_path.exists(), "ONNX file was not written to disk" diff --git a/tests/models/test_efficient_vit.py b/tests/models/test_efficient_vit.py new file mode 100644 index 0000000..a357a43 --- /dev/null +++ b/tests/models/test_efficient_vit.py @@ -0,0 +1,74 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import pytest +import torch + +from kornia.core._compat import torch_version_lt +from kornia.models.efficient_vit import EfficientViT, EfficientViTConfig +from kornia.models.efficient_vit import backbone as vit + + +class TestEfficientViT: + def _test_smoke(self, device, dtype, img_size: int, expected_resolution: int, model_name: str): + model = getattr(vit, f"efficientvit_backbone_{model_name}")() + model = model.to(device=device, dtype=dtype) + + image = torch.randn(1, 3, img_size, img_size, device=device, dtype=dtype) + + out = model(image) + + assert "input" in out + assert out["input"].shape == image.shape + + assert "stage_final" in out + assert out["stage_final"].shape[-2:] == torch.Size([expected_resolution, expected_resolution]) + + @pytest.mark.parametrize("model_name", ["b3"]) + @pytest.mark.parametrize("img_size,expected_resolution", [(224, 7), (256, 8), (288, 9)]) + @pytest.mark.slow + def test_smoke_slow(self, device, dtype, img_size: int, expected_resolution: int, model_name: str): + self._test_smoke(device, dtype, img_size, expected_resolution, model_name) + + @pytest.mark.parametrize("model_name", ["b0", "b1", "b2"]) + @pytest.mark.parametrize("img_size,expected_resolution", [(224, 7), (256, 8), (288, 9)]) + def test_smoke(self, device, dtype, img_size: int, expected_resolution: int, model_name: str): + self._test_smoke(device, dtype, img_size, expected_resolution, model_name) + + @pytest.mark.slow + @pytest.mark.skipif(torch_version_lt(2, 0, 0), reason="requires torch 2.0.0 or higher") + @pytest.mark.parametrize("model_name", ["l0", "l1", "l2", "l3"]) + @pytest.mark.parametrize("img_size,expected_resolution", [(224, 7), (256, 8), (288, 9), (320, 10), (384, 12)]) + def test_smoke_large(self, device, dtype, img_size: int, expected_resolution: int, model_name: str): + self._test_smoke(device, dtype, img_size, expected_resolution, model_name) + + @pytest.mark.slow + def test_load_pretrained(self, device, dtype): + model = EfficientViT.from_config(EfficientViTConfig()) + model = model.to(device=device, dtype=dtype) + + image = torch.randn(1, 3, 224, 224, device=device, dtype=dtype) + feats = model(image) + assert feats["stage_final"].shape == torch.Size([1, 256, 7, 7]) + + @pytest.mark.parametrize("model_type", ["b1", "b2", "b3"]) + @pytest.mark.parametrize("resolution", [224, 256, 288]) + def test_config(self, model_type, resolution): + config = EfficientViTConfig.from_pretrained(model_type, resolution) + assert model_type in config.checkpoint + assert str(resolution) in config.checkpoint diff --git a/tests/models/test_kimi_vl.py b/tests/models/test_kimi_vl.py new file mode 100644 index 0000000..72ed829 --- /dev/null +++ b/tests/models/test_kimi_vl.py @@ -0,0 +1,181 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.models.kimi_vl import KimiVLConfig, KimiVLModel +from kornia.models.kimi_vl.config import KimiVLProjectorConfig, MoonViTConfig +from kornia.models.kimi_vl.model import KimiVLProjector +from kornia.models.kimi_vl.moonvit import MoonViT, MoonViTAttention, MoonViTEncoder, MoonViTRotaryEmbedding + +from testing.base import BaseTester + + +@pytest.fixture +def config(): + vision_config = MoonViTConfig( + image_size=32, + patch_size=4, + hidden_size=32, + num_hidden_layers=2, + num_attention_heads=4, + intermediate_size=64, + ) + projector_config = KimiVLProjectorConfig( + input_dim=32, # Matches vision_config.hidden_size + hidden_dim=64, + output_dim=64, + ) + return KimiVLConfig(vision_config=vision_config, projector_config=projector_config) + + +@pytest.fixture +def model(device, dtype, config): + return KimiVLModel(config).to(device, dtype) + + +class TestKimiVLModel(BaseTester): + def test_smoke(self, model): + assert model is not None + + @pytest.mark.parametrize("batch_size", [1, 2, 4]) + def test_cardinality(self, device, dtype, model, config, batch_size): + images = torch.randn(batch_size, 3, 32, 32, device=device, dtype=dtype) + + output = model(images) + + # Expected output shape: + # H_patches = 32/4 = 8 + # W_patches = 32/4 = 8 + # After pixel shuffle (downsample 2): H_new = 4, W_new = 4 -> 16 patches + # Output dim = 64 + expected_patches = 16 + assert output.shape == (batch_size, expected_patches, config.projector_config.output_dim) + + def test_variable_resolution(self, device, dtype, model, config): + batch_size = 1 + # 48x48 image -> 12x12 patches -> 6x6 after shuffle -> 36 patches + images = torch.randn(batch_size, 3, 48, 48, device=device, dtype=dtype) + + output = model(images) + assert output.shape == (batch_size, 36, config.projector_config.output_dim) + + def test_attention_mask(self, device, dtype, model, config): + batch_size = 1 + images = torch.randn(batch_size, 3, 32, 32, device=device, dtype=dtype) + + # Create a mask (B, N, N) or (B, 1, N, N) + # N = 64 (8x8 patches) + N = 64 + mask = torch.ones(batch_size, N, N, device=device, dtype=torch.bool) + + # Mask out the last token + mask[:, :, -1] = 0 + + output = model(images, attention_mask=mask) + assert output.shape == (batch_size, 16, config.projector_config.output_dim) + + def test_exception(self, device, dtype, model): + # Test invalid input shape (missing batch dim) + with pytest.raises((RuntimeError, ValueError, IndexError)): + images = torch.randn(3, 32, 32, device=device, dtype=dtype) + model(images) + + def test_gradcheck(self, device, config): + # Convert model to float64 for gradcheck + model = KimiVLModel(config).to(device, torch.float64).train() + batch_size = 1 + images = torch.randn(batch_size, 3, 32, 32, device=device, dtype=torch.float64, requires_grad=True) + + self.gradcheck(model, images, raise_exception=True, fast_mode=True) + + def test_dynamo(self, device, dtype, torch_optimizer, model): + model = model.eval() + batch_size = 1 + images = torch.randn(batch_size, 3, 32, 32, device=device, dtype=dtype) + + model_optimized = torch_optimizer(model) + + with torch.no_grad(): + expected = model(images) + actual = model_optimized(images) + + self.assert_close(actual, expected) + + +class TestKimiVLComponents(BaseTester): + def test_moonvit(self, device, dtype, config): + model = MoonViT(config.vision_config).to(device, dtype) + batch_size = 2 + images = torch.randn(batch_size, 3, 32, 32, device=device, dtype=dtype) + + output = model(images) + # MoonViT output is (B, N, D) + # N = (32/4)^2 = 64 + assert output.shape == (batch_size, 64, config.vision_config.hidden_size) + + def test_moonvit_encoder(self, device, dtype, config): + encoder = MoonViTEncoder(config.vision_config).to(device, dtype) + batch_size = 2 + seq_len = 64 + hidden_size = config.vision_config.hidden_size + + x = torch.randn(batch_size, seq_len, hidden_size, device=device, dtype=dtype) + + # Create dummy cos/sin for RoPE + head_dim = hidden_size // config.vision_config.num_attention_heads + cos = torch.randn(seq_len, head_dim, device=device, dtype=dtype) + sin = torch.randn(seq_len, head_dim, device=device, dtype=dtype) + + output = encoder(x, cos, sin) + assert output.shape == (batch_size, seq_len, hidden_size) + + def test_moonvit_attention(self, device, dtype, config): + attention = MoonViTAttention(config.vision_config).to(device, dtype) + batch_size = 2 + seq_len = 64 + hidden_size = config.vision_config.hidden_size + + x = torch.randn(batch_size, seq_len, hidden_size, device=device, dtype=dtype) + + # Create dummy cos/sin for RoPE + head_dim = hidden_size // config.vision_config.num_attention_heads + cos = torch.randn(seq_len, head_dim, device=device, dtype=dtype) + sin = torch.randn(seq_len, head_dim, device=device, dtype=dtype) + + output = attention(x, cos, sin) + assert output.shape == (batch_size, seq_len, hidden_size) + + def test_moonvit_rotary_embedding(self, device, dtype, config): + dim = config.vision_config.hidden_size // config.vision_config.num_attention_heads + rope = MoonViTRotaryEmbedding(dim).to(device, dtype) + + h, w = 8, 8 + cos, sin = rope(h, w, device) + + seq_len = h * w + assert cos.shape == (seq_len, dim) + assert sin.shape == (seq_len, dim) + + def test_projector(self, device, dtype, config): + model = KimiVLProjector(config.projector_config).to(device, dtype) + batch_size = 2 + input_features = torch.randn(batch_size, 64, 32, device=device, dtype=dtype) + + output = model(input_features, h=8, w=8) + assert output.shape == (batch_size, 16, config.projector_config.output_dim) diff --git a/tests/models/test_mobile_vit.py b/tests/models/test_mobile_vit.py new file mode 100644 index 0000000..16f91c8 --- /dev/null +++ b/tests/models/test_mobile_vit.py @@ -0,0 +1,40 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.models.vit_mobile import MobileViT + +from testing.base import BaseTester + + +class TestMobileViT(BaseTester): + @pytest.mark.parametrize("B", [1, 2]) + @pytest.mark.parametrize("image_size", [(256, 256)]) + @pytest.mark.parametrize("mode", ["xxs", "xs", "s"]) + @pytest.mark.parametrize("patch_size", [(2, 2)]) + def test_smoke(self, device, dtype, B, image_size, mode, patch_size): + ih, iw = image_size + channel = {"xxs": 320, "xs": 384, "s": 640} + + img = torch.rand(B, 3, ih, iw, device=device, dtype=dtype) + mvit = MobileViT(mode=mode, patch_size=patch_size).to(device, dtype) + + out = mvit(img) + assert isinstance(out, torch.Tensor) + assert out.shape == (B, channel[mode], 8, 8) diff --git a/tests/models/test_model_base.py b/tests/models/test_model_base.py new file mode 100644 index 0000000..6ea0c5f --- /dev/null +++ b/tests/models/test_model_base.py @@ -0,0 +1,125 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import os +from unittest.mock import patch + +import pytest +import torch + +from kornia.models.base import ModelBaseMixin + + +class DummyMixin(ModelBaseMixin): + name = "dummy" + + +class TestModelBaseMixinTensorToType: + def test_torch_output_returns_tensor_unchanged(self): + mixin = DummyMixin() + t = torch.rand(1, 3, 8, 8) + out = mixin._tensor_to_type(t, "torch") + assert out is t + + def test_torch_output_returns_list_unchanged(self): + mixin = DummyMixin() + tensors = [torch.rand(1, 3, 8, 8), torch.rand(1, 3, 8, 8)] + out = mixin._tensor_to_type(tensors, "torch") + assert out is tensors + + def test_pil_output_single_tensor(self): + mixin = DummyMixin() + # (C, H, W) tensor in [0, 1] + t = torch.rand(3, 16, 16) + out = mixin._tensor_to_type(t, "pil") + # tensor_to_image returns a PIL Image or ndarray depending on implementation + assert out is not None + + def test_pil_output_list_of_tensors(self): + mixin = DummyMixin() + tensors = [torch.rand(3, 16, 16), torch.rand(3, 16, 16)] + out = mixin._tensor_to_type(tensors, "pil") + assert isinstance(out, list) + assert len(out) == 2 + + def test_unsupported_output_type_raises(self): + mixin = DummyMixin() + t = torch.rand(1, 3, 8, 8) + with pytest.raises(RuntimeError, match=r"Output type.*is not supported"): + mixin._tensor_to_type(t, "numpy") + + +class TestModelBaseMixinSave: + def test_save_single_tensor_calls_write_image_once(self, tmp_path): + mixin = DummyMixin() + t = torch.rand(3, 8, 8) + with patch("kornia.models.base.write_image") as mock_write: + mixin.save(t, str(tmp_path)) + assert mock_write.call_count == 1 + # path is the first positional arg (matches write_image(path_file, image, ...)) + saved_path = mock_write.call_args[0][0] + assert os.path.normcase(str(tmp_path)) in os.path.normcase(saved_path) + + def test_save_list_of_tensors_calls_write_image_per_item(self, tmp_path): + mixin = DummyMixin() + tensors = [torch.rand(3, 8, 8), torch.rand(3, 8, 8), torch.rand(3, 8, 8)] + with patch("kornia.models.base.write_image") as mock_write: + mixin.save(tensors, str(tmp_path)) + assert mock_write.call_count == 3 + + def test_save_creates_directory(self, tmp_path): + mixin = DummyMixin() + t = torch.rand(3, 8, 8) + new_dir = str(tmp_path / "new_subdir") + assert not os.path.exists(new_dir) + with patch("kornia.models.base.write_image"): + mixin.save(t, new_dir) + assert os.path.exists(new_dir) + + +class TestModelBaseMixinSaveOutputs: + def test_save_outputs_with_explicit_dir_single_tensor(self, tmp_path): + mixin = DummyMixin() + t = torch.rand(3, 8, 8) + with patch("kornia.models.base.write_image") as mock_write: + mixin._save_outputs(t, directory=str(tmp_path), suffix="_test") + assert mock_write.call_count == 1 + # path is the first positional arg; verify suffix appears in filename + saved_path = mock_write.call_args[0][0] + assert "_test_" in saved_path + + def test_save_outputs_with_explicit_dir_list(self, tmp_path): + mixin = DummyMixin() + tensors = [torch.rand(3, 8, 8), torch.rand(3, 8, 8)] + with patch("kornia.models.base.write_image") as mock_write: + mixin._save_outputs(tensors, directory=str(tmp_path)) + assert mock_write.call_count == 2 + + def test_save_outputs_none_dir_creates_default(self, tmp_path, monkeypatch): + # Run from tmp_path so we don't pollute the repo + monkeypatch.chdir(tmp_path) + mixin = DummyMixin() + t = torch.rand(3, 8, 8) + with patch("kornia.models.base.write_image"): + mixin._save_outputs(t, directory=None) + kornia_outputs = tmp_path / "kornia_outputs" + assert kornia_outputs.exists() + subdirs = list(kornia_outputs.iterdir()) + assert len(subdirs) == 1 + assert subdirs[0].name.startswith("dummy") diff --git a/tests/models/test_model_common.py b/tests/models/test_model_common.py new file mode 100644 index 0000000..7f41342 --- /dev/null +++ b/tests/models/test_model_common.py @@ -0,0 +1,198 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import torch + +from kornia.models.common import MLP, ConvNormAct, DropPath, LayerNorm2d, window_partition, window_unpartition + + +class TestConvNormAct: + def test_odd_kernel_size(self): + # Odd kernel_size uses symmetric padding (no self.pad attribute added) + layer = ConvNormAct(3, 16, kernel_size=3) + assert not hasattr(layer, "pad") + x = torch.rand(2, 3, 8, 8) + out = layer(x) + assert out.shape == (2, 16, 8, 8) + + def test_even_kernel_size_uses_asymmetric_pad(self): + # Even kernel_size (e.g. 2) triggers the asymmetric padding branch + layer = ConvNormAct(3, 16, kernel_size=2) + assert hasattr(layer, "pad") + x = torch.rand(2, 3, 8, 8) + out = layer(x) + # With kernel_size=2 and stride=1, output H and W should be preserved + assert out.shape == (2, 16, 8, 8) + + def test_act_relu(self): + layer = ConvNormAct(4, 8, kernel_size=1, act="relu") + x = torch.rand(1, 4, 4, 4) + out = layer(x) + assert (out >= 0).all() + + def test_act_silu(self): + layer = ConvNormAct(4, 8, kernel_size=1, act="silu") + x = torch.rand(1, 4, 4, 4) + out = layer(x) + assert out.shape == (1, 8, 4, 4) + + def test_act_none_is_identity(self): + layer = ConvNormAct(4, 8, kernel_size=1, act="none") + x = torch.rand(1, 4, 4, 4) + out = layer(x) + assert out.shape == (1, 8, 4, 4) + + +class TestMLP: + def test_forward_without_sigmoid(self): + mlp = MLP(input_dim=16, hidden_dim=32, output_dim=8, num_layers=3) + x = torch.randn(2, 16) + out = mlp(x) + assert out.shape == (2, 8) + # Output is not constrained to [0, 1] when no sigmoid is applied + assert ((out < 0.0) | (out > 1.0)).any().item() + + def test_forward_with_sigmoid_output(self): + mlp = MLP(input_dim=16, hidden_dim=32, output_dim=8, num_layers=3, sigmoid_output=True) + x = torch.rand(2, 16) + out = mlp(x) + assert out.shape == (2, 8) + # sigmoid squashes to (0, 1) + assert out.min() >= 0.0 + assert out.max() <= 1.0 + + def test_single_layer(self): + mlp = MLP(input_dim=4, hidden_dim=8, output_dim=6, num_layers=1) + x = torch.rand(1, 4) + out = mlp(x) + assert out.shape == (1, 6) + + +class TestDropPath: + def test_inference_mode_passthrough(self): + layer = DropPath(drop_prob=0.5) + layer.eval() + x = torch.ones(4, 8) + out = layer(x) + # In eval mode (not training), no drop should happen + assert torch.equal(out, x) + + def test_zero_drop_prob_passthrough(self): + layer = DropPath(drop_prob=0.0) + layer.train() + x = torch.ones(4, 8) + out = layer(x) + assert torch.equal(out, x) + + def test_training_mode_applies_drop(self): + torch.manual_seed(0) + layer = DropPath(drop_prob=0.99, scale_by_keep=False) + layer.train() + x = torch.ones(100, 8) + out = layer(x) + # With very high drop prob, many rows should be zeroed out + zero_rows = (out.sum(dim=1) == 0).sum().item() + assert zero_rows > 50, f"Expected many zero rows, got {zero_rows}" + + def test_training_mode_scale_by_keep(self): + torch.manual_seed(42) + layer_scaled = DropPath(drop_prob=0.5, scale_by_keep=True) + layer_unscaled = DropPath(drop_prob=0.5, scale_by_keep=False) + layer_scaled.train() + layer_unscaled.train() + x = torch.ones(1000, 4) + out_scaled = layer_scaled(x) + out_unscaled = layer_unscaled(x) + # The scaled version should have a higher mean for surviving rows + # (they get divided by keep_prob = 0.5, so surviving rows have value 2.0) + surviving_scaled = out_scaled[out_scaled.sum(dim=1) != 0].mean() + assert surviving_scaled > 1.5, "scale_by_keep=True should amplify surviving rows" + surviving_unscaled = out_unscaled[out_unscaled.sum(dim=1) != 0].mean() + assert abs(surviving_unscaled.item() - 1.0) < 0.1 + + +class TestLayerNorm2d: + def test_output_shape(self): + layer = LayerNorm2d(num_channels=8) + x = torch.rand(2, 8, 4, 4) + out = layer(x) + assert out.shape == x.shape + + def test_normalizes_channels(self): + layer = LayerNorm2d(num_channels=4) + # All-same input should produce near-zero output (before weight/bias) + x = torch.ones(1, 4, 4, 4) * 5.0 + # The layer has learnable weight (ones) and bias (zeros) by default + out = layer(x) + # Mean of out along channel dim should be ~0 for uniform input + assert out.abs().max() < 1e-5 + + +class TestWindowPartition: + def test_no_padding_needed(self): + # H=8, W=8, window_size=4 -> no padding + x = torch.rand(2, 8, 8, 16) + windows, (Hp, Wp) = window_partition(x, window_size=4) + assert Hp == 8 and Wp == 8 + # 2 batches * (8/4)*(8/4) = 2*4 = 8 windows + assert windows.shape == (8, 4, 4, 16) + + def test_padding_needed(self): + # H=7, W=9, window_size=4 -> padding needed + x = torch.rand(2, 7, 9, 16) + windows, (Hp, Wp) = window_partition(x, window_size=4) + # Hp = 8 (7 padded to next multiple of 4), Wp = 12 + assert Hp == 8 + assert Wp == 12 + # 2 * (8/4)*(12/4) = 2*2*3 = 12 windows + assert windows.shape == (12, 4, 4, 16) + + def test_roundtrip_without_padding(self): + x = torch.rand(2, 8, 8, 16) + windows, pad_hw = window_partition(x, window_size=4) + reconstructed = window_unpartition(windows, window_size=4, pad_hw=pad_hw, hw=(8, 8)) + assert torch.allclose(x, reconstructed) + + def test_roundtrip_with_padding(self): + x = torch.rand(2, 7, 9, 16) + H, W = x.shape[1], x.shape[2] + windows, pad_hw = window_partition(x, window_size=4) + reconstructed = window_unpartition(windows, window_size=4, pad_hw=pad_hw, hw=(H, W)) + assert reconstructed.shape == (2, 7, 9, 16) + assert torch.allclose(x, reconstructed) + + +class TestWindowUnpartition: + def test_no_crop_needed(self): + # Hp==H and Wp==W, so no cropping + x = torch.rand(2, 4, 4, 8) + windows, pad_hw = window_partition(x, window_size=4) + out = window_unpartition(windows, window_size=4, pad_hw=pad_hw, hw=(4, 4)) + assert out.shape == (2, 4, 4, 8) + assert torch.allclose(x, out) + + def test_crop_needed(self): + # Create input with padding scenario: pad_hw != hw + x = torch.rand(2, 6, 6, 8) + windows, pad_hw = window_partition(x, window_size=4) + # pad_hw = (8, 8), original hw = (6, 6) + out = window_unpartition(windows, window_size=4, pad_hw=pad_hw, hw=(6, 6)) + assert out.shape == (2, 6, 6, 8) + # Reconstructed values in the non-padded region should match original + assert torch.allclose(x, out) diff --git a/tests/models/test_naflex.py b/tests/models/test_naflex.py new file mode 100644 index 0000000..8ccd78d --- /dev/null +++ b/tests/models/test_naflex.py @@ -0,0 +1,98 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import pytest +import torch +from torch import Tensor + +from kornia.models.processors.naflex import NaFlex + +from testing.base import BaseTester + + +class TestNaFlex(BaseTester): + @pytest.fixture + def model(self) -> NaFlex: + """Create a NaFlex model with mock embeddings for testing. + + Returns: + NaFlex instance with mock patch embedding function and position embedding. + """ + + def mock_patch_embedding(x: Tensor) -> Tensor: + """Mock patch embedding function simulating Conv2d output. + + Dynamically calculates output size based on patch_size=16. + """ + B, _, H, W = x.shape + h_out = H // 16 + w_out = W // 16 + return torch.randn(B, 768, h_out, w_out, dtype=x.dtype, device=x.device) + + position_embedding = torch.randn(196, 768) + return NaFlex( + patch_embedding_fcn=mock_patch_embedding, + position_embedding=position_embedding, + ) + + def test_smoke(self, model: NaFlex, device: torch.device, dtype: torch.dtype) -> None: + """Test basic forward pass with standard input.""" + model = model.to(device) + input_data = torch.randn(1, 3, 224, 224, device=device, dtype=dtype) + out = model(input_data) + assert isinstance(out, Tensor) + assert out.shape == (1, 196, 768) + + def test_cardinality(self, model: NaFlex, device: torch.device, dtype: torch.dtype) -> None: + """Test output cardinality with non-square input resolution. + + For 224x320 input with 16x16 patches, expect 14x20=280 patches. + """ + model = model.to(device) + input_data = torch.randn(1, 3, 224, 320, device=device, dtype=dtype) + out = model(input_data) + assert out.shape[0] == 1 + assert out.shape[1] == 280 + assert out.shape[2] == 768 + + def test_exception(self, device: torch.device, dtype: torch.dtype) -> None: + """Test that invalid position embeddings raise appropriate errors.""" + + def fake_patch_fcn(x: Tensor) -> Tensor: + return torch.randn(1, 100, 768, device=device, dtype=dtype) + + bad_pos_embed = torch.randn(200, 768, device=device, dtype=dtype) + wrapper_bad = NaFlex(fake_patch_fcn, bad_pos_embed) + input_data = torch.randn(1, 3, 224, 224, device=device, dtype=dtype) + + with pytest.raises(ValueError, match="Original positional embedding is not a square grid"): + wrapper_bad(input_data) + + def test_interpolation(self, device: torch.device, dtype: torch.dtype) -> None: + """Test positional embedding interpolation for different input sizes.""" + + def mock_patch_embedding_dynamic(x: Tensor) -> Tensor: + """Dynamic mock for resizing tests.""" + B, _, H, W = x.shape + h_out = H // 16 + w_out = W // 16 + return torch.randn(B, 768, h_out, w_out, dtype=x.dtype, device=x.device) + + position_embedding = torch.randn(196, 768) + model = NaFlex(mock_patch_embedding_dynamic, position_embedding).to(device) + input_448 = torch.randn(1, 3, 448, 448, device=device, dtype=dtype) + out = model(input_448) + assert out.shape == (1, 784, 768) diff --git a/tests/models/test_object_detector.py b/tests/models/test_object_detector.py new file mode 100644 index 0000000..76f1402 --- /dev/null +++ b/tests/models/test_object_detector.py @@ -0,0 +1,103 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import warnings +from pathlib import Path + +import pytest +import torch + +import kornia +from kornia.core._compat import torch_version_lt +from kornia.models.rt_detr import RTDETR, DETRPostProcessor, RTDETRConfig + +from testing.base import BaseTester + + +class TestObjectDetector(BaseTester): + def test_smoke(self, device, dtype): + batch_size = 3 + confidence = 0.3 + config = RTDETRConfig("resnet50d", 10, head_num_queries=10) + model = RTDETR.from_config(config).to(device, dtype).eval() + pre_processor = kornia.models.processors.ResizePreProcessor(32, 32) + post_processor = DETRPostProcessor(confidence, num_top_queries=3).to(device, dtype).eval() + detector = kornia.contrib.object_detection.ObjectDetector(model, pre_processor, post_processor) + + sizes = torch.randint(5, 10, (batch_size, 2)) * 32 + imgs = [torch.randn(3, h, w, device=device, dtype=dtype) for h, w in sizes] + pre_processor_out = pre_processor(imgs) + detections = detector(imgs) + + assert pre_processor_out[0].shape[-1] == 32 + assert pre_processor_out[0].shape[-2] == 32 + assert len(detections) == batch_size + for dets in detections: + assert dets.shape[1] == 6, dets.shape + assert torch.all(dets[:, 0].int() == dets[:, 0]) + assert torch.all(dets[:, 1] >= 0.3) + + @pytest.mark.slow + @pytest.mark.skipif(torch_version_lt(2, 0, 0), reason="Unsupported ONNX opset version: 16") + @pytest.mark.parametrize("variant", ("resnet50d", "hgnetv2_l")) + def test_onnx(self, device, dtype, tmp_path: Path, variant: str): + config = RTDETRConfig(variant, 1) + model = RTDETR.from_config(config).to(device=device, dtype=dtype).eval() + pre_processor = kornia.models.processors.ResizePreProcessor(640, 640) + post_processor = DETRPostProcessor(0.3, num_top_queries=3) + detector = kornia.contrib.object_detection.ObjectDetector(model, pre_processor, post_processor) + + data = torch.rand(1, 3, 400, 640, device=device, dtype=dtype) + + model_path = tmp_path / "rtdetr.onnx" + + dynamic_axes = {"images": {0: "N"}} + # Suppress onnxscript deprecation warnings (Python 3.15 compatibility) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning, module="onnxscript.converter") + torch.onnx.export( + detector, + data, + model_path, + input_names=["images"], + output_names=["detections"], + dynamic_axes=dynamic_axes, + opset_version=17, + ) + + assert model_path.is_file() + + def test_results_from_detections(self, device, dtype): + # label_id, confidence, data + detections = torch.tensor( + [ + [0, 0.9, 0.0, 0.0, 1.0, 1.0], + [1, 0.8, 0.0, 0.0, 1.0, 1.0], + [2, 0.7, 0.0, 0.0, 1.0, 1.0], + [3, 0.6, 0.0, 0.0, 1.0, 1.0], + [4, 0.5, 0.0, 0.0, 1.0, 1.0], + ], + device=device, + dtype=dtype, + ) + + detector_results: list = kornia.contrib.object_detection.results_from_detections(detections, format="xywh") + + assert len(detector_results) == 5 + for j, det in enumerate(detector_results): + for i in range(4): + assert det.bbox.data[i] == float(detections[j, i + 2]) diff --git a/tests/models/test_paligemma.py b/tests/models/test_paligemma.py new file mode 100644 index 0000000..d714a3e --- /dev/null +++ b/tests/models/test_paligemma.py @@ -0,0 +1,62 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.models.paligemma import PaliGemma, PaliGemmaConfig +from kornia.models.paligemma.modeling_paligemma import GemmaAttention, GemmaMLP + + +class TestPaliGemmaModules: + @pytest.fixture + def config(self): + conf = PaliGemmaConfig() + conf.hidden_size = 32 + conf.intermediate_size = 64 + conf.num_hidden_layers = 1 + conf.num_attention_heads = 4 + conf.head_dim = 8 + conf.vocab_size = 100 + + conf.vision_config.image_size = 32 + conf.vision_config.patch_size = 16 + return conf + + def test_mlp(self, config): + model = GemmaMLP(config) + x = torch.randn(1, 10, config.hidden_size) + output = model(x) + assert output.shape == (1, 10, config.hidden_size) + + def test_attention(self, config): + model = GemmaAttention(config) + x = torch.randn(1, 10, config.hidden_size) + position_ids = torch.arange(10).unsqueeze(0) + output = model(x, position_ids=position_ids) + assert output.shape == (1, 10, config.hidden_size) + + def test_forward(self, config): + model = PaliGemma(config) + model.eval() + + pixel_values = torch.randn(1, 3, 32, 32) + input_ids = torch.randint(0, config.vocab_size, (1, 5)) + + logits = model(input_ids=input_ids, pixel_values=pixel_values) + + assert logits.shape == (1, 9, config.vocab_size) diff --git a/tests/models/test_prompter.py b/tests/models/test_prompter.py new file mode 100644 index 0000000..518cb41 --- /dev/null +++ b/tests/models/test_prompter.py @@ -0,0 +1,153 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.contrib.visual_prompter import VisualPrompter +from kornia.core._compat import torch_version +from kornia.models.sam import SamConfig + +from testing.base import BaseTester + + +class TestVisualPrompter(BaseTester): + @pytest.mark.slow + def test_smoke(self, device, dtype): + data = torch.rand(3, 77, 128, device=device, dtype=dtype) + prompter = VisualPrompter(SamConfig("vit_b"), device, dtype) + + prompter.set_image(data) + assert prompter.is_image_set + + prompter.reset_image() + assert not prompter.is_image_set + + @pytest.mark.slow + @pytest.mark.parametrize("batch_size", [1, 4]) + @pytest.mark.parametrize("N", [2, 5]) + @pytest.mark.parametrize("multimask_output", [True, False]) + def test_cardinality(self, device, batch_size, N, multimask_output): + # SAM: don't supports float64 + dtype = torch.float32 + data = torch.rand(3, 77, 128, device=device, dtype=dtype) + prompter = VisualPrompter(SamConfig("vit_b"), device, dtype) + + keypoints = torch.randint(0, min(data.shape[-2:]), (batch_size, N, 2), device=device).to(dtype=dtype) + labels = torch.randint(0, 1, (batch_size, N), device=device).to(dtype=dtype) + + prompter.set_image(data) + + out = prompter.predict(keypoints, labels, multimask_output=multimask_output) + + C = 3 if multimask_output else 1 + assert out.logits.shape == (batch_size, C, 256, 256) + assert out.scores.shape == (batch_size, C) + + def test_exception(self): + prompter = VisualPrompter(SamConfig("vit_b")) + data = torch.rand(1, 2, 3, 256, 256) + + # Wrong shape for the image + from kornia.core.exceptions import ShapeError + + with pytest.raises(ShapeError) as errinfo: + prompter.set_image(data, [], False) + assert ( + "Shape mismatch" in str(errinfo.value) + or "Shape dimension mismatch" in str(errinfo.value) + or "Expected shape" in str(errinfo.value) + ) + + # predict without set an image + with pytest.raises(Exception) as errinfo: + prompter.predict() + assert "An image must be set with `self.set_image(...)`" in str(errinfo) + + # Valid masks + with pytest.raises(ShapeError) as errinfo: + prompter._valid_masks(data) + assert ( + "Shape mismatch" in str(errinfo.value) + or "Shape dimension mismatch" in str(errinfo.value) + or "Expected shape" in str(errinfo.value) + ) + + # Valid boxes + with pytest.raises(ShapeError) as errinfo: + prompter._valid_boxes(data) + assert ( + "Shape mismatch" in str(errinfo.value) + or "Shape dimension mismatch" in str(errinfo.value) + or "Expected shape" in str(errinfo.value) + ) + + # Valid keypoints + with pytest.raises(ShapeError) as errinfo: + prompter._valid_keypoints(data, None) + assert ( + "Shape mismatch" in str(errinfo.value) + or "Shape dimension mismatch" in str(errinfo.value) + or "Expected shape" in str(errinfo.value) + ) + + with pytest.raises(ShapeError) as errinfo: + prompter._valid_keypoints(torch.rand(1, 1, 2), data) + assert ( + "Shape mismatch" in str(errinfo.value) + or "Shape dimension mismatch" in str(errinfo.value) + or "Expected shape" in str(errinfo.value) + ) + + with pytest.raises(Exception) as errinfo: + prompter._valid_keypoints(torch.rand(1, 1, 2), torch.rand(2, 1)) + assert "The keypoints and labels should have the same batch size" in str(errinfo) + + @pytest.mark.skip(reason="Unnecessary test") + def test_gradcheck(self, device): ... + + @pytest.mark.skip(reason="Unnecessary test") + def test_module(self): ... + + @pytest.mark.skipif(torch_version() in {"2.1.2", "2.0.1"}, reason="Not working well") + def test_dynamo(self, device, torch_optimizer): + dtype = torch.float32 + batch_size = 1 + N = 2 + data = torch.rand(3, 77, 128, device=device, dtype=dtype) + keypoints = torch.randint(0, min(data.shape[-2:]), (batch_size, N, 2), device=device, dtype=dtype) + labels = torch.randint(0, 1, (batch_size, N), device=device, dtype=dtype) + + prompter = VisualPrompter(SamConfig("vit_b"), device, dtype) + prompter.set_image(data) + + expected = prompter.predict(keypoints=keypoints, keypoints_labels=labels) + prompter.reset_image() + + prompter.compile() + prompter.set_image(data) + actual = prompter.predict(keypoints=keypoints, keypoints_labels=labels) + + # TODO (joao): explore the reason for the discrepancy between cuda/cpu + rtol = None + atol = None + if "cuda" in device.type: + rtol = 1e-3 + atol = 1e-3 + + self.assert_close(expected.logits, actual.logits, rtol=rtol, atol=atol) + self.assert_close(expected.scores, actual.scores, rtol=rtol, atol=atol) diff --git a/tests/models/test_rt_detr.py b/tests/models/test_rt_detr.py new file mode 100644 index 0000000..3c51919 --- /dev/null +++ b/tests/models/test_rt_detr.py @@ -0,0 +1,138 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from functools import partial + +import pytest +import torch + +from kornia.models.rt_detr.architecture.hgnetv2 import PPHGNetV2 +from kornia.models.rt_detr.architecture.hybrid_encoder import HybridEncoder, RepVggBlock +from kornia.models.rt_detr.architecture.resnet_d import ResNetD +from kornia.models.rt_detr.architecture.rtdetr_head import RTDETRHead +from kornia.models.rt_detr.model import RTDETR, RTDETRConfig + +from testing.base import BaseTester, assert_close + + +@pytest.mark.parametrize( + "backbone_factory", + (partial(ResNetD.from_config, 18), partial(ResNetD.from_config, 50), partial(PPHGNetV2.from_config, "L")), +) +@pytest.mark.slow +def test_backbone(backbone_factory, device, dtype): + backbone = backbone_factory().to(device, dtype) + assert hasattr(backbone, "out_channels") + assert len(backbone.out_channels) == 3 + + N, C, H, W = 2, 3, 224, 256 + imgs = torch.randn(N, C, H, W, device=device, dtype=dtype) + fmaps = backbone(imgs) + + assert len(fmaps) == 3 + downscale = 8 + for fmap, ch in zip(fmaps, backbone.out_channels): + assert fmap.shape == (N, ch, H // downscale, W // downscale) + downscale *= 2 + + +def test_neck(device, dtype): + N = 2 + in_channels = [64, 128, 256] + sizes = [(32, 24), (16, 12), (8, 6)] + hidden_dim = 64 + neck = HybridEncoder(in_channels, hidden_dim, 128).to(device, dtype) + fmaps = [torch.randn(N, ch_in, h, w, device=device, dtype=dtype) for ch_in, (h, w) in zip(in_channels, sizes)] + + outs = neck(fmaps) + assert len(outs) == len(fmaps) + for out, (h, w) in zip(outs, sizes): + assert out.shape == (N, hidden_dim, h, w) + + +def test_head(device, dtype): + N = 2 + in_channels = [32, 64, 128] + sizes = [(32, 24), (16, 12), (8, 6)] + num_classes = 5 + num_queries = 10 + decoder = RTDETRHead(num_classes, 32, num_queries, in_channels, 2).to(device, dtype).eval() + fmaps = [torch.randn(N, ch_in, h, w, device=device, dtype=dtype) for ch_in, (h, w) in zip(in_channels, sizes)] + + logits, boxes = decoder(fmaps) + assert logits.shape == (N, num_queries, num_classes) + assert boxes.shape == (N, num_queries, 4) + + +def test_regvgg_optimize_for_deployment(device, dtype): + module = RepVggBlock(64, 64).to(device, dtype).eval() + x = torch.randn(2, 64, 9, 9, device=device, dtype=dtype) + + expected = module(x) + module.optimize_for_deployment() + actual = module(x) + assert_close(actual, expected, atol=1e-3, rtol=1e-2) + + +class TestRTDETR(BaseTester): + @pytest.mark.slow # This will be slow for the bigger variants + @pytest.mark.parametrize("variant", ("resnet18d", "resnet34d", "resnet50d", "resnet101d", "hgnetv2_l", "hgnetv2_x")) + def test_smoke(self, variant, device, dtype): + model = RTDETR.from_config(RTDETRConfig(variant, 10)).to(device, dtype).eval() + images = torch.randn(2, 3, 224, 256, device=device, dtype=dtype) + out = model(images) + + assert isinstance(out, tuple) + assert len(out) == 2 + + @pytest.mark.parametrize("shape", ((1, 3, 96, 128), (2, 3, 224, 256))) + def test_cardinality(self, shape, device, dtype): + num_classes = 10 + num_queries = 10 + config = RTDETRConfig("resnet18d", num_classes, head_num_queries=num_queries) + model = RTDETR.from_config(config).to(device, dtype).eval() + + images = torch.randn(shape, device=device, dtype=dtype) + logits, boxes = model(images) + + assert logits.shape == (shape[0], num_queries, num_classes) + assert boxes.shape == (shape[0], num_queries, 4) + + @pytest.mark.skip("Unnecessary") + def test_exception(self): ... + + @pytest.mark.skip("Unnecessary") + def test_gradcheck(self): ... + + @pytest.mark.skip("Unnecessary") + def test_module(self): ... + + @pytest.mark.skip("Needs more investigation") + @pytest.mark.parametrize("variant", ("resnet50d", "hgnetv2_l")) + def test_dynamo(self, variant, device, dtype, torch_optimizer): + # NOTE: This test passes on Mac M1 CPU, PyTorch 2.0.0, + # but fails on GitHub Actions Ubuntu-latest CPU, PyTorch 2.0.0. + # Perhaps random weights cause outputs to be much more different? + # Using pre-trained weights might see a smaller difference. + model = RTDETR.from_config(RTDETRConfig(variant, 10, head_num_queries=10)).to(device, dtype).eval() + model_optimized = torch_optimizer(model) + + img = torch.rand(1, 3, 224, 256, device=device, dtype=dtype) + expected = model(img) + actual = model_optimized(img) + + self.assert_close(actual, expected) diff --git a/tests/models/test_rtdetr_onnx.py b/tests/models/test_rtdetr_onnx.py new file mode 100644 index 0000000..80d2456 --- /dev/null +++ b/tests/models/test_rtdetr_onnx.py @@ -0,0 +1,43 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""ONNX export tests for RT-DETR.""" + +import pytest + +from kornia.models.rt_detr.model import RTDETR, RTDETRConfig + + +def test_rtdetr_to_onnx(tmp_path): + """RT-DETR exports to ONNX with correct dual output names.""" + onnx = pytest.importorskip("onnx") + pytest.importorskip("onnxscript") + + config = RTDETRConfig.from_name("rtdetr_r18vd", num_classes=10) + model = RTDETR.from_config(config) + model.eval() + + onnx_path = tmp_path / "rtdetr.onnx" + op = model.to_onnx(save=True, onnx_name=str(onnx_path), pseudo_shape=[1, 3, 640, 640]) + + assert isinstance(op, onnx.ModelProto) + assert onnx_path.exists(), "ONNX file was not written to disk" + + # Verify the output nodes are named correctly + output_names = [o.name for o in op.graph.output] + assert "pred_logits" in output_names, f"Expected 'pred_logits' in outputs, got {output_names}" + assert "pred_boxes" in output_names, f"Expected 'pred_boxes' in outputs, got {output_names}" diff --git a/tests/models/test_sam.py b/tests/models/test_sam.py new file mode 100644 index 0000000..653fcb6 --- /dev/null +++ b/tests/models/test_sam.py @@ -0,0 +1,119 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch +import torch.nn.functional as F + +from kornia.models.sam import Sam, SamConfig + +from testing.base import BaseTester + + +def _pad_rb(x, size): + """Pads right bottom.""" + pad_h = size - x.shape[-2] + pad_w = size - x.shape[-1] + return F.pad(x, (0, pad_w, 0, pad_h)) + + +class TestSam(BaseTester): + @pytest.mark.slow + @pytest.mark.parametrize("model_type", ["vit_b", "mobile_sam"]) + def test_smoke(self, device, model_type): + model = Sam.from_config(SamConfig(model_type)).to(device) + assert isinstance(model, Sam) + + img_size = model.image_encoder.img_size + data = torch.randn(1, 3, img_size, img_size, device=device) + keypoints = torch.randint(0, img_size, (1, 2, 2), device=device, dtype=torch.float) + labels = torch.randint(0, 1, (1, 2), device=device, dtype=torch.float) + + model(data, [{"points": (keypoints, labels)}], False) + + @pytest.mark.slow + @pytest.mark.parametrize("batch_size", [1, 3]) + @pytest.mark.parametrize("N", [2, 5]) + @pytest.mark.parametrize("multimask_output", [True, False]) + def test_cardinality(self, device, batch_size, N, multimask_output): + # SAM: don't supports float64 + dtype = torch.float32 + data = torch.rand(1, 3, 77, 128, device=device, dtype=dtype) + model = Sam.from_config(SamConfig("vit_b")) + model = model.to(device=device, dtype=dtype) + data = _pad_rb(data, model.image_encoder.img_size) + keypoints = torch.randint(0, min(data.shape[-2:]), (batch_size, N, 2), device=device).to(dtype=dtype) + labels = torch.randint(0, 1, (batch_size, N), device=device).to(dtype=dtype) + + out = model(data, [{"points": (keypoints, labels)}], multimask_output) + + C = 3 if multimask_output else 1 + assert len(out) == data.size(0) + assert out[0].logits.shape == (batch_size, C, 256, 256) + + def test_exception(self): + model = Sam.from_config(SamConfig("mobile_sam")) + + from kornia.core.exceptions import ShapeError + + with pytest.raises(ShapeError) as errinfo: + data = torch.rand(3, 1, 2) + model(data, [], False) + assert "Shape dimension mismatch" in str(errinfo.value) or "Expected shape" in str(errinfo.value) + + with pytest.raises(Exception) as errinfo: + data = torch.rand(2, 3, 1, 2) + model(data, [{}], False) + assert "The number of images (`B`) should match with the length of prompts!" in str(errinfo) + + @pytest.mark.slow + @pytest.mark.parametrize("model_type", ["vit_b", "vit_l", "vit_h", "mobile_sam"]) + def test_config(self, device, model_type): + model = Sam.from_config(SamConfig(model_type)) + model = model.to(device=device) + + assert isinstance(model, Sam) + assert next(model.parameters()).device == device + + @pytest.mark.skip(reason="Unsupported at moment -- the code is not tested for training and had `torch.no_grad`") + def test_gradcheck(self, device): ... + + @pytest.mark.skip(reason="Unnecessary test") + def test_module(self): ... + + @pytest.mark.skip(reason="Needs to be reviewed.") + def test_dynamo(self, device, torch_optimizer): + dtype = torch.float32 + img = torch.rand(1, 3, 128, 75, device=device, dtype=dtype) + + op = Sam.from_config(SamConfig("vit_b")) + op = op.to(device=device, dtype=dtype) + + op_optimized = torch_optimizer(op) + + img = _pad_rb(img, op.image_encoder.img_size) + + expected = op(img, [{}], False) + actual = op_optimized(img, [{}], False) + + self.assert_close(expected[0].logits, actual[0].logits) + self.assert_close(expected[0].scores, actual[0].scores) + + @pytest.mark.slow + @pytest.mark.parametrize("model_type", ["vit_b", "mobile_sam"]) + def test_pretrained(self, model_type): + Sam.from_config(SamConfig(model_type, pretrained=True)) diff --git a/tests/models/test_sam_onnx.py b/tests/models/test_sam_onnx.py new file mode 100644 index 0000000..61fe262 --- /dev/null +++ b/tests/models/test_sam_onnx.py @@ -0,0 +1,55 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""ONNX export tests for SAM (image encoder subgraph).""" + +import pytest + +from kornia.models.sam.model import Sam, SamConfig + + +def test_sam_has_to_onnx(): + """Sam exposes a to_onnx() method via ONNXExportMixin.""" + assert hasattr(Sam, "to_onnx") + + +def test_sam_encoder_to_onnx(tmp_path): + """Sam.to_onnx() exports the image encoder subgraph with correct output name.""" + onnx = pytest.importorskip("onnx") + pytest.importorskip("onnxscript") + + # Create a tiny pseudo-SAM model to prevent CI timeouts during ONNX trace + config = SamConfig( + model_type="vit_b", + encoder_embed_dim=16, + encoder_depth=1, + encoder_num_heads=1, + encoder_global_attn_indexes=[0], + ) + model = Sam.from_config(config) + model.eval() + + onnx_path = tmp_path / "sam_encoder.onnx" + # Export only the image encoder (Sam.to_onnx overrides the full-model export). + # Use a small spatial size to keep the test fast; only batch dimension is dynamic. + op = model.to_onnx(save=True, onnx_name=str(onnx_path), pseudo_shape=[1, 3, 1024, 1024]) + + assert isinstance(op, onnx.ModelProto) + assert onnx_path.exists(), "ONNX file was not written to disk" + + output_names = [o.name for o in op.graph.output] + assert "image_embeddings" in output_names, f"Expected 'image_embeddings', got {output_names}" diff --git a/tests/models/test_siglip2.py b/tests/models/test_siglip2.py new file mode 100644 index 0000000..e6539c7 --- /dev/null +++ b/tests/models/test_siglip2.py @@ -0,0 +1,365 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Tests for SigLip2 model.""" + +import pytest +import torch + +from kornia.models.siglip2 import SigLip2Config, SigLip2Model, SigLip2Result +from kornia.models.siglip2.attention import SigLip2Attention +from kornia.models.siglip2.config import SigLip2TextConfig, SigLip2VisionConfig +from kornia.models.siglip2.preprocessor import SigLip2ImagePreprocessor +from kornia.models.siglip2.text_encoder import SigLip2TextEmbeddings, SigLip2TextEncoder, SigLip2TextModel +from kornia.models.siglip2.vision_encoder import SigLip2VisionEmbeddings, SigLip2VisionEncoder, SigLip2VisionModel + +from testing.base import BaseTester + + +@pytest.fixture +def config(): + """Fixture for SigLip2Config.""" + return SigLip2Config() + + +@pytest.fixture +def model(device, dtype, config): + """Fixture for SigLip2Model.""" + return SigLip2Model(config).to(device, dtype).eval() + + +def _create_input_ids(batch_size, seq_len, config, device): + """Create input_ids with smaller range to avoid memory issues with large vocab.""" + return torch.randint(0, min(100, config.text_config.vocab_size), (batch_size, seq_len), device=device) + + +class TestSigLip2Model(BaseTester): + """Test suite for SigLip2 model.""" + + def test_smoke(self, device, dtype, config): + """Test basic model instantiation.""" + model = SigLip2Model(config).to(device, dtype) + assert model is not None + + @pytest.mark.parametrize("batch_size", [1, 2, 4]) + def test_cardinality(self, device, dtype, model, config, batch_size): + """Test output shapes with different inputs and batch sizes.""" + pixel_values = torch.randn(batch_size, 3, 224, 224, device=device, dtype=dtype) + seq_len = 10 + input_ids = _create_input_ids(batch_size, seq_len, config, device) + + with torch.no_grad(): + # Image only + output: SigLip2Result = model(pixel_values=pixel_values) + assert output.image_embeds is not None + assert output.image_embeds.shape == (batch_size, config.projection_dim) + assert output.text_embeds is None + + # Text only + output: SigLip2Result = model(input_ids=input_ids) + assert output.text_embeds is not None + assert output.text_embeds.shape == (batch_size, config.projection_dim) + assert output.image_embeds is None + + # Joint + output: SigLip2Result = model(pixel_values=pixel_values, input_ids=input_ids) + assert output.image_embeds.shape == (batch_size, config.projection_dim) + assert output.text_embeds.shape == (batch_size, config.projection_dim) + assert output.logits_per_image.shape == (batch_size, batch_size) + assert output.logits_per_text.shape == (batch_size, batch_size) + + def test_exception(self, device, dtype, model, config): + """Test exception handling.""" + # Test invalid pixel_values shape (wrong number of dimensions) + with pytest.raises((RuntimeError, ValueError, IndexError)): + invalid_pixel_values = torch.randn(3, 224, 224, device=device, dtype=dtype) # Missing batch dimension + model.get_image_features(invalid_pixel_values) + + # Test invalid attention mask shape + with pytest.raises((RuntimeError, ValueError, IndexError)): + input_ids = _create_input_ids(2, 10, config, device) + invalid_attention_mask = torch.ones(2, 5, device=device) # Wrong sequence length + model.get_text_features(input_ids, attention_mask=invalid_attention_mask) + + # Test input_ids with wrong number of dimensions + with pytest.raises((RuntimeError, ValueError, IndexError)): + invalid_input_ids = torch.randint(0, 100, (10,), device=device) # Missing batch dimension + model.get_text_features(invalid_input_ids) + + def test_get_image_features(self, device, dtype, model, config): + """Test get_image_features method.""" + + batch_size = 2 + pixel_values = torch.randn(batch_size, 3, 224, 224, device=device, dtype=dtype) + + with torch.no_grad(): + features = model.get_image_features(pixel_values) + + assert features.shape == (batch_size, config.projection_dim) + # Check normalization + norms = features.norm(dim=-1) + self.assert_close(norms, torch.ones_like(norms), rtol=1e-5, atol=1e-5) + + def test_get_text_features(self, device, dtype, model, config): + """Test get_text_features method.""" + + batch_size = 2 + seq_len = 10 + input_ids = _create_input_ids(batch_size, seq_len, config, device) + attention_mask = torch.ones(batch_size, seq_len, device=device) + + with torch.no_grad(): + features = model.get_text_features(input_ids, attention_mask=attention_mask) + + assert features.shape == (batch_size, config.projection_dim) + # Check normalization + norms = features.norm(dim=-1) + self.assert_close(norms, torch.ones_like(norms), rtol=1e-5, atol=1e-5) + + def test_attention_mask_handling(self, device, dtype, model, config): + """Test attention mask handling in text encoder.""" + + batch_size = 2 + seq_len = 10 + input_ids = _create_input_ids(batch_size, seq_len, config, device) + + # Create attention mask with different lengths + attention_mask = torch.ones(batch_size, seq_len, device=device) + attention_mask[0, 5:] = 0 # First sequence has 5 tokens + attention_mask[1, 8:] = 0 # Second sequence has 8 tokens + + with torch.no_grad(): + features = model.get_text_features(input_ids, attention_mask=attention_mask) + + assert features.shape == (batch_size, config.projection_dim) + + def test_return_loss(self, device, dtype, model, config): + """Test forward pass with return_loss=True and verify logit_scale clamping.""" + import math + + batch_size = 2 + pixel_values = torch.randn(batch_size, 3, 224, 224, device=device, dtype=dtype) + seq_len = 10 + input_ids = _create_input_ids(batch_size, seq_len, config, device) + + with torch.no_grad(): + output = model(pixel_values=pixel_values, input_ids=input_ids, return_loss=True) + + assert output.loss is not None + assert output.loss.item() >= 0.0 # Loss should be non-negative + + # Test logit_scale clamping with extreme values + with torch.no_grad(): + # Test max clamping + model.logit_scale.data.fill_(100.0) + output_max = model(pixel_values=pixel_values, input_ids=input_ids) + assert torch.isfinite(output_max.logits_per_image).all(), "Max clamp: logits contain non-finite values" + assert math.isclose(output_max.logit_scale.item(), config.logit_scale_max, rel_tol=1e-5, abs_tol=1e-5), ( + f"Max clamp failed: {output_max.logit_scale.item()} != {config.logit_scale_max}" + ) + + # Test min clamping + model.logit_scale.data.fill_(-10.0) + output_min = model(pixel_values=pixel_values, input_ids=input_ids) + assert output_min.logit_scale.item() >= 1.0, f"Min clamp failed: {output_min.logit_scale.item()} < 1.0" + + def test_gradcheck(self, device, dtype, config): + """Test gradient computation correctness.""" + # Convert model to float64 for gradcheck + model = SigLip2Model(config).to(device, torch.float64).train() + batch_size = 1 + pixel_values = torch.randn(batch_size, 3, 224, 224, device=device, dtype=torch.float64, requires_grad=True) + seq_len = 5 + input_ids = _create_input_ids(batch_size, seq_len, config, device).to(torch.int64) + + # Only check gradients for pixel_values (input_ids are indices, not differentiable) + def func(pixel_vals): + # Use input_ids as closure variable, not as gradcheck input + return model.get_image_features(pixel_vals) + model.get_text_features(input_ids) + + self.gradcheck(func, pixel_values, raise_exception=True, fast_mode=True) + + def test_dynamo(self, device, dtype, torch_optimizer, model, config): + """Test torch.compile compatibility.""" + batch_size = 1 + pixel_values = torch.randn(batch_size, 3, 224, 224, device=device, dtype=dtype) + seq_len = 10 + input_ids = _create_input_ids(batch_size, seq_len, config, device) + + model_optimized = torch_optimizer(model) + + with torch.no_grad(): + expected = model(pixel_values=pixel_values, input_ids=input_ids) + actual = model_optimized(pixel_values=pixel_values, input_ids=input_ids) + + self.assert_close(actual.image_embeds, expected.image_embeds) + self.assert_close(actual.text_embeds, expected.text_embeds) + + +class TestSigLip2Components(BaseTester): + """Test suite for SigLip2 individual components.""" + + def test_vision_embeddings(self, device, dtype): + """Test SigLip2VisionEmbeddings.""" + config = SigLip2VisionConfig(image_size=224, patch_size=16, hidden_size=768) + embeddings = SigLip2VisionEmbeddings(config).to(device, dtype) + + batch_size = 2 + pixel_values = torch.randn(batch_size, 3, 224, 224, device=device, dtype=dtype) + + with torch.no_grad(): + output = embeddings(pixel_values) + + num_patches = (224 // 16) ** 2 + assert output.shape == (batch_size, num_patches, config.hidden_size) + + def test_vision_encoder(self, device, dtype): + """Test SigLip2VisionEncoder.""" + config = SigLip2VisionConfig( + image_size=224, patch_size=16, hidden_size=768, num_hidden_layers=2, num_attention_heads=12 + ) + encoder = SigLip2VisionEncoder(config).to(device, dtype) + embeddings = SigLip2VisionEmbeddings(config).to(device, dtype) + + batch_size = 2 + pixel_values = torch.randn(batch_size, 3, 224, 224, device=device, dtype=dtype) + + with torch.no_grad(): + # Encoder expects embeddings, not raw pixel values + hidden_states = embeddings(pixel_values) + output = encoder(hidden_states) + + num_patches = (224 // 16) ** 2 + assert output[0].shape == (batch_size, num_patches, config.hidden_size) + + def test_vision_model(self, device, dtype): + """Test SigLip2VisionModel.""" + config = SigLip2VisionConfig( + image_size=224, patch_size=16, hidden_size=768, num_hidden_layers=2, num_attention_heads=12 + ) + model = SigLip2VisionModel(config).to(device, dtype) + + batch_size = 2 + pixel_values = torch.randn(batch_size, 3, 224, 224, device=device, dtype=dtype) + + with torch.no_grad(): + pooled_output, last_hidden_state = model(pixel_values) + + assert pooled_output.shape == (batch_size, config.hidden_size) + num_patches = (224 // 16) ** 2 + assert last_hidden_state.shape == (batch_size, num_patches, config.hidden_size) + + def test_text_embeddings(self, device, dtype): + """Test SigLip2TextEmbeddings.""" + config = SigLip2TextConfig(vocab_size=1000, hidden_size=768, max_position_embeddings=512) + embeddings = SigLip2TextEmbeddings(config).to(device, dtype) + + batch_size = 2 + seq_len = 10 + input_ids = torch.randint(0, config.vocab_size, (batch_size, seq_len), device=device) + + with torch.no_grad(): + output = embeddings(input_ids) + + assert output.shape == (batch_size, seq_len, config.hidden_size) + + def test_text_encoder(self, device, dtype): + """Test SigLip2TextEncoder.""" + config = SigLip2TextConfig( + vocab_size=1000, + hidden_size=768, + num_hidden_layers=2, + num_attention_heads=12, + max_position_embeddings=512, + ) + encoder = SigLip2TextEncoder(config).to(device, dtype) + embeddings = SigLip2TextEmbeddings(config).to(device, dtype) + + batch_size = 2 + seq_len = 10 + input_ids = torch.randint(0, config.vocab_size, (batch_size, seq_len), device=device) + attention_mask = torch.ones(batch_size, seq_len, device=device) + + with torch.no_grad(): + # Encoder expects hidden_states (embeddings), not input_ids + hidden_states = embeddings(input_ids) + output = encoder(hidden_states, attention_mask=attention_mask) + + assert output[0].shape == (batch_size, seq_len, config.hidden_size) + + def test_text_model(self, device, dtype): + """Test SigLip2TextModel.""" + config = SigLip2TextConfig( + vocab_size=1000, + hidden_size=768, + num_hidden_layers=2, + num_attention_heads=12, + max_position_embeddings=512, + ) + model = SigLip2TextModel(config).to(device, dtype) + + batch_size = 2 + seq_len = 10 + input_ids = torch.randint(0, config.vocab_size, (batch_size, seq_len), device=device) + attention_mask = torch.ones(batch_size, seq_len, device=device) + + with torch.no_grad(): + pooled_output, last_hidden_state = model(input_ids=input_ids, attention_mask=attention_mask) + + assert pooled_output.shape == (batch_size, config.hidden_size) + assert last_hidden_state.shape == (batch_size, seq_len, config.hidden_size) + + def test_attention(self, device, dtype): + """Test SigLip2Attention.""" + hidden_size = 768 + num_heads = 12 + attention = SigLip2Attention(hidden_size=hidden_size, num_heads=num_heads).to(device, dtype) + + batch_size = 2 + seq_len = 10 + hidden_states = torch.randn(batch_size, seq_len, hidden_size, device=device, dtype=dtype) + attention_mask = torch.ones(batch_size, seq_len, device=device) + + with torch.no_grad(): + output = attention(hidden_states, attention_mask=attention_mask) + + # Attention returns a single tensor, not a tuple + assert output.shape == (batch_size, seq_len, hidden_size) + + @pytest.mark.parametrize("batch_size", [1, 2, 4]) + @pytest.mark.parametrize("input_size", [(256, 256), (300, 400), (512, 512)]) + @pytest.mark.parametrize("image_size", [(224, 224), (256, 256), (384, 384)]) + def test_image_preprocessor(self, device, dtype, batch_size, input_size, image_size): + """Test SigLip2ImagePreprocessor with different configurations.""" + preprocessor = SigLip2ImagePreprocessor(image_size=image_size).to(device, dtype) + + # Test with batch of images (4D tensor) + images = torch.randint(0, 255, (batch_size, 3, *input_size), device=device, dtype=dtype) + with torch.no_grad(): + output = preprocessor(images) + assert output.shape == (batch_size, 3, image_size[0], image_size[1]) + + def test_image_preprocessor_single_image(self, device, dtype): + """Test SigLip2ImagePreprocessor with single image (3D tensor).""" + image_size = (224, 224) + preprocessor = SigLip2ImagePreprocessor(image_size=image_size).to(device, dtype) + + # Test with single image (3D tensor) - preprocessor adds batch dimension + image = torch.randint(0, 255, (3, 256, 256), device=device, dtype=dtype) + with torch.no_grad(): + output = preprocessor(image) + assert output.shape == (1, 3, image_size[0], image_size[1]) diff --git a/tests/models/test_small_sr.py b/tests/models/test_small_sr.py new file mode 100644 index 0000000..1640918 --- /dev/null +++ b/tests/models/test_small_sr.py @@ -0,0 +1,295 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +import pytest +import torch + +from kornia.config import kornia_config +from kornia.models.small_sr import SmallSRNet, SmallSRNetWrapper + +from testing.base import BaseTester + + +class TestSmallSRNet(BaseTester): + """Test suite for SmallSRNet - the core super-resolution model.""" + + @pytest.mark.parametrize("upscale_factor", [2, 3, 4]) + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_smoke(self, device, dtype, upscale_factor, batch_size): + """Test that SmallSRNet can be instantiated and run with different upscale factors.""" + model = SmallSRNet(upscale_factor=upscale_factor, pretrained=False).to(device, dtype) + assert model is not None + + # Input is single channel (Y channel from YCbCr) + x = torch.randn(batch_size, 1, 224, 224, device=device, dtype=dtype) + output = model(x) + assert output is not None + + def test_exception_invalid_input_channels(self, device, dtype): + """Test that SmallSRNet raises an error with wrong number of input channels.""" + model = SmallSRNet(upscale_factor=3, pretrained=False).to(device, dtype) + + # SmallSRNet expects 1 channel input (Y channel), not 3 + with pytest.raises(RuntimeError): + x = torch.randn(1, 3, 224, 224, device=device, dtype=dtype) + model(x) + + def test_smoke_upscale_factor_one(self, device, dtype): + """Test that upscale_factor=1 works correctly (PixelShuffle acts as identity).""" + # SmallSRNet doesn't validate upscale_factor, it just uses it in PixelShuffle + # With upscale_factor=1, PixelShuffle acts as identity (no upscaling) + model = SmallSRNet(upscale_factor=1, pretrained=False).to(device, dtype) + assert model is not None + + # Test forward pass - should work without crashing + x = torch.randn(1, 1, 64, 64, device=device, dtype=dtype) + output = model(x) + + # With upscale_factor=1, output dimensions should match input dimensions + assert output.shape == x.shape, f"Expected {x.shape}, got {output.shape}" + assert torch.isfinite(output).all() + + @pytest.mark.parametrize("upscale_factor", [2, 3, 4]) + @pytest.mark.parametrize("batch_size", [1, 2, 4]) + @pytest.mark.parametrize("height,width", [(64, 64), (128, 128), (224, 224)]) + def test_cardinality(self, device, dtype, upscale_factor, batch_size, height, width): + """Test that output shape matches expected upscaled dimensions.""" + model = SmallSRNet(upscale_factor=upscale_factor, pretrained=False).to(device, dtype) + + x = torch.randn(batch_size, 1, height, width, device=device, dtype=dtype) + output = model(x) + + expected_shape = (batch_size, 1, height * upscale_factor, width * upscale_factor) + assert output.shape == expected_shape, f"Expected {expected_shape}, got {output.shape}" + + @pytest.mark.parametrize("upscale_factor", [2, 3, 4]) + def test_feature_forward_pass(self, device, dtype, upscale_factor): + """Test the forward pass produces valid outputs with expected value ranges.""" + model = SmallSRNet(upscale_factor=upscale_factor, pretrained=False).to(device, dtype) + + x = torch.randn(1, 1, 64, 64, device=device, dtype=dtype) + output = model(x) + + # Check output is not all zeros + assert not torch.allclose(output, torch.zeros_like(output)) + + # Check output is finite + assert torch.isfinite(output).all() + + @pytest.mark.slow + @pytest.mark.parametrize("upscale_factor", [2, 3, 4]) + def test_feature_pretrained_loading(self, device, dtype, upscale_factor): + """Test that pretrained weights can be loaded (only upscale_factor=3 has pretrained weights).""" + if upscale_factor == 3: + # Check if pretrained weights are cached to avoid network download attempts in offline CI + cache_path = os.path.join(kornia_config.hub_onnx_dir, "small_sr.pth") + if not os.path.exists(cache_path): + pytest.skip(f"Pretrained weights not cached at {cache_path}. Skipping to avoid network download.") + + # download and load pretrained weights + model = SmallSRNet(upscale_factor=upscale_factor, pretrained=True).to(device, dtype) + + x = torch.randn(1, 1, 224, 224, device=device, dtype=dtype) + output = model(x) + + # Pretrained model should be in eval mode + assert not model.training + + # Output should be valid + assert torch.isfinite(output).all() + else: + # For other upscale factors, pretrained weights may not exist + # Model should still initialize without error + model = SmallSRNet(upscale_factor=upscale_factor, pretrained=False).to(device, dtype) + assert model is not None + + @pytest.mark.parametrize("upscale_factor", [2, 3, 4]) + def test_gradcheck(self, device, upscale_factor): + """Test that gradients are computed correctly.""" + model = SmallSRNet(upscale_factor=upscale_factor, pretrained=False).to(device, torch.float64) + model.train() + + x = torch.randn(1, 1, 16, 16, device=device, dtype=torch.float64, requires_grad=True) + + self.gradcheck(model, (x,), nondet_tol=1e-4) + + @pytest.mark.parametrize("upscale_factor", [2, 3, 4]) + def test_dynamo(self, device, dtype, torch_optimizer, upscale_factor): + """Test that the model works with torch.compile.""" + model = SmallSRNet(upscale_factor=upscale_factor, pretrained=False).to(device, dtype) + x = torch.randn(1, 1, 64, 64, device=device, dtype=dtype) + + op = model + op_optimized = torch_optimizer(model) + + actual = op(x) + expected = op_optimized(x) + + self.assert_close(actual, expected, rtol=1e-4, atol=1e-4) + + +class TestSmallSRNetWrapper(BaseTester): + """Test suite for SmallSRNetWrapper - the RGB-input wrapper with color space conversion.""" + + @pytest.mark.parametrize("upscale_factor", [2, 3, 4]) + @pytest.mark.parametrize("batch_size", [1, 2]) + def test_smoke(self, device, dtype, upscale_factor, batch_size): + """Test that SmallSRNetWrapper can be instantiated and run.""" + model = SmallSRNetWrapper(upscale_factor=upscale_factor, pretrained=False).to(device, dtype) + assert model is not None + + # Input is RGB image + x = torch.randn(batch_size, 3, 224, 224, device=device, dtype=dtype) + output = model(x) + assert output is not None + + def test_exception_invalid_input_channels(self, device, dtype): + """Test that SmallSRNetWrapper raises an error with wrong number of input channels.""" + model = SmallSRNetWrapper(upscale_factor=3, pretrained=False).to(device, dtype) + + # SmallSRNetWrapper expects 3 channel RGB input, rgb_to_ycbcr will raise ValueError + with pytest.raises(ValueError, match="Input size must have a shape of"): + x = torch.randn(1, 1, 224, 224, device=device, dtype=dtype) + model(x) + + def test_exception_negative_values(self, device, dtype): + """Test handling of invalid pixel value ranges (should still process but may produce unexpected results).""" + model = SmallSRNetWrapper(upscale_factor=3, pretrained=False).to(device, dtype) + x = torch.randn(1, 3, 64, 64, device=device, dtype=dtype) - 1.0 # Negative values + output = model(x) + + # Should not crash, output should be finite + assert torch.isfinite(output).all() + + @pytest.mark.parametrize("upscale_factor", [2, 3, 4]) + @pytest.mark.parametrize("batch_size", [1, 2, 4]) + @pytest.mark.parametrize("height,width", [(64, 64), (128, 128), (224, 224)]) + def test_cardinality(self, device, dtype, upscale_factor, batch_size, height, width): + """Test that output shape matches expected upscaled dimensions for RGB images.""" + model = SmallSRNetWrapper(upscale_factor=upscale_factor, pretrained=False).to(device, dtype) + + x = torch.randn(batch_size, 3, height, width, device=device, dtype=dtype) + output = model(x) + + expected_shape = (batch_size, 3, height * upscale_factor, width * upscale_factor) + assert output.shape == expected_shape, f"Expected {expected_shape}, got {output.shape}" + + @pytest.mark.parametrize("upscale_factor", [2, 3, 4]) + def test_feature_rgb_processing(self, device, dtype, upscale_factor): + """Test that RGB images are processed correctly through color space conversions.""" + model = SmallSRNetWrapper(upscale_factor=upscale_factor, pretrained=False).to(device, dtype) + x = torch.rand(1, 3, 64, 64, device=device, dtype=dtype) # RGB in [0, 1] + output = model(x) + + # Check output is not all zeros + assert not torch.allclose(output, torch.zeros_like(output)) + + # Check output is finite + assert torch.isfinite(output).all() + + # Output should have 3 channels (RGB) + assert output.shape[1] == 3 + + @pytest.mark.slow + def test_feature_pretrained_upscale_3x(self, device, dtype): + """Test upscaling with pretrained weights (only available for upscale_factor=3).""" + # Check if pretrained weights are cached to avoid network download attempts in offline CI + cache_path = os.path.join(kornia_config.hub_onnx_dir, "small_sr.pth") + if not os.path.exists(cache_path): + pytest.skip(f"Pretrained weights not cached at {cache_path}. Skipping to avoid network download.") + + model = SmallSRNetWrapper(upscale_factor=3, pretrained=True).to(device, dtype) + x = torch.rand(1, 3, 224, 224, device=device, dtype=dtype) + output = model(x) + + # Check output shape + assert output.shape == (1, 3, 224 * 3, 224 * 3) + + # Pretrained model's inner SmallSRNet should be in eval mode + assert not model.model.training + + # Output should be valid + assert torch.isfinite(output).all() + + @pytest.mark.parametrize("upscale_factor", [2, 3, 4]) + def test_feature_color_space_conversion(self, device, dtype, upscale_factor): + """Test that color space conversions (RGB->YCbCr->RGB) work correctly.""" + model = SmallSRNetWrapper(upscale_factor=upscale_factor, pretrained=False).to(device, dtype) + model.eval() + + x = torch.rand(1, 3, 32, 32, device=device, dtype=dtype) + + with torch.no_grad(): + # Get wrapper output + wrapper_output = model(x) + + # Manually replicate the pipeline to verify correctness + ycbcr = model.rgb_to_ycbcr(x) + y, cb, cr = ycbcr.split(1, dim=1) + out_y = model.model(y) + out_cb = torch.nn.functional.interpolate(cb, scale_factor=upscale_factor, mode="bicubic") + out_cr = torch.nn.functional.interpolate(cr, scale_factor=upscale_factor, mode="bicubic") + out_ycbcr = torch.cat([out_y, out_cb, out_cr], dim=1) + expected_output = model.ycbcr_to_rgb(out_ycbcr) + + # Wrapper output should match manual pipeline + self.assert_close(wrapper_output, expected_output, rtol=1e-4, atol=1e-4) + + @pytest.mark.parametrize("upscale_factor", [2, 3, 4]) + def test_gradcheck(self, device, upscale_factor): + """Test that gradients are computed correctly through the wrapper.""" + model = SmallSRNetWrapper(upscale_factor=upscale_factor, pretrained=False).to(device, torch.float64) + model.train() + + x = torch.randn(1, 3, 16, 16, device=device, dtype=torch.float64, requires_grad=True) + + self.gradcheck(model, (x,), nondet_tol=1e-4) + + @pytest.mark.parametrize("upscale_factor", [2, 3, 4]) + def test_dynamo(self, device, dtype, torch_optimizer, upscale_factor): + """Test that the wrapper works with torch.compile.""" + model = SmallSRNetWrapper(upscale_factor=upscale_factor, pretrained=False).to(device, dtype) + x = torch.randn(1, 3, 64, 64, device=device, dtype=dtype) + + op = model + op_optimized = torch_optimizer(model) + + actual = op(x) + expected = op_optimized(x) + + self.assert_close(actual, expected, rtol=1e-4, atol=1e-4) + + def test_feature_consistency_across_batch_sizes(self, device, dtype): + """Test that processing images individually vs in batch produces consistent results.""" + model = SmallSRNetWrapper(upscale_factor=3, pretrained=False).to(device, dtype) + model.eval() + + # Create two identical images + x1 = torch.rand(1, 3, 64, 64, device=device, dtype=dtype) + x2 = x1.clone() + x_batch = torch.cat([x1, x2], dim=0) + + with torch.no_grad(): + output1 = model(x1) + output2 = model(x2) + output_batch = model(x_batch) + + # Individual processing should match batch processing + self.assert_close(output1, output_batch[0:1], rtol=1e-4, atol=1e-4) + self.assert_close(output2, output_batch[1:2], rtol=1e-4, atol=1e-4) diff --git a/tests/models/test_structures.py b/tests/models/test_structures.py new file mode 100644 index 0000000..8202b6e --- /dev/null +++ b/tests/models/test_structures.py @@ -0,0 +1,139 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import pytest +import torch + +from kornia.core.exceptions import BaseError +from kornia.models.structures import Prompts, SegmentationResults + + +class TestSegmentationResults: + def _make_results(self, B=2, C=3, H=16, W=16, threshold=0.0): + logits = torch.randn(B, C, H, W) + scores = torch.rand(B, C) + return SegmentationResults(logits=logits, scores=scores, mask_threshold=threshold) + + def test_binary_masks_uses_logits_when_no_original(self): + r = self._make_results() + masks = r.binary_masks + assert masks.shape == r.logits.shape + assert masks.dtype == torch.bool + assert torch.equal(masks, r.logits > r.mask_threshold) + + def test_binary_masks_uses_original_res_logits_when_set(self): + r = self._make_results() + # Simulate having called original_res_logits() + fake_hires = torch.randn(2, 3, 32, 32) + 10.0 # all positive -> all True + r._original_res_logits = fake_hires + masks = r.binary_masks + assert masks.shape == (2, 3, 32, 32) + assert masks.all() + + def test_original_res_logits_without_encoder_resize(self): + r = self._make_results(B=1, C=1, H=8, W=8) + # No encoder resize (image_size_encoder=None), just crop and resize + result = r.original_res_logits(input_size=(8, 8), original_size=(32, 32), image_size_encoder=None) + assert result.shape == (1, 1, 32, 32) + assert r._original_res_logits is not None + + def test_original_res_logits_with_encoder_resize(self): + r = self._make_results(B=1, C=1, H=8, W=8) + # With encoder resize: first resize to (16, 16), then crop, then resize to (32, 32) + result = r.original_res_logits(input_size=(16, 16), original_size=(32, 32), image_size_encoder=(16, 16)) + assert result.shape == (1, 1, 32, 32) + + def test_original_res_logits_crops_padding(self): + # Logits have extra spatial dimension due to padding + r = self._make_results(B=1, C=1, H=10, W=10) + # Crop to 8x8, then resize to 4x4 + result = r.original_res_logits(input_size=(8, 8), original_size=(4, 4), image_size_encoder=None) + assert result.shape == (1, 1, 4, 4) + + def test_squeeze_without_original_res_logits(self): + r = self._make_results(B=1, C=3, H=8, W=8) + squeezed = r.squeeze(dim=0) + assert squeezed.logits.shape == (3, 8, 8) + assert squeezed.scores.shape == (3,) + + def test_squeeze_with_original_res_logits(self): + r = self._make_results(B=1, C=3, H=8, W=8) + r._original_res_logits = torch.randn(1, 3, 32, 32) + squeezed = r.squeeze(dim=0) + assert squeezed.logits.shape == (3, 8, 8) + assert isinstance(squeezed._original_res_logits, torch.Tensor) + assert squeezed._original_res_logits.shape == (3, 32, 32) + + def test_binary_masks_threshold(self): + logits = torch.tensor([[[[0.5, -0.5], [0.3, 0.1]]]]) + scores = torch.ones(1, 1) + r = SegmentationResults(logits=logits, scores=scores, mask_threshold=0.2) + masks = r.binary_masks + # 0.5 > 0.2 -> True, -0.5 > 0.2 -> False, 0.3 > 0.2 -> True, 0.1 > 0.2 -> False + expected = torch.tensor([[[[True, False], [True, False]]]]) + assert torch.equal(masks, expected) + + +class TestPrompts: + def test_no_prompts(self): + p = Prompts() + assert p.points is None + assert p.boxes is None + assert p.masks is None + assert p.keypoints is None + assert p.keypoints_labels is None + + def test_keypoints_from_tuple(self): + coords = torch.rand(2, 5, 2) + labels = torch.randint(0, 2, (2, 5)).float() + p = Prompts(points=(coords, labels)) + assert torch.equal(p.keypoints, coords) + assert torch.equal(p.keypoints_labels, labels) + + def test_keypoints_none_when_points_none(self): + p = Prompts(points=None) + assert p.keypoints is None + assert p.keypoints_labels is None + + def test_boxes_only(self): + boxes = torch.rand(2, 4) + p = Prompts(boxes=boxes) + assert torch.equal(p.boxes, boxes) + assert p.keypoints is None + + def test_keypoints_and_boxes_matching_batch(self): + coords = torch.rand(3, 5, 2) + labels = torch.rand(3, 5) + boxes = torch.rand(3, 4) + # Should not raise: batch sizes match + p = Prompts(points=(coords, labels), boxes=boxes) + assert p.keypoints.shape[0] == 3 + assert p.boxes.shape[0] == 3 + + def test_keypoints_and_boxes_mismatched_batch_raises(self): + coords = torch.rand(2, 5, 2) + labels = torch.rand(2, 5) + boxes = torch.rand(3, 4) # different batch size + with pytest.raises(BaseError, match="same batch size"): + Prompts(points=(coords, labels), boxes=boxes) + + def test_masks_only(self): + masks = torch.rand(2, 1, 32, 32) + p = Prompts(masks=masks) + assert torch.equal(p.masks, masks) diff --git a/tests/models/test_tiny_vit.py b/tests/models/test_tiny_vit.py new file mode 100644 index 0000000..4794614 --- /dev/null +++ b/tests/models/test_tiny_vit.py @@ -0,0 +1,86 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys + +import pytest +import torch + +from kornia.models.tiny_vit import TinyViT + +from testing.base import BaseTester + + +class TestTinyViT(BaseTester): + @pytest.mark.parametrize("img_size", [224, 256]) + def test_smoke(self, device, dtype, img_size): + model = TinyViT(img_size=img_size).to(device=device, dtype=dtype) + data = torch.randn(1, 3, img_size, img_size, device=device, dtype=dtype) + + out = model(data) + assert isinstance(out, torch.Tensor) + + @pytest.mark.slow + @pytest.mark.parametrize("num_classes", [10, 100]) + @pytest.mark.parametrize("batch_size", [1, 3]) + def test_cardinality(self, device, dtype, batch_size, num_classes): + model = TinyViT(num_classes=num_classes).to(device=device, dtype=dtype) + data = torch.rand(batch_size, 3, model.img_size, model.img_size, device=device, dtype=dtype) + + out = model(data) + assert out.shape == (batch_size, num_classes) + + @pytest.mark.skip("not implemented") + def test_exception(self): ... + + @pytest.mark.skip("not implemented") + def test_gradcheck(self): ... + + @pytest.mark.skip("not implemented") + def test_module(self): ... + + @pytest.mark.skipif(sys.version_info.major == 3 and sys.version_info.minor == 8, reason="not working for py3.8") + def test_dynamo(self, device, dtype, torch_optimizer): + op = TinyViT().to(device=device, dtype=dtype) + img = torch.rand(1, 3, op.img_size, op.img_size, device=device, dtype=dtype) + + op_optimized = torch_optimizer(op) + self.assert_close(op(img), op_optimized(img)) + + @pytest.mark.slow + @pytest.mark.parametrize("pretrained", [False, True]) + @pytest.mark.parametrize("variant", ["5m", "11m", "21m"]) + def test_from_config(self, variant, pretrained): + model = TinyViT.from_config(variant, pretrained=pretrained) + assert isinstance(model, TinyViT) + + @pytest.mark.slow + @pytest.mark.parametrize("num_classes", [1000, 8]) + @pytest.mark.parametrize("img_size", [224, 256]) + def test_pretrained(self, img_size, num_classes): + model = TinyViT.from_config("5m", img_size=img_size, num_classes=num_classes, pretrained=True) + assert isinstance(model, TinyViT) + + @pytest.mark.slow + def test_mobile_sam_backbone(self, device, dtype): + img_size = 1024 + batch_size = 1 + model = TinyViT.from_config("5m", img_size=img_size, mobile_sam=True).to(device=device, dtype=dtype) + data = torch.randn(batch_size, 3, img_size, img_size, device=device, dtype=dtype) + + out = model(data) + assert out.shape == (batch_size, 256, img_size // 16, img_size // 16) diff --git a/tests/models/test_vision_transformer.py b/tests/models/test_vision_transformer.py new file mode 100644 index 0000000..6e87c8b --- /dev/null +++ b/tests/models/test_vision_transformer.py @@ -0,0 +1,62 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.models.vit import VisionTransformer + +from testing.base import BaseTester + + +class TestVisionTransformer(BaseTester): + @pytest.mark.parametrize("B", [1, 2]) + @pytest.mark.parametrize("H", [1, 3, 8]) + @pytest.mark.parametrize("D", [240, 768]) + @pytest.mark.parametrize("image_size", [32, 224]) + def test_smoke(self, device, dtype, B, H, D, image_size): + patch_size = 16 + T = image_size**2 // patch_size**2 + 1 # tokens size + + img = torch.rand(B, 3, image_size, image_size, device=device, dtype=dtype) + vit = VisionTransformer(image_size=image_size, num_heads=H, embed_dim=D).to(device, dtype) + + out = vit(img) + assert isinstance(out, torch.Tensor) + assert out.shape == (B, T, D) + + feats = vit.encoder_results + assert isinstance(feats, list) + assert len(feats) == 12 + for f in feats: + assert f.shape == (B, T, D) + + @pytest.mark.parametrize("H", [3, 8]) + @pytest.mark.parametrize("D", [245, 1001]) + @pytest.mark.parametrize("image_size", [32, 224]) + def test_exception(self, device, dtype, H, D, image_size): + with pytest.raises(ValueError): + VisionTransformer(image_size=image_size, num_heads=H, embed_dim=D).to(device, dtype) + + def test_backbone(self, device, dtype): + def backbone_mock(x): + return torch.ones(1, 128, 14, 14, device=device, dtype=dtype) + + img = torch.rand(1, 3, 32, 32, device=device, dtype=dtype) + vit = VisionTransformer(backbone=backbone_mock, num_heads=8).to(device, dtype) + out = vit(img) + assert out.shape == (1, 197, 128) diff --git a/tests/morphology/test_bottom_hat.py b/tests/morphology/test_bottom_hat.py new file mode 100644 index 0000000..66b9583 --- /dev/null +++ b/tests/morphology/test_bottom_hat.py @@ -0,0 +1,127 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.morphology import bottom_hat + +from testing.base import BaseTester, assert_close +from testing.parametrized_tester import parametrized_test + + +@parametrized_test( + smoke_inputs=lambda device, dtype: ( + torch.rand(1, 3, 4, 4, device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + cardinality_tests=[ + { + "inputs": lambda device, dtype: ( + torch.ones((1, 3, 4, 4), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([1, 3, 4, 4]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((2, 3, 2, 4), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([2, 3, 2, 4]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((3, 3, 4, 1), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([3, 3, 4, 1]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((3, 2, 5, 5), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([3, 2, 5, 5]), + }, + ], + gradcheck_inputs=lambda device: ( + torch.rand(2, 3, 4, 4, requires_grad=True, device=device, dtype=torch.float64), + torch.rand(3, 3, requires_grad=True, device=device, dtype=torch.float64), + ), +) +class TestBottomHat(BaseTester): + def setup_method(self) -> None: + self.func = bottom_hat + + def test_kernel(self, device, dtype): + tensor = torch.tensor([[0.5, 1.0, 0.3], [0.7, 0.3, 0.8], [0.4, 0.9, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + kernel = torch.tensor([[0.0, 1.0, 0.0], [1.0, 1.0, 1.0], [0.0, 1.0, 0.0]], device=device, dtype=dtype) + expected = torch.tensor([[0.2, 0.0, 0.5], [0.0, 0.4, 0.0], [0.3, 0.0, 0.6]], device=device, dtype=dtype)[ + None, None, :, : + ] + assert_close(bottom_hat(tensor, kernel), expected, atol=1e-3, rtol=1e-3) + + def test_structural_element(self, device, dtype): + tensor = torch.tensor([[0.5, 1.0, 0.3], [0.7, 0.3, 0.8], [0.4, 0.9, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + structural_element = torch.tensor( + [[-1.0, 0.0, -1.0], [0.0, 0.0, 0.0], [-1.0, 0.0, -1.0]], device=device, dtype=dtype + ) + expected = torch.tensor([[0.2, 0.0, 0.5], [0.0, 0.4, 0.0], [0.3, 0.0, 0.6]], device=device, dtype=dtype)[ + None, None, :, : + ] + assert_close( + bottom_hat(tensor, torch.ones_like(structural_element), structuring_element=structural_element), + expected, + atol=1e-3, + rtol=1e-3, + ) + + def test_exception(self, device, dtype): + sample = torch.ones(1, 1, 3, 4, device=device, dtype=dtype) + kernel = torch.ones(3, 3, device=device, dtype=dtype) + + with pytest.raises(TypeError): + assert bottom_hat([0.0], kernel) + + with pytest.raises(TypeError): + assert bottom_hat(sample, [0.0]) + + with pytest.raises(ValueError): + test = torch.ones(2, 3, 4, device=device, dtype=dtype) + assert bottom_hat(test, kernel) + + with pytest.raises(ValueError): + test = torch.ones(2, 3, 4, device=device, dtype=dtype) + assert bottom_hat(sample, test) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + op = bottom_hat + op_script = torch.jit.script(op) + + sample = torch.rand(1, 2, 7, 7, device=device, dtype=dtype) + kernel = torch.ones(3, 3, device=device, dtype=dtype) + + actual = op_script(sample, kernel) + expected = op(sample, kernel) + + assert_close(actual, expected) diff --git a/tests/morphology/test_closing.py b/tests/morphology/test_closing.py new file mode 100644 index 0000000..69abd02 --- /dev/null +++ b/tests/morphology/test_closing.py @@ -0,0 +1,127 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.morphology import closing + +from testing.base import BaseTester, assert_close +from testing.parametrized_tester import parametrized_test + + +@parametrized_test( + smoke_inputs=lambda device, dtype: ( + torch.rand(1, 3, 4, 4, device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + cardinality_tests=[ + { + "inputs": lambda device, dtype: ( + torch.ones((1, 3, 4, 4), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([1, 3, 4, 4]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((2, 3, 2, 4), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([2, 3, 2, 4]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((3, 3, 4, 1), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([3, 3, 4, 1]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((3, 2, 5, 5), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([3, 2, 5, 5]), + }, + ], + gradcheck_inputs=lambda device: ( + torch.rand(2, 3, 4, 4, requires_grad=True, device=device, dtype=torch.float64), + torch.rand(3, 3, requires_grad=True, device=device, dtype=torch.float64), + ), +) +class TestClosing(BaseTester): + def setup_method(self) -> None: + self.func = closing + + def test_kernel(self, device, dtype): + tensor = torch.tensor([[0.5, 1.0, 0.3], [0.7, 0.3, 0.8], [0.4, 0.9, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + kernel = torch.tensor([[0.0, 1.0, 0.0], [1.0, 1.0, 1.0], [0.0, 1.0, 0.0]], device=device, dtype=dtype) + expected = torch.tensor([[0.7, 1.0, 0.8], [0.7, 0.7, 0.8], [0.7, 0.9, 0.8]], device=device, dtype=dtype)[ + None, None, :, : + ] + assert_close(closing(tensor, kernel), expected, atol=1e-4, rtol=1e-4) + + def test_structural_element(self, device, dtype): + tensor = torch.tensor([[0.5, 1.0, 0.3], [0.7, 0.3, 0.8], [0.4, 0.9, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + structural_element = torch.tensor( + [[-1.0, 0.0, -1.0], [0.0, 0.0, 0.0], [-1.0, 0.0, -1.0]], device=device, dtype=dtype + ) + expected = torch.tensor([[0.7, 1.0, 0.8], [0.7, 0.7, 0.8], [0.7, 0.9, 0.8]], device=device, dtype=dtype)[ + None, None, :, : + ] + assert_close( + closing(tensor, torch.ones_like(structural_element), structuring_element=structural_element), + expected, + atol=1e-3, + rtol=1e-3, + ) + + def test_exception(self, device, dtype): + tensor = torch.ones(1, 1, 3, 4, device=device, dtype=dtype) + kernel = torch.ones(3, 3, device=device, dtype=dtype) + + with pytest.raises(TypeError): + assert closing([0.0], kernel) + + with pytest.raises(TypeError): + assert closing(tensor, [0.0]) + + with pytest.raises(ValueError): + test = torch.ones(2, 3, 4, device=device, dtype=dtype) + assert closing(test, kernel) + + with pytest.raises(ValueError): + test = torch.ones(2, 3, 4, device=device, dtype=dtype) + assert closing(tensor, test) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + op = closing + op_script = torch.jit.script(op) + + tensor = torch.rand(1, 2, 7, 7, device=device, dtype=dtype) + kernel = torch.ones(3, 3, device=device, dtype=dtype) + + actual = op_script(tensor, kernel) + expected = op(tensor, kernel) + + assert_close(actual, expected) diff --git a/tests/morphology/test_dilation.py b/tests/morphology/test_dilation.py new file mode 100644 index 0000000..24cc894 --- /dev/null +++ b/tests/morphology/test_dilation.py @@ -0,0 +1,167 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.morphology import dilation + +from testing.base import BaseTester, assert_close +from testing.parametrized_tester import parametrized_test + + +@parametrized_test( + smoke_inputs=lambda device, dtype: ( + torch.rand(1, 3, 4, 4, device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + cardinality_tests=[ + { + "inputs": lambda device, dtype: ( + torch.ones((1, 3, 4, 4), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([1, 3, 4, 4]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((2, 3, 2, 4), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([2, 3, 2, 4]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((3, 3, 4, 1), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([3, 3, 4, 1]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((3, 2, 5, 5), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([3, 2, 5, 5]), + }, + ], + gradcheck_inputs=lambda device: ( + torch.rand(2, 3, 4, 4, requires_grad=True, device=device, dtype=torch.float64), + torch.rand(3, 3, requires_grad=True, device=device, dtype=torch.float64), + ), +) +class TestDilate(BaseTester): + def setup_method(self) -> None: + self.func = dilation + + def test_kernel(self, device, dtype): + tensor = torch.tensor([[0.5, 1.0, 0.3], [0.7, 0.3, 0.8], [0.4, 0.9, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + kernel = torch.tensor([[0.0, 1.0, 0.0], [1.0, 1.0, 1.0], [0.0, 1.0, 0.0]], device=device, dtype=dtype) + expected = torch.tensor([[1.0, 1.0, 1.0], [0.7, 1.0, 0.8], [0.9, 0.9, 0.9]], device=device, dtype=dtype)[ + None, None, :, : + ] + assert_close(dilation(tensor, kernel, engine="unfold"), expected, atol=1e-4, rtol=1e-4) + assert_close(dilation(tensor, kernel, engine="convolution"), expected, atol=1e-3, rtol=1e-3) + + def test_structural_element(self, device, dtype): + tensor = torch.tensor([[0.5, 1.0, 0.3], [0.7, 0.3, 0.8], [0.4, 0.9, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + structural_element = torch.tensor( + [[-1.0, 0.0, -1.0], [0.0, 0.0, 0.0], [-1.0, 0.0, -1.0]], device=device, dtype=dtype + ) + expected = torch.tensor([[1.0, 1.0, 1.0], [0.7, 1.0, 0.8], [0.9, 0.9, 0.9]], device=device, dtype=dtype)[ + None, None, :, : + ] + assert_close( + dilation( + tensor, torch.ones_like(structural_element), structuring_element=structural_element, engine="unfold" + ), + expected, + atol=1e-3, + rtol=1e-3, + ) + assert_close( + dilation( + tensor, + torch.ones_like(structural_element), + structuring_element=structural_element, + engine="convolution", + ), + expected, + atol=1e-3, + rtol=1e-3, + ) + + def test_flip(self, device, dtype): + tensor = torch.tensor([[0.5, 1.0, 0.3], [0.7, 0.3, 0.8], [0.4, 0.9, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + kernel = torch.tensor([[0.0, 1.0, 1.0], [0.0, 1.0, 1.0], [0.0, 1.0, 1.0]], device=device, dtype=dtype) + expected = torch.tensor([[0.7, 1.0, 1.0], [0.7, 1.0, 1.0], [0.7, 0.9, 0.9]], device=device, dtype=dtype)[ + None, None, :, : + ] + assert_close(dilation(tensor, kernel), expected, atol=1e-3, rtol=1e-3) + + def test_exception(self, device, dtype): + tensor = torch.ones(1, 1, 3, 4, device=device, dtype=dtype) + kernel = torch.ones(3, 3, device=device, dtype=dtype) + + with pytest.raises(TypeError): + assert dilation([0.0], kernel) + + with pytest.raises(TypeError): + assert dilation(tensor, [0.0]) + + with pytest.raises(ValueError): + test = torch.ones(2, 3, 4, device=device, dtype=dtype) + assert dilation(test, kernel) + + with pytest.raises(ValueError): + test = torch.ones(2, 3, 4, device=device, dtype=dtype) + assert dilation(tensor, test) + + with pytest.raises(NotImplementedError, match="unknown"): + dilation(tensor, kernel, engine="invalid_engine") + + def test_custom_origin(self, device, dtype): + # Custom origin shifts the structuring element anchor point + tensor = torch.zeros(1, 1, 5, 5, device=device, dtype=dtype) + tensor[0, 0, 2, 2] = 1.0 # single hot pixel in center + kernel = torch.ones(3, 3, device=device, dtype=dtype) + + # Default origin (center): dilation spreads symmetrically + out_default = dilation(tensor, kernel) + # Custom origin (top-left corner): effect shifts + out_custom = dilation(tensor, kernel, origin=[0, 0]) + # Results should differ when the anchor point changes + assert not torch.equal(out_default, out_custom) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + op = dilation + op_script = torch.jit.script(op) + + tensor = torch.rand(1, 2, 7, 7, device=device, dtype=dtype) + kernel = torch.ones(3, 3, device=device, dtype=dtype) + + actual = op_script(tensor, kernel) + expected = op(tensor, kernel) + + assert_close(actual, expected) diff --git a/tests/morphology/test_erosion.py b/tests/morphology/test_erosion.py new file mode 100644 index 0000000..a225465 --- /dev/null +++ b/tests/morphology/test_erosion.py @@ -0,0 +1,150 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.morphology import erosion + +from testing.base import BaseTester, assert_close +from testing.parametrized_tester import parametrized_test + + +@parametrized_test( + smoke_inputs=lambda device, dtype: ( + torch.rand(1, 3, 4, 4, device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + cardinality_tests=[ + { + "inputs": lambda device, dtype: ( + torch.ones((1, 3, 4, 4), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([1, 3, 4, 4]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((2, 3, 2, 4), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([2, 3, 2, 4]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((3, 3, 4, 1), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([3, 3, 4, 1]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((3, 2, 5, 5), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([3, 2, 5, 5]), + }, + ], + gradcheck_inputs=lambda device: ( + torch.rand(2, 3, 4, 4, requires_grad=True, device=device, dtype=torch.float64), + torch.rand(3, 3, requires_grad=True, device=device, dtype=torch.float64), + ), +) +class TestErode(BaseTester): + def setup_method(self) -> None: + self.func = erosion + + def test_kernel(self, device, dtype): + tensor = torch.tensor([[0.5, 1.0, 0.3], [0.7, 0.3, 0.8], [0.4, 0.9, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + kernel = torch.tensor([[0.0, 1.0, 0.0], [1.0, 1.0, 1.0], [0.0, 1.0, 0.0]], device=device, dtype=dtype) + expected = torch.tensor([[0.5, 0.3, 0.3], [0.3, 0.3, 0.2], [0.4, 0.2, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + assert_close(erosion(tensor, kernel), expected, atol=1e-4, rtol=1e-4) + assert_close(erosion(tensor, kernel, engine="convolution"), expected, atol=1e-3, rtol=1e-3) + + def test_structural_element(self, device, dtype): + tensor = torch.tensor([[0.5, 1.0, 0.3], [0.7, 0.3, 0.8], [0.4, 0.9, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + structural_element = torch.tensor( + [[-1.0, 0.0, -1.0], [0.0, 0.0, 0.0], [-1.0, 0.0, -1.0]], device=device, dtype=dtype + ) + expected = torch.tensor([[0.5, 0.3, 0.3], [0.3, 0.3, 0.2], [0.4, 0.2, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + assert_close( + erosion(tensor, torch.ones_like(structural_element), structuring_element=structural_element), + expected, + atol=1e-4, + rtol=1e-4, + ) + assert_close( + erosion( + tensor, + torch.ones_like(structural_element), + structuring_element=structural_element, + engine="convolution", + ), + expected, + atol=1e-3, + rtol=1e-3, + ) + + def test_flip(self, device, dtype): + tensor = torch.tensor([[0.5, 1.0, 0.3], [0.7, 0.3, 0.8], [0.4, 0.9, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + kernel = torch.tensor([[0.0, 1.0, 1.0], [0.0, 1.0, 1.0], [0.0, 1.0, 1.0]], device=device, dtype=dtype) + expected = torch.tensor([[0.3, 0.3, 0.3], [0.3, 0.2, 0.2], [0.3, 0.2, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + assert_close(erosion(tensor, kernel, engine="unfold"), expected, atol=1e-4, rtol=1e-4) + assert_close(erosion(tensor, kernel, engine="convolution"), expected, atol=1e-3, rtol=1e-3) + + def test_exception(self, device, dtype): + tensor = torch.ones(1, 1, 3, 4, device=device, dtype=dtype) + kernel = torch.ones(3, 3, device=device, dtype=dtype) + + with pytest.raises(TypeError): + assert erosion([0.0], kernel) + + with pytest.raises(TypeError): + assert erosion(tensor, [0.0]) + + with pytest.raises(ValueError): + test = torch.ones(2, 3, 4, device=device, dtype=dtype) + assert erosion(test, kernel) + + with pytest.raises(ValueError): + test = torch.ones(2, 3, 4, device=device, dtype=dtype) + assert erosion(tensor, test) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + op = erosion + op_script = torch.jit.script(op) + + tensor = torch.rand(1, 2, 7, 7, device=device, dtype=dtype) + kernel = torch.ones(3, 3, device=device, dtype=dtype) + + actual = op_script(tensor, kernel) + expected = op(tensor, kernel) + + assert_close(actual, expected) diff --git a/tests/morphology/test_gradient.py b/tests/morphology/test_gradient.py new file mode 100644 index 0000000..001ac29 --- /dev/null +++ b/tests/morphology/test_gradient.py @@ -0,0 +1,127 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.morphology import gradient + +from testing.base import BaseTester, assert_close +from testing.parametrized_tester import parametrized_test + + +@parametrized_test( + smoke_inputs=lambda device, dtype: ( + torch.rand(1, 3, 4, 4, device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + cardinality_tests=[ + { + "inputs": lambda device, dtype: ( + torch.ones((1, 3, 4, 4), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([1, 3, 4, 4]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((2, 3, 2, 4), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([2, 3, 2, 4]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((3, 3, 4, 1), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([3, 3, 4, 1]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((3, 2, 5, 5), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([3, 2, 5, 5]), + }, + ], + gradcheck_inputs=lambda device: ( + torch.rand(2, 3, 4, 4, requires_grad=True, device=device, dtype=torch.float64), + torch.rand(3, 3, requires_grad=True, device=device, dtype=torch.float64), + ), +) +class TestGradient(BaseTester): + def setup_method(self) -> None: + self.func = gradient + + def test_kernel(self, device, dtype): + tensor = torch.tensor([[0.5, 1.0, 0.3], [0.7, 0.3, 0.8], [0.4, 0.9, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + kernel = torch.tensor([[0.0, 1.0, 0.0], [1.0, 1.0, 1.0], [0.0, 1.0, 0.0]], device=device, dtype=dtype) + expected = torch.tensor([[0.5, 0.7, 0.7], [0.4, 0.7, 0.6], [0.5, 0.7, 0.7]], device=device, dtype=dtype)[ + None, None, :, : + ] + assert_close(gradient(tensor, kernel), expected, atol=1e-3, rtol=1e-3) + + def test_structural_element(self, device, dtype): + tensor = torch.tensor([[0.5, 1.0, 0.3], [0.7, 0.3, 0.8], [0.4, 0.9, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + structural_element = torch.tensor( + [[-1.0, 0.0, -1.0], [0.0, 0.0, 0.0], [-1.0, 0.0, -1.0]], device=device, dtype=dtype + ) + expected = torch.tensor([[0.5, 0.7, 0.7], [0.4, 0.7, 0.6], [0.5, 0.7, 0.7]], device=device, dtype=dtype)[ + None, None, :, : + ] + assert_close( + gradient(tensor, torch.ones_like(structural_element), structuring_element=structural_element), + expected, + atol=1e-3, + rtol=1e-3, + ) + + def test_exception(self, device, dtype): + tensor = torch.ones(1, 1, 3, 4, device=device, dtype=dtype) + kernel = torch.ones(3, 3, device=device, dtype=dtype) + + with pytest.raises(TypeError): + assert gradient([0.0], kernel) + + with pytest.raises(TypeError): + assert gradient(tensor, [0.0]) + + with pytest.raises(ValueError): + test = torch.ones(2, 3, 4, device=device, dtype=dtype) + assert gradient(test, kernel) + + with pytest.raises(ValueError): + test = torch.ones(2, 3, 4, device=device, dtype=dtype) + assert gradient(tensor, test) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + op = gradient + op_script = torch.jit.script(op) + + tensor = torch.rand(1, 2, 7, 7, device=device, dtype=dtype) + kernel = torch.ones(3, 3, device=device, dtype=dtype) + + actual = op_script(tensor, kernel) + expected = op(tensor, kernel) + + assert_close(actual, expected) diff --git a/tests/morphology/test_opening.py b/tests/morphology/test_opening.py new file mode 100644 index 0000000..bc079e1 --- /dev/null +++ b/tests/morphology/test_opening.py @@ -0,0 +1,127 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.morphology import opening + +from testing.base import BaseTester, assert_close +from testing.parametrized_tester import parametrized_test + + +@parametrized_test( + smoke_inputs=lambda device, dtype: ( + torch.rand(1, 3, 4, 4, device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + cardinality_tests=[ + { + "inputs": lambda device, dtype: ( + torch.ones((1, 3, 4, 4), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([1, 3, 4, 4]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((2, 3, 2, 4), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([2, 3, 2, 4]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((3, 3, 4, 1), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([3, 3, 4, 1]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((3, 2, 5, 5), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([3, 2, 5, 5]), + }, + ], + gradcheck_inputs=lambda device: ( + torch.rand(2, 3, 4, 4, requires_grad=True, device=device, dtype=torch.float64), + torch.rand(3, 3, requires_grad=True, device=device, dtype=torch.float64), + ), +) +class TestOpening(BaseTester): + def setup_method(self) -> None: + self.func = opening + + def test_kernel(self, device, dtype): + tensor = torch.tensor([[0.5, 1.0, 0.3], [0.7, 0.3, 0.8], [0.4, 0.9, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + kernel = torch.tensor([[0.0, 1.0, 0.0], [1.0, 1.0, 1.0], [0.0, 1.0, 0.0]], device=device, dtype=dtype) + expected = torch.tensor([[0.5, 0.5, 0.3], [0.5, 0.3, 0.3], [0.4, 0.4, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + assert_close(opening(tensor, kernel), expected, atol=1e-4, rtol=1e-4) + + def test_structural_element(self, device, dtype): + tensor = torch.tensor([[0.5, 1.0, 0.3], [0.7, 0.3, 0.8], [0.4, 0.9, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + structural_element = torch.tensor( + [[-1.0, 0.0, -10.0], [0.0, 0.0, 0.0], [-1.0, 0.0, -1.0]], device=device, dtype=dtype + ) + expected = torch.tensor([[0.5, 0.5, 0.3], [0.5, 0.3, 0.3], [0.4, 0.4, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + assert_close( + opening(tensor, torch.ones_like(structural_element), structuring_element=structural_element), + expected, + atol=1e-3, + rtol=1e-3, + ) + + def test_exception(self, device, dtype): + tensor = torch.ones(1, 1, 3, 4, device=device, dtype=dtype) + kernel = torch.ones(3, 3, device=device, dtype=dtype) + + with pytest.raises(TypeError): + assert opening([0.0], kernel) + + with pytest.raises(TypeError): + assert opening(tensor, [0.0]) + + with pytest.raises(ValueError): + test = torch.ones(2, 3, 4, device=device, dtype=dtype) + assert opening(test, kernel) + + with pytest.raises(ValueError): + test = torch.ones(2, 3, 4, device=device, dtype=dtype) + assert opening(tensor, test) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + op = opening + op_script = torch.jit.script(op) + + tensor = torch.rand(1, 2, 7, 7, device=device, dtype=dtype) + kernel = torch.ones(3, 3, device=device, dtype=dtype) + + actual = op_script(tensor, kernel) + expected = op(tensor, kernel) + + assert_close(actual, expected) diff --git a/tests/morphology/test_top_hat.py b/tests/morphology/test_top_hat.py new file mode 100644 index 0000000..9509647 --- /dev/null +++ b/tests/morphology/test_top_hat.py @@ -0,0 +1,127 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.morphology import top_hat + +from testing.base import BaseTester, assert_close +from testing.parametrized_tester import parametrized_test + + +@parametrized_test( + smoke_inputs=lambda device, dtype: ( + torch.rand(1, 3, 4, 4, device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + cardinality_tests=[ + { + "inputs": lambda device, dtype: ( + torch.ones((1, 3, 4, 4), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([1, 3, 4, 4]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((2, 3, 2, 4), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([2, 3, 2, 4]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((3, 3, 4, 1), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([3, 3, 4, 1]), + }, + { + "inputs": lambda device, dtype: ( + torch.ones((3, 2, 5, 5), device=device, dtype=dtype), + torch.ones((3, 3), device=device, dtype=dtype), + ), + "expected_shape": torch.Size([3, 2, 5, 5]), + }, + ], + gradcheck_inputs=lambda device: ( + torch.rand(2, 3, 4, 4, requires_grad=True, device=device, dtype=torch.float64), + torch.rand(3, 3, requires_grad=True, device=device, dtype=torch.float64), + ), +) +class TestTopHat(BaseTester): + def setup_method(self) -> None: + self.func = top_hat + + def test_kernel(self, device, dtype): + tensor = torch.tensor([[0.5, 1.0, 0.3], [0.7, 0.3, 0.8], [0.4, 0.9, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + kernel = torch.tensor([[0.0, 1.0, 0.0], [1.0, 1.0, 1.0], [0.0, 1.0, 0.0]], device=device, dtype=dtype) + expected = torch.tensor([[0.0, 0.5, 0.0], [0.2, 0.0, 0.5], [0.0, 0.5, 0.0]], device=device, dtype=dtype)[ + None, None, :, : + ] + assert_close(top_hat(tensor, kernel), expected, atol=1e-3, rtol=1e-3) + + def test_structural_element(self, device, dtype): + tensor = torch.tensor([[0.5, 1.0, 0.3], [0.7, 0.3, 0.8], [0.4, 0.9, 0.2]], device=device, dtype=dtype)[ + None, None, :, : + ] + structural_element = torch.tensor( + [[-1.0, 0.0, -1.0], [0.0, 0.0, 0.0], [-1.0, 0.0, -1.0]], device=device, dtype=dtype + ) + expected = torch.tensor([[0.0, 0.5, 0.0], [0.2, 0.0, 0.5], [0.0, 0.5, 0.0]], device=device, dtype=dtype)[ + None, None, :, : + ] + assert_close( + top_hat(tensor, torch.ones_like(structural_element), structuring_element=structural_element), + expected, + atol=1e-3, + rtol=1e-3, + ) + + def test_exception(self, device, dtype): + sample = torch.ones(1, 1, 3, 4, device=device, dtype=dtype) + kernel = torch.ones(3, 3, device=device, dtype=dtype) + + with pytest.raises(TypeError): + assert top_hat([0.0], kernel) + + with pytest.raises(TypeError): + assert top_hat(sample, [0.0]) + + with pytest.raises(ValueError): + test = torch.ones(2, 3, 4, device=device, dtype=dtype) + assert top_hat(test, kernel) + + with pytest.raises(ValueError): + test = torch.ones(2, 3, 4, device=device, dtype=dtype) + assert top_hat(sample, test) + + @pytest.mark.jit() + def test_jit(self, device, dtype): + op = top_hat + op_script = torch.jit.script(op) + + sample = torch.rand(1, 2, 7, 7, device=device, dtype=dtype) + kernel = torch.ones(3, 3, device=device, dtype=dtype) + + actual = op_script(sample, kernel) + expected = op(sample, kernel) + + assert_close(actual, expected) diff --git a/tests/onnx/__init__.py b/tests/onnx/__init__.py new file mode 100644 index 0000000..4d273d5 --- /dev/null +++ b/tests/onnx/__init__.py @@ -0,0 +1,16 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/onnx/check.py b/tests/onnx/check.py new file mode 100644 index 0000000..c655ceb --- /dev/null +++ b/tests/onnx/check.py @@ -0,0 +1,33 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# limitations under the License. +import sys +from pathlib import Path + +script_dir = Path(__file__).parent.parent.parent +sys.path.insert(0, str(script_dir)) +# ruff: noqa: E402 +import test_resize_onnx + +test_resize_onnx.test_resize_dynamo_with_binding() +print("Test 1 passed") +test_resize_onnx.test_resize_upscale_dynamo() +print("Test 2 passed") +test_resize_onnx.test_resize_downscale_dynamo() +print("Test 3 passed") +test_resize_onnx.test_resize_nearest_dynamo() +print("Test 4 passed") +print("ALL TESTS PASSED") diff --git a/tests/onnx/test_resize_onnx.py b/tests/onnx/test_resize_onnx.py new file mode 100644 index 0000000..903b03c --- /dev/null +++ b/tests/onnx/test_resize_onnx.py @@ -0,0 +1,152 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import tempfile + +import numpy as np +import pytest +import torch + +from kornia.geometry.transform import Resize + +onnx = pytest.importorskip("onnx") +ort = pytest.importorskip("onnxruntime") + + +def test_resize_dynamo_with_binding(): + model = Resize((32, 32), interpolation="bilinear") + model.eval() + + x = torch.randn(1, 3, 64, 64) + with torch.no_grad(): + torch_out = model(x) + + fd, temp_path = tempfile.mkstemp(suffix=".onnx") + os.close(fd) + + try: + torch.onnx.export(model, x, temp_path, dynamo=True, opset_version=18) + + ort_session = ort.InferenceSession(temp_path) + input_name = ort_session.get_inputs()[0].name + output_name = ort_session.get_outputs()[0].name + + binding = ort_session.io_binding() + binding.bind_cpu_input(input_name, x.numpy()) + binding.bind_output(output_name) + + ort_session.run_with_iobinding(binding) + ort_out = binding.copy_outputs_to_cpu()[0] + + np.testing.assert_allclose(torch_out.numpy(), ort_out, rtol=1e-4, atol=1e-4) + finally: + if os.path.exists(temp_path): + os.unlink(temp_path) + + +def test_resize_upscale_dynamo(): + """Test upscaling with dynamo.""" + model = Resize((128, 128)) + model.eval() + x = torch.randn(1, 3, 64, 64) + + with torch.no_grad(): + torch_out = model(x) + + fd, temp_path = tempfile.mkstemp(suffix=".onnx") + os.close(fd) + + try: + torch.onnx.export(model, x, temp_path, dynamo=True, opset_version=18) + + ort_session = ort.InferenceSession(temp_path) + input_name = ort_session.get_inputs()[0].name + output_name = ort_session.get_outputs()[0].name + + binding = ort_session.io_binding() + binding.bind_cpu_input(input_name, x.numpy()) + binding.bind_output(output_name) + ort_session.run_with_iobinding(binding) + ort_out = binding.copy_outputs_to_cpu()[0] + + np.testing.assert_allclose(torch_out.numpy(), ort_out, rtol=1e-4, atol=1e-4) + finally: + if os.path.exists(temp_path): + os.unlink(temp_path) + + +def test_resize_downscale_dynamo(): + """Test downscaling with dynamo.""" + model = Resize((16, 16)) + model.eval() + x = torch.randn(1, 3, 64, 64) + + with torch.no_grad(): + torch_out = model(x) + + fd, temp_path = tempfile.mkstemp(suffix=".onnx") + os.close(fd) + + try: + torch.onnx.export(model, x, temp_path, dynamo=True, opset_version=18) + + ort_session = ort.InferenceSession(temp_path) + input_name = ort_session.get_inputs()[0].name + output_name = ort_session.get_outputs()[0].name + + binding = ort_session.io_binding() + binding.bind_cpu_input(input_name, x.numpy()) + binding.bind_output(output_name) + ort_session.run_with_iobinding(binding) + ort_out = binding.copy_outputs_to_cpu()[0] + + np.testing.assert_allclose(torch_out.numpy(), ort_out, rtol=1e-4, atol=1e-4) + finally: + if os.path.exists(temp_path): + os.unlink(temp_path) + + +def test_resize_nearest_dynamo(): + """Test nearest neighbor interpolation with dynamo.""" + model = Resize((32, 32), interpolation="nearest") + model.eval() + x = torch.randn(1, 3, 64, 64) + + with torch.no_grad(): + torch_out = model(x) + + fd, temp_path = tempfile.mkstemp(suffix=".onnx") + os.close(fd) + + try: + torch.onnx.export(model, x, temp_path, dynamo=True, opset_version=18) + + ort_session = ort.InferenceSession(temp_path) + input_name = ort_session.get_inputs()[0].name + output_name = ort_session.get_outputs()[0].name + + binding = ort_session.io_binding() + binding.bind_cpu_input(input_name, x.numpy()) + binding.bind_output(output_name) + ort_session.run_with_iobinding(binding) + ort_out = binding.copy_outputs_to_cpu()[0] + + np.testing.assert_allclose(torch_out.numpy(), ort_out, rtol=1e-4, atol=1e-4) + finally: + if os.path.exists(temp_path): + os.unlink(temp_path) diff --git a/tests/onnx/test_sequential.py b/tests/onnx/test_sequential.py new file mode 100644 index 0000000..1ca9345 --- /dev/null +++ b/tests/onnx/test_sequential.py @@ -0,0 +1,108 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest + +onnx = pytest.importorskip("onnx") + +from kornia.onnx.sequential import ONNXSequential # noqa: E402 + + +class TestONNXSequential: + @pytest.fixture + def mock_model_proto(self): + from onnx.helper import make_graph, make_model, make_node, make_tensor_value_info + + # Create a minimal ONNX model with an input and output + input_info = make_tensor_value_info("input", onnx.TensorProto.FLOAT, [1, 2]) + output_info = make_tensor_value_info("output", onnx.TensorProto.FLOAT, [1, 2]) + node = make_node("Identity", ["input"], ["output"]) + graph = make_graph([node], "test_graph", [input_info], [output_info]) + op = onnx.OperatorSetIdProto() + op.version = 17 + model = make_model(graph, opset_imports=[op], ir_version=9) + return model + + @pytest.fixture + def onnx_sequential(self, mock_model_proto): + return ONNXSequential(mock_model_proto) + + def test_init(self, onnx_sequential, mock_model_proto): + assert len(onnx_sequential.operators) == 1 + assert onnx_sequential.operators[0] == mock_model_proto + + def test_load_op(self, onnx_sequential, mock_model_proto): + # Test loading a ModelProto object + model = onnx_sequential._load_op(mock_model_proto) + assert model == mock_model_proto + + def test_combine_models(self, mock_model_proto): + from unittest.mock import patch + + from onnx.helper import make_graph, make_model, make_node, make_tensor_value_info + + # The patch must wrap ONNXSequential() construction so merge_models is mocked + # when _combine() actually calls it. + with patch("onnx.compose.merge_models") as mock_merge_models: + # Create a small ONNX model as the return value of merge_models + input_info = make_tensor_value_info("input", onnx.TensorProto.FLOAT, [1, 2]) + output_info = make_tensor_value_info("output", onnx.TensorProto.FLOAT, [1, 2]) + node = make_node("Identity", ["input"], ["output"]) + graph = make_graph([node], "combined_graph", [input_info], [output_info]) + op = onnx.OperatorSetIdProto() + opset_version = 17 + ir_version = 10 + op.version = opset_version + combined_model = make_model(graph, opset_imports=[op], ir_version=ir_version) + + mock_merge_models.return_value = combined_model + + # Test combining multiple ONNX models with io_maps + onnx_sequential = ONNXSequential( + mock_model_proto, + mock_model_proto, + io_maps=[[("output", "input")]], # list-of-list-of-tuples format + ) + combined_op = onnx_sequential._combined_op + + assert isinstance(combined_op, onnx.ModelProto) + + def test_export_combined_model(self, onnx_sequential): + from unittest.mock import patch + + with patch("onnx.save") as mock_save: + # Test exporting the combined ONNX model + onnx_sequential.export("exported_model.onnx") + mock_save.assert_called_once_with(onnx_sequential._combined_op, "exported_model.onnx") + + def test_create_session(self, onnx_sequential): + from unittest.mock import patch + + with patch("onnxruntime.InferenceSession") as mock_inference_session: + # Test creating an ONNXRuntime session + session = onnx_sequential.create_session() + assert session == mock_inference_session() + + def test_set_get_session(self, onnx_sequential): + from unittest.mock import MagicMock + + import onnxruntime as ort + + # Test setting and getting a custom session + mock_session = MagicMock(spec=ort.InferenceSession) + onnx_sequential.set_session(mock_session) + assert onnx_sequential.get_session() == mock_session diff --git a/tests/onnx/test_utils.py b/tests/onnx/test_utils.py new file mode 100644 index 0000000..b7302bd --- /dev/null +++ b/tests/onnx/test_utils.py @@ -0,0 +1,217 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import urllib + +import pytest + +onnx = pytest.importorskip("onnx") + +from kornia.onnx.utils import ONNXLoader # noqa: E402 + + +class TestONNXLoader: + def test_get_file_path(self): + # Test getting local file path for caching + model_name = "some_model" + expected_path = os.path.join(".kornia_hub", "some_model.onnx") + assert ONNXLoader._get_file_path(model_name, None, suffix=".onnx") == expected_path + + def test_load_model_local(self): + from unittest import mock + + from onnx import ModelProto + + with mock.patch("onnx.load") as mock_onnx_load, mock.patch("os.path.exists") as mock_exists: + model_name = "local_model.onnx" + mock_exists.return_value = True + + # Simulate onnx.load returning a dummy ModelProto + mock_model = mock.Mock(spec=ModelProto) + mock_onnx_load.return_value = mock_model + + model = ONNXLoader.load_model(model_name) + assert model == mock_model + mock_onnx_load.assert_called_once_with(model_name) + + def test_load_model_download(self): + from unittest import mock + + from onnx import ModelProto + + with ( + mock.patch("urllib.request.urlretrieve") as mock_urlretrieve, + mock.patch("os.path.exists") as mock_exists, + mock.patch("onnx.load") as mock_onnx_load, + ): + model_name = "hf://operators/some_model" + mock_exists.return_value = False + mock_urlretrieve.return_value = None # Simulating successful download + + mock_model = mock.Mock(spec=ModelProto) + mock_onnx_load.return_value = mock_model + + model = ONNXLoader.load_model(model_name) + assert model == mock_model + mock_urlretrieve.assert_called_once_with( + "https://huggingface.co/kornia/ONNX_models/resolve/main/operators/some_model.onnx", + os.path.join(".kornia_hub", "onnx_models", "operators", "some_model.onnx"), + ) + + def test_load_model_not_found(self): + model_name = "non_existent_model.onnx" + with pytest.raises(ValueError, match=f"File {model_name} not found"): + ONNXLoader.load_model(model_name) + + def test_download_success(self): + import os + from unittest import mock + + with mock.patch("urllib.request.urlretrieve") as mock_urlretrieve, mock.patch("os.makedirs") as mock_makedirs: + url = "https://huggingface.co/some_model.onnx" + file_path = os.path.join(".test_cache", "some_model.onnx") + + ONNXLoader.download(url, file_path) + + mock_makedirs.assert_called_once_with(os.path.dirname(file_path), exist_ok=True) + mock_urlretrieve.assert_called_once_with(url, file_path) + + def test_download_failure(self): + import os + from unittest import mock + + with mock.patch( + "urllib.request.urlretrieve", + side_effect=urllib.error.HTTPError(url=None, code=404, msg="Not Found", hdrs=None, fp=None), + ) as _: + url = "https://huggingface.co/non_existent_model.onnx" + file_path = os.path.join(".test_cache", "non_existent_model.onnx") + + with pytest.raises(ValueError, match="Error in resolving"): + ONNXLoader.download(url, file_path) + + def test_fetch_repo_contents_success(self): + import os + from unittest import mock + + with mock.patch("requests.get") as mock_get: + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_response.json.return_value = [{"path": os.path.join("operators", "model.onnx")}] + mock_get.return_value = mock_response + + contents = ONNXLoader._fetch_repo_contents("operators") + assert contents == [{"path": os.path.join("operators", "model.onnx")}] + + def test_fetch_repo_contents_failure(self): + from unittest import mock + + with mock.patch("requests.get") as mock_get: + mock_response = mock.Mock() + mock_response.status_code = 404 + mock_get.return_value = mock_response + + with pytest.raises(ValueError, match="Failed to fetch repository contents"): + ONNXLoader._fetch_repo_contents("operators") + + def test_list_operators(self, capsys): + import os + from unittest import mock + + with mock.patch("kornia.onnx.utils.ONNXLoader._fetch_repo_contents") as mock_fetch_repo_contents: + mock_fetch_repo_contents.return_value = [{"path": os.path.join("operators", "some_model.onnx")}] + + ONNXLoader.list_operators() + + captured = capsys.readouterr() + assert ( + os.path.join("operators", "some_model.onnx").replace("\\", "\\\\") in captured.out + ) # .replace() for Windows + + def test_list_models(self, capsys): + import os + from unittest import mock + + with mock.patch("kornia.onnx.utils.ONNXLoader._fetch_repo_contents") as mock_fetch_repo_contents: + mock_fetch_repo_contents.return_value = [{"path": os.path.join("operators", "some_model.onnx")}] + + ONNXLoader.list_models() + + captured = capsys.readouterr() + assert ( + os.path.join("operators", "some_model.onnx").replace("\\", "\\\\") in captured.out + ) # .replace() for Windows + + +def test_io_name_conversion(): + from unittest import mock + + from kornia.onnx.utils import io_name_conversion + + with mock.patch("kornia.core.external.onnx.ModelProto") as mock_model_proto: + # Arrange + mock_model = mock_model_proto() + mock_in_node = mock.Mock() + mock_in_node.name = "input_1" + mock_out_node = mock.Mock() + mock_out_node.name = "output_1" + mock_model.graph.input = [mock_in_node] + mock_model.graph.output = [mock_out_node] + + mock_mid_node = mock.Mock() + mock_mid_node.input = ["input_1"] + mock_mid_node.output = ["output_1"] + mock_model.graph.node = [mock_mid_node] + + mapping = {"input_1": "input", "output_1": "output"} + + # Act + converted_model = io_name_conversion(mock_model, mapping) + + # Assert + assert converted_model.graph.input[0].name == "input" + assert converted_model.graph.output[0].name == "output" + assert converted_model.graph.node[0].input[0] == "input" + assert converted_model.graph.node[0].output[0] == "output" + + +def test_add_metadata(): + from unittest import mock + + from kornia.onnx.utils import add_metadata + + with mock.patch("kornia.core.external.onnx.ModelProto") as mock_model_proto: + # Arrange + mock_model = mock_model_proto() + mock_metadata_props = mock.Mock() + mock_model.metadata_props.add.return_value = mock_metadata_props + + # Act + add_metadata(mock_model, [("test_key", "test_value")]) + + # Assert + calls = [ + mock.call(), # for "source" + mock.call(), # for "version" + mock.call(), # for "test_key" + ] + mock_model.metadata_props.add.assert_has_calls(calls) + assert mock_model.metadata_props.add.call_count == 3 + # Check if version was added + # (Since it's a mock, we just check if any call set value to kornia.__version__) + # Metadata logic: metadata_props.key = key; metadata_props.value = str(value) diff --git a/tests/performance/test_imgwarp_speed.py b/tests/performance/test_imgwarp_speed.py new file mode 100644 index 0000000..dc27b10 --- /dev/null +++ b/tests/performance/test_imgwarp_speed.py @@ -0,0 +1,45 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from time import time + +import pytest +import torch + +import kornia + +shapes = [(512, 3, 256, 256), (256, 1, 64, 64)] +PSs = [224, 32] + + +@pytest.mark.xfail(reason="May cause memory issues.") +def test_performance_speed(device, dtype): + if device.type != "cuda" or not torch.cuda.is_available(): + pytest.skip("Cuda not available in system,") + + print("Benchmarking warp_affine") + for input_shape in shapes: + for PS in PSs: + BS = input_shape[0] + data = torch.rand(input_shape).to(device) + As = torch.eye(3).unsqueeze(0).repeat(BS, 1, 1)[:, :2, :].to(device) + As += 0.1 * torch.rand(As.size()).to(device) + torch.cuda.synchronize(device) + t = time() + _ = kornia.warp_affine(data, As, (PS, PS)) + print(f"inp={input_shape}, PS={PS}, dev={device}, {time() - t}, sec") + torch.cuda.synchronize(device) diff --git a/tests/performance/test_project_points_speed.py b/tests/performance/test_project_points_speed.py new file mode 100644 index 0000000..7f45bdb --- /dev/null +++ b/tests/performance/test_project_points_speed.py @@ -0,0 +1,43 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from time import time + +import pytest +import torch + +import kornia + +points_shapes = [(64, 1024**2, 3), (8192, 8192, 3), (1024**2, 64, 3)] + +# TODO: remove xfail once we have enough gpu bandwidth in the CI + + +@pytest.mark.xfail(reason="May cause memory issues.") +def test_performance_speed(device, dtype): + if device.type != "cuda" or not torch.cuda.is_available(): + pytest.skip("Cuda not available in system,") + + print("Benchmarking project_points") + for input_shape in points_shapes: + data = torch.rand(input_shape).to(device) + pose = torch.rand((1, 4, 4)).to(device) + torch.cuda.synchronize(device) + t = time() + kornia.geometry.transform_points(pose, data) + torch.cuda.synchronize(device) + print(f"inp={input_shape}, dev={device}, {time() - t}, sec") diff --git a/tests/sensors/camera/test_camera_model.py b/tests/sensors/camera/test_camera_model.py new file mode 100644 index 0000000..32936ce --- /dev/null +++ b/tests/sensors/camera/test_camera_model.py @@ -0,0 +1,113 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.geometry.vector import Vector3 +from kornia.image import ImageSize +from kornia.sensors.camera import CameraModel, CameraModelType + +from testing.base import BaseTester + + +class TestPinholeCamera(BaseTester): + def _make_rand_data(self, batch_size, device, dtype): + params = torch.rand(batch_size, 4).to(dtype).to(device) + image_sizes = torch.randint(1, 100, (batch_size, 2)).to(dtype).to(device) + return params, ImageSize(image_sizes[:, 0], image_sizes[:, 1]) + + def test_smoke(self, device, dtype): + params, image_size = self._make_rand_data(1, device, dtype) + cam = CameraModel(image_size, CameraModelType.PINHOLE, params) + assert isinstance(cam, CameraModel) + self.assert_close(cam.params, params) + self.assert_close(image_size.height, cam.height) + self.assert_close(image_size.width, cam.width) + + @pytest.mark.skip(reason="Unnecessary test") + def test_cardinality(self, device, dtype): + pass + + def test_exception(self, device, dtype): + # test for invalid params for different camera models + params = torch.tensor([1.0, 1.0, 1.0, 1.0, 1.0]) + image_size = ImageSize(100, 100) + with pytest.raises(ValueError): + CameraModel(image_size, CameraModelType.PINHOLE, params) + with pytest.raises(ValueError): + CameraModel(image_size, CameraModelType.BROWN_CONRADY, params) + with pytest.raises(ValueError): + CameraModel(image_size, CameraModelType.KANNALA_BRANDT_K3, params) + with pytest.raises(ValueError): + CameraModel(image_size, CameraModelType.ORTHOGRAPHIC, params) + + @pytest.mark.skip(reason="Unnecessary test") + def test_gradcheck(self, device): + pass + + @pytest.mark.skip(reason="Unnecessary test") + def test_jit(self, device, dtype): + pass + + @pytest.mark.skip(reason="Unnecessary test") + def test_module(self, device, dtype): + pass + + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_project_unproject(self, device, dtype, batch_size): + params, image_size = self._make_rand_data(batch_size, device, dtype) + cam = CameraModel(image_size, CameraModelType.PINHOLE, params) + points = torch.rand((batch_size, 3), device=device, dtype=dtype) + projected = cam.project(Vector3(points)) + unprojected = cam.unproject(projected, points[..., 2]) + self.assert_close(points, unprojected.data) + + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_matrix(self, device, dtype, batch_size): + params, image_size = self._make_rand_data(batch_size, device, dtype) + cam = CameraModel(image_size, CameraModelType.PINHOLE, params) + z = torch.zeros(batch_size, dtype=dtype, device=device) + o = torch.ones(batch_size, dtype=dtype, device=device) + K = torch.stack([params[:, 0], z, params[:, 2], z, params[:, 1], params[:, 3], z, z, o], dim=1).reshape( + batch_size, 3, 3 + ) + self.assert_close(cam.matrix(), K) + + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_properties(self, device, dtype, batch_size): + params, image_size = self._make_rand_data(batch_size, device, dtype) + cam = CameraModel(image_size, CameraModelType.PINHOLE, params) + self.assert_close(cam.fx, params[:, 0]) + self.assert_close(cam.fy, params[:, 1]) + self.assert_close(cam.cx, params[:, 2]) + self.assert_close(cam.cy, params[:, 3]) + self.assert_close(cam.width, image_size.width) + self.assert_close(cam.height, image_size.height) + + @pytest.mark.parametrize("batch_size", [1, 2, 5]) + def test_scale(self, device, dtype, batch_size): + params, image_size = self._make_rand_data(batch_size, device, dtype) + cam = CameraModel(image_size, CameraModelType.PINHOLE, params) + scale = torch.rand(batch_size, device=device, dtype=dtype) + scaled_cam = cam.scale(scale) + self.assert_close(cam.fx * scale, scaled_cam.fx) + self.assert_close(cam.fy * scale, scaled_cam.fy) + self.assert_close(cam.cx * scale, scaled_cam.cx) + self.assert_close(cam.cy * scale, scaled_cam.cy) + self.assert_close(cam.width * scale, scaled_cam.width) + self.assert_close(cam.height * scale, scaled_cam.height) diff --git a/tests/sensors/camera/test_distortion_model.py b/tests/sensors/camera/test_distortion_model.py new file mode 100644 index 0000000..74637f9 --- /dev/null +++ b/tests/sensors/camera/test_distortion_model.py @@ -0,0 +1,68 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.geometry.vector import Vector2 +from kornia.sensors.camera.distortion_model import AffineTransform + +from testing.base import BaseTester + + +class TestAffineTransform(BaseTester): + @pytest.mark.skip(reason="Unnecessary test") + def test_smoke(self, device, dtype): + pass + + @pytest.mark.skip(reason="Unnecessary test") + def test_cardinality(self, device, dtype): + pass + + @pytest.mark.skip(reason="Unnecessary test") + def test_exception(self, device, dtype): + pass + + @pytest.mark.skip(reason="Unnecessary test") + def test_gradcheck(self, device): + pass + + @pytest.mark.skip(reason="Unnecessary test") + def test_jit(self, device, dtype): + pass + + @pytest.mark.skip(reason="Unnecessary test") + def test_module(self, device, dtype): + pass + + def test_distort(self, device, dtype): + distortion = AffineTransform() + points = torch.tensor([[1.0, 1.0], [1.0, 5.0], [2.0, 4.0], [3.0, 9.0]], device=device, dtype=dtype) + params = torch.tensor([[328.0, 328.0, 150.0, 150.0]], device=device, dtype=dtype) + expected = torch.tensor( + [[478.0, 478.0], [478.0, 1790.0], [806.0, 1462.0], [1134.0, 3102.0]], device=device, dtype=dtype + ) + self.assert_close(distortion.distort(params, Vector2(points)).data, expected) + + def test_undistort(self, device, dtype): + distortion = AffineTransform() + points = torch.tensor( + [[478.0, 478.0], [478.0, 1790.0], [806.0, 1462.0], [1134.0, 3102.0]], device=device, dtype=dtype + ) + params = torch.tensor([[328.0, 328.0, 150.0, 150.0]], device=device, dtype=dtype) + expected = torch.tensor([[1.0, 1.0], [1.0, 5.0], [2.0, 4.0], [3.0, 9.0]], device=device, dtype=dtype) + self.assert_close(distortion.undistort(params, Vector2(points)).data, expected) diff --git a/tests/sensors/camera/test_projection_model.py b/tests/sensors/camera/test_projection_model.py new file mode 100644 index 0000000..29d6fa4 --- /dev/null +++ b/tests/sensors/camera/test_projection_model.py @@ -0,0 +1,76 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + +from kornia.geometry.vector import Vector2, Vector3 +from kornia.sensors.camera.projection_model import Z1Projection + +from testing.base import BaseTester + + +class TestProjection(BaseTester): + @pytest.mark.skip(reason="Unnecessary test") + def test_smoke(self, device, dtype): + pass + + @pytest.mark.skip(reason="Unnecessary test") + def test_cardinality(self, device, dtype): + pass + + @pytest.mark.skip(reason="Unnecessary test") + def test_exception(self, device, dtype): + pass + + @pytest.mark.skip(reason="Unnecessary test") + def test_gradcheck(self, device): + pass + + @pytest.mark.skip(reason="Unnecessary test") + def test_jit(self, device, dtype): + pass + + @pytest.mark.skip(reason="Unnecessary test") + def test_module(self, device, dtype): + pass + + def test_project(self, device, dtype): + projection = Z1Projection() + points = torch.tensor( + [[0.0, 0.0, 1.0], [1.0, 1.0, 1.0], [6.0, 6.0, 2.0], [9.0, 9.0, 3.0]], + device=device, + dtype=dtype, + ) + expected = torch.tensor([[0.0, 0.0], [1.0, 1.0], [3.0, 3.0], [3.0, 3.0]], device=device, dtype=dtype) + self.assert_close(projection.project(Vector3(points)).data, expected) + + def test_unproject(self, device, dtype): + projection = Z1Projection() + points = torch.tensor([[0.0, 0.0], [1.0, 1.0], [3.0, 3.0], [3.0, 3.0]], device=device, dtype=dtype) + expected = torch.tensor( + [[0.0, 0.0, 1.0], [1.0, 1.0, 1.0], [6.0, 6.0, 2.0], [9.0, 9.0, 3.0]], + device=device, + dtype=dtype, + ) + self.assert_close( + projection.unproject( + Vector2(points), + torch.tensor([1.0, 1.0, 2.0, 3.0], device=device, dtype=dtype), + ).data, + expected, + ) diff --git a/tests/smoke_test.py b/tests/smoke_test.py new file mode 100644 index 0000000..882b065 --- /dev/null +++ b/tests/smoke_test.py @@ -0,0 +1,25 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import torch + + +@pytest.mark.parametrize("batch_size", [1, 2, 5]) +def test_smoke(batch_size): + x = torch.rand(batch_size, 2, 3) + assert x.shape == (batch_size, 2, 3), x.shape diff --git a/tests/test_import.py b/tests/test_import.py new file mode 100644 index 0000000..5cc435b --- /dev/null +++ b/tests/test_import.py @@ -0,0 +1,53 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import subprocess +import sys +import textwrap + + +def test_import_without_jit_script_deprecation(): + """Importing kornia must not eagerly call ``torch.jit.script``. + + ``torch.jit.script`` is deprecated (and unsupported on Python 3.14+). Decorating + functions with ``@torch.jit.script`` runs the deprecated path at import time, which + makes ``python -W error -c "import kornia"`` fail. This guards against reintroducing + an import-time scripting call. See https://github.com/kornia/kornia/issues/3727. + + A fresh interpreter is used so the result is independent of kornia already being + imported by the test session. Only the ``torch.jit.script`` deprecation is promoted + to an error to avoid coupling the test to unrelated third-party warnings. + """ + script = textwrap.dedent( + """ + import warnings + + warnings.filterwarnings("error", message=r".*torch\\.jit\\.script.*") + warnings.filterwarnings("error", category=DeprecationWarning, module=r"kornia.*") + warnings.filterwarnings("error", category=FutureWarning, module=r"kornia.*") + + import kornia # noqa: F401 + """ + ) + # Trusted, fixed command (the current interpreter running a literal script); no external input. + result = subprocess.run( # noqa: S603 + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, f"importing kornia raised a deprecation warning:\n{result.stderr}" diff --git a/tests/tracking/test_planar_tracking.py b/tests/tracking/test_planar_tracking.py new file mode 100644 index 0000000..878d982 --- /dev/null +++ b/tests/tracking/test_planar_tracking.py @@ -0,0 +1,246 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from unittest.mock import MagicMock + +import pytest +import torch + +from kornia.core._compat import torch_version_le +from kornia.feature import DescriptorMatcher, GFTTAffNetHardNet, LocalFeatureMatcher, SIFTFeature +from kornia.geometry import rescale, transform_points +from kornia.tracking import HomographyTracker + +from testing.base import BaseTester + + +def _make_tracker(minimum_inliers_num: int = 5) -> HomographyTracker: + """Return a HomographyTracker whose heavy sub-modules are replaced with lightweight mocks.""" + initial_matcher = MagicMock() + fast_matcher = MagicMock() + ransac = MagicMock() + # Set extract_features to None so isinstance(..., nn.Module) guard skips feature pre-extraction + initial_matcher.extract_features = None + fast_matcher.extract_features = None + return HomographyTracker( + initial_matcher=initial_matcher, + fast_matcher=fast_matcher, + ransac=ransac, + minimum_inliers_num=minimum_inliers_num, + ) + + +def _match_dict(n_keypoints: int, device: torch.device, dtype: torch.dtype) -> dict: + """Produce a fake match dict with n_keypoints matches for batch 0.""" + return { + "keypoints0": torch.rand(n_keypoints, 2, device=device, dtype=dtype), + "keypoints1": torch.rand(n_keypoints, 2, device=device, dtype=dtype), + "batch_indexes": torch.zeros(n_keypoints, dtype=torch.long, device=device), + } + + +class TestHomographyTrackerUnit: + """Fast unit tests for HomographyTracker using mocked sub-modules.""" + + def test_reset_tracking_clears_homography(self): + tracker = _make_tracker() + image = torch.rand(1, 1, 8, 8) + tracker.set_target(image) + tracker.previous_homography = torch.eye(3) + tracker.reset_tracking() + assert tracker.previous_homography is None + + def test_set_target_without_extract_features(self): + tracker = _make_tracker() + image = torch.rand(1, 1, 8, 8) + tracker.set_target(image) + assert torch.equal(tracker.target, image) + # Representations stay empty when extract_features not present + assert tracker.target_initial_representation == {} + assert tracker.target_fast_representation == {} + + def test_set_target_with_extract_features(self): + from torch import nn + + fake_feats = {"desc": torch.rand(1, 4)} + + class _FakeExtract(nn.Module): + def forward(self, x): + return fake_feats + + initial_matcher = MagicMock() + fast_matcher = MagicMock() + initial_matcher.extract_features = _FakeExtract() + fast_matcher.extract_features = _FakeExtract() + tracker = HomographyTracker( + initial_matcher=initial_matcher, + fast_matcher=fast_matcher, + ransac=MagicMock(), + ) + image = torch.rand(1, 1, 8, 8) + tracker.set_target(image) + assert tracker.target_initial_representation == fake_feats + assert tracker.target_fast_representation == fake_feats + + def test_no_match_returns_empty_tensor_and_false(self): + tracker = _make_tracker() + image = torch.rand(1, 1, 8, 8) + tracker.set_target(image) + H, success = tracker.no_match() + assert not success + assert H.shape == (3, 3) + assert tracker.inliers_num == 0 + + def test_match_initial_too_few_keypoints(self): + # Fewer than minimum_inliers_num keypoints → no_match + tracker = _make_tracker(minimum_inliers_num=10) + image = torch.rand(1, 1, 8, 8) + tracker.set_target(image) + tracker.initial_matcher.return_value = _match_dict(3, torch.device("cpu"), torch.float32) + _, success = tracker.match_initial(torch.rand(1, 1, 8, 8)) + assert not success + + def test_match_initial_too_few_inliers(self): + # Enough keypoints but RANSAC reports few inliers → no_match + tracker = _make_tracker(minimum_inliers_num=5) + image = torch.rand(1, 1, 8, 8) + tracker.set_target(image) + tracker.initial_matcher.return_value = _match_dict(20, torch.device("cpu"), torch.float32) + # RANSAC returns homography + inlier mask with only 2 inliers + inliers = torch.zeros(20, dtype=torch.bool) + inliers[:2] = True + tracker.ransac.return_value = (torch.eye(3), inliers) + _, success = tracker.match_initial(torch.rand(1, 1, 8, 8)) + assert not success + + def test_match_initial_success(self): + tracker = _make_tracker(minimum_inliers_num=5) + image = torch.rand(1, 1, 8, 8) + tracker.set_target(image) + tracker.initial_matcher.return_value = _match_dict(20, torch.device("cpu"), torch.float32) + inliers = torch.ones(20, dtype=torch.bool) + H_expected = torch.eye(3) * 2 + tracker.ransac.return_value = (H_expected, inliers) + H, success = tracker.match_initial(torch.rand(1, 1, 8, 8)) + assert success + assert tracker.previous_homography is not None + assert torch.allclose(H, H_expected) + + def test_forward_routes_to_match_initial_when_no_previous(self): + tracker = _make_tracker(minimum_inliers_num=5) + image = torch.rand(1, 1, 8, 8) + tracker.set_target(image) + tracker.initial_matcher.return_value = _match_dict(20, torch.device("cpu"), torch.float32) + inliers = torch.ones(20, dtype=torch.bool) + tracker.ransac.return_value = (torch.eye(3), inliers) + assert tracker.previous_homography is None + _, success = tracker(torch.rand(1, 1, 8, 8)) + assert success + + def test_forward_routes_to_track_next_frame_when_previous_set(self): + tracker = _make_tracker(minimum_inliers_num=5) + image = torch.rand(1, 1, 8, 8) + tracker.set_target(image) + tracker.previous_homography = torch.eye(3) + tracker.fast_matcher.return_value = _match_dict(20, torch.device("cpu"), torch.float32) + inliers = torch.ones(20, dtype=torch.bool) + tracker.ransac.return_value = (torch.eye(3), inliers) + _, success = tracker(torch.rand(1, 1, 8, 8)) + assert success + + def test_track_next_frame_too_few_keypoints_resets(self): + tracker = _make_tracker(minimum_inliers_num=10) + image = torch.rand(1, 1, 8, 8) + tracker.set_target(image) + tracker.previous_homography = torch.eye(3) + tracker.fast_matcher.return_value = _match_dict(3, torch.device("cpu"), torch.float32) + _, success = tracker.track_next_frame(torch.rand(1, 1, 8, 8)) + assert not success + assert tracker.previous_homography is None # reset_tracking was called + + def test_track_next_frame_too_few_inliers_resets(self): + tracker = _make_tracker(minimum_inliers_num=5) + image = torch.rand(1, 1, 8, 8) + tracker.set_target(image) + tracker.previous_homography = torch.eye(3) + tracker.fast_matcher.return_value = _match_dict(20, torch.device("cpu"), torch.float32) + inliers = torch.zeros(20, dtype=torch.bool) + inliers[:2] = True + tracker.ransac.return_value = (torch.eye(3), inliers) + _, success = tracker.track_next_frame(torch.rand(1, 1, 8, 8)) + assert not success + assert tracker.previous_homography is None # reset_tracking was called + + +@pytest.fixture() +def data_url(): + url = "https://github.com/kornia/data_test/blob/main/loftr_outdoor_and_homography_data.pt?raw=true" + return url + + +class TestHomographyTracker(BaseTester): + @pytest.mark.slow + def test_smoke(self, device): + tracker = HomographyTracker().to(device) + assert tracker is not None + + @pytest.mark.slow + def test_nomatch(self, device, dtype, data_url): + data = torch.hub.load_state_dict_from_url(data_url) + + # This is not unit test, but that is quite good integration test + matcher = LocalFeatureMatcher(SIFTFeature(100), DescriptorMatcher("smnn", 0.95)).to(device, dtype) + tracker = HomographyTracker(matcher, matcher, minimum_inliers_num=100) + for k in data.keys(): + if isinstance(data[k], torch.Tensor): + data[k] = data[k].to(device, dtype) + tracker.set_target(data["image0"]) + torch.random.manual_seed(0) + _, success = tracker(torch.zeros_like(data["image0"])) + assert not success + + @pytest.mark.slow + @pytest.mark.skipif(torch_version_le(1, 9, 1), reason="Fails for bached torch.linalg.solve") + def test_real(self, device, dtype, data_url): + data = torch.hub.load_state_dict_from_url(data_url) + # This is not unit test, but that is quite good integration test + for k in data.keys(): + if isinstance(data[k], torch.Tensor): + data[k] = data[k].to(device, dtype) + + data["image0"] = rescale(data["image0"], 0.5, interpolation="bilinear", align_corners=False) + data["image1"] = rescale(data["image1"], 0.5, interpolation="bilinear", align_corners=False) + + matcher = LocalFeatureMatcher(GFTTAffNetHardNet(1000), DescriptorMatcher("snn", 0.8)).to(device, dtype) + torch.manual_seed(8) # issue kornia#2027 + tracker = HomographyTracker(matcher, matcher).to(device, dtype) + + with torch.no_grad(): + tracker.set_target(data["image0"]) + torch.manual_seed(8) # issue kornia#2027 + homography, success = tracker(data["image1"]) + assert success + pts_src = data["pts0"].to(device, dtype) / 2.0 + pts_dst = data["pts1"].to(device, dtype) / 2.0 + # Reprojection error of 5px is OK + self.assert_close(transform_points(homography[None], pts_src[None]), pts_dst[None], rtol=5e-2, atol=5) + + with torch.no_grad(): + torch.manual_seed(6) + homography, success = tracker(data["image1"]) + assert success + self.assert_close(transform_points(homography[None], pts_src[None]), pts_dst[None], rtol=5e-2, atol=5) diff --git a/tests/utils/test_deprecated.py b/tests/utils/test_deprecated.py new file mode 100644 index 0000000..522e41c --- /dev/null +++ b/tests/utils/test_deprecated.py @@ -0,0 +1,117 @@ +# LICENSE HEADER MANAGED BY add-license-header +# +# Copyright 2018 Kornia Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import warnings + +import numpy as np +import pytest +import torch + +from kornia import utils + + +class TestDeprecatedWrappers: + """Verify that every re-exported function in kornia.utils emits a DeprecationWarning.""" + + def test_create_meshgrid_warns(self): + with pytest.warns(DeprecationWarning, match="kornia.geometry.create_meshgrid"): + out = utils.create_meshgrid(4, 4) + assert out.shape == (1, 4, 4, 2) + + def test_create_meshgrid3d_warns(self): + with pytest.warns(DeprecationWarning, match="kornia.geometry.create_meshgrid3d"): + out = utils.create_meshgrid3d(2, 3, 4) + assert out.shape == (1, 2, 3, 4, 3) + + def test_draw_line_warns(self): + img = torch.zeros(3, 16, 16) + p1 = torch.tensor([0, 0]) + p2 = torch.tensor([7, 7]) + color = torch.tensor([1.0, 0.0, 0.0]) + with pytest.warns(DeprecationWarning, match="kornia.image.draw_line"): + out = utils.draw_line(img, p1, p2, color) + assert out.shape == img.shape + + def test_draw_rectangle_warns(self): + img = torch.zeros(1, 3, 16, 16) + rect = torch.tensor([[[2, 2, 10, 10]]], dtype=torch.float32) + with pytest.warns(DeprecationWarning, match="kornia.image.draw_rectangle"): + out = utils.draw_rectangle(img, rect) + assert out.shape == img.shape + + def test_draw_point2d_warns(self): + img = torch.zeros(1, 3, 16, 16) + points = torch.tensor([[4, 4]], dtype=torch.long) + color = torch.tensor([0.0, 1.0, 0.0]) + with pytest.warns(DeprecationWarning, match="kornia.image.draw_point2d"): + out = utils.draw_point2d(img[0], points, color) + assert out.shape == img[0].shape + + def test_draw_convex_polygon_warns(self): + img = torch.zeros(1, 3, 16, 16) + polygon = torch.tensor([[[2.0, 2.0], [14.0, 2.0], [14.0, 14.0], [2.0, 14.0]]]) + color = torch.tensor([[1.0, 0.0, 0.0]]) + with pytest.warns(DeprecationWarning, match="kornia.image.draw_convex_polygon"): + out = utils.draw_convex_polygon(img, polygon, color) + assert out.shape == img.shape + + def test_image_to_tensor_warns(self): + arr = np.zeros((8, 8, 3), dtype=np.uint8) + with pytest.warns(DeprecationWarning, match="kornia.image.image_to_tensor"): + t = utils.image_to_tensor(arr) + assert isinstance(t, torch.Tensor) + + def test_tensor_to_image_warns(self): + t = torch.zeros(3, 8, 8) + with pytest.warns(DeprecationWarning, match="kornia.image.tensor_to_image"): + arr = utils.tensor_to_image(t) + assert arr is not None + + def test_one_hot_warns(self): + labels = torch.tensor([0, 1, 2]) + with pytest.warns(DeprecationWarning, match="kornia.losses.one_hot"): + out = utils.one_hot(labels, num_classes=3, device=torch.device("cpu"), dtype=torch.float32) + assert out.shape == (3, 3) + + def test_all_wrappers_in_all(self): + expected = { + "create_meshgrid", + "create_meshgrid3d", + "draw_convex_polygon", + "draw_line", + "draw_point2d", + "draw_rectangle", + "image_to_string", + "image_to_tensor", + "load_pointcloud_ply", + "one_hot", + "print_image", + "save_pointcloud_ply", + "tensor_to_image", + } + assert expected.issubset(set(utils.__all__)) + + def test_multiple_calls_each_warn(self): + """Deprecation warning must be raised every time, not just the first call.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + utils.create_meshgrid(2, 2) + utils.create_meshgrid(2, 2) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert len(dep_warnings) == 2 diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..5a6c71b --- /dev/null +++ b/uv.lock @@ -0,0 +1,2694 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" +resolution-markers = [ + "python_full_version >= '3.13' and platform_machine != 's390x' and sys_platform == 'darwin'", + "python_full_version >= '3.13' and platform_machine == 's390x' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and platform_machine != 's390x' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and platform_machine == 's390x' and sys_platform == 'darwin'", + "python_full_version >= '3.13' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_machine != 's390x' and sys_platform == 'linux') or (python_full_version >= '3.13' and platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version >= '3.13' and platform_machine == 's390x' and sys_platform != 'darwin'", + "(python_full_version == '3.12.*' and platform_machine != 'aarch64' and platform_machine != 's390x' and sys_platform == 'linux') or (python_full_version == '3.12.*' and platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version == '3.12.*' and platform_machine == 's390x' and sys_platform != 'darwin'", + "python_full_version < '3.12' and platform_machine != 's390x' and sys_platform == 'darwin'", + "python_full_version < '3.12' and platform_machine == 's390x' and sys_platform == 'darwin'", + "python_full_version < '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version < '3.12' and platform_machine != 'aarch64' and platform_machine != 's390x' and sys_platform == 'linux') or (python_full_version < '3.12' and platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version < '3.12' and platform_machine == 's390x' and sys_platform != 'darwin'", +] + +[[package]] +name = "accessible-pygments" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bc/c1/bbac6a50d02774f91572938964c582fff4270eee73ab822a4aeea4d8b11b/accessible_pygments-0.0.5.tar.gz", hash = "sha256:40918d3e6a2b619ad424cb91e556bd3bd8865443d9f22f1dcdf79e33c8046872", size = 1377899, upload-time = "2024-05-10T11:23:10.216Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/3f/95338030883d8c8b91223b4e21744b04d11b161a3ef117295d8241f50ab4/accessible_pygments-0.0.5-py3-none-any.whl", hash = "sha256:88ae3211e68a1d0b011504b2ffc1691feafce124b845bd072ab6f9f66f34d4b7", size = 1395903, upload-time = "2024-05-10T11:23:08.421Z" }, +] + +[[package]] +name = "alabaster" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/f8/d9c74d0daf3f742840fd818d69cfae176fa332022fd44e3469487d5a9420/alabaster-1.0.0.tar.gz", hash = "sha256:c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e", size = 24210, upload-time = "2024-07-26T18:15:03.762Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b", size = 13929, upload-time = "2024-07-26T18:15:02.05Z" }, +] + +[[package]] +name = "annotated-doc" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, +] + +[[package]] +name = "anyio" +version = "4.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/ce/8a777047513153587e5434fd752e89334ac33e379aa3497db860eeb60377/anyio-4.12.0.tar.gz", hash = "sha256:73c693b567b0c55130c104d0b43a9baf3aa6a31fc6110116509f27bf75e21ec0", size = 228266, upload-time = "2025-11-28T23:37:38.911Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/9c/36c5c37947ebfb8c7f22e0eb6e4d188ee2d53aa3880f3f2744fb894f0cb1/anyio-4.12.0-py3-none-any.whl", hash = "sha256:dad2376a628f98eeca4881fc56cd06affd18f659b17a747d3ff0307ced94b1bb", size = 113362, upload-time = "2025-11-28T23:36:57.897Z" }, +] + +[[package]] +name = "astor" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/21/75b771132fee241dfe601d39ade629548a9626d1d39f333fde31bc46febe/astor-0.8.1.tar.gz", hash = "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e", size = 35090, upload-time = "2019-12-10T01:50:35.51Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/88/97eef84f48fa04fbd6750e62dcceafba6c63c81b7ac1420856c8dcc0a3f9/astor-0.8.1-py2.py3-none-any.whl", hash = "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5", size = 27488, upload-time = "2019-12-10T01:50:33.628Z" }, +] + +[[package]] +name = "babel" +version = "2.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/6b/d52e42361e1aa00709585ecc30b3f9684b3ab62530771402248b1b1d6240/babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d", size = 9951852, upload-time = "2025-02-01T15:17:41.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2", size = 10182537, upload-time = "2025-02-01T15:17:37.39Z" }, +] + +[[package]] +name = "beautifulsoup4" +version = "4.14.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c3/b0/1c6a16426d389813b48d95e26898aff79abbde42ad353958ad95cc8c9b21/beautifulsoup4-4.14.3.tar.gz", hash = "sha256:6292b1c5186d356bba669ef9f7f051757099565ad9ada5dd630bd9de5fa7fb86", size = 627737, upload-time = "2025-11-30T15:08:26.084Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" }, +] + +[[package]] +name = "certifi" +version = "2025.11.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/8c/58f469717fa48465e4a50c014a0400602d3c437d7c0c468e17ada824da3a/certifi-2025.11.12.tar.gz", hash = "sha256:d8ab5478f2ecd78af242878415affce761ca6bc54a22a27e026d7c25357c3316", size = 160538, upload-time = "2025-11-12T02:54:51.517Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/7d/9bc192684cea499815ff478dfcdc13835ddf401365057044fb721ec6bddb/certifi-2025.11.12-py3-none-any.whl", hash = "sha256:97de8790030bbd5c2d96b7ec782fc2f7820ef8dba6db909ccf95449f2d062d4b", size = 159438, upload-time = "2025-11-12T02:54:49.735Z" }, +] + +[[package]] +name = "cffi" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/4a/3dfd5f7850cbf0d06dc84ba9aa00db766b52ca38d8b86e3a38314d52498c/cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", size = 184344, upload-time = "2025-09-08T23:22:26.456Z" }, + { url = "https://files.pythonhosted.org/packages/4f/8b/f0e4c441227ba756aafbe78f117485b25bb26b1c059d01f137fa6d14896b/cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", size = 180560, upload-time = "2025-09-08T23:22:28.197Z" }, + { url = "https://files.pythonhosted.org/packages/b1/b7/1200d354378ef52ec227395d95c2576330fd22a869f7a70e88e1447eb234/cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", size = 209613, upload-time = "2025-09-08T23:22:29.475Z" }, + { url = "https://files.pythonhosted.org/packages/b8/56/6033f5e86e8cc9bb629f0077ba71679508bdf54a9a5e112a3c0b91870332/cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", size = 216476, upload-time = "2025-09-08T23:22:31.063Z" }, + { url = "https://files.pythonhosted.org/packages/dc/7f/55fecd70f7ece178db2f26128ec41430d8720f2d12ca97bf8f0a628207d5/cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5", size = 203374, upload-time = "2025-09-08T23:22:32.507Z" }, + { url = "https://files.pythonhosted.org/packages/84/ef/a7b77c8bdc0f77adc3b46888f1ad54be8f3b7821697a7b89126e829e676a/cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664", size = 202597, upload-time = "2025-09-08T23:22:34.132Z" }, + { url = "https://files.pythonhosted.org/packages/d7/91/500d892b2bf36529a75b77958edfcd5ad8e2ce4064ce2ecfeab2125d72d1/cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", size = 215574, upload-time = "2025-09-08T23:22:35.443Z" }, + { url = "https://files.pythonhosted.org/packages/44/64/58f6255b62b101093d5df22dcb752596066c7e89dd725e0afaed242a61be/cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", size = 218971, upload-time = "2025-09-08T23:22:36.805Z" }, + { url = "https://files.pythonhosted.org/packages/ab/49/fa72cebe2fd8a55fbe14956f9970fe8eb1ac59e5df042f603ef7c8ba0adc/cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", size = 211972, upload-time = "2025-09-08T23:22:38.436Z" }, + { url = "https://files.pythonhosted.org/packages/0b/28/dd0967a76aab36731b6ebfe64dec4e981aff7e0608f60c2d46b46982607d/cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", size = 217078, upload-time = "2025-09-08T23:22:39.776Z" }, + { url = "https://files.pythonhosted.org/packages/2b/c0/015b25184413d7ab0a410775fdb4a50fca20f5589b5dab1dbbfa3baad8ce/cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5", size = 172076, upload-time = "2025-09-08T23:22:40.95Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8f/dc5531155e7070361eb1b7e4c1a9d896d0cb21c49f807a6c03fd63fc877e/cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5", size = 182820, upload-time = "2025-09-08T23:22:42.463Z" }, + { url = "https://files.pythonhosted.org/packages/95/5c/1b493356429f9aecfd56bc171285a4c4ac8697f76e9bbbbb105e537853a1/cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d", size = 177635, upload-time = "2025-09-08T23:22:43.623Z" }, + { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, + { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, + { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, + { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, + { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, + { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, + { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, + { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, + { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, + { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230, upload-time = "2025-09-08T23:23:00.879Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043, upload-time = "2025-09-08T23:23:02.231Z" }, + { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" }, + { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" }, + { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" }, + { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" }, + { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909, upload-time = "2025-09-08T23:23:14.32Z" }, + { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402, upload-time = "2025-09-08T23:23:15.535Z" }, + { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780, upload-time = "2025-09-08T23:23:16.761Z" }, + { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320, upload-time = "2025-09-08T23:23:18.087Z" }, + { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487, upload-time = "2025-09-08T23:23:19.622Z" }, + { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" }, + { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" }, + { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" }, + { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" }, + { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" }, + { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328, upload-time = "2025-09-08T23:23:44.61Z" }, + { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650, upload-time = "2025-09-08T23:23:45.848Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687, upload-time = "2025-09-08T23:23:47.105Z" }, + { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773, upload-time = "2025-09-08T23:23:29.347Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013, upload-time = "2025-09-08T23:23:30.63Z" }, + { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" }, + { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" }, + { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" }, + { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" }, + { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" }, + { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487, upload-time = "2025-09-08T23:23:40.423Z" }, + { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726, upload-time = "2025-09-08T23:23:41.742Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, +] + +[[package]] +name = "cfgv" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132", size = 7334, upload-time = "2025-11-19T20:55:51.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/69/33ddede1939fdd074bce5434295f38fae7136463422fe4fd3e0e89b98062/charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", size = 129418, upload-time = "2025-10-14T04:42:32.879Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/27/c6491ff4954e58a10f69ad90aca8a1b6fe9c5d3c6f380907af3c37435b59/charset_normalizer-3.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8", size = 206988, upload-time = "2025-10-14T04:40:33.79Z" }, + { url = "https://files.pythonhosted.org/packages/94/59/2e87300fe67ab820b5428580a53cad894272dbb97f38a7a814a2a1ac1011/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0", size = 147324, upload-time = "2025-10-14T04:40:34.961Z" }, + { url = "https://files.pythonhosted.org/packages/07/fb/0cf61dc84b2b088391830f6274cb57c82e4da8bbc2efeac8c025edb88772/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3", size = 142742, upload-time = "2025-10-14T04:40:36.105Z" }, + { url = "https://files.pythonhosted.org/packages/62/8b/171935adf2312cd745d290ed93cf16cf0dfe320863ab7cbeeae1dcd6535f/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc", size = 160863, upload-time = "2025-10-14T04:40:37.188Z" }, + { url = "https://files.pythonhosted.org/packages/09/73/ad875b192bda14f2173bfc1bc9a55e009808484a4b256748d931b6948442/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897", size = 157837, upload-time = "2025-10-14T04:40:38.435Z" }, + { url = "https://files.pythonhosted.org/packages/6d/fc/de9cce525b2c5b94b47c70a4b4fb19f871b24995c728e957ee68ab1671ea/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381", size = 151550, upload-time = "2025-10-14T04:40:40.053Z" }, + { url = "https://files.pythonhosted.org/packages/55/c2/43edd615fdfba8c6f2dfbd459b25a6b3b551f24ea21981e23fb768503ce1/charset_normalizer-3.4.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815", size = 149162, upload-time = "2025-10-14T04:40:41.163Z" }, + { url = "https://files.pythonhosted.org/packages/03/86/bde4ad8b4d0e9429a4e82c1e8f5c659993a9a863ad62c7df05cf7b678d75/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0", size = 150019, upload-time = "2025-10-14T04:40:42.276Z" }, + { url = "https://files.pythonhosted.org/packages/1f/86/a151eb2af293a7e7bac3a739b81072585ce36ccfb4493039f49f1d3cae8c/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161", size = 143310, upload-time = "2025-10-14T04:40:43.439Z" }, + { url = "https://files.pythonhosted.org/packages/b5/fe/43dae6144a7e07b87478fdfc4dbe9efd5defb0e7ec29f5f58a55aeef7bf7/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4", size = 162022, upload-time = "2025-10-14T04:40:44.547Z" }, + { url = "https://files.pythonhosted.org/packages/80/e6/7aab83774f5d2bca81f42ac58d04caf44f0cc2b65fc6db2b3b2e8a05f3b3/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89", size = 149383, upload-time = "2025-10-14T04:40:46.018Z" }, + { url = "https://files.pythonhosted.org/packages/4f/e8/b289173b4edae05c0dde07f69f8db476a0b511eac556dfe0d6bda3c43384/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569", size = 159098, upload-time = "2025-10-14T04:40:47.081Z" }, + { url = "https://files.pythonhosted.org/packages/d8/df/fe699727754cae3f8478493c7f45f777b17c3ef0600e28abfec8619eb49c/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224", size = 152991, upload-time = "2025-10-14T04:40:48.246Z" }, + { url = "https://files.pythonhosted.org/packages/1a/86/584869fe4ddb6ffa3bd9f491b87a01568797fb9bd8933f557dba9771beaf/charset_normalizer-3.4.4-cp311-cp311-win32.whl", hash = "sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a", size = 99456, upload-time = "2025-10-14T04:40:49.376Z" }, + { url = "https://files.pythonhosted.org/packages/65/f6/62fdd5feb60530f50f7e38b4f6a1d5203f4d16ff4f9f0952962c044e919a/charset_normalizer-3.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016", size = 106978, upload-time = "2025-10-14T04:40:50.844Z" }, + { url = "https://files.pythonhosted.org/packages/7a/9d/0710916e6c82948b3be62d9d398cb4fcf4e97b56d6a6aeccd66c4b2f2bd5/charset_normalizer-3.4.4-cp311-cp311-win_arm64.whl", hash = "sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1", size = 99969, upload-time = "2025-10-14T04:40:52.272Z" }, + { url = "https://files.pythonhosted.org/packages/f3/85/1637cd4af66fa687396e757dec650f28025f2a2f5a5531a3208dc0ec43f2/charset_normalizer-3.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394", size = 208425, upload-time = "2025-10-14T04:40:53.353Z" }, + { url = "https://files.pythonhosted.org/packages/9d/6a/04130023fef2a0d9c62d0bae2649b69f7b7d8d24ea5536feef50551029df/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25", size = 148162, upload-time = "2025-10-14T04:40:54.558Z" }, + { url = "https://files.pythonhosted.org/packages/78/29/62328d79aa60da22c9e0b9a66539feae06ca0f5a4171ac4f7dc285b83688/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef", size = 144558, upload-time = "2025-10-14T04:40:55.677Z" }, + { url = "https://files.pythonhosted.org/packages/86/bb/b32194a4bf15b88403537c2e120b817c61cd4ecffa9b6876e941c3ee38fe/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d", size = 161497, upload-time = "2025-10-14T04:40:57.217Z" }, + { url = "https://files.pythonhosted.org/packages/19/89/a54c82b253d5b9b111dc74aca196ba5ccfcca8242d0fb64146d4d3183ff1/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8", size = 159240, upload-time = "2025-10-14T04:40:58.358Z" }, + { url = "https://files.pythonhosted.org/packages/c0/10/d20b513afe03acc89ec33948320a5544d31f21b05368436d580dec4e234d/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86", size = 153471, upload-time = "2025-10-14T04:40:59.468Z" }, + { url = "https://files.pythonhosted.org/packages/61/fa/fbf177b55bdd727010f9c0a3c49eefa1d10f960e5f09d1d887bf93c2e698/charset_normalizer-3.4.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a", size = 150864, upload-time = "2025-10-14T04:41:00.623Z" }, + { url = "https://files.pythonhosted.org/packages/05/12/9fbc6a4d39c0198adeebbde20b619790e9236557ca59fc40e0e3cebe6f40/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f", size = 150647, upload-time = "2025-10-14T04:41:01.754Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/6a9a593d52e3e8c5d2b167daf8c6b968808efb57ef4c210acb907c365bc4/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc", size = 145110, upload-time = "2025-10-14T04:41:03.231Z" }, + { url = "https://files.pythonhosted.org/packages/30/42/9a52c609e72471b0fc54386dc63c3781a387bb4fe61c20231a4ebcd58bdd/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf", size = 162839, upload-time = "2025-10-14T04:41:04.715Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5b/c0682bbf9f11597073052628ddd38344a3d673fda35a36773f7d19344b23/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15", size = 150667, upload-time = "2025-10-14T04:41:05.827Z" }, + { url = "https://files.pythonhosted.org/packages/e4/24/a41afeab6f990cf2daf6cb8c67419b63b48cf518e4f56022230840c9bfb2/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9", size = 160535, upload-time = "2025-10-14T04:41:06.938Z" }, + { url = "https://files.pythonhosted.org/packages/2a/e5/6a4ce77ed243c4a50a1fecca6aaaab419628c818a49434be428fe24c9957/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0", size = 154816, upload-time = "2025-10-14T04:41:08.101Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ef/89297262b8092b312d29cdb2517cb1237e51db8ecef2e9af5edbe7b683b1/charset_normalizer-3.4.4-cp312-cp312-win32.whl", hash = "sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26", size = 99694, upload-time = "2025-10-14T04:41:09.23Z" }, + { url = "https://files.pythonhosted.org/packages/3d/2d/1e5ed9dd3b3803994c155cd9aacb60c82c331bad84daf75bcb9c91b3295e/charset_normalizer-3.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525", size = 107131, upload-time = "2025-10-14T04:41:10.467Z" }, + { url = "https://files.pythonhosted.org/packages/d0/d9/0ed4c7098a861482a7b6a95603edce4c0d9db2311af23da1fb2b75ec26fc/charset_normalizer-3.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3", size = 100390, upload-time = "2025-10-14T04:41:11.915Z" }, + { url = "https://files.pythonhosted.org/packages/97/45/4b3a1239bbacd321068ea6e7ac28875b03ab8bc0aa0966452db17cd36714/charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794", size = 208091, upload-time = "2025-10-14T04:41:13.346Z" }, + { url = "https://files.pythonhosted.org/packages/7d/62/73a6d7450829655a35bb88a88fca7d736f9882a27eacdca2c6d505b57e2e/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed", size = 147936, upload-time = "2025-10-14T04:41:14.461Z" }, + { url = "https://files.pythonhosted.org/packages/89/c5/adb8c8b3d6625bef6d88b251bbb0d95f8205831b987631ab0c8bb5d937c2/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72", size = 144180, upload-time = "2025-10-14T04:41:15.588Z" }, + { url = "https://files.pythonhosted.org/packages/91/ed/9706e4070682d1cc219050b6048bfd293ccf67b3d4f5a4f39207453d4b99/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328", size = 161346, upload-time = "2025-10-14T04:41:16.738Z" }, + { url = "https://files.pythonhosted.org/packages/d5/0d/031f0d95e4972901a2f6f09ef055751805ff541511dc1252ba3ca1f80cf5/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede", size = 158874, upload-time = "2025-10-14T04:41:17.923Z" }, + { url = "https://files.pythonhosted.org/packages/f5/83/6ab5883f57c9c801ce5e5677242328aa45592be8a00644310a008d04f922/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894", size = 153076, upload-time = "2025-10-14T04:41:19.106Z" }, + { url = "https://files.pythonhosted.org/packages/75/1e/5ff781ddf5260e387d6419959ee89ef13878229732732ee73cdae01800f2/charset_normalizer-3.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1", size = 150601, upload-time = "2025-10-14T04:41:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/d7/57/71be810965493d3510a6ca79b90c19e48696fb1ff964da319334b12677f0/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490", size = 150376, upload-time = "2025-10-14T04:41:21.398Z" }, + { url = "https://files.pythonhosted.org/packages/e5/d5/c3d057a78c181d007014feb7e9f2e65905a6c4ef182c0ddf0de2924edd65/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44", size = 144825, upload-time = "2025-10-14T04:41:22.583Z" }, + { url = "https://files.pythonhosted.org/packages/e6/8c/d0406294828d4976f275ffbe66f00266c4b3136b7506941d87c00cab5272/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133", size = 162583, upload-time = "2025-10-14T04:41:23.754Z" }, + { url = "https://files.pythonhosted.org/packages/d7/24/e2aa1f18c8f15c4c0e932d9287b8609dd30ad56dbe41d926bd846e22fb8d/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3", size = 150366, upload-time = "2025-10-14T04:41:25.27Z" }, + { url = "https://files.pythonhosted.org/packages/e4/5b/1e6160c7739aad1e2df054300cc618b06bf784a7a164b0f238360721ab86/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e", size = 160300, upload-time = "2025-10-14T04:41:26.725Z" }, + { url = "https://files.pythonhosted.org/packages/7a/10/f882167cd207fbdd743e55534d5d9620e095089d176d55cb22d5322f2afd/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc", size = 154465, upload-time = "2025-10-14T04:41:28.322Z" }, + { url = "https://files.pythonhosted.org/packages/89/66/c7a9e1b7429be72123441bfdbaf2bc13faab3f90b933f664db506dea5915/charset_normalizer-3.4.4-cp313-cp313-win32.whl", hash = "sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac", size = 99404, upload-time = "2025-10-14T04:41:29.95Z" }, + { url = "https://files.pythonhosted.org/packages/c4/26/b9924fa27db384bdcd97ab83b4f0a8058d96ad9626ead570674d5e737d90/charset_normalizer-3.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14", size = 107092, upload-time = "2025-10-14T04:41:31.188Z" }, + { url = "https://files.pythonhosted.org/packages/af/8f/3ed4bfa0c0c72a7ca17f0380cd9e4dd842b09f664e780c13cff1dcf2ef1b/charset_normalizer-3.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2", size = 100408, upload-time = "2025-10-14T04:41:32.624Z" }, + { url = "https://files.pythonhosted.org/packages/2a/35/7051599bd493e62411d6ede36fd5af83a38f37c4767b92884df7301db25d/charset_normalizer-3.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd", size = 207746, upload-time = "2025-10-14T04:41:33.773Z" }, + { url = "https://files.pythonhosted.org/packages/10/9a/97c8d48ef10d6cd4fcead2415523221624bf58bcf68a802721a6bc807c8f/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb", size = 147889, upload-time = "2025-10-14T04:41:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/979224a919a1b606c82bd2c5fa49b5c6d5727aa47b4312bb27b1734f53cd/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e", size = 143641, upload-time = "2025-10-14T04:41:36.116Z" }, + { url = "https://files.pythonhosted.org/packages/ba/33/0ad65587441fc730dc7bd90e9716b30b4702dc7b617e6ba4997dc8651495/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14", size = 160779, upload-time = "2025-10-14T04:41:37.229Z" }, + { url = "https://files.pythonhosted.org/packages/67/ed/331d6b249259ee71ddea93f6f2f0a56cfebd46938bde6fcc6f7b9a3d0e09/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191", size = 159035, upload-time = "2025-10-14T04:41:38.368Z" }, + { url = "https://files.pythonhosted.org/packages/67/ff/f6b948ca32e4f2a4576aa129d8bed61f2e0543bf9f5f2b7fc3758ed005c9/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838", size = 152542, upload-time = "2025-10-14T04:41:39.862Z" }, + { url = "https://files.pythonhosted.org/packages/16/85/276033dcbcc369eb176594de22728541a925b2632f9716428c851b149e83/charset_normalizer-3.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6", size = 149524, upload-time = "2025-10-14T04:41:41.319Z" }, + { url = "https://files.pythonhosted.org/packages/9e/f2/6a2a1f722b6aba37050e626530a46a68f74e63683947a8acff92569f979a/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e", size = 150395, upload-time = "2025-10-14T04:41:42.539Z" }, + { url = "https://files.pythonhosted.org/packages/60/bb/2186cb2f2bbaea6338cad15ce23a67f9b0672929744381e28b0592676824/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c", size = 143680, upload-time = "2025-10-14T04:41:43.661Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a5/bf6f13b772fbb2a90360eb620d52ed8f796f3c5caee8398c3b2eb7b1c60d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090", size = 162045, upload-time = "2025-10-14T04:41:44.821Z" }, + { url = "https://files.pythonhosted.org/packages/df/c5/d1be898bf0dc3ef9030c3825e5d3b83f2c528d207d246cbabe245966808d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152", size = 149687, upload-time = "2025-10-14T04:41:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/a5/42/90c1f7b9341eef50c8a1cb3f098ac43b0508413f33affd762855f67a410e/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828", size = 160014, upload-time = "2025-10-14T04:41:47.631Z" }, + { url = "https://files.pythonhosted.org/packages/76/be/4d3ee471e8145d12795ab655ece37baed0929462a86e72372fd25859047c/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec", size = 154044, upload-time = "2025-10-14T04:41:48.81Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6f/8f7af07237c34a1defe7defc565a9bc1807762f672c0fde711a4b22bf9c0/charset_normalizer-3.4.4-cp314-cp314-win32.whl", hash = "sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9", size = 99940, upload-time = "2025-10-14T04:41:49.946Z" }, + { url = "https://files.pythonhosted.org/packages/4b/51/8ade005e5ca5b0d80fb4aff72a3775b325bdc3d27408c8113811a7cbe640/charset_normalizer-3.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c", size = 107104, upload-time = "2025-10-14T04:41:51.051Z" }, + { url = "https://files.pythonhosted.org/packages/da/5f/6b8f83a55bb8278772c5ae54a577f3099025f9ade59d0136ac24a0df4bde/charset_normalizer-3.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2", size = 100743, upload-time = "2025-10-14T04:41:52.122Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" }, +] + +[[package]] +name = "click" +version = "8.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/75/31212c6bf2503fdf920d87fee5d7a86a2e3bcf444984126f13d8e4016804/click-8.3.2.tar.gz", hash = "sha256:14162b8b3b3550a7d479eafa77dfd3c38d9dc8951f6f69c78913a8f9a7540fd5", size = 302856, upload-time = "2026-04-03T19:14:45.118Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/20/71885d8b97d4f3dde17b1fdb92dbd4908b00541c5a3379787137285f602e/click-8.3.2-py3-none-any.whl", hash = "sha256:1924d2c27c5653561cd2cae4548d1406039cb79b858b747cfea24924bbc1616d", size = 108379, upload-time = "2026-04-03T19:14:43.505Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coloredlogs" +version = "15.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "humanfriendly" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/eed8f27100517e8c0e6b923d5f0845d0cb99763da6fdee00478f91db7325/coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0", size = 278520, upload-time = "2021-06-11T10:22:45.202Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934", size = 46018, upload-time = "2021-06-11T10:22:42.561Z" }, +] + +[[package]] +name = "contourpy" +version = "1.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/2e/c4390a31919d8a78b90e8ecf87cd4b4c4f05a5b48d05ec17db8e5404c6f4/contourpy-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:709a48ef9a690e1343202916450bc48b9e51c049b089c7f79a267b46cffcdaa1", size = 288773, upload-time = "2025-07-26T12:01:02.277Z" }, + { url = "https://files.pythonhosted.org/packages/0d/44/c4b0b6095fef4dc9c420e041799591e3b63e9619e3044f7f4f6c21c0ab24/contourpy-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:23416f38bfd74d5d28ab8429cc4d63fa67d5068bd711a85edb1c3fb0c3e2f381", size = 270149, upload-time = "2025-07-26T12:01:04.072Z" }, + { url = "https://files.pythonhosted.org/packages/30/2e/dd4ced42fefac8470661d7cb7e264808425e6c5d56d175291e93890cce09/contourpy-1.3.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:929ddf8c4c7f348e4c0a5a3a714b5c8542ffaa8c22954862a46ca1813b667ee7", size = 329222, upload-time = "2025-07-26T12:01:05.688Z" }, + { url = "https://files.pythonhosted.org/packages/f2/74/cc6ec2548e3d276c71389ea4802a774b7aa3558223b7bade3f25787fafc2/contourpy-1.3.3-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e999574eddae35f1312c2b4b717b7885d4edd6cb46700e04f7f02db454e67c1", size = 377234, upload-time = "2025-07-26T12:01:07.054Z" }, + { url = "https://files.pythonhosted.org/packages/03/b3/64ef723029f917410f75c09da54254c5f9ea90ef89b143ccadb09df14c15/contourpy-1.3.3-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0bf67e0e3f482cb69779dd3061b534eb35ac9b17f163d851e2a547d56dba0a3a", size = 380555, upload-time = "2025-07-26T12:01:08.801Z" }, + { url = "https://files.pythonhosted.org/packages/5f/4b/6157f24ca425b89fe2eb7e7be642375711ab671135be21e6faa100f7448c/contourpy-1.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51e79c1f7470158e838808d4a996fa9bac72c498e93d8ebe5119bc1e6becb0db", size = 355238, upload-time = "2025-07-26T12:01:10.319Z" }, + { url = "https://files.pythonhosted.org/packages/98/56/f914f0dd678480708a04cfd2206e7c382533249bc5001eb9f58aa693e200/contourpy-1.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:598c3aaece21c503615fd59c92a3598b428b2f01bfb4b8ca9c4edeecc2438620", size = 1326218, upload-time = "2025-07-26T12:01:12.659Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d7/4a972334a0c971acd5172389671113ae82aa7527073980c38d5868ff1161/contourpy-1.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:322ab1c99b008dad206d406bb61d014cf0174df491ae9d9d0fac6a6fda4f977f", size = 1392867, upload-time = "2025-07-26T12:01:15.533Z" }, + { url = "https://files.pythonhosted.org/packages/75/3e/f2cc6cd56dc8cff46b1a56232eabc6feea52720083ea71ab15523daab796/contourpy-1.3.3-cp311-cp311-win32.whl", hash = "sha256:fd907ae12cd483cd83e414b12941c632a969171bf90fc937d0c9f268a31cafff", size = 183677, upload-time = "2025-07-26T12:01:17.088Z" }, + { url = "https://files.pythonhosted.org/packages/98/4b/9bd370b004b5c9d8045c6c33cf65bae018b27aca550a3f657cdc99acdbd8/contourpy-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:3519428f6be58431c56581f1694ba8e50626f2dd550af225f82fb5f5814d2a42", size = 225234, upload-time = "2025-07-26T12:01:18.256Z" }, + { url = "https://files.pythonhosted.org/packages/d9/b6/71771e02c2e004450c12b1120a5f488cad2e4d5b590b1af8bad060360fe4/contourpy-1.3.3-cp311-cp311-win_arm64.whl", hash = "sha256:15ff10bfada4bf92ec8b31c62bf7c1834c244019b4a33095a68000d7075df470", size = 193123, upload-time = "2025-07-26T12:01:19.848Z" }, + { url = "https://files.pythonhosted.org/packages/be/45/adfee365d9ea3d853550b2e735f9d66366701c65db7855cd07621732ccfc/contourpy-1.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b08a32ea2f8e42cf1d4be3169a98dd4be32bafe4f22b6c4cb4ba810fa9e5d2cb", size = 293419, upload-time = "2025-07-26T12:01:21.16Z" }, + { url = "https://files.pythonhosted.org/packages/53/3e/405b59cfa13021a56bba395a6b3aca8cec012b45bf177b0eaf7a202cde2c/contourpy-1.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:556dba8fb6f5d8742f2923fe9457dbdd51e1049c4a43fd3986a0b14a1d815fc6", size = 273979, upload-time = "2025-07-26T12:01:22.448Z" }, + { url = "https://files.pythonhosted.org/packages/d4/1c/a12359b9b2ca3a845e8f7f9ac08bdf776114eb931392fcad91743e2ea17b/contourpy-1.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92d9abc807cf7d0e047b95ca5d957cf4792fcd04e920ca70d48add15c1a90ea7", size = 332653, upload-time = "2025-07-26T12:01:24.155Z" }, + { url = "https://files.pythonhosted.org/packages/63/12/897aeebfb475b7748ea67b61e045accdfcf0d971f8a588b67108ed7f5512/contourpy-1.3.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2e8faa0ed68cb29af51edd8e24798bb661eac3bd9f65420c1887b6ca89987c8", size = 379536, upload-time = "2025-07-26T12:01:25.91Z" }, + { url = "https://files.pythonhosted.org/packages/43/8a/a8c584b82deb248930ce069e71576fc09bd7174bbd35183b7943fb1064fd/contourpy-1.3.3-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:626d60935cf668e70a5ce6ff184fd713e9683fb458898e4249b63be9e28286ea", size = 384397, upload-time = "2025-07-26T12:01:27.152Z" }, + { url = "https://files.pythonhosted.org/packages/cc/8f/ec6289987824b29529d0dfda0d74a07cec60e54b9c92f3c9da4c0ac732de/contourpy-1.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d00e655fcef08aba35ec9610536bfe90267d7ab5ba944f7032549c55a146da1", size = 362601, upload-time = "2025-07-26T12:01:28.808Z" }, + { url = "https://files.pythonhosted.org/packages/05/0a/a3fe3be3ee2dceb3e615ebb4df97ae6f3828aa915d3e10549ce016302bd1/contourpy-1.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:451e71b5a7d597379ef572de31eeb909a87246974d960049a9848c3bc6c41bf7", size = 1331288, upload-time = "2025-07-26T12:01:31.198Z" }, + { url = "https://files.pythonhosted.org/packages/33/1d/acad9bd4e97f13f3e2b18a3977fe1b4a37ecf3d38d815333980c6c72e963/contourpy-1.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:459c1f020cd59fcfe6650180678a9993932d80d44ccde1fa1868977438f0b411", size = 1403386, upload-time = "2025-07-26T12:01:33.947Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8f/5847f44a7fddf859704217a99a23a4f6417b10e5ab1256a179264561540e/contourpy-1.3.3-cp312-cp312-win32.whl", hash = "sha256:023b44101dfe49d7d53932be418477dba359649246075c996866106da069af69", size = 185018, upload-time = "2025-07-26T12:01:35.64Z" }, + { url = "https://files.pythonhosted.org/packages/19/e8/6026ed58a64563186a9ee3f29f41261fd1828f527dd93d33b60feca63352/contourpy-1.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:8153b8bfc11e1e4d75bcb0bff1db232f9e10b274e0929de9d608027e0d34ff8b", size = 226567, upload-time = "2025-07-26T12:01:36.804Z" }, + { url = "https://files.pythonhosted.org/packages/d1/e2/f05240d2c39a1ed228d8328a78b6f44cd695f7ef47beb3e684cf93604f86/contourpy-1.3.3-cp312-cp312-win_arm64.whl", hash = "sha256:07ce5ed73ecdc4a03ffe3e1b3e3c1166db35ae7584be76f65dbbe28a7791b0cc", size = 193655, upload-time = "2025-07-26T12:01:37.999Z" }, + { url = "https://files.pythonhosted.org/packages/68/35/0167aad910bbdb9599272bd96d01a9ec6852f36b9455cf2ca67bd4cc2d23/contourpy-1.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:177fb367556747a686509d6fef71d221a4b198a3905fe824430e5ea0fda54eb5", size = 293257, upload-time = "2025-07-26T12:01:39.367Z" }, + { url = "https://files.pythonhosted.org/packages/96/e4/7adcd9c8362745b2210728f209bfbcf7d91ba868a2c5f40d8b58f54c509b/contourpy-1.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d002b6f00d73d69333dac9d0b8d5e84d9724ff9ef044fd63c5986e62b7c9e1b1", size = 274034, upload-time = "2025-07-26T12:01:40.645Z" }, + { url = "https://files.pythonhosted.org/packages/73/23/90e31ceeed1de63058a02cb04b12f2de4b40e3bef5e082a7c18d9c8ae281/contourpy-1.3.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:348ac1f5d4f1d66d3322420f01d42e43122f43616e0f194fc1c9f5d830c5b286", size = 334672, upload-time = "2025-07-26T12:01:41.942Z" }, + { url = "https://files.pythonhosted.org/packages/ed/93/b43d8acbe67392e659e1d984700e79eb67e2acb2bd7f62012b583a7f1b55/contourpy-1.3.3-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:655456777ff65c2c548b7c454af9c6f33f16c8884f11083244b5819cc214f1b5", size = 381234, upload-time = "2025-07-26T12:01:43.499Z" }, + { url = "https://files.pythonhosted.org/packages/46/3b/bec82a3ea06f66711520f75a40c8fc0b113b2a75edb36aa633eb11c4f50f/contourpy-1.3.3-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:644a6853d15b2512d67881586bd03f462c7ab755db95f16f14d7e238f2852c67", size = 385169, upload-time = "2025-07-26T12:01:45.219Z" }, + { url = "https://files.pythonhosted.org/packages/4b/32/e0f13a1c5b0f8572d0ec6ae2f6c677b7991fafd95da523159c19eff0696a/contourpy-1.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4debd64f124ca62069f313a9cb86656ff087786016d76927ae2cf37846b006c9", size = 362859, upload-time = "2025-07-26T12:01:46.519Z" }, + { url = "https://files.pythonhosted.org/packages/33/71/e2a7945b7de4e58af42d708a219f3b2f4cff7386e6b6ab0a0fa0033c49a9/contourpy-1.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a15459b0f4615b00bbd1e91f1b9e19b7e63aea7483d03d804186f278c0af2659", size = 1332062, upload-time = "2025-07-26T12:01:48.964Z" }, + { url = "https://files.pythonhosted.org/packages/12/fc/4e87ac754220ccc0e807284f88e943d6d43b43843614f0a8afa469801db0/contourpy-1.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca0fdcd73925568ca027e0b17ab07aad764be4706d0a925b89227e447d9737b7", size = 1403932, upload-time = "2025-07-26T12:01:51.979Z" }, + { url = "https://files.pythonhosted.org/packages/a6/2e/adc197a37443f934594112222ac1aa7dc9a98faf9c3842884df9a9d8751d/contourpy-1.3.3-cp313-cp313-win32.whl", hash = "sha256:b20c7c9a3bf701366556e1b1984ed2d0cedf999903c51311417cf5f591d8c78d", size = 185024, upload-time = "2025-07-26T12:01:53.245Z" }, + { url = "https://files.pythonhosted.org/packages/18/0b/0098c214843213759692cc638fce7de5c289200a830e5035d1791d7a2338/contourpy-1.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:1cadd8b8969f060ba45ed7c1b714fe69185812ab43bd6b86a9123fe8f99c3263", size = 226578, upload-time = "2025-07-26T12:01:54.422Z" }, + { url = "https://files.pythonhosted.org/packages/8a/9a/2f6024a0c5995243cd63afdeb3651c984f0d2bc727fd98066d40e141ad73/contourpy-1.3.3-cp313-cp313-win_arm64.whl", hash = "sha256:fd914713266421b7536de2bfa8181aa8c699432b6763a0ea64195ebe28bff6a9", size = 193524, upload-time = "2025-07-26T12:01:55.73Z" }, + { url = "https://files.pythonhosted.org/packages/c0/b3/f8a1a86bd3298513f500e5b1f5fd92b69896449f6cab6a146a5d52715479/contourpy-1.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:88df9880d507169449d434c293467418b9f6cbe82edd19284aa0409e7fdb933d", size = 306730, upload-time = "2025-07-26T12:01:57.051Z" }, + { url = "https://files.pythonhosted.org/packages/3f/11/4780db94ae62fc0c2053909b65dc3246bd7cecfc4f8a20d957ad43aa4ad8/contourpy-1.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:d06bb1f751ba5d417047db62bca3c8fde202b8c11fb50742ab3ab962c81e8216", size = 287897, upload-time = "2025-07-26T12:01:58.663Z" }, + { url = "https://files.pythonhosted.org/packages/ae/15/e59f5f3ffdd6f3d4daa3e47114c53daabcb18574a26c21f03dc9e4e42ff0/contourpy-1.3.3-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e4e6b05a45525357e382909a4c1600444e2a45b4795163d3b22669285591c1ae", size = 326751, upload-time = "2025-07-26T12:02:00.343Z" }, + { url = "https://files.pythonhosted.org/packages/0f/81/03b45cfad088e4770b1dcf72ea78d3802d04200009fb364d18a493857210/contourpy-1.3.3-cp313-cp313t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab3074b48c4e2cf1a960e6bbeb7f04566bf36b1861d5c9d4d8ac04b82e38ba20", size = 375486, upload-time = "2025-07-26T12:02:02.128Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ba/49923366492ffbdd4486e970d421b289a670ae8cf539c1ea9a09822b371a/contourpy-1.3.3-cp313-cp313t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6c3d53c796f8647d6deb1abe867daeb66dcc8a97e8455efa729516b997b8ed99", size = 388106, upload-time = "2025-07-26T12:02:03.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/52/5b00ea89525f8f143651f9f03a0df371d3cbd2fccd21ca9b768c7a6500c2/contourpy-1.3.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50ed930df7289ff2a8d7afeb9603f8289e5704755c7e5c3bbd929c90c817164b", size = 352548, upload-time = "2025-07-26T12:02:05.165Z" }, + { url = "https://files.pythonhosted.org/packages/32/1d/a209ec1a3a3452d490f6b14dd92e72280c99ae3d1e73da74f8277d4ee08f/contourpy-1.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4feffb6537d64b84877da813a5c30f1422ea5739566abf0bd18065ac040e120a", size = 1322297, upload-time = "2025-07-26T12:02:07.379Z" }, + { url = "https://files.pythonhosted.org/packages/bc/9e/46f0e8ebdd884ca0e8877e46a3f4e633f6c9c8c4f3f6e72be3fe075994aa/contourpy-1.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2b7e9480ffe2b0cd2e787e4df64270e3a0440d9db8dc823312e2c940c167df7e", size = 1391023, upload-time = "2025-07-26T12:02:10.171Z" }, + { url = "https://files.pythonhosted.org/packages/b9/70/f308384a3ae9cd2209e0849f33c913f658d3326900d0ff5d378d6a1422d2/contourpy-1.3.3-cp313-cp313t-win32.whl", hash = "sha256:283edd842a01e3dcd435b1c5116798d661378d83d36d337b8dde1d16a5fc9ba3", size = 196157, upload-time = "2025-07-26T12:02:11.488Z" }, + { url = "https://files.pythonhosted.org/packages/b2/dd/880f890a6663b84d9e34a6f88cded89d78f0091e0045a284427cb6b18521/contourpy-1.3.3-cp313-cp313t-win_amd64.whl", hash = "sha256:87acf5963fc2b34825e5b6b048f40e3635dd547f590b04d2ab317c2619ef7ae8", size = 240570, upload-time = "2025-07-26T12:02:12.754Z" }, + { url = "https://files.pythonhosted.org/packages/80/99/2adc7d8ffead633234817ef8e9a87115c8a11927a94478f6bb3d3f4d4f7d/contourpy-1.3.3-cp313-cp313t-win_arm64.whl", hash = "sha256:3c30273eb2a55024ff31ba7d052dde990d7d8e5450f4bbb6e913558b3d6c2301", size = 199713, upload-time = "2025-07-26T12:02:14.4Z" }, + { url = "https://files.pythonhosted.org/packages/72/8b/4546f3ab60f78c514ffb7d01a0bd743f90de36f0019d1be84d0a708a580a/contourpy-1.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fde6c716d51c04b1c25d0b90364d0be954624a0ee9d60e23e850e8d48353d07a", size = 292189, upload-time = "2025-07-26T12:02:16.095Z" }, + { url = "https://files.pythonhosted.org/packages/fd/e1/3542a9cb596cadd76fcef413f19c79216e002623158befe6daa03dbfa88c/contourpy-1.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cbedb772ed74ff5be440fa8eee9bd49f64f6e3fc09436d9c7d8f1c287b121d77", size = 273251, upload-time = "2025-07-26T12:02:17.524Z" }, + { url = "https://files.pythonhosted.org/packages/b1/71/f93e1e9471d189f79d0ce2497007731c1e6bf9ef6d1d61b911430c3db4e5/contourpy-1.3.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22e9b1bd7a9b1d652cd77388465dc358dafcd2e217d35552424aa4f996f524f5", size = 335810, upload-time = "2025-07-26T12:02:18.9Z" }, + { url = "https://files.pythonhosted.org/packages/91/f9/e35f4c1c93f9275d4e38681a80506b5510e9327350c51f8d4a5a724d178c/contourpy-1.3.3-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a22738912262aa3e254e4f3cb079a95a67132fc5a063890e224393596902f5a4", size = 382871, upload-time = "2025-07-26T12:02:20.418Z" }, + { url = "https://files.pythonhosted.org/packages/b5/71/47b512f936f66a0a900d81c396a7e60d73419868fba959c61efed7a8ab46/contourpy-1.3.3-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:afe5a512f31ee6bd7d0dda52ec9864c984ca3d66664444f2d72e0dc4eb832e36", size = 386264, upload-time = "2025-07-26T12:02:21.916Z" }, + { url = "https://files.pythonhosted.org/packages/04/5f/9ff93450ba96b09c7c2b3f81c94de31c89f92292f1380261bd7195bea4ea/contourpy-1.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f64836de09927cba6f79dcd00fdd7d5329f3fccc633468507079c829ca4db4e3", size = 363819, upload-time = "2025-07-26T12:02:23.759Z" }, + { url = "https://files.pythonhosted.org/packages/3e/a6/0b185d4cc480ee494945cde102cb0149ae830b5fa17bf855b95f2e70ad13/contourpy-1.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1fd43c3be4c8e5fd6e4f2baeae35ae18176cf2e5cced681cca908addf1cdd53b", size = 1333650, upload-time = "2025-07-26T12:02:26.181Z" }, + { url = "https://files.pythonhosted.org/packages/43/d7/afdc95580ca56f30fbcd3060250f66cedbde69b4547028863abd8aa3b47e/contourpy-1.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6afc576f7b33cf00996e5c1102dc2a8f7cc89e39c0b55df93a0b78c1bd992b36", size = 1404833, upload-time = "2025-07-26T12:02:28.782Z" }, + { url = "https://files.pythonhosted.org/packages/e2/e2/366af18a6d386f41132a48f033cbd2102e9b0cf6345d35ff0826cd984566/contourpy-1.3.3-cp314-cp314-win32.whl", hash = "sha256:66c8a43a4f7b8df8b71ee1840e4211a3c8d93b214b213f590e18a1beca458f7d", size = 189692, upload-time = "2025-07-26T12:02:30.128Z" }, + { url = "https://files.pythonhosted.org/packages/7d/c2/57f54b03d0f22d4044b8afb9ca0e184f8b1afd57b4f735c2fa70883dc601/contourpy-1.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:cf9022ef053f2694e31d630feaacb21ea24224be1c3ad0520b13d844274614fd", size = 232424, upload-time = "2025-07-26T12:02:31.395Z" }, + { url = "https://files.pythonhosted.org/packages/18/79/a9416650df9b525737ab521aa181ccc42d56016d2123ddcb7b58e926a42c/contourpy-1.3.3-cp314-cp314-win_arm64.whl", hash = "sha256:95b181891b4c71de4bb404c6621e7e2390745f887f2a026b2d99e92c17892339", size = 198300, upload-time = "2025-07-26T12:02:32.956Z" }, + { url = "https://files.pythonhosted.org/packages/1f/42/38c159a7d0f2b7b9c04c64ab317042bb6952b713ba875c1681529a2932fe/contourpy-1.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:33c82d0138c0a062380332c861387650c82e4cf1747aaa6938b9b6516762e772", size = 306769, upload-time = "2025-07-26T12:02:34.2Z" }, + { url = "https://files.pythonhosted.org/packages/c3/6c/26a8205f24bca10974e77460de68d3d7c63e282e23782f1239f226fcae6f/contourpy-1.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ea37e7b45949df430fe649e5de8351c423430046a2af20b1c1961cae3afcda77", size = 287892, upload-time = "2025-07-26T12:02:35.807Z" }, + { url = "https://files.pythonhosted.org/packages/66/06/8a475c8ab718ebfd7925661747dbb3c3ee9c82ac834ccb3570be49d129f4/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d304906ecc71672e9c89e87c4675dc5c2645e1f4269a5063b99b0bb29f232d13", size = 326748, upload-time = "2025-07-26T12:02:37.193Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a3/c5ca9f010a44c223f098fccd8b158bb1cb287378a31ac141f04730dc49be/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ca658cd1a680a5c9ea96dc61cdbae1e85c8f25849843aa799dfd3cb370ad4fbe", size = 375554, upload-time = "2025-07-26T12:02:38.894Z" }, + { url = "https://files.pythonhosted.org/packages/80/5b/68bd33ae63fac658a4145088c1e894405e07584a316738710b636c6d0333/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ab2fd90904c503739a75b7c8c5c01160130ba67944a7b77bbf36ef8054576e7f", size = 388118, upload-time = "2025-07-26T12:02:40.642Z" }, + { url = "https://files.pythonhosted.org/packages/40/52/4c285a6435940ae25d7410a6c36bda5145839bc3f0beb20c707cda18b9d2/contourpy-1.3.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7301b89040075c30e5768810bc96a8e8d78085b47d8be6e4c3f5a0b4ed478a0", size = 352555, upload-time = "2025-07-26T12:02:42.25Z" }, + { url = "https://files.pythonhosted.org/packages/24/ee/3e81e1dd174f5c7fefe50e85d0892de05ca4e26ef1c9a59c2a57e43b865a/contourpy-1.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2a2a8b627d5cc6b7c41a4beff6c5ad5eb848c88255fda4a8745f7e901b32d8e4", size = 1322295, upload-time = "2025-07-26T12:02:44.668Z" }, + { url = "https://files.pythonhosted.org/packages/3c/b2/6d913d4d04e14379de429057cd169e5e00f6c2af3bb13e1710bcbdb5da12/contourpy-1.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fd6ec6be509c787f1caf6b247f0b1ca598bef13f4ddeaa126b7658215529ba0f", size = 1391027, upload-time = "2025-07-26T12:02:47.09Z" }, + { url = "https://files.pythonhosted.org/packages/93/8a/68a4ec5c55a2971213d29a9374913f7e9f18581945a7a31d1a39b5d2dfe5/contourpy-1.3.3-cp314-cp314t-win32.whl", hash = "sha256:e74a9a0f5e3fff48fb5a7f2fd2b9b70a3fe014a67522f79b7cca4c0c7e43c9ae", size = 202428, upload-time = "2025-07-26T12:02:48.691Z" }, + { url = "https://files.pythonhosted.org/packages/fa/96/fd9f641ffedc4fa3ace923af73b9d07e869496c9cc7a459103e6e978992f/contourpy-1.3.3-cp314-cp314t-win_amd64.whl", hash = "sha256:13b68d6a62db8eafaebb8039218921399baf6e47bf85006fd8529f2a08ef33fc", size = 250331, upload-time = "2025-07-26T12:02:50.137Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8c/469afb6465b853afff216f9528ffda78a915ff880ed58813ba4faf4ba0b6/contourpy-1.3.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b7448cb5a725bb1e35ce88771b86fba35ef418952474492cf7c764059933ff8b", size = 203831, upload-time = "2025-07-26T12:02:51.449Z" }, + { url = "https://files.pythonhosted.org/packages/a5/29/8dcfe16f0107943fa92388c23f6e05cff0ba58058c4c95b00280d4c75a14/contourpy-1.3.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:cd5dfcaeb10f7b7f9dc8941717c6c2ade08f587be2226222c12b25f0483ed497", size = 278809, upload-time = "2025-07-26T12:02:52.74Z" }, + { url = "https://files.pythonhosted.org/packages/85/a9/8b37ef4f7dafeb335daee3c8254645ef5725be4d9c6aa70b50ec46ef2f7e/contourpy-1.3.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:0c1fc238306b35f246d61a1d416a627348b5cf0648648a031e14bb8705fcdfe8", size = 261593, upload-time = "2025-07-26T12:02:54.037Z" }, + { url = "https://files.pythonhosted.org/packages/0a/59/ebfb8c677c75605cc27f7122c90313fd2f375ff3c8d19a1694bda74aaa63/contourpy-1.3.3-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70f9aad7de812d6541d29d2bbf8feb22ff7e1c299523db288004e3157ff4674e", size = 302202, upload-time = "2025-07-26T12:02:55.947Z" }, + { url = "https://files.pythonhosted.org/packages/3c/37/21972a15834d90bfbfb009b9d004779bd5a07a0ec0234e5ba8f64d5736f4/contourpy-1.3.3-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5ed3657edf08512fc3fe81b510e35c2012fbd3081d2e26160f27ca28affec989", size = 329207, upload-time = "2025-07-26T12:02:57.468Z" }, + { url = "https://files.pythonhosted.org/packages/0c/58/bd257695f39d05594ca4ad60df5bcb7e32247f9951fd09a9b8edb82d1daa/contourpy-1.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:3d1a3799d62d45c18bafd41c5fa05120b96a28079f2393af559b843d1a966a77", size = 225315, upload-time = "2025-07-26T12:02:58.801Z" }, +] + +[[package]] +name = "coverage" +version = "7.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/45/2c665ca77ec32ad67e25c77daf1cee28ee4558f3bc571cdbaf88a00b9f23/coverage-7.13.0.tar.gz", hash = "sha256:a394aa27f2d7ff9bc04cf703817773a59ad6dfbd577032e690f961d2460ee936", size = 820905, upload-time = "2025-12-08T13:14:38.055Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/dc/888bf90d8b1c3d0b4020a40e52b9f80957d75785931ec66c7dfaccc11c7d/coverage-7.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0dfa3855031070058add1a59fdfda0192fd3e8f97e7c81de0596c145dea51820", size = 218104, upload-time = "2025-12-08T13:12:33.333Z" }, + { url = "https://files.pythonhosted.org/packages/8d/ea/069d51372ad9c380214e86717e40d1a743713a2af191cfba30a0911b0a4a/coverage-7.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4fdb6f54f38e334db97f72fa0c701e66d8479af0bc3f9bfb5b90f1c30f54500f", size = 218606, upload-time = "2025-12-08T13:12:34.498Z" }, + { url = "https://files.pythonhosted.org/packages/68/09/77b1c3a66c2aa91141b6c4471af98e5b1ed9b9e6d17255da5eb7992299e3/coverage-7.13.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7e442c013447d1d8d195be62852270b78b6e255b79b8675bad8479641e21fd96", size = 248999, upload-time = "2025-12-08T13:12:36.02Z" }, + { url = "https://files.pythonhosted.org/packages/0a/32/2e2f96e9d5691eaf1181d9040f850b8b7ce165ea10810fd8e2afa534cef7/coverage-7.13.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1ed5630d946859de835a85e9a43b721123a8a44ec26e2830b296d478c7fd4259", size = 250925, upload-time = "2025-12-08T13:12:37.221Z" }, + { url = "https://files.pythonhosted.org/packages/7b/45/b88ddac1d7978859b9a39a8a50ab323186148f1d64bc068f86fc77706321/coverage-7.13.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7f15a931a668e58087bc39d05d2b4bf4b14ff2875b49c994bbdb1c2217a8daeb", size = 253032, upload-time = "2025-12-08T13:12:38.763Z" }, + { url = "https://files.pythonhosted.org/packages/71/cb/e15513f94c69d4820a34b6bf3d2b1f9f8755fa6021be97c7065442d7d653/coverage-7.13.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:30a3a201a127ea57f7e14ba43c93c9c4be8b7d17a26e03bb49e6966d019eede9", size = 249134, upload-time = "2025-12-08T13:12:40.382Z" }, + { url = "https://files.pythonhosted.org/packages/09/61/d960ff7dc9e902af3310ce632a875aaa7860f36d2bc8fc8b37ee7c1b82a5/coverage-7.13.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7a485ff48fbd231efa32d58f479befce52dcb6bfb2a88bb7bf9a0b89b1bc8030", size = 250731, upload-time = "2025-12-08T13:12:41.992Z" }, + { url = "https://files.pythonhosted.org/packages/98/34/c7c72821794afc7c7c2da1db8f00c2c98353078aa7fb6b5ff36aac834b52/coverage-7.13.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:22486cdafba4f9e471c816a2a5745337742a617fef68e890d8baf9f3036d7833", size = 248795, upload-time = "2025-12-08T13:12:43.331Z" }, + { url = "https://files.pythonhosted.org/packages/0a/5b/e0f07107987a43b2def9aa041c614ddb38064cbf294a71ef8c67d43a0cdd/coverage-7.13.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:263c3dbccc78e2e331e59e90115941b5f53e85cfcc6b3b2fbff1fd4e3d2c6ea8", size = 248514, upload-time = "2025-12-08T13:12:44.546Z" }, + { url = "https://files.pythonhosted.org/packages/71/c2/c949c5d3b5e9fc6dd79e1b73cdb86a59ef14f3709b1d72bf7668ae12e000/coverage-7.13.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e5330fa0cc1f5c3c4c3bb8e101b742025933e7848989370a1d4c8c5e401ea753", size = 249424, upload-time = "2025-12-08T13:12:45.759Z" }, + { url = "https://files.pythonhosted.org/packages/11/f1/bbc009abd6537cec0dffb2cc08c17a7f03de74c970e6302db4342a6e05af/coverage-7.13.0-cp311-cp311-win32.whl", hash = "sha256:0f4872f5d6c54419c94c25dd6ae1d015deeb337d06e448cd890a1e89a8ee7f3b", size = 220597, upload-time = "2025-12-08T13:12:47.378Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f6/d9977f2fb51c10fbaed0718ce3d0a8541185290b981f73b1d27276c12d91/coverage-7.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:51a202e0f80f241ccb68e3e26e19ab5b3bf0f813314f2c967642f13ebcf1ddfe", size = 221536, upload-time = "2025-12-08T13:12:48.7Z" }, + { url = "https://files.pythonhosted.org/packages/be/ad/3fcf43fd96fb43e337a3073dea63ff148dcc5c41ba7a14d4c7d34efb2216/coverage-7.13.0-cp311-cp311-win_arm64.whl", hash = "sha256:d2a9d7f1c11487b1c69367ab3ac2d81b9b3721f097aa409a3191c3e90f8f3dd7", size = 220206, upload-time = "2025-12-08T13:12:50.365Z" }, + { url = "https://files.pythonhosted.org/packages/9b/f1/2619559f17f31ba00fc40908efd1fbf1d0a5536eb75dc8341e7d660a08de/coverage-7.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0b3d67d31383c4c68e19a88e28fc4c2e29517580f1b0ebec4a069d502ce1e0bf", size = 218274, upload-time = "2025-12-08T13:12:52.095Z" }, + { url = "https://files.pythonhosted.org/packages/2b/11/30d71ae5d6e949ff93b2a79a2c1b4822e00423116c5c6edfaeef37301396/coverage-7.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:581f086833d24a22c89ae0fe2142cfaa1c92c930adf637ddf122d55083fb5a0f", size = 218638, upload-time = "2025-12-08T13:12:53.418Z" }, + { url = "https://files.pythonhosted.org/packages/79/c2/fce80fc6ded8d77e53207489d6065d0fed75db8951457f9213776615e0f5/coverage-7.13.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0a3a30f0e257df382f5f9534d4ce3d4cf06eafaf5192beb1a7bd066cb10e78fb", size = 250129, upload-time = "2025-12-08T13:12:54.744Z" }, + { url = "https://files.pythonhosted.org/packages/5b/b6/51b5d1eb6fcbb9a1d5d6984e26cbe09018475c2922d554fd724dd0f056ee/coverage-7.13.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:583221913fbc8f53b88c42e8dbb8fca1d0f2e597cb190ce45916662b8b9d9621", size = 252885, upload-time = "2025-12-08T13:12:56.401Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f8/972a5affea41de798691ab15d023d3530f9f56a72e12e243f35031846ff7/coverage-7.13.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f5d9bd30756fff3e7216491a0d6d520c448d5124d3d8e8f56446d6412499e74", size = 253974, upload-time = "2025-12-08T13:12:57.718Z" }, + { url = "https://files.pythonhosted.org/packages/8a/56/116513aee860b2c7968aa3506b0f59b22a959261d1dbf3aea7b4450a7520/coverage-7.13.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a23e5a1f8b982d56fa64f8e442e037f6ce29322f1f9e6c2344cd9e9f4407ee57", size = 250538, upload-time = "2025-12-08T13:12:59.254Z" }, + { url = "https://files.pythonhosted.org/packages/d6/75/074476d64248fbadf16dfafbf93fdcede389ec821f74ca858d7c87d2a98c/coverage-7.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9b01c22bc74a7fb44066aaf765224c0d933ddf1f5047d6cdfe4795504a4493f8", size = 251912, upload-time = "2025-12-08T13:13:00.604Z" }, + { url = "https://files.pythonhosted.org/packages/f2/d2/aa4f8acd1f7c06024705c12609d8698c51b27e4d635d717cd1934c9668e2/coverage-7.13.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:898cce66d0836973f48dda4e3514d863d70142bdf6dfab932b9b6a90ea5b222d", size = 250054, upload-time = "2025-12-08T13:13:01.892Z" }, + { url = "https://files.pythonhosted.org/packages/19/98/8df9e1af6a493b03694a1e8070e024e7d2cdc77adedc225a35e616d505de/coverage-7.13.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:3ab483ea0e251b5790c2aac03acde31bff0c736bf8a86829b89382b407cd1c3b", size = 249619, upload-time = "2025-12-08T13:13:03.236Z" }, + { url = "https://files.pythonhosted.org/packages/d8/71/f8679231f3353018ca66ef647fa6fe7b77e6bff7845be54ab84f86233363/coverage-7.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1d84e91521c5e4cb6602fe11ece3e1de03b2760e14ae4fcf1a4b56fa3c801fcd", size = 251496, upload-time = "2025-12-08T13:13:04.511Z" }, + { url = "https://files.pythonhosted.org/packages/04/86/9cb406388034eaf3c606c22094edbbb82eea1fa9d20c0e9efadff20d0733/coverage-7.13.0-cp312-cp312-win32.whl", hash = "sha256:193c3887285eec1dbdb3f2bd7fbc351d570ca9c02ca756c3afbc71b3c98af6ef", size = 220808, upload-time = "2025-12-08T13:13:06.422Z" }, + { url = "https://files.pythonhosted.org/packages/1c/59/af483673df6455795daf5f447c2f81a3d2fcfc893a22b8ace983791f6f34/coverage-7.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:4f3e223b2b2db5e0db0c2b97286aba0036ca000f06aca9b12112eaa9af3d92ae", size = 221616, upload-time = "2025-12-08T13:13:07.95Z" }, + { url = "https://files.pythonhosted.org/packages/64/b0/959d582572b30a6830398c60dd419c1965ca4b5fb38ac6b7093a0d50ca8d/coverage-7.13.0-cp312-cp312-win_arm64.whl", hash = "sha256:086cede306d96202e15a4b77ace8472e39d9f4e5f9fd92dd4fecdfb2313b2080", size = 220261, upload-time = "2025-12-08T13:13:09.581Z" }, + { url = "https://files.pythonhosted.org/packages/7c/cc/bce226595eb3bf7d13ccffe154c3c487a22222d87ff018525ab4dd2e9542/coverage-7.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:28ee1c96109974af104028a8ef57cec21447d42d0e937c0275329272e370ebcf", size = 218297, upload-time = "2025-12-08T13:13:10.977Z" }, + { url = "https://files.pythonhosted.org/packages/3b/9f/73c4d34600aae03447dff3d7ad1d0ac649856bfb87d1ca7d681cfc913f9e/coverage-7.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d1e97353dcc5587b85986cda4ff3ec98081d7e84dd95e8b2a6d59820f0545f8a", size = 218673, upload-time = "2025-12-08T13:13:12.562Z" }, + { url = "https://files.pythonhosted.org/packages/63/ab/8fa097db361a1e8586535ae5073559e6229596b3489ec3ef2f5b38df8cb2/coverage-7.13.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:99acd4dfdfeb58e1937629eb1ab6ab0899b131f183ee5f23e0b5da5cba2fec74", size = 249652, upload-time = "2025-12-08T13:13:13.909Z" }, + { url = "https://files.pythonhosted.org/packages/90/3a/9bfd4de2ff191feb37ef9465855ca56a6f2f30a3bca172e474130731ac3d/coverage-7.13.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ff45e0cd8451e293b63ced93161e189780baf444119391b3e7d25315060368a6", size = 252251, upload-time = "2025-12-08T13:13:15.553Z" }, + { url = "https://files.pythonhosted.org/packages/df/61/b5d8105f016e1b5874af0d7c67542da780ccd4a5f2244a433d3e20ceb1ad/coverage-7.13.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f4f72a85316d8e13234cafe0a9f81b40418ad7a082792fa4165bd7d45d96066b", size = 253492, upload-time = "2025-12-08T13:13:16.849Z" }, + { url = "https://files.pythonhosted.org/packages/f3/b8/0fad449981803cc47a4694768b99823fb23632150743f9c83af329bb6090/coverage-7.13.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:11c21557d0e0a5a38632cbbaca5f008723b26a89d70db6315523df6df77d6232", size = 249850, upload-time = "2025-12-08T13:13:18.142Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e9/8d68337c3125014d918cf4327d5257553a710a2995a6a6de2ac77e5aa429/coverage-7.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:76541dc8d53715fb4f7a3a06b34b0dc6846e3c69bc6204c55653a85dd6220971", size = 251633, upload-time = "2025-12-08T13:13:19.56Z" }, + { url = "https://files.pythonhosted.org/packages/55/14/d4112ab26b3a1bc4b3c1295d8452dcf399ed25be4cf649002fb3e64b2d93/coverage-7.13.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6e9e451dee940a86789134b6b0ffbe31c454ade3b849bb8a9d2cca2541a8e91d", size = 249586, upload-time = "2025-12-08T13:13:20.883Z" }, + { url = "https://files.pythonhosted.org/packages/2c/a9/22b0000186db663b0d82f86c2f1028099ae9ac202491685051e2a11a5218/coverage-7.13.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5c67dace46f361125e6b9cace8fe0b729ed8479f47e70c89b838d319375c8137", size = 249412, upload-time = "2025-12-08T13:13:22.22Z" }, + { url = "https://files.pythonhosted.org/packages/a1/2e/42d8e0d9e7527fba439acdc6ed24a2b97613b1dc85849b1dd935c2cffef0/coverage-7.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f59883c643cb19630500f57016f76cfdcd6845ca8c5b5ea1f6e17f74c8e5f511", size = 251191, upload-time = "2025-12-08T13:13:23.899Z" }, + { url = "https://files.pythonhosted.org/packages/a4/af/8c7af92b1377fd8860536aadd58745119252aaaa71a5213e5a8e8007a9f5/coverage-7.13.0-cp313-cp313-win32.whl", hash = "sha256:58632b187be6f0be500f553be41e277712baa278147ecb7559983c6d9faf7ae1", size = 220829, upload-time = "2025-12-08T13:13:25.182Z" }, + { url = "https://files.pythonhosted.org/packages/58/f9/725e8bf16f343d33cbe076c75dc8370262e194ff10072c0608b8e5cf33a3/coverage-7.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:73419b89f812f498aca53f757dd834919b48ce4799f9d5cad33ca0ae442bdb1a", size = 221640, upload-time = "2025-12-08T13:13:26.836Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ff/e98311000aa6933cc79274e2b6b94a2fe0fe3434fca778eba82003675496/coverage-7.13.0-cp313-cp313-win_arm64.whl", hash = "sha256:eb76670874fdd6091eedcc856128ee48c41a9bbbb9c3f1c7c3cf169290e3ffd6", size = 220269, upload-time = "2025-12-08T13:13:28.116Z" }, + { url = "https://files.pythonhosted.org/packages/cf/cf/bbaa2e1275b300343ea865f7d424cc0a2e2a1df6925a070b2b2d5d765330/coverage-7.13.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6e63ccc6e0ad8986386461c3c4b737540f20426e7ec932f42e030320896c311a", size = 218990, upload-time = "2025-12-08T13:13:29.463Z" }, + { url = "https://files.pythonhosted.org/packages/21/1d/82f0b3323b3d149d7672e7744c116e9c170f4957e0c42572f0366dbb4477/coverage-7.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:494f5459ffa1bd45e18558cd98710c36c0b8fbfa82a5eabcbe671d80ecffbfe8", size = 219340, upload-time = "2025-12-08T13:13:31.524Z" }, + { url = "https://files.pythonhosted.org/packages/fb/e3/fe3fd4702a3832a255f4d43013eacb0ef5fc155a5960ea9269d8696db28b/coverage-7.13.0-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:06cac81bf10f74034e055e903f5f946e3e26fc51c09fc9f584e4a1605d977053", size = 260638, upload-time = "2025-12-08T13:13:32.965Z" }, + { url = "https://files.pythonhosted.org/packages/ad/01/63186cb000307f2b4da463f72af9b85d380236965574c78e7e27680a2593/coverage-7.13.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f2ffc92b46ed6e6760f1d47a71e56b5664781bc68986dbd1836b2b70c0ce2071", size = 262705, upload-time = "2025-12-08T13:13:34.378Z" }, + { url = "https://files.pythonhosted.org/packages/7c/a1/c0dacef0cc865f2455d59eed3548573ce47ed603205ffd0735d1d78b5906/coverage-7.13.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0602f701057c6823e5db1b74530ce85f17c3c5be5c85fc042ac939cbd909426e", size = 265125, upload-time = "2025-12-08T13:13:35.73Z" }, + { url = "https://files.pythonhosted.org/packages/ef/92/82b99223628b61300bd382c205795533bed021505eab6dd86e11fb5d7925/coverage-7.13.0-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:25dc33618d45456ccb1d37bce44bc78cf269909aa14c4db2e03d63146a8a1493", size = 259844, upload-time = "2025-12-08T13:13:37.69Z" }, + { url = "https://files.pythonhosted.org/packages/cf/2c/89b0291ae4e6cd59ef042708e1c438e2290f8c31959a20055d8768349ee2/coverage-7.13.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:71936a8b3b977ddd0b694c28c6a34f4fff2e9dd201969a4ff5d5fc7742d614b0", size = 262700, upload-time = "2025-12-08T13:13:39.525Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f9/a5f992efae1996245e796bae34ceb942b05db275e4b34222a9a40b9fbd3b/coverage-7.13.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:936bc20503ce24770c71938d1369461f0c5320830800933bc3956e2a4ded930e", size = 260321, upload-time = "2025-12-08T13:13:41.172Z" }, + { url = "https://files.pythonhosted.org/packages/4c/89/a29f5d98c64fedbe32e2ac3c227fbf78edc01cc7572eee17d61024d89889/coverage-7.13.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:af0a583efaacc52ae2521f8d7910aff65cdb093091d76291ac5820d5e947fc1c", size = 259222, upload-time = "2025-12-08T13:13:43.282Z" }, + { url = "https://files.pythonhosted.org/packages/b3/c3/940fe447aae302a6701ee51e53af7e08b86ff6eed7631e5740c157ee22b9/coverage-7.13.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f1c23e24a7000da892a312fb17e33c5f94f8b001de44b7cf8ba2e36fbd15859e", size = 261411, upload-time = "2025-12-08T13:13:44.72Z" }, + { url = "https://files.pythonhosted.org/packages/eb/31/12a4aec689cb942a89129587860ed4d0fd522d5fda81237147fde554b8ae/coverage-7.13.0-cp313-cp313t-win32.whl", hash = "sha256:5f8a0297355e652001015e93be345ee54393e45dc3050af4a0475c5a2b767d46", size = 221505, upload-time = "2025-12-08T13:13:46.332Z" }, + { url = "https://files.pythonhosted.org/packages/65/8c/3b5fe3259d863572d2b0827642c50c3855d26b3aefe80bdc9eba1f0af3b0/coverage-7.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6abb3a4c52f05e08460bd9acf04fec027f8718ecaa0d09c40ffbc3fbd70ecc39", size = 222569, upload-time = "2025-12-08T13:13:47.79Z" }, + { url = "https://files.pythonhosted.org/packages/b0/39/f71fa8316a96ac72fc3908839df651e8eccee650001a17f2c78cdb355624/coverage-7.13.0-cp313-cp313t-win_arm64.whl", hash = "sha256:3ad968d1e3aa6ce5be295ab5fe3ae1bf5bb4769d0f98a80a0252d543a2ef2e9e", size = 220841, upload-time = "2025-12-08T13:13:49.243Z" }, + { url = "https://files.pythonhosted.org/packages/f8/4b/9b54bedda55421449811dcd5263a2798a63f48896c24dfb92b0f1b0845bd/coverage-7.13.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:453b7ec753cf5e4356e14fe858064e5520c460d3bbbcb9c35e55c0d21155c256", size = 218343, upload-time = "2025-12-08T13:13:50.811Z" }, + { url = "https://files.pythonhosted.org/packages/59/df/c3a1f34d4bba2e592c8979f924da4d3d4598b0df2392fbddb7761258e3dc/coverage-7.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af827b7cbb303e1befa6c4f94fd2bf72f108089cfa0f8abab8f4ca553cf5ca5a", size = 218672, upload-time = "2025-12-08T13:13:52.284Z" }, + { url = "https://files.pythonhosted.org/packages/07/62/eec0659e47857698645ff4e6ad02e30186eb8afd65214fd43f02a76537cb/coverage-7.13.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9987a9e4f8197a1000280f7cc089e3ea2c8b3c0a64d750537809879a7b4ceaf9", size = 249715, upload-time = "2025-12-08T13:13:53.791Z" }, + { url = "https://files.pythonhosted.org/packages/23/2d/3c7ff8b2e0e634c1f58d095f071f52ed3c23ff25be524b0ccae8b71f99f8/coverage-7.13.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3188936845cd0cb114fa6a51842a304cdbac2958145d03be2377ec41eb285d19", size = 252225, upload-time = "2025-12-08T13:13:55.274Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ac/fb03b469d20e9c9a81093575003f959cf91a4a517b783aab090e4538764b/coverage-7.13.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2bdb3babb74079f021696cb46b8bb5f5661165c385d3a238712b031a12355be", size = 253559, upload-time = "2025-12-08T13:13:57.161Z" }, + { url = "https://files.pythonhosted.org/packages/29/62/14afa9e792383c66cc0a3b872a06ded6e4ed1079c7d35de274f11d27064e/coverage-7.13.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7464663eaca6adba4175f6c19354feea61ebbdd735563a03d1e472c7072d27bb", size = 249724, upload-time = "2025-12-08T13:13:58.692Z" }, + { url = "https://files.pythonhosted.org/packages/31/b7/333f3dab2939070613696ab3ee91738950f0467778c6e5a5052e840646b7/coverage-7.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8069e831f205d2ff1f3d355e82f511eb7c5522d7d413f5db5756b772ec8697f8", size = 251582, upload-time = "2025-12-08T13:14:00.642Z" }, + { url = "https://files.pythonhosted.org/packages/81/cb/69162bda9381f39b2287265d7e29ee770f7c27c19f470164350a38318764/coverage-7.13.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:6fb2d5d272341565f08e962cce14cdf843a08ac43bd621783527adb06b089c4b", size = 249538, upload-time = "2025-12-08T13:14:02.556Z" }, + { url = "https://files.pythonhosted.org/packages/e0/76/350387b56a30f4970abe32b90b2a434f87d29f8b7d4ae40d2e8a85aacfb3/coverage-7.13.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5e70f92ef89bac1ac8a99b3324923b4749f008fdbd7aa9cb35e01d7a284a04f9", size = 249349, upload-time = "2025-12-08T13:14:04.015Z" }, + { url = "https://files.pythonhosted.org/packages/86/0d/7f6c42b8d59f4c7e43ea3059f573c0dcfed98ba46eb43c68c69e52ae095c/coverage-7.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4b5de7d4583e60d5fd246dd57fcd3a8aa23c6e118a8c72b38adf666ba8e7e927", size = 251011, upload-time = "2025-12-08T13:14:05.505Z" }, + { url = "https://files.pythonhosted.org/packages/d7/f1/4bb2dff379721bb0b5c649d5c5eaf438462cad824acf32eb1b7ca0c7078e/coverage-7.13.0-cp314-cp314-win32.whl", hash = "sha256:a6c6e16b663be828a8f0b6c5027d36471d4a9f90d28444aa4ced4d48d7d6ae8f", size = 221091, upload-time = "2025-12-08T13:14:07.127Z" }, + { url = "https://files.pythonhosted.org/packages/ba/44/c239da52f373ce379c194b0ee3bcc121020e397242b85f99e0afc8615066/coverage-7.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:0900872f2fdb3ee5646b557918d02279dc3af3dfb39029ac4e945458b13f73bc", size = 221904, upload-time = "2025-12-08T13:14:08.542Z" }, + { url = "https://files.pythonhosted.org/packages/89/1f/b9f04016d2a29c2e4a0307baefefad1a4ec5724946a2b3e482690486cade/coverage-7.13.0-cp314-cp314-win_arm64.whl", hash = "sha256:3a10260e6a152e5f03f26db4a407c4c62d3830b9af9b7c0450b183615f05d43b", size = 220480, upload-time = "2025-12-08T13:14:10.958Z" }, + { url = "https://files.pythonhosted.org/packages/16/d4/364a1439766c8e8647860584171c36010ca3226e6e45b1753b1b249c5161/coverage-7.13.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:9097818b6cc1cfb5f174e3263eba4a62a17683bcfe5c4b5d07f4c97fa51fbf28", size = 219074, upload-time = "2025-12-08T13:14:13.345Z" }, + { url = "https://files.pythonhosted.org/packages/ce/f4/71ba8be63351e099911051b2089662c03d5671437a0ec2171823c8e03bec/coverage-7.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0018f73dfb4301a89292c73be6ba5f58722ff79f51593352759c1790ded1cabe", size = 219342, upload-time = "2025-12-08T13:14:15.02Z" }, + { url = "https://files.pythonhosted.org/packages/5e/25/127d8ed03d7711a387d96f132589057213e3aef7475afdaa303412463f22/coverage-7.13.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:166ad2a22ee770f5656e1257703139d3533b4a0b6909af67c6b4a3adc1c98657", size = 260713, upload-time = "2025-12-08T13:14:16.907Z" }, + { url = "https://files.pythonhosted.org/packages/fd/db/559fbb6def07d25b2243663b46ba9eb5a3c6586c0c6f4e62980a68f0ee1c/coverage-7.13.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f6aaef16d65d1787280943f1c8718dc32e9cf141014e4634d64446702d26e0ff", size = 262825, upload-time = "2025-12-08T13:14:18.68Z" }, + { url = "https://files.pythonhosted.org/packages/37/99/6ee5bf7eff884766edb43bd8736b5e1c5144d0fe47498c3779326fe75a35/coverage-7.13.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e999e2dcc094002d6e2c7bbc1fb85b58ba4f465a760a8014d97619330cdbbbf3", size = 265233, upload-time = "2025-12-08T13:14:20.55Z" }, + { url = "https://files.pythonhosted.org/packages/d8/90/92f18fe0356ea69e1f98f688ed80cec39f44e9f09a1f26a1bbf017cc67f2/coverage-7.13.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:00c3d22cf6fb1cf3bf662aaaa4e563be8243a5ed2630339069799835a9cc7f9b", size = 259779, upload-time = "2025-12-08T13:14:22.367Z" }, + { url = "https://files.pythonhosted.org/packages/90/5d/b312a8b45b37a42ea7d27d7d3ff98ade3a6c892dd48d1d503e773503373f/coverage-7.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:22ccfe8d9bb0d6134892cbe1262493a8c70d736b9df930f3f3afae0fe3ac924d", size = 262700, upload-time = "2025-12-08T13:14:24.309Z" }, + { url = "https://files.pythonhosted.org/packages/63/f8/b1d0de5c39351eb71c366f872376d09386640840a2e09b0d03973d791e20/coverage-7.13.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:9372dff5ea15930fea0445eaf37bbbafbc771a49e70c0aeed8b4e2c2614cc00e", size = 260302, upload-time = "2025-12-08T13:14:26.068Z" }, + { url = "https://files.pythonhosted.org/packages/aa/7c/d42f4435bc40c55558b3109a39e2d456cddcec37434f62a1f1230991667a/coverage-7.13.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:69ac2c492918c2461bc6ace42d0479638e60719f2a4ef3f0815fa2df88e9f940", size = 259136, upload-time = "2025-12-08T13:14:27.604Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d3/23413241dc04d47cfe19b9a65b32a2edd67ecd0b817400c2843ebc58c847/coverage-7.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:739c6c051a7540608d097b8e13c76cfa85263ced467168dc6b477bae3df7d0e2", size = 261467, upload-time = "2025-12-08T13:14:29.09Z" }, + { url = "https://files.pythonhosted.org/packages/13/e6/6e063174500eee216b96272c0d1847bf215926786f85c2bd024cf4d02d2f/coverage-7.13.0-cp314-cp314t-win32.whl", hash = "sha256:fe81055d8c6c9de76d60c94ddea73c290b416e061d40d542b24a5871bad498b7", size = 221875, upload-time = "2025-12-08T13:14:31.106Z" }, + { url = "https://files.pythonhosted.org/packages/3b/46/f4fb293e4cbe3620e3ac2a3e8fd566ed33affb5861a9b20e3dd6c1896cbc/coverage-7.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:445badb539005283825959ac9fa4a28f712c214b65af3a2c464f1adc90f5fcbc", size = 222982, upload-time = "2025-12-08T13:14:33.1Z" }, + { url = "https://files.pythonhosted.org/packages/68/62/5b3b9018215ed9733fbd1ae3b2ed75c5de62c3b55377a52cae732e1b7805/coverage-7.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:de7f6748b890708578fc4b7bb967d810aeb6fcc9bff4bb77dbca77dab2f9df6a", size = 221016, upload-time = "2025-12-08T13:14:34.601Z" }, + { url = "https://files.pythonhosted.org/packages/8d/4c/1968f32fb9a2604645827e11ff84a31e59d532e01995f904723b4f5328b3/coverage-7.13.0-py3-none-any.whl", hash = "sha256:850d2998f380b1e266459ca5b47bc9e7daf9af1d070f66317972f382d46f1904", size = 210068, upload-time = "2025-12-08T13:14:36.236Z" }, +] + +[package.optional-dependencies] +toml = [ + { name = "tomli", marker = "python_full_version <= '3.11'" }, +] + +[[package]] +name = "cryptography" +version = "46.0.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/47/93/ac8f3d5ff04d54bc814e961a43ae5b0b146154c89c61b47bb07557679b18/cryptography-46.0.7.tar.gz", hash = "sha256:e4cfd68c5f3e0bfdad0d38e023239b96a2fe84146481852dffbcca442c245aa5", size = 750652, upload-time = "2026-04-08T01:57:54.692Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/5d/4a8f770695d73be252331e60e526291e3df0c9b27556a90a6b47bccca4c2/cryptography-46.0.7-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:ea42cbe97209df307fdc3b155f1b6fa2577c0defa8f1f7d3be7d31d189108ad4", size = 7179869, upload-time = "2026-04-08T01:56:17.157Z" }, + { url = "https://files.pythonhosted.org/packages/5f/45/6d80dc379b0bbc1f9d1e429f42e4cb9e1d319c7a8201beffd967c516ea01/cryptography-46.0.7-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b36a4695e29fe69215d75960b22577197aca3f7a25b9cf9d165dcfe9d80bc325", size = 4275492, upload-time = "2026-04-08T01:56:19.36Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9a/1765afe9f572e239c3469f2cb429f3ba7b31878c893b246b4b2994ffe2fe/cryptography-46.0.7-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5ad9ef796328c5e3c4ceed237a183f5d41d21150f972455a9d926593a1dcb308", size = 4426670, upload-time = "2026-04-08T01:56:21.415Z" }, + { url = "https://files.pythonhosted.org/packages/8f/3e/af9246aaf23cd4ee060699adab1e47ced3f5f7e7a8ffdd339f817b446462/cryptography-46.0.7-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:73510b83623e080a2c35c62c15298096e2a5dc8d51c3b4e1740211839d0dea77", size = 4280275, upload-time = "2026-04-08T01:56:23.539Z" }, + { url = "https://files.pythonhosted.org/packages/0f/54/6bbbfc5efe86f9d71041827b793c24811a017c6ac0fd12883e4caa86b8ed/cryptography-46.0.7-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cbd5fb06b62bd0721e1170273d3f4d5a277044c47ca27ee257025146c34cbdd1", size = 4928402, upload-time = "2026-04-08T01:56:25.624Z" }, + { url = "https://files.pythonhosted.org/packages/2d/cf/054b9d8220f81509939599c8bdbc0c408dbd2bdd41688616a20731371fe0/cryptography-46.0.7-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:420b1e4109cc95f0e5700eed79908cef9268265c773d3a66f7af1eef53d409ef", size = 4459985, upload-time = "2026-04-08T01:56:27.309Z" }, + { url = "https://files.pythonhosted.org/packages/f9/46/4e4e9c6040fb01c7467d47217d2f882daddeb8828f7df800cb806d8a2288/cryptography-46.0.7-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:24402210aa54baae71d99441d15bb5a1919c195398a87b563df84468160a65de", size = 3990652, upload-time = "2026-04-08T01:56:29.095Z" }, + { url = "https://files.pythonhosted.org/packages/36/5f/313586c3be5a2fbe87e4c9a254207b860155a8e1f3cca99f9910008e7d08/cryptography-46.0.7-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:8a469028a86f12eb7d2fe97162d0634026d92a21f3ae0ac87ed1c4a447886c83", size = 4279805, upload-time = "2026-04-08T01:56:30.928Z" }, + { url = "https://files.pythonhosted.org/packages/69/33/60dfc4595f334a2082749673386a4d05e4f0cf4df8248e63b2c3437585f2/cryptography-46.0.7-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9694078c5d44c157ef3162e3bf3946510b857df5a3955458381d1c7cfc143ddb", size = 4892883, upload-time = "2026-04-08T01:56:32.614Z" }, + { url = "https://files.pythonhosted.org/packages/c7/0b/333ddab4270c4f5b972f980adef4faa66951a4aaf646ca067af597f15563/cryptography-46.0.7-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:42a1e5f98abb6391717978baf9f90dc28a743b7d9be7f0751a6f56a75d14065b", size = 4459756, upload-time = "2026-04-08T01:56:34.306Z" }, + { url = "https://files.pythonhosted.org/packages/d2/14/633913398b43b75f1234834170947957c6b623d1701ffc7a9600da907e89/cryptography-46.0.7-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:91bbcb08347344f810cbe49065914fe048949648f6bd5c2519f34619142bbe85", size = 4410244, upload-time = "2026-04-08T01:56:35.977Z" }, + { url = "https://files.pythonhosted.org/packages/10/f2/19ceb3b3dc14009373432af0c13f46aa08e3ce334ec6eff13492e1812ccd/cryptography-46.0.7-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5d1c02a14ceb9148cc7816249f64f623fbfee39e8c03b3650d842ad3f34d637e", size = 4674868, upload-time = "2026-04-08T01:56:38.034Z" }, + { url = "https://files.pythonhosted.org/packages/1a/bb/a5c213c19ee94b15dfccc48f363738633a493812687f5567addbcbba9f6f/cryptography-46.0.7-cp311-abi3-win32.whl", hash = "sha256:d23c8ca48e44ee015cd0a54aeccdf9f09004eba9fc96f38c911011d9ff1bd457", size = 3026504, upload-time = "2026-04-08T01:56:39.666Z" }, + { url = "https://files.pythonhosted.org/packages/2b/02/7788f9fefa1d060ca68717c3901ae7fffa21ee087a90b7f23c7a603c32ae/cryptography-46.0.7-cp311-abi3-win_amd64.whl", hash = "sha256:397655da831414d165029da9bc483bed2fe0e75dde6a1523ec2fe63f3c46046b", size = 3488363, upload-time = "2026-04-08T01:56:41.893Z" }, + { url = "https://files.pythonhosted.org/packages/7b/56/15619b210e689c5403bb0540e4cb7dbf11a6bf42e483b7644e471a2812b3/cryptography-46.0.7-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:d151173275e1728cf7839aaa80c34fe550c04ddb27b34f48c232193df8db5842", size = 7119671, upload-time = "2026-04-08T01:56:44Z" }, + { url = "https://files.pythonhosted.org/packages/74/66/e3ce040721b0b5599e175ba91ab08884c75928fbeb74597dd10ef13505d2/cryptography-46.0.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:db0f493b9181c7820c8134437eb8b0b4792085d37dbb24da050476ccb664e59c", size = 4268551, upload-time = "2026-04-08T01:56:46.071Z" }, + { url = "https://files.pythonhosted.org/packages/03/11/5e395f961d6868269835dee1bafec6a1ac176505a167f68b7d8818431068/cryptography-46.0.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ebd6daf519b9f189f85c479427bbd6e9c9037862cf8fe89ee35503bd209ed902", size = 4408887, upload-time = "2026-04-08T01:56:47.718Z" }, + { url = "https://files.pythonhosted.org/packages/40/53/8ed1cf4c3b9c8e611e7122fb56f1c32d09e1fff0f1d77e78d9ff7c82653e/cryptography-46.0.7-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b7b412817be92117ec5ed95f880defe9cf18a832e8cafacf0a22337dc1981b4d", size = 4271354, upload-time = "2026-04-08T01:56:49.312Z" }, + { url = "https://files.pythonhosted.org/packages/50/46/cf71e26025c2e767c5609162c866a78e8a2915bbcfa408b7ca495c6140c4/cryptography-46.0.7-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:fbfd0e5f273877695cb93baf14b185f4878128b250cc9f8e617ea0c025dfb022", size = 4905845, upload-time = "2026-04-08T01:56:50.916Z" }, + { url = "https://files.pythonhosted.org/packages/c0/ea/01276740375bac6249d0a971ebdf6b4dc9ead0ee0a34ef3b5a88c1a9b0d4/cryptography-46.0.7-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:ffca7aa1d00cf7d6469b988c581598f2259e46215e0140af408966a24cf086ce", size = 4444641, upload-time = "2026-04-08T01:56:52.882Z" }, + { url = "https://files.pythonhosted.org/packages/3d/4c/7d258f169ae71230f25d9f3d06caabcff8c3baf0978e2b7d65e0acac3827/cryptography-46.0.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:60627cf07e0d9274338521205899337c5d18249db56865f943cbe753aa96f40f", size = 3967749, upload-time = "2026-04-08T01:56:54.597Z" }, + { url = "https://files.pythonhosted.org/packages/b5/2a/2ea0767cad19e71b3530e4cad9605d0b5e338b6a1e72c37c9c1ceb86c333/cryptography-46.0.7-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:80406c3065e2c55d7f49a9550fe0c49b3f12e5bfff5dedb727e319e1afb9bf99", size = 4270942, upload-time = "2026-04-08T01:56:56.416Z" }, + { url = "https://files.pythonhosted.org/packages/41/3d/fe14df95a83319af25717677e956567a105bb6ab25641acaa093db79975d/cryptography-46.0.7-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:c5b1ccd1239f48b7151a65bc6dd54bcfcc15e028c8ac126d3fada09db0e07ef1", size = 4871079, upload-time = "2026-04-08T01:56:58.31Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/4a479e0f36f8f378d397f4eab4c850b4ffb79a2f0d58704b8fa0703ddc11/cryptography-46.0.7-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:d5f7520159cd9c2154eb61eb67548ca05c5774d39e9c2c4339fd793fe7d097b2", size = 4443999, upload-time = "2026-04-08T01:57:00.508Z" }, + { url = "https://files.pythonhosted.org/packages/28/17/b59a741645822ec6d04732b43c5d35e4ef58be7bfa84a81e5ae6f05a1d33/cryptography-46.0.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fcd8eac50d9138c1d7fc53a653ba60a2bee81a505f9f8850b6b2888555a45d0e", size = 4399191, upload-time = "2026-04-08T01:57:02.654Z" }, + { url = "https://files.pythonhosted.org/packages/59/6a/bb2e166d6d0e0955f1e9ff70f10ec4b2824c9cfcdb4da772c7dd69cc7d80/cryptography-46.0.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65814c60f8cc400c63131584e3e1fad01235edba2614b61fbfbfa954082db0ee", size = 4655782, upload-time = "2026-04-08T01:57:04.592Z" }, + { url = "https://files.pythonhosted.org/packages/95/b6/3da51d48415bcb63b00dc17c2eff3a651b7c4fed484308d0f19b30e8cb2c/cryptography-46.0.7-cp314-cp314t-win32.whl", hash = "sha256:fdd1736fed309b4300346f88f74cd120c27c56852c3838cab416e7a166f67298", size = 3002227, upload-time = "2026-04-08T01:57:06.91Z" }, + { url = "https://files.pythonhosted.org/packages/32/a8/9f0e4ed57ec9cebe506e58db11ae472972ecb0c659e4d52bbaee80ca340a/cryptography-46.0.7-cp314-cp314t-win_amd64.whl", hash = "sha256:e06acf3c99be55aa3b516397fe42f5855597f430add9c17fa46bf2e0fb34c9bb", size = 3475332, upload-time = "2026-04-08T01:57:08.807Z" }, + { url = "https://files.pythonhosted.org/packages/a7/7f/cd42fc3614386bc0c12f0cb3c4ae1fc2bbca5c9662dfed031514911d513d/cryptography-46.0.7-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:462ad5cb1c148a22b2e3bcc5ad52504dff325d17daf5df8d88c17dda1f75f2a4", size = 7165618, upload-time = "2026-04-08T01:57:10.645Z" }, + { url = "https://files.pythonhosted.org/packages/a5/d0/36a49f0262d2319139d2829f773f1b97ef8aef7f97e6e5bd21455e5a8fb5/cryptography-46.0.7-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:84d4cced91f0f159a7ddacad249cc077e63195c36aac40b4150e7a57e84fffe7", size = 4270628, upload-time = "2026-04-08T01:57:12.885Z" }, + { url = "https://files.pythonhosted.org/packages/8a/6c/1a42450f464dda6ffbe578a911f773e54dd48c10f9895a23a7e88b3e7db5/cryptography-46.0.7-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:128c5edfe5e5938b86b03941e94fac9ee793a94452ad1365c9fc3f4f62216832", size = 4415405, upload-time = "2026-04-08T01:57:14.923Z" }, + { url = "https://files.pythonhosted.org/packages/9a/92/4ed714dbe93a066dc1f4b4581a464d2d7dbec9046f7c8b7016f5286329e2/cryptography-46.0.7-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5e51be372b26ef4ba3de3c167cd3d1022934bc838ae9eaad7e644986d2a3d163", size = 4272715, upload-time = "2026-04-08T01:57:16.638Z" }, + { url = "https://files.pythonhosted.org/packages/b7/e6/a26b84096eddd51494bba19111f8fffe976f6a09f132706f8f1bf03f51f7/cryptography-46.0.7-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cdf1a610ef82abb396451862739e3fc93b071c844399e15b90726ef7470eeaf2", size = 4918400, upload-time = "2026-04-08T01:57:19.021Z" }, + { url = "https://files.pythonhosted.org/packages/c7/08/ffd537b605568a148543ac3c2b239708ae0bd635064bab41359252ef88ed/cryptography-46.0.7-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1d25aee46d0c6f1a501adcddb2d2fee4b979381346a78558ed13e50aa8a59067", size = 4450634, upload-time = "2026-04-08T01:57:21.185Z" }, + { url = "https://files.pythonhosted.org/packages/16/01/0cd51dd86ab5b9befe0d031e276510491976c3a80e9f6e31810cce46c4ad/cryptography-46.0.7-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:cdfbe22376065ffcf8be74dc9a909f032df19bc58a699456a21712d6e5eabfd0", size = 3985233, upload-time = "2026-04-08T01:57:22.862Z" }, + { url = "https://files.pythonhosted.org/packages/92/49/819d6ed3a7d9349c2939f81b500a738cb733ab62fbecdbc1e38e83d45e12/cryptography-46.0.7-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:abad9dac36cbf55de6eb49badd4016806b3165d396f64925bf2999bcb67837ba", size = 4271955, upload-time = "2026-04-08T01:57:24.814Z" }, + { url = "https://files.pythonhosted.org/packages/80/07/ad9b3c56ebb95ed2473d46df0847357e01583f4c52a85754d1a55e29e4d0/cryptography-46.0.7-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:935ce7e3cfdb53e3536119a542b839bb94ec1ad081013e9ab9b7cfd478b05006", size = 4879888, upload-time = "2026-04-08T01:57:26.88Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c7/201d3d58f30c4c2bdbe9b03844c291feb77c20511cc3586daf7edc12a47b/cryptography-46.0.7-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:35719dc79d4730d30f1c2b6474bd6acda36ae2dfae1e3c16f2051f215df33ce0", size = 4449961, upload-time = "2026-04-08T01:57:29.068Z" }, + { url = "https://files.pythonhosted.org/packages/a5/ef/649750cbf96f3033c3c976e112265c33906f8e462291a33d77f90356548c/cryptography-46.0.7-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7bbc6ccf49d05ac8f7d7b5e2e2c33830d4fe2061def88210a126d130d7f71a85", size = 4401696, upload-time = "2026-04-08T01:57:31.029Z" }, + { url = "https://files.pythonhosted.org/packages/41/52/a8908dcb1a389a459a29008c29966c1d552588d4ae6d43f3a1a4512e0ebe/cryptography-46.0.7-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a1529d614f44b863a7b480c6d000fe93b59acee9c82ffa027cfadc77521a9f5e", size = 4664256, upload-time = "2026-04-08T01:57:33.144Z" }, + { url = "https://files.pythonhosted.org/packages/4b/fa/f0ab06238e899cc3fb332623f337a7364f36f4bb3f2534c2bb95a35b132c/cryptography-46.0.7-cp38-abi3-win32.whl", hash = "sha256:f247c8c1a1fb45e12586afbb436ef21ff1e80670b2861a90353d9b025583d246", size = 3013001, upload-time = "2026-04-08T01:57:34.933Z" }, + { url = "https://files.pythonhosted.org/packages/d2/f1/00ce3bde3ca542d1acd8f8cfa38e446840945aa6363f9b74746394b14127/cryptography-46.0.7-cp38-abi3-win_amd64.whl", hash = "sha256:506c4ff91eff4f82bdac7633318a526b1d1309fc07ca76a3ad182cb5b686d6d3", size = 3472985, upload-time = "2026-04-08T01:57:36.714Z" }, + { url = "https://files.pythonhosted.org/packages/63/0c/dca8abb64e7ca4f6b2978769f6fea5ad06686a190cec381f0a796fdcaaba/cryptography-46.0.7-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fc9ab8856ae6cf7c9358430e49b368f3108f050031442eaeb6b9d87e4dcf4e4f", size = 3476879, upload-time = "2026-04-08T01:57:38.664Z" }, + { url = "https://files.pythonhosted.org/packages/3a/ea/075aac6a84b7c271578d81a2f9968acb6e273002408729f2ddff517fed4a/cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d3b99c535a9de0adced13d159c5a9cf65c325601aa30f4be08afd680643e9c15", size = 4219700, upload-time = "2026-04-08T01:57:40.625Z" }, + { url = "https://files.pythonhosted.org/packages/6c/7b/1c55db7242b5e5612b29fc7a630e91ee7a6e3c8e7bf5406d22e206875fbd/cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d02c738dacda7dc2a74d1b2b3177042009d5cab7c7079db74afc19e56ca1b455", size = 4385982, upload-time = "2026-04-08T01:57:42.725Z" }, + { url = "https://files.pythonhosted.org/packages/cb/da/9870eec4b69c63ef5925bf7d8342b7e13bc2ee3d47791461c4e49ca212f4/cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:04959522f938493042d595a736e7dbdff6eb6cc2339c11465b3ff89343b65f65", size = 4219115, upload-time = "2026-04-08T01:57:44.939Z" }, + { url = "https://files.pythonhosted.org/packages/f4/72/05aa5832b82dd341969e9a734d1812a6aadb088d9eb6f0430fc337cc5a8f/cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:3986ac1dee6def53797289999eabe84798ad7817f3e97779b5061a95b0ee4968", size = 4385479, upload-time = "2026-04-08T01:57:46.86Z" }, + { url = "https://files.pythonhosted.org/packages/20/2a/1b016902351a523aa2bd446b50a5bc1175d7a7d1cf90fe2ef904f9b84ebc/cryptography-46.0.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:258514877e15963bd43b558917bc9f54cf7cf866c38aa576ebf47a77ddbc43a4", size = 3412829, upload-time = "2026-04-08T01:57:48.874Z" }, +] + +[[package]] +name = "cycler" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/95/a3dbbb5028f35eafb79008e7522a75244477d2838f38cbb722248dabc2a8/cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c", size = 7615, upload-time = "2023-10-07T05:32:18.335Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, +] + +[[package]] +name = "diffusers" +version = "0.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "httpx" }, + { name = "huggingface-hub" }, + { name = "importlib-metadata" }, + { name = "numpy" }, + { name = "pillow" }, + { name = "regex" }, + { name = "requests" }, + { name = "safetensors" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/88/45/ccb2e2180ddf475a0f931dac6a50346310e4c464ce3cccb8a65d1fc1e16d/diffusers-0.36.0.tar.gz", hash = "sha256:a9cde8721b415bde6a678f2d02abb85396487e1b0e0d2b4abb462d14a9825ab0", size = 3795088, upload-time = "2025-12-08T10:14:34.255Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/50/281f92cb1f83854dbd79b6e958b3bc5018607e2542971d41604ba7a14b2f/diffusers-0.36.0-py3-none-any.whl", hash = "sha256:525d42abc74bfc3b2db594999961295c054b48ef40a11724dacf50e6abd1af98", size = 4597884, upload-time = "2025-12-08T10:14:31.979Z" }, +] + +[[package]] +name = "dill" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/12/80/630b4b88364e9a8c8c5797f4602d0f76ef820909ee32f0bacb9f90654042/dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0", size = 186976, upload-time = "2025-04-16T00:41:48.867Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/3d/9373ad9c56321fdab5b41197068e1d8c25883b3fea29dd361f9b55116869/dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049", size = 119668, upload-time = "2025-04-16T00:41:47.671Z" }, +] + +[[package]] +name = "distlib" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, +] + +[[package]] +name = "docutils" +version = "0.21.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/ed/aefcc8cd0ba62a0560c3c18c33925362d46c6075480bfa4df87b28e169a9/docutils-0.21.2.tar.gz", hash = "sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f", size = 2204444, upload-time = "2024-04-23T18:57:18.24Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408, upload-time = "2024-04-23T18:57:14.835Z" }, +] + +[[package]] +name = "einops" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/81/df4fbe24dff8ba3934af99044188e20a98ed441ad17a274539b74e82e126/einops-0.8.1.tar.gz", hash = "sha256:de5d960a7a761225532e0f1959e5315ebeafc0cd43394732f103ca44b9837e84", size = 54805, upload-time = "2025-02-09T03:17:00.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/62/9773de14fe6c45c23649e98b83231fffd7b9892b6cf863251dc2afa73643/einops-0.8.1-py3-none-any.whl", hash = "sha256:919387eb55330f5757c6bea9165c5ff5cfe63a642682ea788a6d472576d81737", size = 64359, upload-time = "2025-02-09T03:17:01.998Z" }, +] + +[[package]] +name = "filelock" +version = "3.20.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/65/ce7f1b70157833bf3cb851b556a37d4547ceafc158aa9b34b36782f23696/filelock-3.20.3.tar.gz", hash = "sha256:18c57ee915c7ec61cff0ecf7f0f869936c7c30191bb0cf406f1341778d0834e1", size = 19485, upload-time = "2026-01-09T17:55:05.421Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl", hash = "sha256:4b0dda527ee31078689fc205ec4f1c1bf7d56cf88b6dc9426c4f230e46c2dce1", size = 16701, upload-time = "2026-01-09T17:55:04.334Z" }, +] + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/2d/d2a548598be01649e2d46231d151a6c56d10b964d94043a335ae56ea2d92/flatbuffers-25.12.19-py2.py3-none-any.whl", hash = "sha256:7634f50c427838bb021c2d66a3d1168e9d199b0607e6329399f04846d42e20b4", size = 26661, upload-time = "2025-12-19T23:16:13.622Z" }, +] + +[[package]] +name = "fonttools" +version = "4.61.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/ca/cf17b88a8df95691275a3d77dc0a5ad9907f328ae53acbe6795da1b2f5ed/fonttools-4.61.1.tar.gz", hash = "sha256:6675329885c44657f826ef01d9e4fb33b9158e9d93c537d84ad8399539bc6f69", size = 3565756, upload-time = "2025-12-12T17:31:24.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/12/bf9f4eaa2fad039356cc627587e30ed008c03f1cebd3034376b5ee8d1d44/fonttools-4.61.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c6604b735bb12fef8e0efd5578c9fb5d3d8532d5001ea13a19cddf295673ee09", size = 2852213, upload-time = "2025-12-12T17:29:46.675Z" }, + { url = "https://files.pythonhosted.org/packages/ac/49/4138d1acb6261499bedde1c07f8c2605d1d8f9d77a151e5507fd3ef084b6/fonttools-4.61.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5ce02f38a754f207f2f06557523cd39a06438ba3aafc0639c477ac409fc64e37", size = 2401689, upload-time = "2025-12-12T17:29:48.769Z" }, + { url = "https://files.pythonhosted.org/packages/e5/fe/e6ce0fe20a40e03aef906af60aa87668696f9e4802fa283627d0b5ed777f/fonttools-4.61.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77efb033d8d7ff233385f30c62c7c79271c8885d5c9657d967ede124671bbdfb", size = 5058809, upload-time = "2025-12-12T17:29:51.701Z" }, + { url = "https://files.pythonhosted.org/packages/79/61/1ca198af22f7dd22c17ab86e9024ed3c06299cfdb08170640e9996d501a0/fonttools-4.61.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:75c1a6dfac6abd407634420c93864a1e274ebc1c7531346d9254c0d8f6ca00f9", size = 5036039, upload-time = "2025-12-12T17:29:53.659Z" }, + { url = "https://files.pythonhosted.org/packages/99/cc/fa1801e408586b5fce4da9f5455af8d770f4fc57391cd5da7256bb364d38/fonttools-4.61.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0de30bfe7745c0d1ffa2b0b7048fb7123ad0d71107e10ee090fa0b16b9452e87", size = 5034714, upload-time = "2025-12-12T17:29:55.592Z" }, + { url = "https://files.pythonhosted.org/packages/bf/aa/b7aeafe65adb1b0a925f8f25725e09f078c635bc22754f3fecb7456955b0/fonttools-4.61.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:58b0ee0ab5b1fc9921eccfe11d1435added19d6494dde14e323f25ad2bc30c56", size = 5158648, upload-time = "2025-12-12T17:29:57.861Z" }, + { url = "https://files.pythonhosted.org/packages/99/f9/08ea7a38663328881384c6e7777bbefc46fd7d282adfd87a7d2b84ec9d50/fonttools-4.61.1-cp311-cp311-win32.whl", hash = "sha256:f79b168428351d11e10c5aeb61a74e1851ec221081299f4cf56036a95431c43a", size = 2280681, upload-time = "2025-12-12T17:29:59.943Z" }, + { url = "https://files.pythonhosted.org/packages/07/ad/37dd1ae5fa6e01612a1fbb954f0927681f282925a86e86198ccd7b15d515/fonttools-4.61.1-cp311-cp311-win_amd64.whl", hash = "sha256:fe2efccb324948a11dd09d22136fe2ac8a97d6c1347cf0b58a911dcd529f66b7", size = 2331951, upload-time = "2025-12-12T17:30:02.254Z" }, + { url = "https://files.pythonhosted.org/packages/6f/16/7decaa24a1bd3a70c607b2e29f0adc6159f36a7e40eaba59846414765fd4/fonttools-4.61.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f3cb4a569029b9f291f88aafc927dd53683757e640081ca8c412781ea144565e", size = 2851593, upload-time = "2025-12-12T17:30:04.225Z" }, + { url = "https://files.pythonhosted.org/packages/94/98/3c4cb97c64713a8cf499b3245c3bf9a2b8fd16a3e375feff2aed78f96259/fonttools-4.61.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41a7170d042e8c0024703ed13b71893519a1a6d6e18e933e3ec7507a2c26a4b2", size = 2400231, upload-time = "2025-12-12T17:30:06.47Z" }, + { url = "https://files.pythonhosted.org/packages/b7/37/82dbef0f6342eb01f54bca073ac1498433d6ce71e50c3c3282b655733b31/fonttools-4.61.1-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:10d88e55330e092940584774ee5e8a6971b01fc2f4d3466a1d6c158230880796", size = 4954103, upload-time = "2025-12-12T17:30:08.432Z" }, + { url = "https://files.pythonhosted.org/packages/6c/44/f3aeac0fa98e7ad527f479e161aca6c3a1e47bb6996b053d45226fe37bf2/fonttools-4.61.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:15acc09befd16a0fb8a8f62bc147e1a82817542d72184acca9ce6e0aeda9fa6d", size = 5004295, upload-time = "2025-12-12T17:30:10.56Z" }, + { url = "https://files.pythonhosted.org/packages/14/e8/7424ced75473983b964d09f6747fa09f054a6d656f60e9ac9324cf40c743/fonttools-4.61.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e6bcdf33aec38d16508ce61fd81838f24c83c90a1d1b8c68982857038673d6b8", size = 4944109, upload-time = "2025-12-12T17:30:12.874Z" }, + { url = "https://files.pythonhosted.org/packages/c8/8b/6391b257fa3d0b553d73e778f953a2f0154292a7a7a085e2374b111e5410/fonttools-4.61.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5fade934607a523614726119164ff621e8c30e8fa1ffffbbd358662056ba69f0", size = 5093598, upload-time = "2025-12-12T17:30:15.79Z" }, + { url = "https://files.pythonhosted.org/packages/d9/71/fd2ea96cdc512d92da5678a1c98c267ddd4d8c5130b76d0f7a80f9a9fde8/fonttools-4.61.1-cp312-cp312-win32.whl", hash = "sha256:75da8f28eff26defba42c52986de97b22106cb8f26515b7c22443ebc9c2d3261", size = 2269060, upload-time = "2025-12-12T17:30:18.058Z" }, + { url = "https://files.pythonhosted.org/packages/80/3b/a3e81b71aed5a688e89dfe0e2694b26b78c7d7f39a5ffd8a7d75f54a12a8/fonttools-4.61.1-cp312-cp312-win_amd64.whl", hash = "sha256:497c31ce314219888c0e2fce5ad9178ca83fe5230b01a5006726cdf3ac9f24d9", size = 2319078, upload-time = "2025-12-12T17:30:22.862Z" }, + { url = "https://files.pythonhosted.org/packages/4b/cf/00ba28b0990982530addb8dc3e9e6f2fa9cb5c20df2abdda7baa755e8fe1/fonttools-4.61.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8c56c488ab471628ff3bfa80964372fc13504ece601e0d97a78ee74126b2045c", size = 2846454, upload-time = "2025-12-12T17:30:24.938Z" }, + { url = "https://files.pythonhosted.org/packages/5a/ca/468c9a8446a2103ae645d14fee3f610567b7042aba85031c1c65e3ef7471/fonttools-4.61.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:dc492779501fa723b04d0ab1f5be046797fee17d27700476edc7ee9ae535a61e", size = 2398191, upload-time = "2025-12-12T17:30:27.343Z" }, + { url = "https://files.pythonhosted.org/packages/a3/4b/d67eedaed19def5967fade3297fed8161b25ba94699efc124b14fb68cdbc/fonttools-4.61.1-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:64102ca87e84261419c3747a0d20f396eb024bdbeb04c2bfb37e2891f5fadcb5", size = 4928410, upload-time = "2025-12-12T17:30:29.771Z" }, + { url = "https://files.pythonhosted.org/packages/b0/8d/6fb3494dfe61a46258cd93d979cf4725ded4eb46c2a4ca35e4490d84daea/fonttools-4.61.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c1b526c8d3f615a7b1867f38a9410849c8f4aef078535742198e942fba0e9bd", size = 4984460, upload-time = "2025-12-12T17:30:32.073Z" }, + { url = "https://files.pythonhosted.org/packages/f7/f1/a47f1d30b3dc00d75e7af762652d4cbc3dff5c2697a0dbd5203c81afd9c3/fonttools-4.61.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41ed4b5ec103bd306bb68f81dc166e77409e5209443e5773cb4ed837bcc9b0d3", size = 4925800, upload-time = "2025-12-12T17:30:34.339Z" }, + { url = "https://files.pythonhosted.org/packages/a7/01/e6ae64a0981076e8a66906fab01539799546181e32a37a0257b77e4aa88b/fonttools-4.61.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b501c862d4901792adaec7c25b1ecc749e2662543f68bb194c42ba18d6eec98d", size = 5067859, upload-time = "2025-12-12T17:30:36.593Z" }, + { url = "https://files.pythonhosted.org/packages/73/aa/28e40b8d6809a9b5075350a86779163f074d2b617c15d22343fce81918db/fonttools-4.61.1-cp313-cp313-win32.whl", hash = "sha256:4d7092bb38c53bbc78e9255a59158b150bcdc115a1e3b3ce0b5f267dc35dd63c", size = 2267821, upload-time = "2025-12-12T17:30:38.478Z" }, + { url = "https://files.pythonhosted.org/packages/1a/59/453c06d1d83dc0951b69ef692d6b9f1846680342927df54e9a1ca91c6f90/fonttools-4.61.1-cp313-cp313-win_amd64.whl", hash = "sha256:21e7c8d76f62ab13c9472ccf74515ca5b9a761d1bde3265152a6dc58700d895b", size = 2318169, upload-time = "2025-12-12T17:30:40.951Z" }, + { url = "https://files.pythonhosted.org/packages/32/8f/4e7bf82c0cbb738d3c2206c920ca34ca74ef9dabde779030145d28665104/fonttools-4.61.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fff4f534200a04b4a36e7ae3cb74493afe807b517a09e99cb4faa89a34ed6ecd", size = 2846094, upload-time = "2025-12-12T17:30:43.511Z" }, + { url = "https://files.pythonhosted.org/packages/71/09/d44e45d0a4f3a651f23a1e9d42de43bc643cce2971b19e784cc67d823676/fonttools-4.61.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d9203500f7c63545b4ce3799319fe4d9feb1a1b89b28d3cb5abd11b9dd64147e", size = 2396589, upload-time = "2025-12-12T17:30:45.681Z" }, + { url = "https://files.pythonhosted.org/packages/89/18/58c64cafcf8eb677a99ef593121f719e6dcbdb7d1c594ae5a10d4997ca8a/fonttools-4.61.1-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fa646ecec9528bef693415c79a86e733c70a4965dd938e9a226b0fc64c9d2e6c", size = 4877892, upload-time = "2025-12-12T17:30:47.709Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ec/9e6b38c7ba1e09eb51db849d5450f4c05b7e78481f662c3b79dbde6f3d04/fonttools-4.61.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:11f35ad7805edba3aac1a3710d104592df59f4b957e30108ae0ba6c10b11dd75", size = 4972884, upload-time = "2025-12-12T17:30:49.656Z" }, + { url = "https://files.pythonhosted.org/packages/5e/87/b5339da8e0256734ba0dbbf5b6cdebb1dd79b01dc8c270989b7bcd465541/fonttools-4.61.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b931ae8f62db78861b0ff1ac017851764602288575d65b8e8ff1963fed419063", size = 4924405, upload-time = "2025-12-12T17:30:51.735Z" }, + { url = "https://files.pythonhosted.org/packages/0b/47/e3409f1e1e69c073a3a6fd8cb886eb18c0bae0ee13db2c8d5e7f8495e8b7/fonttools-4.61.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b148b56f5de675ee16d45e769e69f87623a4944f7443850bf9a9376e628a89d2", size = 5035553, upload-time = "2025-12-12T17:30:54.823Z" }, + { url = "https://files.pythonhosted.org/packages/bf/b6/1f6600161b1073a984294c6c031e1a56ebf95b6164249eecf30012bb2e38/fonttools-4.61.1-cp314-cp314-win32.whl", hash = "sha256:9b666a475a65f4e839d3d10473fad6d47e0a9db14a2f4a224029c5bfde58ad2c", size = 2271915, upload-time = "2025-12-12T17:30:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/52/7b/91e7b01e37cc8eb0e1f770d08305b3655e4f002fc160fb82b3390eabacf5/fonttools-4.61.1-cp314-cp314-win_amd64.whl", hash = "sha256:4f5686e1fe5fce75d82d93c47a438a25bf0d1319d2843a926f741140b2b16e0c", size = 2323487, upload-time = "2025-12-12T17:30:59.804Z" }, + { url = "https://files.pythonhosted.org/packages/39/5c/908ad78e46c61c3e3ed70c3b58ff82ab48437faf84ec84f109592cabbd9f/fonttools-4.61.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e76ce097e3c57c4bcb67c5aa24a0ecdbd9f74ea9219997a707a4061fbe2707aa", size = 2929571, upload-time = "2025-12-12T17:31:02.574Z" }, + { url = "https://files.pythonhosted.org/packages/bd/41/975804132c6dea64cdbfbaa59f3518a21c137a10cccf962805b301ac6ab2/fonttools-4.61.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:9cfef3ab326780c04d6646f68d4b4742aae222e8b8ea1d627c74e38afcbc9d91", size = 2435317, upload-time = "2025-12-12T17:31:04.974Z" }, + { url = "https://files.pythonhosted.org/packages/b0/5a/aef2a0a8daf1ebaae4cfd83f84186d4a72ee08fd6a8451289fcd03ffa8a4/fonttools-4.61.1-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a75c301f96db737e1c5ed5fd7d77d9c34466de16095a266509e13da09751bd19", size = 4882124, upload-time = "2025-12-12T17:31:07.456Z" }, + { url = "https://files.pythonhosted.org/packages/80/33/d6db3485b645b81cea538c9d1c9219d5805f0877fda18777add4671c5240/fonttools-4.61.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:91669ccac46bbc1d09e9273546181919064e8df73488ea087dcac3e2968df9ba", size = 5100391, upload-time = "2025-12-12T17:31:09.732Z" }, + { url = "https://files.pythonhosted.org/packages/6c/d6/675ba631454043c75fcf76f0ca5463eac8eb0666ea1d7badae5fea001155/fonttools-4.61.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c33ab3ca9d3ccd581d58e989d67554e42d8d4ded94ab3ade3508455fe70e65f7", size = 4978800, upload-time = "2025-12-12T17:31:11.681Z" }, + { url = "https://files.pythonhosted.org/packages/7f/33/d3ec753d547a8d2bdaedd390d4a814e8d5b45a093d558f025c6b990b554c/fonttools-4.61.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:664c5a68ec406f6b1547946683008576ef8b38275608e1cee6c061828171c118", size = 5006426, upload-time = "2025-12-12T17:31:13.764Z" }, + { url = "https://files.pythonhosted.org/packages/b4/40/cc11f378b561a67bea850ab50063366a0d1dd3f6d0a30ce0f874b0ad5664/fonttools-4.61.1-cp314-cp314t-win32.whl", hash = "sha256:aed04cabe26f30c1647ef0e8fbb207516fd40fe9472e9439695f5c6998e60ac5", size = 2335377, upload-time = "2025-12-12T17:31:16.49Z" }, + { url = "https://files.pythonhosted.org/packages/e4/ff/c9a2b66b39f8628531ea58b320d66d951267c98c6a38684daa8f50fb02f8/fonttools-4.61.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2180f14c141d2f0f3da43f3a81bc8aa4684860f6b0e6f9e165a4831f24e6a23b", size = 2400613, upload-time = "2025-12-12T17:31:18.769Z" }, + { url = "https://files.pythonhosted.org/packages/c7/4e/ce75a57ff3aebf6fc1f4e9d508b8e5810618a33d900ad6c19eb30b290b97/fonttools-4.61.1-py3-none-any.whl", hash = "sha256:17d2bf5d541add43822bcf0c43d7d847b160c9bb01d15d5007d84e2217aaa371", size = 1148996, upload-time = "2025-12-12T17:31:21.03Z" }, +] + +[[package]] +name = "fsspec" +version = "2025.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/27/954057b0d1f53f086f681755207dda6de6c660ce133c829158e8e8fe7895/fsspec-2025.12.0.tar.gz", hash = "sha256:c505de011584597b1060ff778bb664c1bc022e87921b0e4f10cc9c44f9635973", size = 309748, upload-time = "2025-12-03T15:23:42.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/c7/b64cae5dba3a1b138d7123ec36bb5ccd39d39939f18454407e5468f4763f/fsspec-2025.12.0-py3-none-any.whl", hash = "sha256:8bf1fe301b7d8acfa6e8571e3b1c3d158f909666642431cc78a1b7b4dbc5ec5b", size = 201422, upload-time = "2025-12-03T15:23:41.434Z" }, +] + +[[package]] +name = "furo" +version = "2025.12.19" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "accessible-pygments" }, + { name = "beautifulsoup4" }, + { name = "pygments" }, + { name = "sphinx" }, + { name = "sphinx-basic-ng" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ec/20/5f5ad4da6a5a27c80f2ed2ee9aee3f9e36c66e56e21c00fde467b2f8f88f/furo-2025.12.19.tar.gz", hash = "sha256:188d1f942037d8b37cd3985b955839fea62baa1730087dc29d157677c857e2a7", size = 1661473, upload-time = "2025-12-19T17:34:40.889Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/b2/50e9b292b5cac13e9e81272c7171301abc753a60460d21505b606e15cf21/furo-2025.12.19-py3-none-any.whl", hash = "sha256:bb0ead5309f9500130665a26bee87693c41ce4dbdff864dbfb6b0dae4673d24f", size = 339262, upload-time = "2025-12-19T17:34:38.905Z" }, +] + +[[package]] +name = "gast" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/f6/e73969782a2ecec280f8a176f2476149dd9dba69d5f8779ec6108a7721e6/gast-0.7.0.tar.gz", hash = "sha256:0bb14cd1b806722e91ddbab6fb86bba148c22b40e7ff11e248974e04c8adfdae", size = 33630, upload-time = "2025-11-29T15:30:05.266Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/33/f1c6a276de27b7d7339a34749cc33fa87f077f921969c47185d34a887ae2/gast-0.7.0-py3-none-any.whl", hash = "sha256:99cbf1365633a74099f69c59bd650476b96baa5ef196fec88032b00b31ba36f7", size = 22966, upload-time = "2025-11-29T15:30:03.983Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "hf-xet" +version = "1.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/92/ec9ad04d0b5728dca387a45af7bc98fbb0d73b2118759f5f6038b61a57e8/hf_xet-1.4.3.tar.gz", hash = "sha256:8ddedb73c8c08928c793df2f3401ec26f95be7f7e516a7bee2fbb546f6676113", size = 670477, upload-time = "2026-03-31T22:40:07.874Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/43/724d307b34e353da0abd476e02f72f735cdd2bc86082dee1b32ea0bfee1d/hf_xet-1.4.3-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:7551659ba4f1e1074e9623996f28c3873682530aee0a846b7f2f066239228144", size = 3800935, upload-time = "2026-03-31T22:39:49.618Z" }, + { url = "https://files.pythonhosted.org/packages/2b/d2/8bee5996b699262edb87dbb54118d287c0e1b2fc78af7cdc41857ba5e3c4/hf_xet-1.4.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bee693ada985e7045997f05f081d0e12c4c08bd7626dc397f8a7c487e6c04f7f", size = 3558942, upload-time = "2026-03-31T22:39:47.938Z" }, + { url = "https://files.pythonhosted.org/packages/c3/a1/e993d09cbe251196fb60812b09a58901c468127b7259d2bf0f68bf6088eb/hf_xet-1.4.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:21644b404bb0100fe3857892f752c4d09642586fd988e61501c95bbf44b393a3", size = 4207657, upload-time = "2026-03-31T22:39:39.69Z" }, + { url = "https://files.pythonhosted.org/packages/64/44/9eb6d21e5c34c63e5e399803a6932fa983cabdf47c0ecbcfe7ea97684b8c/hf_xet-1.4.3-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:987f09cfe418237812896a6736b81b1af02a3a6dcb4b4944425c4c4fca7a7cf8", size = 3986765, upload-time = "2026-03-31T22:39:37.936Z" }, + { url = "https://files.pythonhosted.org/packages/ea/7b/8ad6f16fdb82f5f7284a34b5ec48645bd575bdcd2f6f0d1644775909c486/hf_xet-1.4.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:60cf7fc43a99da0a853345cf86d23738c03983ee5249613a6305d3e57a5dca74", size = 4188162, upload-time = "2026-03-31T22:39:58.382Z" }, + { url = "https://files.pythonhosted.org/packages/1b/c4/39d6e136cbeea9ca5a23aad4b33024319222adbdc059ebcda5fc7d9d5ff4/hf_xet-1.4.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2815a49a7a59f3e2edf0cf113ae88e8cb2ca2a221bf353fb60c609584f4884d4", size = 4424525, upload-time = "2026-03-31T22:40:00.225Z" }, + { url = "https://files.pythonhosted.org/packages/46/f2/adc32dae6bdbc367853118b9878139ac869419a4ae7ba07185dc31251b76/hf_xet-1.4.3-cp313-cp313t-win_amd64.whl", hash = "sha256:42ee323265f1e6a81b0e11094564fb7f7e0ec75b5105ffd91ae63f403a11931b", size = 3671610, upload-time = "2026-03-31T22:40:10.42Z" }, + { url = "https://files.pythonhosted.org/packages/e2/19/25d897dcc3f81953e0c2cde9ec186c7a0fee413eb0c9a7a9130d87d94d3a/hf_xet-1.4.3-cp313-cp313t-win_arm64.whl", hash = "sha256:27c976ba60079fb8217f485b9c5c7fcd21c90b0367753805f87cb9f3cdc4418a", size = 3528529, upload-time = "2026-03-31T22:40:09.106Z" }, + { url = "https://files.pythonhosted.org/packages/ec/36/3e8f85ca9fe09b8de2b2e10c63b3b3353d7dda88a0b3d426dffbe7b8313b/hf_xet-1.4.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:5251d5ece3a81815bae9abab41cf7ddb7bcb8f56411bce0827f4a3071c92fdc6", size = 3801019, upload-time = "2026-03-31T22:39:56.651Z" }, + { url = "https://files.pythonhosted.org/packages/b5/9c/defb6cb1de28bccb7bd8d95f6e60f72a3d3fa4cb3d0329c26fb9a488bfe7/hf_xet-1.4.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1feb0f3abeacee143367c326a128a2e2b60868ec12a36c225afb1d6c5a05e6d2", size = 3558746, upload-time = "2026-03-31T22:39:54.766Z" }, + { url = "https://files.pythonhosted.org/packages/c1/bd/8d001191893178ff8e826e46ad5299446e62b93cd164e17b0ffea08832ec/hf_xet-1.4.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8b301fc150290ca90b4fccd079829b84bb4786747584ae08b94b4577d82fb791", size = 4207692, upload-time = "2026-03-31T22:39:46.246Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/6790b402803250e9936435613d3a78b9aaeee7973439f0918848dde58309/hf_xet-1.4.3-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:d972fbe95ddc0d3c0fc49b31a8a69f47db35c1e3699bf316421705741aab6653", size = 3986281, upload-time = "2026-03-31T22:39:44.648Z" }, + { url = "https://files.pythonhosted.org/packages/51/56/ea62552fe53db652a9099eda600b032d75554d0e86c12a73824bfedef88b/hf_xet-1.4.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c5b48db1ee344a805a1b9bd2cda9b6b65fe77ed3787bd6e87ad5521141d317cd", size = 4187414, upload-time = "2026-03-31T22:40:04.951Z" }, + { url = "https://files.pythonhosted.org/packages/7d/f5/bc1456d4638061bea997e6d2db60a1a613d7b200e0755965ec312dc1ef79/hf_xet-1.4.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:22bdc1f5fb8b15bf2831440b91d1c9bbceeb7e10c81a12e8d75889996a5c9da8", size = 4424368, upload-time = "2026-03-31T22:40:06.347Z" }, + { url = "https://files.pythonhosted.org/packages/e4/76/ab597bae87e1f06d18d3ecb8ed7f0d3c9a37037fc32ce76233d369273c64/hf_xet-1.4.3-cp314-cp314t-win_amd64.whl", hash = "sha256:0392c79b7cf48418cd61478c1a925246cf10639f4cd9d94368d8ca1e8df9ea07", size = 3672280, upload-time = "2026-03-31T22:40:16.401Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/2e462d34e23a09a74d73785dbed71cc5dbad82a72eee2ad60a72a554155d/hf_xet-1.4.3-cp314-cp314t-win_arm64.whl", hash = "sha256:681c92a07796325778a79d76c67011764ecc9042a8c3579332b61b63ae512075", size = 3528945, upload-time = "2026-03-31T22:40:14.995Z" }, + { url = "https://files.pythonhosted.org/packages/ac/9f/9c23e4a447b8f83120798f9279d0297a4d1360bdbf59ef49ebec78fe2545/hf_xet-1.4.3-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d0da85329eaf196e03e90b84c2d0aca53bd4573d097a75f99609e80775f98025", size = 3805048, upload-time = "2026-03-31T22:39:53.105Z" }, + { url = "https://files.pythonhosted.org/packages/0b/f8/7aacb8e5f4a7899d39c787b5984e912e6c18b11be136ef13947d7a66d265/hf_xet-1.4.3-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:e23717ce4186b265f69afa66e6f0069fe7efbf331546f5c313d00e123dc84583", size = 3562178, upload-time = "2026-03-31T22:39:51.295Z" }, + { url = "https://files.pythonhosted.org/packages/df/9a/a24b26dc8a65f0ecc0fe5be981a19e61e7ca963b85e062c083f3a9100529/hf_xet-1.4.3-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc360b70c815bf340ed56c7b8c63aacf11762a4b099b2fe2c9bd6d6068668c08", size = 4212320, upload-time = "2026-03-31T22:39:42.922Z" }, + { url = "https://files.pythonhosted.org/packages/53/60/46d493db155d2ee2801b71fb1b0fd67696359047fdd8caee2c914cc50c79/hf_xet-1.4.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:39f2d2e9654cd9b4319885733993807aab6de9dfbd34c42f0b78338d6617421f", size = 3991546, upload-time = "2026-03-31T22:39:41.335Z" }, + { url = "https://files.pythonhosted.org/packages/bc/f5/067363e1c96c6b17256910830d1b54099d06287e10f4ec6ec4e7e08371fc/hf_xet-1.4.3-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:49ad8a8cead2b56051aa84d7fce3e1335efe68df3cf6c058f22a65513885baac", size = 4193200, upload-time = "2026-03-31T22:40:01.936Z" }, + { url = "https://files.pythonhosted.org/packages/42/4b/53951592882d9c23080c7644542fda34a3813104e9e11fa1a7d82d419cb8/hf_xet-1.4.3-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7716d62015477a70ea272d2d68cd7cad140f61c52ee452e133e139abfe2c17ba", size = 4429392, upload-time = "2026-03-31T22:40:03.492Z" }, + { url = "https://files.pythonhosted.org/packages/8a/21/75a6c175b4e79662ad8e62f46a40ce341d8d6b206b06b4320d07d55b188c/hf_xet-1.4.3-cp37-abi3-win_amd64.whl", hash = "sha256:6b591fcad34e272a5b02607485e4f2a1334aebf1bc6d16ce8eb1eb8978ac2021", size = 3677359, upload-time = "2026-03-31T22:40:13.619Z" }, + { url = "https://files.pythonhosted.org/packages/8a/7c/44314ecd0e89f8b2b51c9d9e5e7a60a9c1c82024ac471d415860557d3cd8/hf_xet-1.4.3-cp37-abi3-win_arm64.whl", hash = "sha256:7c2c7e20bcfcc946dc67187c203463f5e932e395845d098cc2a93f5b67ca0b47", size = 3533664, upload-time = "2026-03-31T22:40:12.152Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "huggingface-hub" +version = "1.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "tqdm" }, + { name = "typer" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/40/68d9b286b125d9318ae95c8f8b206e8672e7244b0eea61ebb4a88037638c/huggingface_hub-1.9.1.tar.gz", hash = "sha256:442af372207cc24dcb089caf507fcd7dbc1217c11d6059a06f6b90afe64e8bd2", size = 750355, upload-time = "2026-04-07T13:47:59.167Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/af/10a89c54937dccf6c10792770f362d96dd67aedfde108e6e1fd7a0836789/huggingface_hub-1.9.1-py3-none-any.whl", hash = "sha256:8dae771b969b318203727a6c6c5209d25e661f6f0dd010fc09cc4a12cf81c657", size = 637356, upload-time = "2026-04-07T13:47:57.239Z" }, +] + +[[package]] +name = "humanfriendly" +version = "10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyreadline3", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702, upload-time = "2021-09-17T21:40:43.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477", size = 86794, upload-time = "2021-09-17T21:40:39.897Z" }, +] + +[[package]] +name = "identify" +version = "2.6.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/e7/685de97986c916a6d93b3876139e00eef26ad5bbbd61925d670ae8013449/identify-2.6.15.tar.gz", hash = "sha256:e4f4864b96c6557ef2a1e1c951771838f4edc9df3a72ec7118b338801b11c7bf", size = 99311, upload-time = "2025-10-02T17:43:40.631Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/1c/e5fd8f973d4f375adb21565739498e2e9a1e54c858a97b9a8ccfdc81da9b/identify-2.6.15-py2.py3-none-any.whl", hash = "sha256:1181ef7608e00704db228516541eb83a88a9f94433a8c80bb9b5bd54b1d81757", size = 99183, upload-time = "2025-10-02T17:43:39.137Z" }, +] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "imagesize" +version = "1.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/84/62473fb57d61e31fef6e36d64a179c8781605429fd927b5dd608c997be31/imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a", size = 1280026, upload-time = "2022-07-01T12:21:05.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b", size = 8769, upload-time = "2022-07-01T12:21:02.467Z" }, +] + +[[package]] +name = "importlib-metadata" +version = "8.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f3/49/3b30cad09e7771a4982d9975a8cbf64f00d4a1ececb53297f1d9a7be1b10/importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb", size = 57107, upload-time = "2025-12-21T10:00:19.278Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/5e/f8e9a1d23b9c20a551a8a02ea3637b4642e22c2626e3a13a9a29cdea99eb/importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151", size = 27865, upload-time = "2025-12-21T10:00:18.329Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "ivy" +version = "1.0.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astor" }, + { name = "cryptography" }, + { name = "dill" }, + { name = "einops" }, + { name = "gast" }, + { name = "networkx" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "requests" }, + { name = "ruff" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/33/c6ba417c58d8a55f02335701711398fc0c28dda9e94003ec983ebca0f005/ivy-1.0.0.5.tar.gz", hash = "sha256:e5af36d1d45c4b73ed91c5fcabe76257434bf23dcbff798c22365cf397ebe9de", size = 1951056, upload-time = "2025-06-16T00:12:16.04Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/b3/d6bc38e1dd15f1af13ce905fd164646b9a7a40f5211634262eb19a38ecbd/ivy-1.0.0.5-py3-none-any.whl", hash = "sha256:2370ebbd2ac46dcb44b6a89a0a5cd64e98cac9289708d8c946da3e7acf9b5747", size = 2503058, upload-time = "2025-06-16T00:12:14.124Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "kiwisolver" +version = "1.4.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/3c/85844f1b0feb11ee581ac23fe5fce65cd049a200c1446708cc1b7f922875/kiwisolver-1.4.9.tar.gz", hash = "sha256:c3b22c26c6fd6811b0ae8363b95ca8ce4ea3c202d3d0975b2914310ceb1bcc4d", size = 97564, upload-time = "2025-08-10T21:27:49.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/ab/c80b0d5a9d8a1a65f4f815f2afff9798b12c3b9f31f1d304dd233dd920e2/kiwisolver-1.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:eb14a5da6dc7642b0f3a18f13654847cd8b7a2550e2645a5bda677862b03ba16", size = 124167, upload-time = "2025-08-10T21:25:53.403Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c0/27fe1a68a39cf62472a300e2879ffc13c0538546c359b86f149cc19f6ac3/kiwisolver-1.4.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:39a219e1c81ae3b103643d2aedb90f1ef22650deb266ff12a19e7773f3e5f089", size = 66579, upload-time = "2025-08-10T21:25:54.79Z" }, + { url = "https://files.pythonhosted.org/packages/31/a2/a12a503ac1fd4943c50f9822678e8015a790a13b5490354c68afb8489814/kiwisolver-1.4.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2405a7d98604b87f3fc28b1716783534b1b4b8510d8142adca34ee0bc3c87543", size = 65309, upload-time = "2025-08-10T21:25:55.76Z" }, + { url = "https://files.pythonhosted.org/packages/66/e1/e533435c0be77c3f64040d68d7a657771194a63c279f55573188161e81ca/kiwisolver-1.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:dc1ae486f9abcef254b5618dfb4113dd49f94c68e3e027d03cf0143f3f772b61", size = 1435596, upload-time = "2025-08-10T21:25:56.861Z" }, + { url = "https://files.pythonhosted.org/packages/67/1e/51b73c7347f9aabdc7215aa79e8b15299097dc2f8e67dee2b095faca9cb0/kiwisolver-1.4.9-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a1f570ce4d62d718dce3f179ee78dac3b545ac16c0c04bb363b7607a949c0d1", size = 1246548, upload-time = "2025-08-10T21:25:58.246Z" }, + { url = "https://files.pythonhosted.org/packages/21/aa/72a1c5d1e430294f2d32adb9542719cfb441b5da368d09d268c7757af46c/kiwisolver-1.4.9-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb27e7b78d716c591e88e0a09a2139c6577865d7f2e152488c2cc6257f460872", size = 1263618, upload-time = "2025-08-10T21:25:59.857Z" }, + { url = "https://files.pythonhosted.org/packages/a3/af/db1509a9e79dbf4c260ce0cfa3903ea8945f6240e9e59d1e4deb731b1a40/kiwisolver-1.4.9-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:15163165efc2f627eb9687ea5f3a28137217d217ac4024893d753f46bce9de26", size = 1317437, upload-time = "2025-08-10T21:26:01.105Z" }, + { url = "https://files.pythonhosted.org/packages/e0/f2/3ea5ee5d52abacdd12013a94130436e19969fa183faa1e7c7fbc89e9a42f/kiwisolver-1.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bdee92c56a71d2b24c33a7d4c2856bd6419d017e08caa7802d2963870e315028", size = 2195742, upload-time = "2025-08-10T21:26:02.675Z" }, + { url = "https://files.pythonhosted.org/packages/6f/9b/1efdd3013c2d9a2566aa6a337e9923a00590c516add9a1e89a768a3eb2fc/kiwisolver-1.4.9-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:412f287c55a6f54b0650bd9b6dce5aceddb95864a1a90c87af16979d37c89771", size = 2290810, upload-time = "2025-08-10T21:26:04.009Z" }, + { url = "https://files.pythonhosted.org/packages/fb/e5/cfdc36109ae4e67361f9bc5b41323648cb24a01b9ade18784657e022e65f/kiwisolver-1.4.9-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:2c93f00dcba2eea70af2be5f11a830a742fe6b579a1d4e00f47760ef13be247a", size = 2461579, upload-time = "2025-08-10T21:26:05.317Z" }, + { url = "https://files.pythonhosted.org/packages/62/86/b589e5e86c7610842213994cdea5add00960076bef4ae290c5fa68589cac/kiwisolver-1.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f117e1a089d9411663a3207ba874f31be9ac8eaa5b533787024dc07aeb74f464", size = 2268071, upload-time = "2025-08-10T21:26:06.686Z" }, + { url = "https://files.pythonhosted.org/packages/3b/c6/f8df8509fd1eee6c622febe54384a96cfaf4d43bf2ccec7a0cc17e4715c9/kiwisolver-1.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:be6a04e6c79819c9a8c2373317d19a96048e5a3f90bec587787e86a1153883c2", size = 73840, upload-time = "2025-08-10T21:26:07.94Z" }, + { url = "https://files.pythonhosted.org/packages/e2/2d/16e0581daafd147bc11ac53f032a2b45eabac897f42a338d0a13c1e5c436/kiwisolver-1.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:0ae37737256ba2de764ddc12aed4956460277f00c4996d51a197e72f62f5eec7", size = 65159, upload-time = "2025-08-10T21:26:09.048Z" }, + { url = "https://files.pythonhosted.org/packages/86/c9/13573a747838aeb1c76e3267620daa054f4152444d1f3d1a2324b78255b5/kiwisolver-1.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ac5a486ac389dddcc5bef4f365b6ae3ffff2c433324fb38dd35e3fab7c957999", size = 123686, upload-time = "2025-08-10T21:26:10.034Z" }, + { url = "https://files.pythonhosted.org/packages/51/ea/2ecf727927f103ffd1739271ca19c424d0e65ea473fbaeea1c014aea93f6/kiwisolver-1.4.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f2ba92255faa7309d06fe44c3a4a97efe1c8d640c2a79a5ef728b685762a6fd2", size = 66460, upload-time = "2025-08-10T21:26:11.083Z" }, + { url = "https://files.pythonhosted.org/packages/5b/5a/51f5464373ce2aeb5194508298a508b6f21d3867f499556263c64c621914/kiwisolver-1.4.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a2899935e724dd1074cb568ce7ac0dce28b2cd6ab539c8e001a8578eb106d14", size = 64952, upload-time = "2025-08-10T21:26:12.058Z" }, + { url = "https://files.pythonhosted.org/packages/70/90/6d240beb0f24b74371762873e9b7f499f1e02166a2d9c5801f4dbf8fa12e/kiwisolver-1.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f6008a4919fdbc0b0097089f67a1eb55d950ed7e90ce2cc3e640abadd2757a04", size = 1474756, upload-time = "2025-08-10T21:26:13.096Z" }, + { url = "https://files.pythonhosted.org/packages/12/42/f36816eaf465220f683fb711efdd1bbf7a7005a2473d0e4ed421389bd26c/kiwisolver-1.4.9-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:67bb8b474b4181770f926f7b7d2f8c0248cbcb78b660fdd41a47054b28d2a752", size = 1276404, upload-time = "2025-08-10T21:26:14.457Z" }, + { url = "https://files.pythonhosted.org/packages/2e/64/bc2de94800adc830c476dce44e9b40fd0809cddeef1fde9fcf0f73da301f/kiwisolver-1.4.9-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2327a4a30d3ee07d2fbe2e7933e8a37c591663b96ce42a00bc67461a87d7df77", size = 1294410, upload-time = "2025-08-10T21:26:15.73Z" }, + { url = "https://files.pythonhosted.org/packages/5f/42/2dc82330a70aa8e55b6d395b11018045e58d0bb00834502bf11509f79091/kiwisolver-1.4.9-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7a08b491ec91b1d5053ac177afe5290adacf1f0f6307d771ccac5de30592d198", size = 1343631, upload-time = "2025-08-10T21:26:17.045Z" }, + { url = "https://files.pythonhosted.org/packages/22/fd/f4c67a6ed1aab149ec5a8a401c323cee7a1cbe364381bb6c9c0d564e0e20/kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d8fc5c867c22b828001b6a38d2eaeb88160bf5783c6cb4a5e440efc981ce286d", size = 2224963, upload-time = "2025-08-10T21:26:18.737Z" }, + { url = "https://files.pythonhosted.org/packages/45/aa/76720bd4cb3713314677d9ec94dcc21ced3f1baf4830adde5bb9b2430a5f/kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3b3115b2581ea35bb6d1f24a4c90af37e5d9b49dcff267eeed14c3893c5b86ab", size = 2321295, upload-time = "2025-08-10T21:26:20.11Z" }, + { url = "https://files.pythonhosted.org/packages/80/19/d3ec0d9ab711242f56ae0dc2fc5d70e298bb4a1f9dfab44c027668c673a1/kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:858e4c22fb075920b96a291928cb7dea5644e94c0ee4fcd5af7e865655e4ccf2", size = 2487987, upload-time = "2025-08-10T21:26:21.49Z" }, + { url = "https://files.pythonhosted.org/packages/39/e9/61e4813b2c97e86b6fdbd4dd824bf72d28bcd8d4849b8084a357bc0dd64d/kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ed0fecd28cc62c54b262e3736f8bb2512d8dcfdc2bcf08be5f47f96bf405b145", size = 2291817, upload-time = "2025-08-10T21:26:22.812Z" }, + { url = "https://files.pythonhosted.org/packages/a0/41/85d82b0291db7504da3c2defe35c9a8a5c9803a730f297bd823d11d5fb77/kiwisolver-1.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:f68208a520c3d86ea51acf688a3e3002615a7f0238002cccc17affecc86a8a54", size = 73895, upload-time = "2025-08-10T21:26:24.37Z" }, + { url = "https://files.pythonhosted.org/packages/e2/92/5f3068cf15ee5cb624a0c7596e67e2a0bb2adee33f71c379054a491d07da/kiwisolver-1.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:2c1a4f57df73965f3f14df20b80ee29e6a7930a57d2d9e8491a25f676e197c60", size = 64992, upload-time = "2025-08-10T21:26:25.732Z" }, + { url = "https://files.pythonhosted.org/packages/31/c1/c2686cda909742ab66c7388e9a1a8521a59eb89f8bcfbee28fc980d07e24/kiwisolver-1.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5d0432ccf1c7ab14f9949eec60c5d1f924f17c037e9f8b33352fa05799359b8", size = 123681, upload-time = "2025-08-10T21:26:26.725Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f0/f44f50c9f5b1a1860261092e3bc91ecdc9acda848a8b8c6abfda4a24dd5c/kiwisolver-1.4.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efb3a45b35622bb6c16dbfab491a8f5a391fe0e9d45ef32f4df85658232ca0e2", size = 66464, upload-time = "2025-08-10T21:26:27.733Z" }, + { url = "https://files.pythonhosted.org/packages/2d/7a/9d90a151f558e29c3936b8a47ac770235f436f2120aca41a6d5f3d62ae8d/kiwisolver-1.4.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1a12cf6398e8a0a001a059747a1cbf24705e18fe413bc22de7b3d15c67cffe3f", size = 64961, upload-time = "2025-08-10T21:26:28.729Z" }, + { url = "https://files.pythonhosted.org/packages/e9/e9/f218a2cb3a9ffbe324ca29a9e399fa2d2866d7f348ec3a88df87fc248fc5/kiwisolver-1.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b67e6efbf68e077dd71d1a6b37e43e1a99d0bff1a3d51867d45ee8908b931098", size = 1474607, upload-time = "2025-08-10T21:26:29.798Z" }, + { url = "https://files.pythonhosted.org/packages/d9/28/aac26d4c882f14de59041636292bc838db8961373825df23b8eeb807e198/kiwisolver-1.4.9-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5656aa670507437af0207645273ccdfee4f14bacd7f7c67a4306d0dcaeaf6eed", size = 1276546, upload-time = "2025-08-10T21:26:31.401Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ad/8bfc1c93d4cc565e5069162f610ba2f48ff39b7de4b5b8d93f69f30c4bed/kiwisolver-1.4.9-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bfc08add558155345129c7803b3671cf195e6a56e7a12f3dde7c57d9b417f525", size = 1294482, upload-time = "2025-08-10T21:26:32.721Z" }, + { url = "https://files.pythonhosted.org/packages/da/f1/6aca55ff798901d8ce403206d00e033191f63d82dd708a186e0ed2067e9c/kiwisolver-1.4.9-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:40092754720b174e6ccf9e845d0d8c7d8e12c3d71e7fc35f55f3813e96376f78", size = 1343720, upload-time = "2025-08-10T21:26:34.032Z" }, + { url = "https://files.pythonhosted.org/packages/d1/91/eed031876c595c81d90d0f6fc681ece250e14bf6998c3d7c419466b523b7/kiwisolver-1.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:497d05f29a1300d14e02e6441cf0f5ee81c1ff5a304b0d9fb77423974684e08b", size = 2224907, upload-time = "2025-08-10T21:26:35.824Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ec/4d1925f2e49617b9cca9c34bfa11adefad49d00db038e692a559454dfb2e/kiwisolver-1.4.9-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:bdd1a81a1860476eb41ac4bc1e07b3f07259e6d55bbf739b79c8aaedcf512799", size = 2321334, upload-time = "2025-08-10T21:26:37.534Z" }, + { url = "https://files.pythonhosted.org/packages/43/cb/450cd4499356f68802750c6ddc18647b8ea01ffa28f50d20598e0befe6e9/kiwisolver-1.4.9-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:e6b93f13371d341afee3be9f7c5964e3fe61d5fa30f6a30eb49856935dfe4fc3", size = 2488313, upload-time = "2025-08-10T21:26:39.191Z" }, + { url = "https://files.pythonhosted.org/packages/71/67/fc76242bd99f885651128a5d4fa6083e5524694b7c88b489b1b55fdc491d/kiwisolver-1.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d75aa530ccfaa593da12834b86a0724f58bff12706659baa9227c2ccaa06264c", size = 2291970, upload-time = "2025-08-10T21:26:40.828Z" }, + { url = "https://files.pythonhosted.org/packages/75/bd/f1a5d894000941739f2ae1b65a32892349423ad49c2e6d0771d0bad3fae4/kiwisolver-1.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:dd0a578400839256df88c16abddf9ba14813ec5f21362e1fe65022e00c883d4d", size = 73894, upload-time = "2025-08-10T21:26:42.33Z" }, + { url = "https://files.pythonhosted.org/packages/95/38/dce480814d25b99a391abbddadc78f7c117c6da34be68ca8b02d5848b424/kiwisolver-1.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:d4188e73af84ca82468f09cadc5ac4db578109e52acb4518d8154698d3a87ca2", size = 64995, upload-time = "2025-08-10T21:26:43.889Z" }, + { url = "https://files.pythonhosted.org/packages/e2/37/7d218ce5d92dadc5ebdd9070d903e0c7cf7edfe03f179433ac4d13ce659c/kiwisolver-1.4.9-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:5a0f2724dfd4e3b3ac5a82436a8e6fd16baa7d507117e4279b660fe8ca38a3a1", size = 126510, upload-time = "2025-08-10T21:26:44.915Z" }, + { url = "https://files.pythonhosted.org/packages/23/b0/e85a2b48233daef4b648fb657ebbb6f8367696a2d9548a00b4ee0eb67803/kiwisolver-1.4.9-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:1b11d6a633e4ed84fc0ddafd4ebfd8ea49b3f25082c04ad12b8315c11d504dc1", size = 67903, upload-time = "2025-08-10T21:26:45.934Z" }, + { url = "https://files.pythonhosted.org/packages/44/98/f2425bc0113ad7de24da6bb4dae1343476e95e1d738be7c04d31a5d037fd/kiwisolver-1.4.9-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:61874cdb0a36016354853593cffc38e56fc9ca5aa97d2c05d3dcf6922cd55a11", size = 66402, upload-time = "2025-08-10T21:26:47.101Z" }, + { url = "https://files.pythonhosted.org/packages/98/d8/594657886df9f34c4177cc353cc28ca7e6e5eb562d37ccc233bff43bbe2a/kiwisolver-1.4.9-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:60c439763a969a6af93b4881db0eed8fadf93ee98e18cbc35bc8da868d0c4f0c", size = 1582135, upload-time = "2025-08-10T21:26:48.665Z" }, + { url = "https://files.pythonhosted.org/packages/5c/c6/38a115b7170f8b306fc929e166340c24958347308ea3012c2b44e7e295db/kiwisolver-1.4.9-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92a2f997387a1b79a75e7803aa7ded2cfbe2823852ccf1ba3bcf613b62ae3197", size = 1389409, upload-time = "2025-08-10T21:26:50.335Z" }, + { url = "https://files.pythonhosted.org/packages/bf/3b/e04883dace81f24a568bcee6eb3001da4ba05114afa622ec9b6fafdc1f5e/kiwisolver-1.4.9-cp313-cp313t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a31d512c812daea6d8b3be3b2bfcbeb091dbb09177706569bcfc6240dcf8b41c", size = 1401763, upload-time = "2025-08-10T21:26:51.867Z" }, + { url = "https://files.pythonhosted.org/packages/9f/80/20ace48e33408947af49d7d15c341eaee69e4e0304aab4b7660e234d6288/kiwisolver-1.4.9-cp313-cp313t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:52a15b0f35dad39862d376df10c5230155243a2c1a436e39eb55623ccbd68185", size = 1453643, upload-time = "2025-08-10T21:26:53.592Z" }, + { url = "https://files.pythonhosted.org/packages/64/31/6ce4380a4cd1f515bdda976a1e90e547ccd47b67a1546d63884463c92ca9/kiwisolver-1.4.9-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a30fd6fdef1430fd9e1ba7b3398b5ee4e2887783917a687d86ba69985fb08748", size = 2330818, upload-time = "2025-08-10T21:26:55.051Z" }, + { url = "https://files.pythonhosted.org/packages/fa/e9/3f3fcba3bcc7432c795b82646306e822f3fd74df0ee81f0fa067a1f95668/kiwisolver-1.4.9-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cc9617b46837c6468197b5945e196ee9ca43057bb7d9d1ae688101e4e1dddf64", size = 2419963, upload-time = "2025-08-10T21:26:56.421Z" }, + { url = "https://files.pythonhosted.org/packages/99/43/7320c50e4133575c66e9f7dadead35ab22d7c012a3b09bb35647792b2a6d/kiwisolver-1.4.9-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:0ab74e19f6a2b027ea4f845a78827969af45ce790e6cb3e1ebab71bdf9f215ff", size = 2594639, upload-time = "2025-08-10T21:26:57.882Z" }, + { url = "https://files.pythonhosted.org/packages/65/d6/17ae4a270d4a987ef8a385b906d2bdfc9fce502d6dc0d3aea865b47f548c/kiwisolver-1.4.9-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dba5ee5d3981160c28d5490f0d1b7ed730c22470ff7f6cc26cfcfaacb9896a07", size = 2391741, upload-time = "2025-08-10T21:26:59.237Z" }, + { url = "https://files.pythonhosted.org/packages/2a/8f/8f6f491d595a9e5912971f3f863d81baddccc8a4d0c3749d6a0dd9ffc9df/kiwisolver-1.4.9-cp313-cp313t-win_arm64.whl", hash = "sha256:0749fd8f4218ad2e851e11cc4dc05c7cbc0cbc4267bdfdb31782e65aace4ee9c", size = 68646, upload-time = "2025-08-10T21:27:00.52Z" }, + { url = "https://files.pythonhosted.org/packages/6b/32/6cc0fbc9c54d06c2969faa9c1d29f5751a2e51809dd55c69055e62d9b426/kiwisolver-1.4.9-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:9928fe1eb816d11ae170885a74d074f57af3a0d65777ca47e9aeb854a1fba386", size = 123806, upload-time = "2025-08-10T21:27:01.537Z" }, + { url = "https://files.pythonhosted.org/packages/b2/dd/2bfb1d4a4823d92e8cbb420fe024b8d2167f72079b3bb941207c42570bdf/kiwisolver-1.4.9-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:d0005b053977e7b43388ddec89fa567f43d4f6d5c2c0affe57de5ebf290dc552", size = 66605, upload-time = "2025-08-10T21:27:03.335Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/00aafdb4e4509c2ca6064646cba9cd4b37933898f426756adb2cb92ebbed/kiwisolver-1.4.9-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2635d352d67458b66fd0667c14cb1d4145e9560d503219034a18a87e971ce4f3", size = 64925, upload-time = "2025-08-10T21:27:04.339Z" }, + { url = "https://files.pythonhosted.org/packages/43/dc/51acc6791aa14e5cb6d8a2e28cefb0dc2886d8862795449d021334c0df20/kiwisolver-1.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:767c23ad1c58c9e827b649a9ab7809fd5fd9db266a9cf02b0e926ddc2c680d58", size = 1472414, upload-time = "2025-08-10T21:27:05.437Z" }, + { url = "https://files.pythonhosted.org/packages/3d/bb/93fa64a81db304ac8a246f834d5094fae4b13baf53c839d6bb6e81177129/kiwisolver-1.4.9-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72d0eb9fba308b8311685c2268cf7d0a0639a6cd027d8128659f72bdd8a024b4", size = 1281272, upload-time = "2025-08-10T21:27:07.063Z" }, + { url = "https://files.pythonhosted.org/packages/70/e6/6df102916960fb8d05069d4bd92d6d9a8202d5a3e2444494e7cd50f65b7a/kiwisolver-1.4.9-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f68e4f3eeca8fb22cc3d731f9715a13b652795ef657a13df1ad0c7dc0e9731df", size = 1298578, upload-time = "2025-08-10T21:27:08.452Z" }, + { url = "https://files.pythonhosted.org/packages/7c/47/e142aaa612f5343736b087864dbaebc53ea8831453fb47e7521fa8658f30/kiwisolver-1.4.9-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d84cd4061ae292d8ac367b2c3fa3aad11cb8625a95d135fe93f286f914f3f5a6", size = 1345607, upload-time = "2025-08-10T21:27:10.125Z" }, + { url = "https://files.pythonhosted.org/packages/54/89/d641a746194a0f4d1a3670fb900d0dbaa786fb98341056814bc3f058fa52/kiwisolver-1.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a60ea74330b91bd22a29638940d115df9dc00af5035a9a2a6ad9399ffb4ceca5", size = 2230150, upload-time = "2025-08-10T21:27:11.484Z" }, + { url = "https://files.pythonhosted.org/packages/aa/6b/5ee1207198febdf16ac11f78c5ae40861b809cbe0e6d2a8d5b0b3044b199/kiwisolver-1.4.9-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ce6a3a4e106cf35c2d9c4fa17c05ce0b180db622736845d4315519397a77beaf", size = 2325979, upload-time = "2025-08-10T21:27:12.917Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ff/b269eefd90f4ae14dcc74973d5a0f6d28d3b9bb1afd8c0340513afe6b39a/kiwisolver-1.4.9-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:77937e5e2a38a7b48eef0585114fe7930346993a88060d0bf886086d2aa49ef5", size = 2491456, upload-time = "2025-08-10T21:27:14.353Z" }, + { url = "https://files.pythonhosted.org/packages/fc/d4/10303190bd4d30de547534601e259a4fbf014eed94aae3e5521129215086/kiwisolver-1.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:24c175051354f4a28c5d6a31c93906dc653e2bf234e8a4bbfb964892078898ce", size = 2294621, upload-time = "2025-08-10T21:27:15.808Z" }, + { url = "https://files.pythonhosted.org/packages/28/e0/a9a90416fce5c0be25742729c2ea52105d62eda6c4be4d803c2a7be1fa50/kiwisolver-1.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:0763515d4df10edf6d06a3c19734e2566368980d21ebec439f33f9eb936c07b7", size = 75417, upload-time = "2025-08-10T21:27:17.436Z" }, + { url = "https://files.pythonhosted.org/packages/1f/10/6949958215b7a9a264299a7db195564e87900f709db9245e4ebdd3c70779/kiwisolver-1.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:0e4e2bf29574a6a7b7f6cb5fa69293b9f96c928949ac4a53ba3f525dffb87f9c", size = 66582, upload-time = "2025-08-10T21:27:18.436Z" }, + { url = "https://files.pythonhosted.org/packages/ec/79/60e53067903d3bc5469b369fe0dfc6b3482e2133e85dae9daa9527535991/kiwisolver-1.4.9-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d976bbb382b202f71c67f77b0ac11244021cfa3f7dfd9e562eefcea2df711548", size = 126514, upload-time = "2025-08-10T21:27:19.465Z" }, + { url = "https://files.pythonhosted.org/packages/25/d1/4843d3e8d46b072c12a38c97c57fab4608d36e13fe47d47ee96b4d61ba6f/kiwisolver-1.4.9-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2489e4e5d7ef9a1c300a5e0196e43d9c739f066ef23270607d45aba368b91f2d", size = 67905, upload-time = "2025-08-10T21:27:20.51Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ae/29ffcbd239aea8b93108de1278271ae764dfc0d803a5693914975f200596/kiwisolver-1.4.9-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e2ea9f7ab7fbf18fffb1b5434ce7c69a07582f7acc7717720f1d69f3e806f90c", size = 66399, upload-time = "2025-08-10T21:27:21.496Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ae/d7ba902aa604152c2ceba5d352d7b62106bedbccc8e95c3934d94472bfa3/kiwisolver-1.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b34e51affded8faee0dfdb705416153819d8ea9250bbbf7ea1b249bdeb5f1122", size = 1582197, upload-time = "2025-08-10T21:27:22.604Z" }, + { url = "https://files.pythonhosted.org/packages/f2/41/27c70d427eddb8bc7e4f16420a20fefc6f480312122a59a959fdfe0445ad/kiwisolver-1.4.9-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8aacd3d4b33b772542b2e01beb50187536967b514b00003bdda7589722d2a64", size = 1390125, upload-time = "2025-08-10T21:27:24.036Z" }, + { url = "https://files.pythonhosted.org/packages/41/42/b3799a12bafc76d962ad69083f8b43b12bf4fe78b097b12e105d75c9b8f1/kiwisolver-1.4.9-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7cf974dd4e35fa315563ac99d6287a1024e4dc2077b8a7d7cd3d2fb65d283134", size = 1402612, upload-time = "2025-08-10T21:27:25.773Z" }, + { url = "https://files.pythonhosted.org/packages/d2/b5/a210ea073ea1cfaca1bb5c55a62307d8252f531beb364e18aa1e0888b5a0/kiwisolver-1.4.9-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:85bd218b5ecfbee8c8a82e121802dcb519a86044c9c3b2e4aef02fa05c6da370", size = 1453990, upload-time = "2025-08-10T21:27:27.089Z" }, + { url = "https://files.pythonhosted.org/packages/5f/ce/a829eb8c033e977d7ea03ed32fb3c1781b4fa0433fbadfff29e39c676f32/kiwisolver-1.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0856e241c2d3df4efef7c04a1e46b1936b6120c9bcf36dd216e3acd84bc4fb21", size = 2331601, upload-time = "2025-08-10T21:27:29.343Z" }, + { url = "https://files.pythonhosted.org/packages/e0/4b/b5e97eb142eb9cd0072dacfcdcd31b1c66dc7352b0f7c7255d339c0edf00/kiwisolver-1.4.9-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:9af39d6551f97d31a4deebeac6f45b156f9755ddc59c07b402c148f5dbb6482a", size = 2422041, upload-time = "2025-08-10T21:27:30.754Z" }, + { url = "https://files.pythonhosted.org/packages/40/be/8eb4cd53e1b85ba4edc3a9321666f12b83113a178845593307a3e7891f44/kiwisolver-1.4.9-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:bb4ae2b57fc1d8cbd1cf7b1d9913803681ffa903e7488012be5b76dedf49297f", size = 2594897, upload-time = "2025-08-10T21:27:32.803Z" }, + { url = "https://files.pythonhosted.org/packages/99/dd/841e9a66c4715477ea0abc78da039832fbb09dac5c35c58dc4c41a407b8a/kiwisolver-1.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:aedff62918805fb62d43a4aa2ecd4482c380dc76cd31bd7c8878588a61bd0369", size = 2391835, upload-time = "2025-08-10T21:27:34.23Z" }, + { url = "https://files.pythonhosted.org/packages/0c/28/4b2e5c47a0da96896fdfdb006340ade064afa1e63675d01ea5ac222b6d52/kiwisolver-1.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:1fa333e8b2ce4d9660f2cda9c0e1b6bafcfb2457a9d259faa82289e73ec24891", size = 79988, upload-time = "2025-08-10T21:27:35.587Z" }, + { url = "https://files.pythonhosted.org/packages/80/be/3578e8afd18c88cdf9cb4cffde75a96d2be38c5a903f1ed0ceec061bd09e/kiwisolver-1.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:4a48a2ce79d65d363597ef7b567ce3d14d68783d2b2263d98db3d9477805ba32", size = 70260, upload-time = "2025-08-10T21:27:36.606Z" }, + { url = "https://files.pythonhosted.org/packages/a3/0f/36d89194b5a32c054ce93e586d4049b6c2c22887b0eb229c61c68afd3078/kiwisolver-1.4.9-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:720e05574713db64c356e86732c0f3c5252818d05f9df320f0ad8380641acea5", size = 60104, upload-time = "2025-08-10T21:27:43.287Z" }, + { url = "https://files.pythonhosted.org/packages/52/ba/4ed75f59e4658fd21fe7dde1fee0ac397c678ec3befba3fe6482d987af87/kiwisolver-1.4.9-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:17680d737d5335b552994a2008fab4c851bcd7de33094a82067ef3a576ff02fa", size = 58592, upload-time = "2025-08-10T21:27:44.314Z" }, + { url = "https://files.pythonhosted.org/packages/33/01/a8ea7c5ea32a9b45ceeaee051a04c8ed4320f5add3c51bfa20879b765b70/kiwisolver-1.4.9-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:85b5352f94e490c028926ea567fc569c52ec79ce131dadb968d3853e809518c2", size = 80281, upload-time = "2025-08-10T21:27:45.369Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/dbd2ecdce306f1d07a1aaf324817ee993aab7aee9db47ceac757deabafbe/kiwisolver-1.4.9-pp311-pypy311_pp73-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:464415881e4801295659462c49461a24fb107c140de781d55518c4b80cb6790f", size = 78009, upload-time = "2025-08-10T21:27:46.376Z" }, + { url = "https://files.pythonhosted.org/packages/da/e9/0d4add7873a73e462aeb45c036a2dead2562b825aa46ba326727b3f31016/kiwisolver-1.4.9-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:fb940820c63a9590d31d88b815e7a3aa5915cad3ce735ab45f0c730b39547de1", size = 73929, upload-time = "2025-08-10T21:27:48.236Z" }, +] + +[[package]] +name = "kornia" +source = { editable = "." } +dependencies = [ + { name = "kornia-rs" }, + { name = "packaging" }, + { name = "torch" }, +] + +[package.optional-dependencies] +dev = [ + { name = "coverage" }, + { name = "diffusers" }, + { name = "ivy" }, + { name = "numpy" }, + { name = "onnx" }, + { name = "onnxruntime" }, + { name = "onnxscript" }, + { name = "pillow" }, + { name = "pre-commit" }, + { name = "pytest" }, + { name = "pytest-cov" }, + { name = "pytest-timeout" }, + { name = "requests" }, + { name = "ruff" }, + { name = "setuptools" }, + { name = "transformers" }, +] +docs = [ + { name = "furo" }, + { name = "ivy" }, + { name = "kornia-moons" }, + { name = "matplotlib" }, + { name = "onnx" }, + { name = "onnxruntime" }, + { name = "opencv-python" }, + { name = "pyyaml" }, + { name = "sphinx" }, + { name = "sphinx-autodoc-defaultargs" }, + { name = "sphinx-autodoc-typehints" }, + { name = "sphinx-copybutton" }, + { name = "sphinx-design" }, + { name = "sphinx-notfound-page" }, + { name = "sphinxcontrib-bibtex" }, + { name = "sphinxcontrib-gtagjs" }, + { name = "sphinxcontrib-youtube" }, +] + +[package.metadata] +requires-dist = [ + { name = "coverage", marker = "extra == 'dev'" }, + { name = "diffusers", marker = "extra == 'dev'" }, + { name = "furo", marker = "extra == 'docs'" }, + { name = "ivy", marker = "extra == 'dev'", specifier = ">=1.0.0.0" }, + { name = "ivy", marker = "extra == 'docs'", specifier = ">=1.0.0.0" }, + { name = "kornia-moons", marker = "extra == 'docs'" }, + { name = "kornia-rs", specifier = ">=0.1.9" }, + { name = "matplotlib", marker = "extra == 'docs'" }, + { name = "numpy", marker = "extra == 'dev'", specifier = "<3" }, + { name = "onnx", marker = "extra == 'dev'" }, + { name = "onnx", marker = "extra == 'docs'" }, + { name = "onnxruntime", marker = "extra == 'dev'" }, + { name = "onnxruntime", marker = "extra == 'docs'" }, + { name = "onnxscript", marker = "extra == 'dev'" }, + { name = "opencv-python", marker = "extra == 'docs'" }, + { name = "packaging" }, + { name = "pillow", marker = "extra == 'dev'" }, + { name = "pre-commit", marker = "extra == 'dev'" }, + { name = "pytest", marker = "extra == 'dev'" }, + { name = "pytest-cov", marker = "extra == 'dev'" }, + { name = "pytest-timeout", marker = "extra == 'dev'" }, + { name = "pyyaml", marker = "extra == 'docs'", specifier = ">=5.1" }, + { name = "requests", marker = "extra == 'dev'" }, + { name = "ruff", marker = "extra == 'dev'" }, + { name = "setuptools", marker = "extra == 'dev'" }, + { name = "sphinx", marker = "extra == 'docs'" }, + { name = "sphinx-autodoc-defaultargs", marker = "extra == 'docs'" }, + { name = "sphinx-autodoc-typehints", marker = "extra == 'docs'" }, + { name = "sphinx-copybutton", marker = "extra == 'docs'", specifier = ">=0.3" }, + { name = "sphinx-design", marker = "extra == 'docs'" }, + { name = "sphinx-notfound-page", marker = "extra == 'docs'" }, + { name = "sphinxcontrib-bibtex", marker = "extra == 'docs'" }, + { name = "sphinxcontrib-gtagjs", marker = "extra == 'docs'" }, + { name = "sphinxcontrib-youtube", marker = "extra == 'docs'" }, + { name = "torch", specifier = ">=2.0.0" }, + { name = "transformers", marker = "extra == 'dev'", specifier = "<5.6" }, +] +provides-extras = ["dev", "docs"] + +[[package]] +name = "kornia-moons" +version = "0.2.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "kornia" }, + { name = "matplotlib" }, + { name = "opencv-python" }, + { name = "torch" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/37/4b/10defe8ac68c3ab2b42c58ed302d36e3ba69a33f092f5798a8248e0e0f68/kornia_moons-0.2.9.tar.gz", hash = "sha256:e6e6b3523296a38f099712a322d708730720cfb97de437aa90fa586a86556455", size = 16816, upload-time = "2023-09-26T12:05:10.931Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/61/2d7b4383eab721c785c318d06e412f49a4e04568b46291d30aa3d2e7dba5/kornia_moons-0.2.9-py3-none-any.whl", hash = "sha256:d35e799b29031365c7b57fee3d84b4be547c0513f7c013e8234e05d33d2b1061", size = 15628, upload-time = "2023-09-26T12:05:08.97Z" }, +] + +[[package]] +name = "kornia-rs" +version = "0.1.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ab/17/8b3518ece01512a575b18f86b346879793d3dea264b314796bbd44d42e11/kornia_rs-0.1.10.tar.gz", hash = "sha256:5fd3fbc65240fa751975f5870b079f98e7fdcaa2885ea577b3da324d8bf01d81", size = 145610, upload-time = "2025-11-08T11:29:32.399Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/25/ab91a87cefd8d92a10749fa5d923366dfd2a2d240d9e57260e4218e9a5af/kornia_rs-0.1.10-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:6757940733f13c52c4f142b9b11e3e9bd12ef9d209e333300602e86e21f5ae2f", size = 2811949, upload-time = "2025-11-08T11:30:19.768Z" }, + { url = "https://files.pythonhosted.org/packages/ae/61/6125a970249e04dd31cf3edf3fb0ceb98ea65269bc416ba48fd70f9a8f5e/kornia_rs-0.1.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:68e90101a34ba2bbce920332b25fd4d25c8c546d9a241b2606a6d886df2dd1ed", size = 2078639, upload-time = "2025-11-08T11:30:06.363Z" }, + { url = "https://files.pythonhosted.org/packages/0a/e4/c3484e5921a08e6368f0565c30646741fd12b46cb45c962d519cac3d12ad/kornia_rs-0.1.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b0adb81858a8963455f2f0da01fcd6ea3296147b918306488edeeaf6bc2a979", size = 2204722, upload-time = "2025-11-08T11:29:33.566Z" }, + { url = "https://files.pythonhosted.org/packages/93/a4/2e6e33da900f19ae6411bfad41d317e56f1ae4f204bd73e61f0881bd5418/kornia_rs-0.1.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c3e237a8428524ad9f86599c0c47b355bc3007669fe297ea3fbd59cd64bc2f7", size = 3042890, upload-time = "2025-11-08T11:29:50.15Z" }, + { url = "https://files.pythonhosted.org/packages/40/48/5e171c98b742139bebd1bd593d768e3c045f824bf0ae14190b63f0ac0acc/kornia_rs-0.1.10-cp311-cp311-win_amd64.whl", hash = "sha256:1d300ea6d4666e47302fba6cc438556d91e37ce41caf291a9a04a8f74c231d0b", size = 2544572, upload-time = "2025-11-08T11:30:32.32Z" }, + { url = "https://files.pythonhosted.org/packages/d8/6c/8248f08c90a10d6b8ca2e74783da8df7fa509f46b64a3b4fbb7dd0ac4e9c/kornia_rs-0.1.10-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f0809277e51156d59be3c39605ba9659e94f7a4cf3b0b6c035ec2f06f6067881", size = 2811606, upload-time = "2025-11-08T11:30:21.346Z" }, + { url = "https://files.pythonhosted.org/packages/83/dc/29e5710cbc5d01c155ee1fd7621db48b94378a7ae394741bb34a6bfb36d9/kornia_rs-0.1.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ecf2ba0291cc1bb178073d56e46b16296a8864a20272b63af02ee88771cb574", size = 2076141, upload-time = "2025-11-08T11:30:07.527Z" }, + { url = "https://files.pythonhosted.org/packages/68/f7/0b3e90b9d0a25e6211c7ac9fa1dfed4db1306a812c359ee49678390a1bdc/kornia_rs-0.1.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d874ca12dd58871f9849672d9bf9fa998398470a88b52d61223ce2133b196662", size = 2205562, upload-time = "2025-11-08T11:29:35.353Z" }, + { url = "https://files.pythonhosted.org/packages/63/d4/315f358b2a2c29d9af3a73f3d1973c2fd8e0cdeb65a57af98643e66fa7c8/kornia_rs-0.1.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f332a2a034cc791006f25c2d85e342a060887145e9236e8e43562badcadededf", size = 3042197, upload-time = "2025-11-08T11:29:51.614Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b8/0ddbdf1d35fec3ef24f5b8cc29eb633ce5ce16c94c9fb090408c1280abe9/kornia_rs-0.1.10-cp312-cp312-win_amd64.whl", hash = "sha256:34111ce1c8abe930079b4b0aeb8d372f876c621a867ed03f77181de685e71a8f", size = 2539656, upload-time = "2025-11-08T11:30:33.908Z" }, + { url = "https://files.pythonhosted.org/packages/90/01/1d658b11635431f8c31f416c90ca99befdc1f4fdd20e91a05b480b9c0ea8/kornia_rs-0.1.10-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:950a943f91c2cff94d80282886b0d48bbc15ef4a7cc4b15ac819724dfdb2f414", size = 2811810, upload-time = "2025-11-08T11:30:22.497Z" }, + { url = "https://files.pythonhosted.org/packages/e4/ed/bd970ded1d819557cc33055d982b1847eb385151ea5b0c915c16ed74f5c0/kornia_rs-0.1.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:63b802aaf95590276d3426edc6d23ff11caf269d2bc2ec37cb6c679b7b2a8ee0", size = 2076195, upload-time = "2025-11-08T11:30:08.726Z" }, + { url = "https://files.pythonhosted.org/packages/c1/10/afd700455105fdba5b043d724f3a65ca36259b89c736a3b71d5a03103808/kornia_rs-0.1.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38087da7cdf2bffe10530c0d53335dd1fc107fae6521f2dd4797c6522b6d11b3", size = 2205781, upload-time = "2025-11-08T11:29:36.8Z" }, + { url = "https://files.pythonhosted.org/packages/25/16/ec8dc3ce1d79660ddd6a186a77037e0c3bf61648e6c72250280b648fb291/kornia_rs-0.1.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa3464de8f9920d87415721c36840ceea23e054dcb54dd9f69189ba9eabce0c7", size = 3042272, upload-time = "2025-11-08T11:29:52.936Z" }, + { url = "https://files.pythonhosted.org/packages/f7/75/62785aba777d35a562a97a987d65840306fab7a8ecd2d928dd8ac779e29b/kornia_rs-0.1.10-cp313-cp313-win_amd64.whl", hash = "sha256:c57d157bebe64c22e2e44c72455b1c7365eee4d767e0c187dc28f22d072ebaf7", size = 2539802, upload-time = "2025-11-08T11:30:35.753Z" }, + { url = "https://files.pythonhosted.org/packages/a5/d5/32b23d110109eb77b2dc952be75411f7e495da9105058e2cb08924a9cc90/kornia_rs-0.1.10-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:0b375f02422ef5986caed612799b4ddcc91f57f303906868b0a8c397a17e7607", size = 2810244, upload-time = "2025-11-08T11:30:23.637Z" }, + { url = "https://files.pythonhosted.org/packages/96/5f/5ecde42b7c18e7df26c413848a98744427c3d370f5eed725b65f0bc356fb/kornia_rs-0.1.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2bcfa438d6b5dbe07d573afc980f2871f6639b2eac5148b8c0bba4f82357b9a", size = 2074220, upload-time = "2025-11-08T11:30:09.972Z" }, + { url = "https://files.pythonhosted.org/packages/18/6c/6fc86eb855bcc723924c3b91de98dc6c0f381987ce582e080b8eade3bc88/kornia_rs-0.1.10-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:021b0a02b2356b12b3954a298f369ed4fe2dd522dcf8b6d72f91bf3bd8eea201", size = 2204672, upload-time = "2025-11-08T11:29:38.777Z" }, + { url = "https://files.pythonhosted.org/packages/19/26/3ac706d1b36761c0f7a36934327079adcb42d761c8c219865123d49fc1b2/kornia_rs-0.1.10-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d9b07e2ae79e423b3248d94afd092e324c5ddfe3157fafc047531cc8bffa6a3", size = 3042797, upload-time = "2025-11-08T11:29:54.719Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f4/d62728d86bc67f5516249b154ff0bdfcf38a854dae284ff0ce62da87af99/kornia_rs-0.1.10-cp313-cp313t-win_amd64.whl", hash = "sha256:b80a037e34d63cb021bcd5fc571e41aff804a2981311f66e883768c6b8e5f8de", size = 2543855, upload-time = "2025-11-08T11:30:37.437Z" }, + { url = "https://files.pythonhosted.org/packages/91/d5/8ed1288a51d2ad71a6c01152ceccdd2d92f21692dfd2304b1ae9383496fa/kornia_rs-0.1.10-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:119eb434d1384257cae6c1ee9444e1aa1b0fda617f6d5a79fef3f145fdac70ac", size = 2809873, upload-time = "2025-11-08T11:30:24.958Z" }, + { url = "https://files.pythonhosted.org/packages/54/2b/fd5f919723aaa69ec5c1e60b10b7904a9126be5b9d6ccc0267fa42ca77e0/kornia_rs-0.1.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60bca692911e5969e51d256299ecc6e90d32b9a2c5bf7bd1c7eb8f096cb9234b", size = 2074360, upload-time = "2025-11-08T11:30:11.327Z" }, + { url = "https://files.pythonhosted.org/packages/43/ec/7987aa5fb7d188180866bd8dafa5bb5b1f00a74ba738bb4e2abe63c589ac/kornia_rs-0.1.10-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61f126644f49ff9947d9402126edacfeeb4b47c0999a7af487d27ce4fc4cbc2a", size = 2206111, upload-time = "2025-11-08T11:29:40.608Z" }, + { url = "https://files.pythonhosted.org/packages/91/08/cb73b7e87a07b2af1146988d159d48722f0a28f550f920397c8964ab7c19/kornia_rs-0.1.10-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:614aeffdb1d39c4041ace0e0fc318b36eb211f6c133912984e946174e67dbb42", size = 3041436, upload-time = "2025-11-08T11:29:55.984Z" }, + { url = "https://files.pythonhosted.org/packages/db/e2/9f50fce2d8e9edd6b2d09908b6d5613f9ead992bf2e80060e080f2e7d64d/kornia_rs-0.1.10-cp314-cp314-win_amd64.whl", hash = "sha256:6de4e73b1c90cc76b7486491866eb9e61e5cf34d3a4016957d4563ac7d3ee39a", size = 2544067, upload-time = "2025-11-08T11:30:38.638Z" }, + { url = "https://files.pythonhosted.org/packages/d1/8f/45895818f3c7a5009737119b075db6b88bbf00938275611bc5d2cfbd0b2a/kornia_rs-0.1.10-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f0db8b41ae03a746bb0dcb970d5ff2fd66213adb4a3b4de1186fe86205698e89", size = 2806089, upload-time = "2025-11-08T11:30:26.117Z" }, + { url = "https://files.pythonhosted.org/packages/38/af/831e79b45702f8b6102438b1ff9b44a912669890cdf209cd275257f6d655/kornia_rs-0.1.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b63ee175125892ef18027bd3a43b447fd53f9bf42cea4d6f699ab4e69cf3f16", size = 2064116, upload-time = "2025-11-08T11:30:13.481Z" }, + { url = "https://files.pythonhosted.org/packages/53/1b/e92606e0fa9a1b52ecf57faf322dcc076ae35315b4e1870d380fd64926d7/kornia_rs-0.1.10-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68eb25ba4639fa5e1cd94a10fb6410c8840c9f0162e5912d834c4a8c7c174493", size = 2197890, upload-time = "2025-11-08T11:29:42.273Z" }, + { url = "https://files.pythonhosted.org/packages/2e/fa/a2adce992b5eb65ef8adfc6f4465989948bfa8b875638e17c214541af25a/kornia_rs-0.1.10-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc18ba839f5c10ceb4757342ee7530cef8a0ecdd20486b8bbe14a56f72fa7037", size = 3040852, upload-time = "2025-11-08T11:29:57.856Z" }, + { url = "https://files.pythonhosted.org/packages/ee/36/40a3e3a235c370f5f61a8f9a7bdedf47d1bdd8f7d7e145e551545babff6b/kornia_rs-0.1.10-cp314-cp314t-win_amd64.whl", hash = "sha256:257eb0a780f990c0c44ac47acb77504dd95b8df0c592fd31354da1228df6678d", size = 2543609, upload-time = "2025-11-08T11:30:40.1Z" }, +] + +[[package]] +name = "latexcodec" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/dd/4270b2c5e2ee49316c3859e62293bd2ea8e382339d63ab7bbe9f39c0ec3b/latexcodec-3.0.1.tar.gz", hash = "sha256:e78a6911cd72f9dec35031c6ec23584de6842bfbc4610a9678868d14cdfb0357", size = 31222, upload-time = "2025-06-17T18:47:34.051Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/40/23569737873cc9637fd488606347e9dd92b9fa37ba4fcda1f98ee5219a97/latexcodec-3.0.1-py3-none-any.whl", hash = "sha256:a9eb8200bff693f0437a69581f7579eb6bca25c4193515c09900ce76451e452e", size = 18532, upload-time = "2025-06-17T18:47:30.726Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, + { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, + { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940, upload-time = "2025-09-27T18:36:22.249Z" }, + { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887, upload-time = "2025-09-27T18:36:23.535Z" }, + { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692, upload-time = "2025-09-27T18:36:24.823Z" }, + { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471, upload-time = "2025-09-27T18:36:25.95Z" }, + { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923, upload-time = "2025-09-27T18:36:27.109Z" }, + { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572, upload-time = "2025-09-27T18:36:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077, upload-time = "2025-09-27T18:36:29.025Z" }, + { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876, upload-time = "2025-09-27T18:36:29.954Z" }, + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + +[[package]] +name = "matplotlib" +version = "3.10.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "contourpy" }, + { name = "cycler" }, + { name = "fonttools" }, + { name = "kiwisolver" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "pyparsing" }, + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8a/76/d3c6e3a13fe484ebe7718d14e269c9569c4eb0020a968a327acb3b9a8fe6/matplotlib-3.10.8.tar.gz", hash = "sha256:2299372c19d56bcd35cf05a2738308758d32b9eaed2371898d8f5bd33f084aa3", size = 34806269, upload-time = "2025-12-10T22:56:51.155Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/86/de7e3a1cdcfc941483af70609edc06b83e7c8a0e0dc9ac325200a3f4d220/matplotlib-3.10.8-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:6be43b667360fef5c754dda5d25a32e6307a03c204f3c0fc5468b78fa87b4160", size = 8251215, upload-time = "2025-12-10T22:55:16.175Z" }, + { url = "https://files.pythonhosted.org/packages/fd/14/baad3222f424b19ce6ad243c71de1ad9ec6b2e4eb1e458a48fdc6d120401/matplotlib-3.10.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a2b336e2d91a3d7006864e0990c83b216fcdca64b5a6484912902cef87313d78", size = 8139625, upload-time = "2025-12-10T22:55:17.712Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a0/7024215e95d456de5883e6732e708d8187d9753a21d32f8ddb3befc0c445/matplotlib-3.10.8-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:efb30e3baaea72ce5928e32bab719ab4770099079d66726a62b11b1ef7273be4", size = 8712614, upload-time = "2025-12-10T22:55:20.8Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f4/b8347351da9a5b3f41e26cf547252d861f685c6867d179a7c9d60ad50189/matplotlib-3.10.8-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d56a1efd5bfd61486c8bc968fa18734464556f0fb8e51690f4ac25d85cbbbbc2", size = 9540997, upload-time = "2025-12-10T22:55:23.258Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c0/c7b914e297efe0bc36917bf216b2acb91044b91e930e878ae12981e461e5/matplotlib-3.10.8-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:238b7ce5717600615c895050239ec955d91f321c209dd110db988500558e70d6", size = 9596825, upload-time = "2025-12-10T22:55:25.217Z" }, + { url = "https://files.pythonhosted.org/packages/6f/d3/a4bbc01c237ab710a1f22b4da72f4ff6d77eb4c7735ea9811a94ae239067/matplotlib-3.10.8-cp311-cp311-win_amd64.whl", hash = "sha256:18821ace09c763ec93aef5eeff087ee493a24051936d7b9ebcad9662f66501f9", size = 8135090, upload-time = "2025-12-10T22:55:27.162Z" }, + { url = "https://files.pythonhosted.org/packages/89/dd/a0b6588f102beab33ca6f5218b31725216577b2a24172f327eaf6417d5c9/matplotlib-3.10.8-cp311-cp311-win_arm64.whl", hash = "sha256:bab485bcf8b1c7d2060b4fcb6fc368a9e6f4cd754c9c2fea281f4be21df394a2", size = 8012377, upload-time = "2025-12-10T22:55:29.185Z" }, + { url = "https://files.pythonhosted.org/packages/9e/67/f997cdcbb514012eb0d10cd2b4b332667997fb5ebe26b8d41d04962fa0e6/matplotlib-3.10.8-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:64fcc24778ca0404ce0cb7b6b77ae1f4c7231cdd60e6778f999ee05cbd581b9a", size = 8260453, upload-time = "2025-12-10T22:55:30.709Z" }, + { url = "https://files.pythonhosted.org/packages/7e/65/07d5f5c7f7c994f12c768708bd2e17a4f01a2b0f44a1c9eccad872433e2e/matplotlib-3.10.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b9a5ca4ac220a0cdd1ba6bcba3608547117d30468fefce49bb26f55c1a3d5c58", size = 8148321, upload-time = "2025-12-10T22:55:33.265Z" }, + { url = "https://files.pythonhosted.org/packages/3e/f3/c5195b1ae57ef85339fd7285dfb603b22c8b4e79114bae5f4f0fcf688677/matplotlib-3.10.8-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3ab4aabc72de4ff77b3ec33a6d78a68227bf1123465887f9905ba79184a1cc04", size = 8716944, upload-time = "2025-12-10T22:55:34.922Z" }, + { url = "https://files.pythonhosted.org/packages/00/f9/7638f5cc82ec8a7aa005de48622eecc3ed7c9854b96ba15bd76b7fd27574/matplotlib-3.10.8-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:24d50994d8c5816ddc35411e50a86ab05f575e2530c02752e02538122613371f", size = 9550099, upload-time = "2025-12-10T22:55:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/57/61/78cd5920d35b29fd2a0fe894de8adf672ff52939d2e9b43cb83cd5ce1bc7/matplotlib-3.10.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:99eefd13c0dc3b3c1b4d561c1169e65fe47aab7b8158754d7c084088e2329466", size = 9613040, upload-time = "2025-12-10T22:55:38.715Z" }, + { url = "https://files.pythonhosted.org/packages/30/4e/c10f171b6e2f44d9e3a2b96efa38b1677439d79c99357600a62cc1e9594e/matplotlib-3.10.8-cp312-cp312-win_amd64.whl", hash = "sha256:dd80ecb295460a5d9d260df63c43f4afbdd832d725a531f008dad1664f458adf", size = 8142717, upload-time = "2025-12-10T22:55:41.103Z" }, + { url = "https://files.pythonhosted.org/packages/f1/76/934db220026b5fef85f45d51a738b91dea7d70207581063cd9bd8fafcf74/matplotlib-3.10.8-cp312-cp312-win_arm64.whl", hash = "sha256:3c624e43ed56313651bc18a47f838b60d7b8032ed348911c54906b130b20071b", size = 8012751, upload-time = "2025-12-10T22:55:42.684Z" }, + { url = "https://files.pythonhosted.org/packages/3d/b9/15fd5541ef4f5b9a17eefd379356cf12175fe577424e7b1d80676516031a/matplotlib-3.10.8-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3f2e409836d7f5ac2f1c013110a4d50b9f7edc26328c108915f9075d7d7a91b6", size = 8261076, upload-time = "2025-12-10T22:55:44.648Z" }, + { url = "https://files.pythonhosted.org/packages/8d/a0/2ba3473c1b66b9c74dc7107c67e9008cb1782edbe896d4c899d39ae9cf78/matplotlib-3.10.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:56271f3dac49a88d7fca5060f004d9d22b865f743a12a23b1e937a0be4818ee1", size = 8148794, upload-time = "2025-12-10T22:55:46.252Z" }, + { url = "https://files.pythonhosted.org/packages/75/97/a471f1c3eb1fd6f6c24a31a5858f443891d5127e63a7788678d14e249aea/matplotlib-3.10.8-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a0a7f52498f72f13d4a25ea70f35f4cb60642b466cbb0a9be951b5bc3f45a486", size = 8718474, upload-time = "2025-12-10T22:55:47.864Z" }, + { url = "https://files.pythonhosted.org/packages/01/be/cd478f4b66f48256f42927d0acbcd63a26a893136456cd079c0cc24fbabf/matplotlib-3.10.8-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:646d95230efb9ca614a7a594d4fcacde0ac61d25e37dd51710b36477594963ce", size = 9549637, upload-time = "2025-12-10T22:55:50.048Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7c/8dc289776eae5109e268c4fb92baf870678dc048a25d4ac903683b86d5bf/matplotlib-3.10.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f89c151aab2e2e23cb3fe0acad1e8b82841fd265379c4cecd0f3fcb34c15e0f6", size = 9613678, upload-time = "2025-12-10T22:55:52.21Z" }, + { url = "https://files.pythonhosted.org/packages/64/40/37612487cc8a437d4dd261b32ca21fe2d79510fe74af74e1f42becb1bdb8/matplotlib-3.10.8-cp313-cp313-win_amd64.whl", hash = "sha256:e8ea3e2d4066083e264e75c829078f9e149fa119d27e19acd503de65e0b13149", size = 8142686, upload-time = "2025-12-10T22:55:54.253Z" }, + { url = "https://files.pythonhosted.org/packages/66/52/8d8a8730e968185514680c2a6625943f70269509c3dcfc0dcf7d75928cb8/matplotlib-3.10.8-cp313-cp313-win_arm64.whl", hash = "sha256:c108a1d6fa78a50646029cb6d49808ff0fc1330fda87fa6f6250c6b5369b6645", size = 8012917, upload-time = "2025-12-10T22:55:56.268Z" }, + { url = "https://files.pythonhosted.org/packages/b5/27/51fe26e1062f298af5ef66343d8ef460e090a27fea73036c76c35821df04/matplotlib-3.10.8-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ad3d9833a64cf48cc4300f2b406c3d0f4f4724a91c0bd5640678a6ba7c102077", size = 8305679, upload-time = "2025-12-10T22:55:57.856Z" }, + { url = "https://files.pythonhosted.org/packages/2c/1e/4de865bc591ac8e3062e835f42dd7fe7a93168d519557837f0e37513f629/matplotlib-3.10.8-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:eb3823f11823deade26ce3b9f40dcb4a213da7a670013929f31d5f5ed1055b22", size = 8198336, upload-time = "2025-12-10T22:55:59.371Z" }, + { url = "https://files.pythonhosted.org/packages/c6/cb/2f7b6e75fb4dce87ef91f60cac4f6e34f4c145ab036a22318ec837971300/matplotlib-3.10.8-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d9050fee89a89ed57b4fb2c1bfac9a3d0c57a0d55aed95949eedbc42070fea39", size = 8731653, upload-time = "2025-12-10T22:56:01.032Z" }, + { url = "https://files.pythonhosted.org/packages/46/b3/bd9c57d6ba670a37ab31fb87ec3e8691b947134b201f881665b28cc039ff/matplotlib-3.10.8-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b44d07310e404ba95f8c25aa5536f154c0a8ec473303535949e52eb71d0a1565", size = 9561356, upload-time = "2025-12-10T22:56:02.95Z" }, + { url = "https://files.pythonhosted.org/packages/c0/3d/8b94a481456dfc9dfe6e39e93b5ab376e50998cddfd23f4ae3b431708f16/matplotlib-3.10.8-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0a33deb84c15ede243aead39f77e990469fff93ad1521163305095b77b72ce4a", size = 9614000, upload-time = "2025-12-10T22:56:05.411Z" }, + { url = "https://files.pythonhosted.org/packages/bd/cd/bc06149fe5585ba800b189a6a654a75f1f127e8aab02fd2be10df7fa500c/matplotlib-3.10.8-cp313-cp313t-win_amd64.whl", hash = "sha256:3a48a78d2786784cc2413e57397981fb45c79e968d99656706018d6e62e57958", size = 8220043, upload-time = "2025-12-10T22:56:07.551Z" }, + { url = "https://files.pythonhosted.org/packages/e3/de/b22cf255abec916562cc04eef457c13e58a1990048de0c0c3604d082355e/matplotlib-3.10.8-cp313-cp313t-win_arm64.whl", hash = "sha256:15d30132718972c2c074cd14638c7f4592bd98719e2308bccea40e0538bc0cb5", size = 8062075, upload-time = "2025-12-10T22:56:09.178Z" }, + { url = "https://files.pythonhosted.org/packages/3c/43/9c0ff7a2f11615e516c3b058e1e6e8f9614ddeca53faca06da267c48345d/matplotlib-3.10.8-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:b53285e65d4fa4c86399979e956235deb900be5baa7fc1218ea67fbfaeaadd6f", size = 8262481, upload-time = "2025-12-10T22:56:10.885Z" }, + { url = "https://files.pythonhosted.org/packages/6f/ca/e8ae28649fcdf039fda5ef554b40a95f50592a3c47e6f7270c9561c12b07/matplotlib-3.10.8-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32f8dce744be5569bebe789e46727946041199030db8aeb2954d26013a0eb26b", size = 8151473, upload-time = "2025-12-10T22:56:12.377Z" }, + { url = "https://files.pythonhosted.org/packages/f1/6f/009d129ae70b75e88cbe7e503a12a4c0670e08ed748a902c2568909e9eb5/matplotlib-3.10.8-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4cf267add95b1c88300d96ca837833d4112756045364f5c734a2276038dae27d", size = 9553896, upload-time = "2025-12-10T22:56:14.432Z" }, + { url = "https://files.pythonhosted.org/packages/f5/26/4221a741eb97967bc1fd5e4c52b9aa5a91b2f4ec05b59f6def4d820f9df9/matplotlib-3.10.8-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2cf5bd12cecf46908f286d7838b2abc6c91cda506c0445b8223a7c19a00df008", size = 9824193, upload-time = "2025-12-10T22:56:16.29Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f3/3abf75f38605772cf48a9daf5821cd4f563472f38b4b828c6fba6fa6d06e/matplotlib-3.10.8-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:41703cc95688f2516b480f7f339d8851a6035f18e100ee6a32bc0b8536a12a9c", size = 9615444, upload-time = "2025-12-10T22:56:18.155Z" }, + { url = "https://files.pythonhosted.org/packages/93/a5/de89ac80f10b8dc615807ee1133cd99ac74082581196d4d9590bea10690d/matplotlib-3.10.8-cp314-cp314-win_amd64.whl", hash = "sha256:83d282364ea9f3e52363da262ce32a09dfe241e4080dcedda3c0db059d3c1f11", size = 8272719, upload-time = "2025-12-10T22:56:20.366Z" }, + { url = "https://files.pythonhosted.org/packages/69/ce/b006495c19ccc0a137b48083168a37bd056392dee02f87dba0472f2797fe/matplotlib-3.10.8-cp314-cp314-win_arm64.whl", hash = "sha256:2c1998e92cd5999e295a731bcb2911c75f597d937341f3030cc24ef2733d78a8", size = 8144205, upload-time = "2025-12-10T22:56:22.239Z" }, + { url = "https://files.pythonhosted.org/packages/68/d9/b31116a3a855bd313c6fcdb7226926d59b041f26061c6c5b1be66a08c826/matplotlib-3.10.8-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:b5a2b97dbdc7d4f353ebf343744f1d1f1cca8aa8bfddb4262fcf4306c3761d50", size = 8305785, upload-time = "2025-12-10T22:56:24.218Z" }, + { url = "https://files.pythonhosted.org/packages/1e/90/6effe8103f0272685767ba5f094f453784057072f49b393e3ea178fe70a5/matplotlib-3.10.8-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3f5c3e4da343bba819f0234186b9004faba952cc420fbc522dc4e103c1985908", size = 8198361, upload-time = "2025-12-10T22:56:26.787Z" }, + { url = "https://files.pythonhosted.org/packages/d7/65/a73188711bea603615fc0baecca1061429ac16940e2385433cc778a9d8e7/matplotlib-3.10.8-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f62550b9a30afde8c1c3ae450e5eb547d579dd69b25c2fc7a1c67f934c1717a", size = 9561357, upload-time = "2025-12-10T22:56:28.953Z" }, + { url = "https://files.pythonhosted.org/packages/f4/3d/b5c5d5d5be8ce63292567f0e2c43dde9953d3ed86ac2de0a72e93c8f07a1/matplotlib-3.10.8-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:495672de149445ec1b772ff2c9ede9b769e3cb4f0d0aa7fa730d7f59e2d4e1c1", size = 9823610, upload-time = "2025-12-10T22:56:31.455Z" }, + { url = "https://files.pythonhosted.org/packages/4d/4b/e7beb6bbd49f6bae727a12b270a2654d13c397576d25bd6786e47033300f/matplotlib-3.10.8-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:595ba4d8fe983b88f0eec8c26a241e16d6376fe1979086232f481f8f3f67494c", size = 9614011, upload-time = "2025-12-10T22:56:33.85Z" }, + { url = "https://files.pythonhosted.org/packages/7c/e6/76f2813d31f032e65f6f797e3f2f6e4aab95b65015924b1c51370395c28a/matplotlib-3.10.8-cp314-cp314t-win_amd64.whl", hash = "sha256:25d380fe8b1dc32cf8f0b1b448470a77afb195438bafdf1d858bfb876f3edf7b", size = 8362801, upload-time = "2025-12-10T22:56:36.107Z" }, + { url = "https://files.pythonhosted.org/packages/5d/49/d651878698a0b67f23aa28e17f45a6d6dd3d3f933fa29087fa4ce5947b5a/matplotlib-3.10.8-cp314-cp314t-win_arm64.whl", hash = "sha256:113bb52413ea508ce954a02c10ffd0d565f9c3bc7f2eddc27dfe1731e71c7b5f", size = 8192560, upload-time = "2025-12-10T22:56:38.008Z" }, + { url = "https://files.pythonhosted.org/packages/04/30/3afaa31c757f34b7725ab9d2ba8b48b5e89c2019c003e7d0ead143aabc5a/matplotlib-3.10.8-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:6da7c2ce169267d0d066adcf63758f0604aa6c3eebf67458930f9d9b79ad1db1", size = 8249198, upload-time = "2025-12-10T22:56:45.584Z" }, + { url = "https://files.pythonhosted.org/packages/48/2f/6334aec331f57485a642a7c8be03cb286f29111ae71c46c38b363230063c/matplotlib-3.10.8-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:9153c3292705be9f9c64498a8872118540c3f4123d1a1c840172edf262c8be4a", size = 8136817, upload-time = "2025-12-10T22:56:47.339Z" }, + { url = "https://files.pythonhosted.org/packages/73/e4/6d6f14b2a759c622f191b2d67e9075a3f56aaccb3be4bb9bb6890030d0a0/matplotlib-3.10.8-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ae029229a57cd1e8fe542485f27e7ca7b23aa9e8944ddb4985d0bc444f1eca2", size = 8713867, upload-time = "2025-12-10T22:56:48.954Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "ml-dtypes" +version = "0.5.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/4a/c27b42ed9b1c7d13d9ba8b6905dece787d6259152f2309338aed29b2447b/ml_dtypes-0.5.4.tar.gz", hash = "sha256:8ab06a50fb9bf9666dd0fe5dfb4676fa2b0ac0f31ecff72a6c3af8e22c063453", size = 692314, upload-time = "2025-11-17T22:32:31.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/5e/712092cfe7e5eb667b8ad9ca7c54442f21ed7ca8979745f1000e24cf8737/ml_dtypes-0.5.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6c7ecb74c4bd71db68a6bea1edf8da8c34f3d9fe218f038814fd1d310ac76c90", size = 679734, upload-time = "2025-11-17T22:31:39.223Z" }, + { url = "https://files.pythonhosted.org/packages/4f/cf/912146dfd4b5c0eea956836c01dcd2fce6c9c844b2691f5152aca196ce4f/ml_dtypes-0.5.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc11d7e8c44a65115d05e2ab9989d1e045125d7be8e05a071a48bc76eb6d6040", size = 5056165, upload-time = "2025-11-17T22:31:41.071Z" }, + { url = "https://files.pythonhosted.org/packages/a9/80/19189ea605017473660e43762dc853d2797984b3c7bf30ce656099add30c/ml_dtypes-0.5.4-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:19b9a53598f21e453ea2fbda8aa783c20faff8e1eeb0d7ab899309a0053f1483", size = 5034975, upload-time = "2025-11-17T22:31:42.758Z" }, + { url = "https://files.pythonhosted.org/packages/b4/24/70bd59276883fdd91600ca20040b41efd4902a923283c4d6edcb1de128d2/ml_dtypes-0.5.4-cp311-cp311-win_amd64.whl", hash = "sha256:7c23c54a00ae43edf48d44066a7ec31e05fdc2eee0be2b8b50dd1903a1db94bb", size = 210742, upload-time = "2025-11-17T22:31:44.068Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c9/64230ef14e40aa3f1cb254ef623bf812735e6bec7772848d19131111ac0d/ml_dtypes-0.5.4-cp311-cp311-win_arm64.whl", hash = "sha256:557a31a390b7e9439056644cb80ed0735a6e3e3bb09d67fd5687e4b04238d1de", size = 160709, upload-time = "2025-11-17T22:31:46.557Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b8/3c70881695e056f8a32f8b941126cf78775d9a4d7feba8abcb52cb7b04f2/ml_dtypes-0.5.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a174837a64f5b16cab6f368171a1a03a27936b31699d167684073ff1c4237dac", size = 676927, upload-time = "2025-11-17T22:31:48.182Z" }, + { url = "https://files.pythonhosted.org/packages/54/0f/428ef6881782e5ebb7eca459689448c0394fa0a80bea3aa9262cba5445ea/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7f7c643e8b1320fd958bf098aa7ecf70623a42ec5154e3be3be673f4c34d900", size = 5028464, upload-time = "2025-11-17T22:31:50.135Z" }, + { url = "https://files.pythonhosted.org/packages/3a/cb/28ce52eb94390dda42599c98ea0204d74799e4d8047a0eb559b6fd648056/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ad459e99793fa6e13bd5b7e6792c8f9190b4e5a1b45c63aba14a4d0a7f1d5ff", size = 5009002, upload-time = "2025-11-17T22:31:52.001Z" }, + { url = "https://files.pythonhosted.org/packages/f5/f0/0cfadd537c5470378b1b32bd859cf2824972174b51b873c9d95cfd7475a5/ml_dtypes-0.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:c1a953995cccb9e25a4ae19e34316671e4e2edaebe4cf538229b1fc7109087b7", size = 212222, upload-time = "2025-11-17T22:31:53.742Z" }, + { url = "https://files.pythonhosted.org/packages/16/2e/9acc86985bfad8f2c2d30291b27cd2bb4c74cea08695bd540906ed744249/ml_dtypes-0.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:9bad06436568442575beb2d03389aa7456c690a5b05892c471215bfd8cf39460", size = 160793, upload-time = "2025-11-17T22:31:55.358Z" }, + { url = "https://files.pythonhosted.org/packages/d9/a1/4008f14bbc616cfb1ac5b39ea485f9c63031c4634ab3f4cf72e7541f816a/ml_dtypes-0.5.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8c760d85a2f82e2bed75867079188c9d18dae2ee77c25a54d60e9cc79be1bc48", size = 676888, upload-time = "2025-11-17T22:31:56.907Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b7/dff378afc2b0d5a7d6cd9d3209b60474d9819d1189d347521e1688a60a53/ml_dtypes-0.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce756d3a10d0c4067172804c9cc276ba9cc0ff47af9078ad439b075d1abdc29b", size = 5036993, upload-time = "2025-11-17T22:31:58.497Z" }, + { url = "https://files.pythonhosted.org/packages/eb/33/40cd74219417e78b97c47802037cf2d87b91973e18bb968a7da48a96ea44/ml_dtypes-0.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:533ce891ba774eabf607172254f2e7260ba5f57bdd64030c9a4fcfbd99815d0d", size = 5010956, upload-time = "2025-11-17T22:31:59.931Z" }, + { url = "https://files.pythonhosted.org/packages/e1/8b/200088c6859d8221454825959df35b5244fa9bdf263fd0249ac5fb75e281/ml_dtypes-0.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:f21c9219ef48ca5ee78402d5cc831bd58ea27ce89beda894428bc67a52da5328", size = 212224, upload-time = "2025-11-17T22:32:01.349Z" }, + { url = "https://files.pythonhosted.org/packages/8f/75/dfc3775cb36367816e678f69a7843f6f03bd4e2bcd79941e01ea960a068e/ml_dtypes-0.5.4-cp313-cp313-win_arm64.whl", hash = "sha256:35f29491a3e478407f7047b8a4834e4640a77d2737e0b294d049746507af5175", size = 160798, upload-time = "2025-11-17T22:32:02.864Z" }, + { url = "https://files.pythonhosted.org/packages/4f/74/e9ddb35fd1dd43b1106c20ced3f53c2e8e7fc7598c15638e9f80677f81d4/ml_dtypes-0.5.4-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:304ad47faa395415b9ccbcc06a0350800bc50eda70f0e45326796e27c62f18b6", size = 702083, upload-time = "2025-11-17T22:32:04.08Z" }, + { url = "https://files.pythonhosted.org/packages/74/f5/667060b0aed1aa63166b22897fdf16dca9eb704e6b4bbf86848d5a181aa7/ml_dtypes-0.5.4-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a0df4223b514d799b8a1629c65ddc351b3efa833ccf7f8ea0cf654a61d1e35d", size = 5354111, upload-time = "2025-11-17T22:32:05.546Z" }, + { url = "https://files.pythonhosted.org/packages/40/49/0f8c498a28c0efa5f5c95a9e374c83ec1385ca41d0e85e7cf40e5d519a21/ml_dtypes-0.5.4-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:531eff30e4d368cb6255bc2328d070e35836aa4f282a0fb5f3a0cd7260257298", size = 5366453, upload-time = "2025-11-17T22:32:07.115Z" }, + { url = "https://files.pythonhosted.org/packages/8c/27/12607423d0a9c6bbbcc780ad19f1f6baa2b68b18ce4bddcdc122c4c68dc9/ml_dtypes-0.5.4-cp313-cp313t-win_amd64.whl", hash = "sha256:cb73dccfc991691c444acc8c0012bee8f2470da826a92e3a20bb333b1a7894e6", size = 225612, upload-time = "2025-11-17T22:32:08.615Z" }, + { url = "https://files.pythonhosted.org/packages/e5/80/5a5929e92c72936d5b19872c5fb8fc09327c1da67b3b68c6a13139e77e20/ml_dtypes-0.5.4-cp313-cp313t-win_arm64.whl", hash = "sha256:3bbbe120b915090d9dd1375e4684dd17a20a2491ef25d640a908281da85e73f1", size = 164145, upload-time = "2025-11-17T22:32:09.782Z" }, + { url = "https://files.pythonhosted.org/packages/72/4e/1339dc6e2557a344f5ba5590872e80346f76f6cb2ac3dd16e4666e88818c/ml_dtypes-0.5.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:2b857d3af6ac0d39db1de7c706e69c7f9791627209c3d6dedbfca8c7e5faec22", size = 673781, upload-time = "2025-11-17T22:32:11.364Z" }, + { url = "https://files.pythonhosted.org/packages/04/f9/067b84365c7e83bda15bba2b06c6ca250ce27b20630b1128c435fb7a09aa/ml_dtypes-0.5.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:805cef3a38f4eafae3a5bf9ebdcdb741d0bcfd9e1bd90eb54abd24f928cd2465", size = 5036145, upload-time = "2025-11-17T22:32:12.783Z" }, + { url = "https://files.pythonhosted.org/packages/c6/bb/82c7dcf38070b46172a517e2334e665c5bf374a262f99a283ea454bece7c/ml_dtypes-0.5.4-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14a4fd3228af936461db66faccef6e4f41c1d82fcc30e9f8d58a08916b1d811f", size = 5010230, upload-time = "2025-11-17T22:32:14.38Z" }, + { url = "https://files.pythonhosted.org/packages/e9/93/2bfed22d2498c468f6bcd0d9f56b033eaa19f33320389314c19ef6766413/ml_dtypes-0.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:8c6a2dcebd6f3903e05d51960a8058d6e131fe69f952a5397e5dbabc841b6d56", size = 221032, upload-time = "2025-11-17T22:32:15.763Z" }, + { url = "https://files.pythonhosted.org/packages/76/a3/9c912fe6ea747bb10fe2f8f54d027eb265db05dfb0c6335e3e063e74e6e8/ml_dtypes-0.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:5a0f68ca8fd8d16583dfa7793973feb86f2fbb56ce3966daf9c9f748f52a2049", size = 163353, upload-time = "2025-11-17T22:32:16.932Z" }, + { url = "https://files.pythonhosted.org/packages/cd/02/48aa7d84cc30ab4ee37624a2fd98c56c02326785750cd212bc0826c2f15b/ml_dtypes-0.5.4-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:bfc534409c5d4b0bf945af29e5d0ab075eae9eecbb549ff8a29280db822f34f9", size = 702085, upload-time = "2025-11-17T22:32:18.175Z" }, + { url = "https://files.pythonhosted.org/packages/5a/e7/85cb99fe80a7a5513253ec7faa88a65306be071163485e9a626fce1b6e84/ml_dtypes-0.5.4-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2314892cdc3fcf05e373d76d72aaa15fda9fb98625effa73c1d646f331fcecb7", size = 5355358, upload-time = "2025-11-17T22:32:19.7Z" }, + { url = "https://files.pythonhosted.org/packages/79/2b/a826ba18d2179a56e144aef69e57fb2ab7c464ef0b2111940ee8a3a223a2/ml_dtypes-0.5.4-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0d2ffd05a2575b1519dc928c0b93c06339eb67173ff53acb00724502cda231cf", size = 5366332, upload-time = "2025-11-17T22:32:21.193Z" }, + { url = "https://files.pythonhosted.org/packages/84/44/f4d18446eacb20ea11e82f133ea8f86e2bf2891785b67d9da8d0ab0ef525/ml_dtypes-0.5.4-cp314-cp314t-win_amd64.whl", hash = "sha256:4381fe2f2452a2d7589689693d3162e876b3ddb0a832cde7a414f8e1adf7eab1", size = 236612, upload-time = "2025-11-17T22:32:22.579Z" }, + { url = "https://files.pythonhosted.org/packages/ad/3f/3d42e9a78fe5edf792a83c074b13b9b770092a4fbf3462872f4303135f09/ml_dtypes-0.5.4-cp314-cp314t-win_arm64.whl", hash = "sha256:11942cbf2cf92157db91e5022633c0d9474d4dfd813a909383bd23ce828a4b7d", size = 168825, upload-time = "2025-11-17T22:32:23.766Z" }, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, +] + +[[package]] +name = "networkx" +version = "3.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, +] + +[[package]] +name = "nodeenv" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, +] + +[[package]] +name = "numpy" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/7a/6a3d14e205d292b738db449d0de649b373a59edb0d0b4493821d0a3e8718/numpy-2.4.0.tar.gz", hash = "sha256:6e504f7b16118198f138ef31ba24d985b124c2c469fe8467007cf30fd992f934", size = 20685720, upload-time = "2025-12-20T16:18:19.023Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/7e/7bae7cbcc2f8132271967aa03e03954fc1e48aa1f3bf32b29ca95fbef352/numpy-2.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:316b2f2584682318539f0bcaca5a496ce9ca78c88066579ebd11fd06f8e4741e", size = 16940166, upload-time = "2025-12-20T16:15:43.434Z" }, + { url = "https://files.pythonhosted.org/packages/0f/27/6c13f5b46776d6246ec884ac5817452672156a506d08a1f2abb39961930a/numpy-2.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a2718c1de8504121714234b6f8241d0019450353276c88b9453c9c3d92e101db", size = 12641781, upload-time = "2025-12-20T16:15:45.701Z" }, + { url = "https://files.pythonhosted.org/packages/14/1c/83b4998d4860d15283241d9e5215f28b40ac31f497c04b12fa7f428ff370/numpy-2.4.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:21555da4ec4a0c942520ead42c3b0dc9477441e085c42b0fbdd6a084869a6f6b", size = 5470247, upload-time = "2025-12-20T16:15:47.943Z" }, + { url = "https://files.pythonhosted.org/packages/54/08/cbce72c835d937795571b0464b52069f869c9e78b0c076d416c5269d2718/numpy-2.4.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:413aa561266a4be2d06cd2b9665e89d9f54c543f418773076a76adcf2af08bc7", size = 6799807, upload-time = "2025-12-20T16:15:49.795Z" }, + { url = "https://files.pythonhosted.org/packages/ff/be/2e647961cd8c980591d75cdcd9e8f647d69fbe05e2a25613dc0a2ea5fb1a/numpy-2.4.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0feafc9e03128074689183031181fac0897ff169692d8492066e949041096548", size = 14701992, upload-time = "2025-12-20T16:15:51.615Z" }, + { url = "https://files.pythonhosted.org/packages/a2/fb/e1652fb8b6fd91ce6ed429143fe2e01ce714711e03e5b762615e7b36172c/numpy-2.4.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8fdfed3deaf1928fb7667d96e0567cdf58c2b370ea2ee7e586aa383ec2cb346", size = 16646871, upload-time = "2025-12-20T16:15:54.129Z" }, + { url = "https://files.pythonhosted.org/packages/62/23/d841207e63c4322842f7cd042ae981cffe715c73376dcad8235fb31debf1/numpy-2.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e06a922a469cae9a57100864caf4f8a97a1026513793969f8ba5b63137a35d25", size = 16487190, upload-time = "2025-12-20T16:15:56.147Z" }, + { url = "https://files.pythonhosted.org/packages/bc/a0/6a842c8421ebfdec0a230e65f61e0dabda6edbef443d999d79b87c273965/numpy-2.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:927ccf5cd17c48f801f4ed43a7e5673a2724bd2171460be3e3894e6e332ef83a", size = 18580762, upload-time = "2025-12-20T16:15:58.524Z" }, + { url = "https://files.pythonhosted.org/packages/0a/d1/c79e0046641186f2134dde05e6181825b911f8bdcef31b19ddd16e232847/numpy-2.4.0-cp311-cp311-win32.whl", hash = "sha256:882567b7ae57c1b1a0250208cc21a7976d8cbcc49d5a322e607e6f09c9e0bd53", size = 6233359, upload-time = "2025-12-20T16:16:00.938Z" }, + { url = "https://files.pythonhosted.org/packages/fc/f0/74965001d231f28184d6305b8cdc1b6fcd4bf23033f6cb039cfe76c9fca7/numpy-2.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:8b986403023c8f3bf8f487c2e6186afda156174d31c175f747d8934dfddf3479", size = 12601132, upload-time = "2025-12-20T16:16:02.484Z" }, + { url = "https://files.pythonhosted.org/packages/65/32/55408d0f46dfebce38017f5bd931affa7256ad6beac1a92a012e1fbc67a7/numpy-2.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:3f3096405acc48887458bbf9f6814d43785ac7ba2a57ea6442b581dedbc60ce6", size = 10573977, upload-time = "2025-12-20T16:16:04.77Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ff/f6400ffec95de41c74b8e73df32e3fff1830633193a7b1e409be7fb1bb8c/numpy-2.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2a8b6bb8369abefb8bd1801b054ad50e02b3275c8614dc6e5b0373c305291037", size = 16653117, upload-time = "2025-12-20T16:16:06.709Z" }, + { url = "https://files.pythonhosted.org/packages/fd/28/6c23e97450035072e8d830a3c411bf1abd1f42c611ff9d29e3d8f55c6252/numpy-2.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2e284ca13d5a8367e43734148622caf0b261b275673823593e3e3634a6490f83", size = 12369711, upload-time = "2025-12-20T16:16:08.758Z" }, + { url = "https://files.pythonhosted.org/packages/bc/af/acbef97b630ab1bb45e6a7d01d1452e4251aa88ce680ac36e56c272120ec/numpy-2.4.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:49ff32b09f5aa0cd30a20c2b39db3e669c845589f2b7fc910365210887e39344", size = 5198355, upload-time = "2025-12-20T16:16:10.902Z" }, + { url = "https://files.pythonhosted.org/packages/c1/c8/4e0d436b66b826f2e53330adaa6311f5cac9871a5b5c31ad773b27f25a74/numpy-2.4.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:36cbfb13c152b1c7c184ddac43765db8ad672567e7bafff2cc755a09917ed2e6", size = 6545298, upload-time = "2025-12-20T16:16:12.607Z" }, + { url = "https://files.pythonhosted.org/packages/ef/27/e1f5d144ab54eac34875e79037011d511ac57b21b220063310cb96c80fbc/numpy-2.4.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:35ddc8f4914466e6fc954c76527aa91aa763682a4f6d73249ef20b418fe6effb", size = 14398387, upload-time = "2025-12-20T16:16:14.257Z" }, + { url = "https://files.pythonhosted.org/packages/67/64/4cb909dd5ab09a9a5d086eff9586e69e827b88a5585517386879474f4cf7/numpy-2.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc578891de1db95b2a35001b695451767b580bb45753717498213c5ff3c41d63", size = 16363091, upload-time = "2025-12-20T16:16:17.32Z" }, + { url = "https://files.pythonhosted.org/packages/9d/9c/8efe24577523ec6809261859737cf117b0eb6fdb655abdfdc81b2e468ce4/numpy-2.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:98e81648e0b36e325ab67e46b5400a7a6d4a22b8a7c8e8bbfe20e7db7906bf95", size = 16176394, upload-time = "2025-12-20T16:16:19.524Z" }, + { url = "https://files.pythonhosted.org/packages/61/f0/1687441ece7b47a62e45a1f82015352c240765c707928edd8aef875d5951/numpy-2.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d57b5046c120561ba8fa8e4030fbb8b822f3063910fa901ffadf16e2b7128ad6", size = 18287378, upload-time = "2025-12-20T16:16:22.866Z" }, + { url = "https://files.pythonhosted.org/packages/d3/6f/f868765d44e6fc466467ed810ba9d8d6db1add7d4a748abfa2a4c99a3194/numpy-2.4.0-cp312-cp312-win32.whl", hash = "sha256:92190db305a6f48734d3982f2c60fa30d6b5ee9bff10f2887b930d7b40119f4c", size = 5955432, upload-time = "2025-12-20T16:16:25.06Z" }, + { url = "https://files.pythonhosted.org/packages/d4/b5/94c1e79fcbab38d1ca15e13777477b2914dd2d559b410f96949d6637b085/numpy-2.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:680060061adb2d74ce352628cb798cfdec399068aa7f07ba9fb818b2b3305f98", size = 12306201, upload-time = "2025-12-20T16:16:26.979Z" }, + { url = "https://files.pythonhosted.org/packages/70/09/c39dadf0b13bb0768cd29d6a3aaff1fb7c6905ac40e9aaeca26b1c086e06/numpy-2.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:39699233bc72dd482da1415dcb06076e32f60eddc796a796c5fb6c5efce94667", size = 10308234, upload-time = "2025-12-20T16:16:29.417Z" }, + { url = "https://files.pythonhosted.org/packages/a7/0d/853fd96372eda07c824d24adf02e8bc92bb3731b43a9b2a39161c3667cc4/numpy-2.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a152d86a3ae00ba5f47b3acf3b827509fd0b6cb7d3259665e63dafbad22a75ea", size = 16649088, upload-time = "2025-12-20T16:16:31.421Z" }, + { url = "https://files.pythonhosted.org/packages/e3/37/cc636f1f2a9f585434e20a3e6e63422f70bfe4f7f6698e941db52ea1ac9a/numpy-2.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39b19251dec4de8ff8496cd0806cbe27bf0684f765abb1f4809554de93785f2d", size = 12364065, upload-time = "2025-12-20T16:16:33.491Z" }, + { url = "https://files.pythonhosted.org/packages/ed/69/0b78f37ca3690969beee54103ce5f6021709134e8020767e93ba691a72f1/numpy-2.4.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:009bd0ea12d3c784b6639a8457537016ce5172109e585338e11334f6a7bb88ee", size = 5192640, upload-time = "2025-12-20T16:16:35.636Z" }, + { url = "https://files.pythonhosted.org/packages/1d/2a/08569f8252abf590294dbb09a430543ec8f8cc710383abfb3e75cc73aeda/numpy-2.4.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:5fe44e277225fd3dff6882d86d3d447205d43532c3627313d17e754fb3905a0e", size = 6541556, upload-time = "2025-12-20T16:16:37.276Z" }, + { url = "https://files.pythonhosted.org/packages/93/e9/a949885a4e177493d61519377952186b6cbfdf1d6002764c664ba28349b5/numpy-2.4.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f935c4493eda9069851058fa0d9e39dbf6286be690066509305e52912714dbb2", size = 14396562, upload-time = "2025-12-20T16:16:38.953Z" }, + { url = "https://files.pythonhosted.org/packages/99/98/9d4ad53b0e9ef901c2ef1d550d2136f5ac42d3fd2988390a6def32e23e48/numpy-2.4.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8cfa5f29a695cb7438965e6c3e8d06e0416060cf0d709c1b1c1653a939bf5c2a", size = 16351719, upload-time = "2025-12-20T16:16:41.503Z" }, + { url = "https://files.pythonhosted.org/packages/28/de/5f3711a38341d6e8dd619f6353251a0cdd07f3d6d101a8fd46f4ef87f895/numpy-2.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ba0cb30acd3ef11c94dc27fbfba68940652492bc107075e7ffe23057f9425681", size = 16176053, upload-time = "2025-12-20T16:16:44.552Z" }, + { url = "https://files.pythonhosted.org/packages/2a/5b/2a3753dc43916501b4183532e7ace862e13211042bceafa253afb5c71272/numpy-2.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:60e8c196cd82cbbd4f130b5290007e13e6de3eca79f0d4d38014769d96a7c475", size = 18277859, upload-time = "2025-12-20T16:16:47.174Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c5/a18bcdd07a941db3076ef489d036ab16d2bfc2eae0cf27e5a26e29189434/numpy-2.4.0-cp313-cp313-win32.whl", hash = "sha256:5f48cb3e88fbc294dc90e215d86fbaf1c852c63dbdb6c3a3e63f45c4b57f7344", size = 5953849, upload-time = "2025-12-20T16:16:49.554Z" }, + { url = "https://files.pythonhosted.org/packages/4f/f1/719010ff8061da6e8a26e1980cf090412d4f5f8060b31f0c45d77dd67a01/numpy-2.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:a899699294f28f7be8992853c0c60741f16ff199205e2e6cdca155762cbaa59d", size = 12302840, upload-time = "2025-12-20T16:16:51.227Z" }, + { url = "https://files.pythonhosted.org/packages/f5/5a/b3d259083ed8b4d335270c76966cb6cf14a5d1b69e1a608994ac57a659e6/numpy-2.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:9198f447e1dc5647d07c9a6bbe2063cc0132728cc7175b39dbc796da5b54920d", size = 10308509, upload-time = "2025-12-20T16:16:53.313Z" }, + { url = "https://files.pythonhosted.org/packages/31/01/95edcffd1bb6c0633df4e808130545c4f07383ab629ac7e316fb44fff677/numpy-2.4.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74623f2ab5cc3f7c886add4f735d1031a1d2be4a4ae63c0546cfd74e7a31ddf6", size = 12491815, upload-time = "2025-12-20T16:16:55.496Z" }, + { url = "https://files.pythonhosted.org/packages/59/ea/5644b8baa92cc1c7163b4b4458c8679852733fa74ca49c942cfa82ded4e0/numpy-2.4.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:0804a8e4ab070d1d35496e65ffd3cf8114c136a2b81f61dfab0de4b218aacfd5", size = 5320321, upload-time = "2025-12-20T16:16:57.468Z" }, + { url = "https://files.pythonhosted.org/packages/26/4e/e10938106d70bc21319bd6a86ae726da37edc802ce35a3a71ecdf1fdfe7f/numpy-2.4.0-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:02a2038eb27f9443a8b266a66911e926566b5a6ffd1a689b588f7f35b81e7dc3", size = 6641635, upload-time = "2025-12-20T16:16:59.379Z" }, + { url = "https://files.pythonhosted.org/packages/b3/8d/a8828e3eaf5c0b4ab116924df82f24ce3416fa38d0674d8f708ddc6c8aac/numpy-2.4.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1889b3a3f47a7b5bee16bc25a2145bd7cb91897f815ce3499db64c7458b6d91d", size = 14456053, upload-time = "2025-12-20T16:17:01.768Z" }, + { url = "https://files.pythonhosted.org/packages/68/a1/17d97609d87d4520aa5ae2dcfb32305654550ac6a35effb946d303e594ce/numpy-2.4.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:85eef4cb5625c47ee6425c58a3502555e10f45ee973da878ac8248ad58c136f3", size = 16401702, upload-time = "2025-12-20T16:17:04.235Z" }, + { url = "https://files.pythonhosted.org/packages/18/32/0f13c1b2d22bea1118356b8b963195446f3af124ed7a5adfa8fdecb1b6ca/numpy-2.4.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6dc8b7e2f4eb184b37655195f421836cfae6f58197b67e3ffc501f1333d993fa", size = 16242493, upload-time = "2025-12-20T16:17:06.856Z" }, + { url = "https://files.pythonhosted.org/packages/ae/23/48f21e3d309fbc137c068a1475358cbd3a901b3987dcfc97a029ab3068e2/numpy-2.4.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:44aba2f0cafd287871a495fb3163408b0bd25bbce135c6f621534a07f4f7875c", size = 18324222, upload-time = "2025-12-20T16:17:09.392Z" }, + { url = "https://files.pythonhosted.org/packages/ac/52/41f3d71296a3dcaa4f456aaa3c6fc8e745b43d0552b6bde56571bb4b4a0f/numpy-2.4.0-cp313-cp313t-win32.whl", hash = "sha256:20c115517513831860c573996e395707aa9fb691eb179200125c250e895fcd93", size = 6076216, upload-time = "2025-12-20T16:17:11.437Z" }, + { url = "https://files.pythonhosted.org/packages/35/ff/46fbfe60ab0710d2a2b16995f708750307d30eccbb4c38371ea9e986866e/numpy-2.4.0-cp313-cp313t-win_amd64.whl", hash = "sha256:b48e35f4ab6f6a7597c46e301126ceba4c44cd3280e3750f85db48b082624fa4", size = 12444263, upload-time = "2025-12-20T16:17:13.182Z" }, + { url = "https://files.pythonhosted.org/packages/a3/e3/9189ab319c01d2ed556c932ccf55064c5d75bb5850d1df7a482ce0badead/numpy-2.4.0-cp313-cp313t-win_arm64.whl", hash = "sha256:4d1cfce39e511069b11e67cd0bd78ceff31443b7c9e5c04db73c7a19f572967c", size = 10378265, upload-time = "2025-12-20T16:17:15.211Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ed/52eac27de39d5e5a6c9aadabe672bc06f55e24a3d9010cd1183948055d76/numpy-2.4.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c95eb6db2884917d86cde0b4d4cf31adf485c8ec36bf8696dd66fa70de96f36b", size = 16647476, upload-time = "2025-12-20T16:17:17.671Z" }, + { url = "https://files.pythonhosted.org/packages/77/c0/990ce1b7fcd4e09aeaa574e2a0a839589e4b08b2ca68070f1acb1fea6736/numpy-2.4.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:65167da969cd1ec3a1df31cb221ca3a19a8aaa25370ecb17d428415e93c1935e", size = 12374563, upload-time = "2025-12-20T16:17:20.216Z" }, + { url = "https://files.pythonhosted.org/packages/37/7c/8c5e389c6ae8f5fd2277a988600d79e9625db3fff011a2d87ac80b881a4c/numpy-2.4.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:3de19cfecd1465d0dcf8a5b5ea8b3155b42ed0b639dba4b71e323d74f2a3be5e", size = 5203107, upload-time = "2025-12-20T16:17:22.47Z" }, + { url = "https://files.pythonhosted.org/packages/e6/94/ca5b3bd6a8a70a5eec9a0b8dd7f980c1eff4b8a54970a9a7fef248ef564f/numpy-2.4.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:6c05483c3136ac4c91b4e81903cb53a8707d316f488124d0398499a4f8e8ef51", size = 6538067, upload-time = "2025-12-20T16:17:24.001Z" }, + { url = "https://files.pythonhosted.org/packages/79/43/993eb7bb5be6761dde2b3a3a594d689cec83398e3f58f4758010f3b85727/numpy-2.4.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:36667db4d6c1cea79c8930ab72fadfb4060feb4bfe724141cd4bd064d2e5f8ce", size = 14411926, upload-time = "2025-12-20T16:17:25.822Z" }, + { url = "https://files.pythonhosted.org/packages/03/75/d4c43b61de473912496317a854dac54f1efec3eeb158438da6884b70bb90/numpy-2.4.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9a818668b674047fd88c4cddada7ab8f1c298812783e8328e956b78dc4807f9f", size = 16354295, upload-time = "2025-12-20T16:17:28.308Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0a/b54615b47ee8736a6461a4bb6749128dd3435c5a759d5663f11f0e9af4ac/numpy-2.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1ee32359fb7543b7b7bd0b2f46294db27e29e7bbdf70541e81b190836cd83ded", size = 16190242, upload-time = "2025-12-20T16:17:30.993Z" }, + { url = "https://files.pythonhosted.org/packages/98/ce/ea207769aacad6246525ec6c6bbd66a2bf56c72443dc10e2f90feed29290/numpy-2.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e493962256a38f58283de033d8af176c5c91c084ea30f15834f7545451c42059", size = 18280875, upload-time = "2025-12-20T16:17:33.327Z" }, + { url = "https://files.pythonhosted.org/packages/17/ef/ec409437aa962ea372ed601c519a2b141701683ff028f894b7466f0ab42b/numpy-2.4.0-cp314-cp314-win32.whl", hash = "sha256:6bbaebf0d11567fa8926215ae731e1d58e6ec28a8a25235b8a47405d301332db", size = 6002530, upload-time = "2025-12-20T16:17:35.729Z" }, + { url = "https://files.pythonhosted.org/packages/5f/4a/5cb94c787a3ed1ac65e1271b968686521169a7b3ec0b6544bb3ca32960b0/numpy-2.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:3d857f55e7fdf7c38ab96c4558c95b97d1c685be6b05c249f5fdafcbd6f9899e", size = 12435890, upload-time = "2025-12-20T16:17:37.599Z" }, + { url = "https://files.pythonhosted.org/packages/48/a0/04b89db963af9de1104975e2544f30de89adbf75b9e75f7dd2599be12c79/numpy-2.4.0-cp314-cp314-win_arm64.whl", hash = "sha256:bb50ce5fb202a26fd5404620e7ef820ad1ab3558b444cb0b55beb7ef66cd2d63", size = 10591892, upload-time = "2025-12-20T16:17:39.649Z" }, + { url = "https://files.pythonhosted.org/packages/53/e5/d74b5ccf6712c06c7a545025a6a71bfa03bdc7e0568b405b0d655232fd92/numpy-2.4.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:355354388cba60f2132df297e2d53053d4063f79077b67b481d21276d61fc4df", size = 12494312, upload-time = "2025-12-20T16:17:41.714Z" }, + { url = "https://files.pythonhosted.org/packages/c2/08/3ca9cc2ddf54dfee7ae9a6479c071092a228c68aef08252aa08dac2af002/numpy-2.4.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:1d8f9fde5f6dc1b6fc34df8162f3b3079365468703fee7f31d4e0cc8c63baed9", size = 5322862, upload-time = "2025-12-20T16:17:44.145Z" }, + { url = "https://files.pythonhosted.org/packages/87/74/0bb63a68394c0c1e52670cfff2e309afa41edbe11b3327d9af29e4383f34/numpy-2.4.0-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:e0434aa22c821f44eeb4c650b81c7fbdd8c0122c6c4b5a576a76d5a35625ecd9", size = 6644986, upload-time = "2025-12-20T16:17:46.203Z" }, + { url = "https://files.pythonhosted.org/packages/06/8f/9264d9bdbcf8236af2823623fe2f3981d740fc3461e2787e231d97c38c28/numpy-2.4.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40483b2f2d3ba7aad426443767ff5632ec3156ef09742b96913787d13c336471", size = 14457958, upload-time = "2025-12-20T16:17:48.017Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d9/f9a69ae564bbc7236a35aa883319364ef5fd41f72aa320cc1cbe66148fe2/numpy-2.4.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d9e6a7664ddd9746e20b7325351fe1a8408d0a2bf9c63b5e898290ddc8f09544", size = 16398394, upload-time = "2025-12-20T16:17:50.409Z" }, + { url = "https://files.pythonhosted.org/packages/34/c7/39241501408dde7f885d241a98caba5421061a2c6d2b2197ac5e3aa842d8/numpy-2.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ecb0019d44f4cdb50b676c5d0cb4b1eae8e15d1ed3d3e6639f986fc92b2ec52c", size = 16241044, upload-time = "2025-12-20T16:17:52.661Z" }, + { url = "https://files.pythonhosted.org/packages/7c/95/cae7effd90e065a95e59fe710eeee05d7328ed169776dfdd9f789e032125/numpy-2.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d0ffd9e2e4441c96a9c91ec1783285d80bf835b677853fc2770a89d50c1e48ac", size = 18321772, upload-time = "2025-12-20T16:17:54.947Z" }, + { url = "https://files.pythonhosted.org/packages/96/df/3c6c279accd2bfb968a76298e5b276310bd55d243df4fa8ac5816d79347d/numpy-2.4.0-cp314-cp314t-win32.whl", hash = "sha256:77f0d13fa87036d7553bf81f0e1fe3ce68d14c9976c9851744e4d3e91127e95f", size = 6148320, upload-time = "2025-12-20T16:17:57.249Z" }, + { url = "https://files.pythonhosted.org/packages/92/8d/f23033cce252e7a75cae853d17f582e86534c46404dea1c8ee094a9d6d84/numpy-2.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b1f5b45829ac1848893f0ddf5cb326110604d6df96cdc255b0bf9edd154104d4", size = 12623460, upload-time = "2025-12-20T16:17:58.963Z" }, + { url = "https://files.pythonhosted.org/packages/a4/4f/1f8475907d1a7c4ef9020edf7f39ea2422ec896849245f00688e4b268a71/numpy-2.4.0-cp314-cp314t-win_arm64.whl", hash = "sha256:23a3e9d1a6f360267e8fbb38ba5db355a6a7e9be71d7fce7ab3125e88bb646c8", size = 10661799, upload-time = "2025-12-20T16:18:01.078Z" }, + { url = "https://files.pythonhosted.org/packages/4b/ef/088e7c7342f300aaf3ee5f2c821c4b9996a1bef2aaf6a49cc8ab4883758e/numpy-2.4.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b54c83f1c0c0f1d748dca0af516062b8829d53d1f0c402be24b4257a9c48ada6", size = 16819003, upload-time = "2025-12-20T16:18:03.41Z" }, + { url = "https://files.pythonhosted.org/packages/ff/ce/a53017b5443b4b84517182d463fc7bcc2adb4faa8b20813f8e5f5aeb5faa/numpy-2.4.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:aabb081ca0ec5d39591fc33018cd4b3f96e1a2dd6756282029986d00a785fba4", size = 12567105, upload-time = "2025-12-20T16:18:05.594Z" }, + { url = "https://files.pythonhosted.org/packages/77/58/5ff91b161f2ec650c88a626c3905d938c89aaadabd0431e6d9c1330c83e2/numpy-2.4.0-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:8eafe7c36c8430b7794edeab3087dec7bf31d634d92f2af9949434b9d1964cba", size = 5395590, upload-time = "2025-12-20T16:18:08.031Z" }, + { url = "https://files.pythonhosted.org/packages/1d/4e/f1a084106df8c2df8132fc437e56987308e0524836aa7733721c8429d4fe/numpy-2.4.0-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:2f585f52b2baf07ff3356158d9268ea095e221371f1074fadea2f42544d58b4d", size = 6709947, upload-time = "2025-12-20T16:18:09.836Z" }, + { url = "https://files.pythonhosted.org/packages/63/09/3d8aeb809c0332c3f642da812ac2e3d74fc9252b3021f8c30c82e99e3f3d/numpy-2.4.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ed06d0fe9cae27d8fb5f400c63ccee72370599c75e683a6358dd3a4fb50aaf", size = 14535119, upload-time = "2025-12-20T16:18:12.105Z" }, + { url = "https://files.pythonhosted.org/packages/fd/7f/68f0fc43a2cbdc6bb239160c754d87c922f60fbaa0fa3cd3d312b8a7f5ee/numpy-2.4.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:57c540ed8fb1f05cb997c6761cd56db72395b0d6985e90571ff660452ade4f98", size = 16475815, upload-time = "2025-12-20T16:18:14.433Z" }, + { url = "https://files.pythonhosted.org/packages/11/73/edeacba3167b1ca66d51b1a5a14697c2c40098b5ffa01811c67b1785a5ab/numpy-2.4.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:a39fb973a726e63223287adc6dafe444ce75af952d711e400f3bf2b36ef55a7b", size = 12489376, upload-time = "2025-12-20T16:18:16.524Z" }, +] + +[[package]] +name = "nvidia-cublas-cu12" +version = "12.8.4.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142", size = 594346921, upload-time = "2025-03-07T01:44:31.254Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182", size = 10248621, upload-time = "2025-03-07T01:40:21.213Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.8.93" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994", size = 88040029, upload-time = "2025-03-07T01:42:13.562Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90", size = 954765, upload-time = "2025-03-07T01:40:01.615Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "9.10.2.21" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and sys_platform == 'linux') or (platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'linux')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" }, +] + +[[package]] +name = "nvidia-cufft-cu12" +version = "11.3.3.83" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and sys_platform == 'linux') or (platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'linux')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" }, +] + +[[package]] +name = "nvidia-cufile-cu12" +version = "1.13.1.3" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc", size = 1197834, upload-time = "2025-03-07T01:45:50.723Z" }, +] + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.9.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9", size = 63619976, upload-time = "2025-03-07T01:46:23.323Z" }, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.7.3.90" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and sys_platform == 'linux') or (platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "nvidia-cusparse-cu12", marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and sys_platform == 'linux') or (platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and sys_platform == 'linux') or (platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'linux')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" }, +] + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.5.8.93" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and sys_platform == 'linux') or (platform_machine != 's390x' and sys_platform != 'darwin' and sys_platform != 'linux')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu12" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623", size = 287193691, upload-time = "2025-02-26T00:15:44.104Z" }, +] + +[[package]] +name = "nvidia-nccl-cu12" +version = "2.27.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ad730cf15cb5d25fe849c6e6ca9eb5b76db16a80f13f425ac68d8e2e55624457", size = 322348229, upload-time = "2025-06-26T04:11:28.385Z" }, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.8.93" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88", size = 39254836, upload-time = "2025-03-07T01:49:55.661Z" }, +] + +[[package]] +name = "nvidia-nvshmem-cu12" +version = "3.3.20" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/6c/99acb2f9eb85c29fc6f3a7ac4dccfd992e22666dd08a642b303311326a97/nvidia_nvshmem_cu12-3.3.20-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d00f26d3f9b2e3c3065be895e3059d6479ea5c638a3f38c9fec49b1b9dd7c1e5", size = 124657145, upload-time = "2025-08-04T20:25:19.995Z" }, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f", size = 89954, upload-time = "2025-03-07T01:42:44.131Z" }, +] + +[[package]] +name = "onnx" +version = "1.21.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ml-dtypes" }, + { name = "numpy" }, + { name = "protobuf" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c5/93/942d2a0f6a70538eea042ce0445c8aefd46559ad153469986f29a743c01c/onnx-1.21.0.tar.gz", hash = "sha256:4d8b67d0aaec5864c87633188b91cc520877477ec0254eda122bef8be43cd764", size = 12074608, upload-time = "2026-03-27T21:33:36.118Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/48/32e383aa6bc40b72a9fd419937aaa647078190c9bfccdc97b316d2dee687/onnx-1.21.0-cp311-cp311-macosx_12_0_universal2.whl", hash = "sha256:2aca19949260875c14866fc77ea0bc37e4e809b24976108762843d328c92d3ce", size = 17968053, upload-time = "2026-03-27T21:32:29.558Z" }, + { url = "https://files.pythonhosted.org/packages/e2/26/5726e8df7d36e96bb3c679912d1a86af42f393d77aa17d6b98a97d4289ce/onnx-1.21.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:82aa6ab51144df07c58c4850cb78d4f1ae969d8c0bf657b28041796d49ba6974", size = 17534821, upload-time = "2026-03-27T21:32:32.351Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2b/021dcd2dd50c3c71b7959d7368526da384a295c162fb4863f36057973f78/onnx-1.21.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10c3185a232089335581fabb98fba4e86d3e8246b8140f2e406082438100ebda", size = 17616664, upload-time = "2026-03-27T21:32:34.921Z" }, + { url = "https://files.pythonhosted.org/packages/12/00/afa32a46fa122a7ed42df1cfe8796922156a3725ba8fc581c4779c96e2fc/onnx-1.21.0-cp311-cp311-win32.whl", hash = "sha256:f53b3c15a3b539c16b99655c43c365622046d68c49b680c48eba4da2a4fb6f27", size = 16289035, upload-time = "2026-03-27T21:32:37.783Z" }, + { url = "https://files.pythonhosted.org/packages/73/8d/483cc980a24d4c0131d0af06d0ff6a37fb08ae90a7848ece8cef645194f1/onnx-1.21.0-cp311-cp311-win_amd64.whl", hash = "sha256:5f78c411743db317a76e5d009f84f7e3d5380411a1567a868e82461a1e5c775d", size = 16443748, upload-time = "2026-03-27T21:32:40.337Z" }, + { url = "https://files.pythonhosted.org/packages/38/78/9d06fd5aaaed1ec9cb8a3b70fbbf00c1bdc18db610771e96379f0ed58112/onnx-1.21.0-cp311-cp311-win_arm64.whl", hash = "sha256:ab6a488dabbb172eebc9f3b3e7ac68763f32b0c571626d4a5004608f866cc83d", size = 16406123, upload-time = "2026-03-27T21:32:45.159Z" }, + { url = "https://files.pythonhosted.org/packages/7d/ae/cb644ec84c25e63575d9d8790fdcc5d1a11d67d3f62f872edb35fa38d158/onnx-1.21.0-cp312-abi3-macosx_12_0_universal2.whl", hash = "sha256:fc2635400fe39ff37ebc4e75342cc54450eadadf39c540ff132c319bf4960095", size = 17965930, upload-time = "2026-03-27T21:32:48.089Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b6/eeb5903586645ef8a49b4b7892580438741acc3df91d7a5bd0f3a59ea9cb/onnx-1.21.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9003d5206c01fa2ff4b46311566865d8e493e1a6998d4009ec6de39843f1b59b", size = 17531344, upload-time = "2026-03-27T21:32:50.837Z" }, + { url = "https://files.pythonhosted.org/packages/a7/00/4823f06357892d1e60d6f34e7299d2ba4ed2108c487cc394f7ce85a3ff14/onnx-1.21.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9261bd580fb8548c9c37b3c6750387eb8f21ea43c63880d37b2c622e1684285", size = 17613697, upload-time = "2026-03-27T21:32:54.222Z" }, + { url = "https://files.pythonhosted.org/packages/23/1d/391f3c567ae068c8ac4f1d1316bae97c9eb45e702f05975fe0e17ad441f0/onnx-1.21.0-cp312-abi3-win32.whl", hash = "sha256:9ea4e824964082811938a9250451d89c4ec474fe42dd36c038bfa5df31993d1e", size = 16287200, upload-time = "2026-03-27T21:32:57.277Z" }, + { url = "https://files.pythonhosted.org/packages/9c/a6/5eefbe5b40ea96de95a766bd2e0e751f35bdea2d4b951991ec9afaa69531/onnx-1.21.0-cp312-abi3-win_amd64.whl", hash = "sha256:458d91948ad9a7729a347550553b49ab6939f9af2cddf334e2116e45467dc61f", size = 16441045, upload-time = "2026-03-27T21:33:00.081Z" }, + { url = "https://files.pythonhosted.org/packages/63/c4/0ed8dc037a39113d2a4d66e0005e07751c299c46b993f1ad5c2c35664c20/onnx-1.21.0-cp312-abi3-win_arm64.whl", hash = "sha256:ca14bc4842fccc3187eb538f07eabeb25a779b39388b006db4356c07403a7bbb", size = 16403134, upload-time = "2026-03-27T21:33:03.987Z" }, + { url = "https://files.pythonhosted.org/packages/f8/89/0e1a9beb536401e2f45ac88735e123f2735e12fc7b56ff6c11727e097526/onnx-1.21.0-cp313-cp313t-macosx_12_0_universal2.whl", hash = "sha256:257d1d1deb6a652913698f1e3f33ef1ca0aa69174892fe38946d4572d89dd94f", size = 17975430, upload-time = "2026-03-27T21:33:07.005Z" }, + { url = "https://files.pythonhosted.org/packages/ec/46/e6dc71a7b3b317265591b20a5f71d0ff5c0d26c24e52283139dc90c66038/onnx-1.21.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7cd7cb8f6459311bdb557cbf6c0ccc6d8ace11c304d1bba0a30b4a4688e245f8", size = 17537435, upload-time = "2026-03-27T21:33:09.765Z" }, + { url = "https://files.pythonhosted.org/packages/49/2e/27affcac63eaf2ef183a44fd1a1354b11da64a6c72fe6f3fdcf5571bcee5/onnx-1.21.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b58a4cfec8d9311b73dc083e4c1fa362069267881144c05139b3eba5dc3a840", size = 17617687, upload-time = "2026-03-27T21:33:12.619Z" }, + { url = "https://files.pythonhosted.org/packages/1c/5c/ac8ed15e941593a3672ce424280b764979026317811f2e8508432bfc3429/onnx-1.21.0-cp313-cp313t-win_amd64.whl", hash = "sha256:1a9baf882562c4cebf79589bebb7cd71a20e30b51158cac3e3bbaf27da6163bd", size = 16449402, upload-time = "2026-03-27T21:33:15.555Z" }, + { url = "https://files.pythonhosted.org/packages/0e/aa/d2231e0dcaad838217afc64c306c8152a080134d2034e247cc973d577674/onnx-1.21.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bba12181566acf49b35875838eba49536a327b2944664b17125577d230c637ad", size = 16408273, upload-time = "2026-03-27T21:33:18.599Z" }, + { url = "https://files.pythonhosted.org/packages/bf/0a/8905b14694def6ad23edf1011fdd581500384062f8c4c567e114be7aa272/onnx-1.21.0-cp314-cp314t-macosx_12_0_universal2.whl", hash = "sha256:7ee9d8fd6a4874a5fa8b44bbcabea104ce752b20469b88bc50c7dcf9030779ad", size = 17975331, upload-time = "2026-03-27T21:33:21.69Z" }, + { url = "https://files.pythonhosted.org/packages/61/28/f4e401e5199d1b9c8b76c7e7ae1169e050515258e877b58fa8bb49d3bdcc/onnx-1.21.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5489f25fe461e7f32128218251a466cabbeeaf1eaa791c79daebf1a80d5a2cc9", size = 17537430, upload-time = "2026-03-27T21:33:24.547Z" }, + { url = "https://files.pythonhosted.org/packages/cf/cf/5d13320eb3660d5af360ea3b43aa9c63a70c92a9b4d1ea0d34501a32fcb8/onnx-1.21.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:db17fc0fec46180b6acbd1d5d8650a04e5527c02b09381da0b5b888d02a204c8", size = 17617662, upload-time = "2026-03-27T21:33:27.418Z" }, + { url = "https://files.pythonhosted.org/packages/4d/50/3eaa1878338247be021e6423696813d61e77e534dccbd15a703a144e703d/onnx-1.21.0-cp314-cp314t-win_amd64.whl", hash = "sha256:19d9971a3e52a12968ae6c70fd0f86c349536de0b0c33922ecdbe52d1972fe60", size = 16463688, upload-time = "2026-03-27T21:33:30.229Z" }, + { url = "https://files.pythonhosted.org/packages/a7/48/38d46b43bbb525e0b6a4c2c4204cc6795d67e45687a2f7403e06d8e7053d/onnx-1.21.0-cp314-cp314t-win_arm64.whl", hash = "sha256:efba467efb316baf2a9452d892c2f982b9b758c778d23e38c7f44fa211b30bb9", size = 16423387, upload-time = "2026-03-27T21:33:33.446Z" }, +] + +[[package]] +name = "onnx-ir" +version = "0.1.13" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ml-dtypes" }, + { name = "numpy" }, + { name = "onnx" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/c2/6db31dc3132e540076f15ed0cdf4a8db7ab75557f4d6c19eda655cac666e/onnx_ir-0.1.13.tar.gz", hash = "sha256:e08f00d30579bdbff2152692a6f1bc1f0523d3321ac6348aadcd40595e56231e", size = 115872, upload-time = "2025-12-17T18:03:13.86Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/b6/f60fd79ff5bc617d49db1378eb7c4c315b21b786502674e4a2d48e64491a/onnx_ir-0.1.13-py3-none-any.whl", hash = "sha256:2791493d1529fdbea60c257dc7bc0933dc812e6d68f4976d8b59aa7b4c2de8cf", size = 133063, upload-time = "2025-12-17T18:03:12.268Z" }, +] + +[[package]] +name = "onnxruntime" +version = "1.23.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coloredlogs" }, + { name = "flatbuffers" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "protobuf" }, + { name = "sympy" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/be/467b00f09061572f022ffd17e49e49e5a7a789056bad95b54dfd3bee73ff/onnxruntime-1.23.2-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:6f91d2c9b0965e86827a5ba01531d5b669770b01775b23199565d6c1f136616c", size = 17196113, upload-time = "2025-10-22T03:47:33.526Z" }, + { url = "https://files.pythonhosted.org/packages/9f/a8/3c23a8f75f93122d2b3410bfb74d06d0f8da4ac663185f91866b03f7da1b/onnxruntime-1.23.2-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:87d8b6eaf0fbeb6835a60a4265fde7a3b60157cf1b2764773ac47237b4d48612", size = 19153857, upload-time = "2025-10-22T03:46:37.578Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d8/506eed9af03d86f8db4880a4c47cd0dffee973ef7e4f4cff9f1d4bcf7d22/onnxruntime-1.23.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbfd2fca76c855317568c1b36a885ddea2272c13cb0e395002c402f2360429a6", size = 15220095, upload-time = "2025-10-22T03:46:24.769Z" }, + { url = "https://files.pythonhosted.org/packages/e9/80/113381ba832d5e777accedc6cb41d10f9eca82321ae31ebb6bcede530cea/onnxruntime-1.23.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da44b99206e77734c5819aa2142c69e64f3b46edc3bd314f6a45a932defc0b3e", size = 17372080, upload-time = "2025-10-22T03:47:00.265Z" }, + { url = "https://files.pythonhosted.org/packages/3a/db/1b4a62e23183a0c3fe441782462c0ede9a2a65c6bbffb9582fab7c7a0d38/onnxruntime-1.23.2-cp311-cp311-win_amd64.whl", hash = "sha256:902c756d8b633ce0dedd889b7c08459433fbcf35e9c38d1c03ddc020f0648c6e", size = 13468349, upload-time = "2025-10-22T03:47:25.783Z" }, + { url = "https://files.pythonhosted.org/packages/1b/9e/f748cd64161213adeef83d0cb16cb8ace1e62fa501033acdd9f9341fff57/onnxruntime-1.23.2-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:b8f029a6b98d3cf5be564d52802bb50a8489ab73409fa9db0bf583eabb7c2321", size = 17195929, upload-time = "2025-10-22T03:47:36.24Z" }, + { url = "https://files.pythonhosted.org/packages/91/9d/a81aafd899b900101988ead7fb14974c8a58695338ab6a0f3d6b0100f30b/onnxruntime-1.23.2-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:218295a8acae83905f6f1aed8cacb8e3eb3bd7513a13fe4ba3b2664a19fc4a6b", size = 19157705, upload-time = "2025-10-22T03:46:40.415Z" }, + { url = "https://files.pythonhosted.org/packages/3c/35/4e40f2fba272a6698d62be2cd21ddc3675edfc1a4b9ddefcc4648f115315/onnxruntime-1.23.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:76ff670550dc23e58ea9bc53b5149b99a44e63b34b524f7b8547469aaa0dcb8c", size = 15226915, upload-time = "2025-10-22T03:46:27.773Z" }, + { url = "https://files.pythonhosted.org/packages/ef/88/9cc25d2bafe6bc0d4d3c1db3ade98196d5b355c0b273e6a5dc09c5d5d0d5/onnxruntime-1.23.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f9b4ae77f8e3c9bee50c27bc1beede83f786fe1d52e99ac85aa8d65a01e9b77", size = 17382649, upload-time = "2025-10-22T03:47:02.782Z" }, + { url = "https://files.pythonhosted.org/packages/c0/b4/569d298f9fc4d286c11c45e85d9ffa9e877af12ace98af8cab52396e8f46/onnxruntime-1.23.2-cp312-cp312-win_amd64.whl", hash = "sha256:25de5214923ce941a3523739d34a520aac30f21e631de53bba9174dc9c004435", size = 13470528, upload-time = "2025-10-22T03:47:28.106Z" }, + { url = "https://files.pythonhosted.org/packages/3d/41/fba0cabccecefe4a1b5fc8020c44febb334637f133acefc7ec492029dd2c/onnxruntime-1.23.2-cp313-cp313-macosx_13_0_arm64.whl", hash = "sha256:2ff531ad8496281b4297f32b83b01cdd719617e2351ffe0dba5684fb283afa1f", size = 17196337, upload-time = "2025-10-22T03:46:35.168Z" }, + { url = "https://files.pythonhosted.org/packages/fe/f9/2d49ca491c6a986acce9f1d1d5fc2099108958cc1710c28e89a032c9cfe9/onnxruntime-1.23.2-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:162f4ca894ec3de1a6fd53589e511e06ecdc3ff646849b62a9da7489dee9ce95", size = 19157691, upload-time = "2025-10-22T03:46:43.518Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a1/428ee29c6eaf09a6f6be56f836213f104618fb35ac6cc586ff0f477263eb/onnxruntime-1.23.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45d127d6e1e9b99d1ebeae9bcd8f98617a812f53f46699eafeb976275744826b", size = 15226898, upload-time = "2025-10-22T03:46:30.039Z" }, + { url = "https://files.pythonhosted.org/packages/f2/2b/b57c8a2466a3126dbe0a792f56ad7290949b02f47b86216cd47d857e4b77/onnxruntime-1.23.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8bace4e0d46480fbeeb7bbe1ffe1f080e6663a42d1086ff95c1551f2d39e7872", size = 17382518, upload-time = "2025-10-22T03:47:05.407Z" }, + { url = "https://files.pythonhosted.org/packages/4a/93/aba75358133b3a941d736816dd392f687e7eab77215a6e429879080b76b6/onnxruntime-1.23.2-cp313-cp313-win_amd64.whl", hash = "sha256:1f9cc0a55349c584f083c1c076e611a7c35d5b867d5d6e6d6c823bf821978088", size = 13470276, upload-time = "2025-10-22T03:47:31.193Z" }, + { url = "https://files.pythonhosted.org/packages/7c/3d/6830fa61c69ca8e905f237001dbfc01689a4e4ab06147020a4518318881f/onnxruntime-1.23.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9d2385e774f46ac38f02b3a91a91e30263d41b2f1f4f26ae34805b2a9ddef466", size = 15229610, upload-time = "2025-10-22T03:46:32.239Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ca/862b1e7a639460f0ca25fd5b6135fb42cf9deea86d398a92e44dfda2279d/onnxruntime-1.23.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2b9233c4947907fd1818d0e581c049c41ccc39b2856cc942ff6d26317cee145", size = 17394184, upload-time = "2025-10-22T03:47:08.127Z" }, +] + +[[package]] +name = "onnxscript" +version = "0.5.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ml-dtypes" }, + { name = "numpy" }, + { name = "onnx" }, + { name = "onnx-ir" }, + { name = "packaging" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2c/f8/358a7d982ea51bc1b0c264f29c08adf096c62ba9f258ba13c954b41c46f5/onnxscript-0.5.7.tar.gz", hash = "sha256:480d572451bc233ed7f742b5005cb0c899594b2fdc28e15167dab26f7fd777ad", size = 596306, upload-time = "2025-12-16T20:47:15.762Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/ec/1656ea93be1e50baf429c20603dce249fa3571f3a180407cee79b1afa013/onnxscript-0.5.7-py3-none-any.whl", hash = "sha256:f94a66059c56d13b44908e9b7fd9dae4b4faa6681c784f3fd4c29cfa863e454e", size = 693353, upload-time = "2025-12-16T20:47:17.897Z" }, +] + +[[package]] +name = "opencv-python" +version = "4.11.0.86" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/17/06/68c27a523103dad5837dc5b87e71285280c4f098c60e4fe8a8db6486ab09/opencv-python-4.11.0.86.tar.gz", hash = "sha256:03d60ccae62304860d232272e4a4fda93c39d595780cb40b161b310244b736a4", size = 95171956, upload-time = "2025-01-16T13:52:24.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/4d/53b30a2a3ac1f75f65a59eb29cf2ee7207ce64867db47036ad61743d5a23/opencv_python-4.11.0.86-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:432f67c223f1dc2824f5e73cdfcd9db0efc8710647d4e813012195dc9122a52a", size = 37326322, upload-time = "2025-01-16T13:52:25.887Z" }, + { url = "https://files.pythonhosted.org/packages/3b/84/0a67490741867eacdfa37bc18df96e08a9d579583b419010d7f3da8ff503/opencv_python-4.11.0.86-cp37-abi3-macosx_13_0_x86_64.whl", hash = "sha256:9d05ef13d23fe97f575153558653e2d6e87103995d54e6a35db3f282fe1f9c66", size = 56723197, upload-time = "2025-01-16T13:55:21.222Z" }, + { url = "https://files.pythonhosted.org/packages/f3/bd/29c126788da65c1fb2b5fb621b7fed0ed5f9122aa22a0868c5e2c15c6d23/opencv_python-4.11.0.86-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b92ae2c8852208817e6776ba1ea0d6b1e0a1b5431e971a2a0ddd2a8cc398202", size = 42230439, upload-time = "2025-01-16T13:51:35.822Z" }, + { url = "https://files.pythonhosted.org/packages/2c/8b/90eb44a40476fa0e71e05a0283947cfd74a5d36121a11d926ad6f3193cc4/opencv_python-4.11.0.86-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b02611523803495003bd87362db3e1d2a0454a6a63025dc6658a9830570aa0d", size = 62986597, upload-time = "2025-01-16T13:52:08.836Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d7/1d5941a9dde095468b288d989ff6539dd69cd429dbf1b9e839013d21b6f0/opencv_python-4.11.0.86-cp37-abi3-win32.whl", hash = "sha256:810549cb2a4aedaa84ad9a1c92fbfdfc14090e2749cedf2c1589ad8359aa169b", size = 29384337, upload-time = "2025-01-16T13:52:13.549Z" }, + { url = "https://files.pythonhosted.org/packages/a4/7d/f1c30a92854540bf789e9cd5dde7ef49bbe63f855b85a2e6b3db8135c591/opencv_python-4.11.0.86-cp37-abi3-win_amd64.whl", hash = "sha256:085ad9b77c18853ea66283e98affefe2de8cc4c1f43eda4c100cf9b2721142ec", size = 39488044, upload-time = "2025-01-16T13:52:21.928Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pillow" +version = "12.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5", size = 46987819, upload-time = "2026-04-01T14:46:17.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/e1/748f5663efe6edcfc4e74b2b93edfb9b8b99b67f21a854c3ae416500a2d9/pillow-12.2.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:8be29e59487a79f173507c30ddf57e733a357f67881430449bb32614075a40ab", size = 5354347, upload-time = "2026-04-01T14:42:44.255Z" }, + { url = "https://files.pythonhosted.org/packages/47/a1/d5ff69e747374c33a3b53b9f98cca7889fce1fd03d79cdc4e1bccc6c5a87/pillow-12.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:71cde9a1e1551df7d34a25462fc60325e8a11a82cc2e2f54578e5e9a1e153d65", size = 4695873, upload-time = "2026-04-01T14:42:46.452Z" }, + { url = "https://files.pythonhosted.org/packages/df/21/e3fbdf54408a973c7f7f89a23b2cb97a7ef30c61ab4142af31eee6aebc88/pillow-12.2.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f490f9368b6fc026f021db16d7ec2fbf7d89e2edb42e8ec09d2c60505f5729c7", size = 6280168, upload-time = "2026-04-01T14:42:49.228Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f1/00b7278c7dd52b17ad4329153748f87b6756ec195ff786c2bdf12518337d/pillow-12.2.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8bd7903a5f2a4545f6fd5935c90058b89d30045568985a71c79f5fd6edf9b91e", size = 8088188, upload-time = "2026-04-01T14:42:51.735Z" }, + { url = "https://files.pythonhosted.org/packages/ad/cf/220a5994ef1b10e70e85748b75649d77d506499352be135a4989c957b701/pillow-12.2.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3997232e10d2920a68d25191392e3a4487d8183039e1c74c2297f00ed1c50705", size = 6394401, upload-time = "2026-04-01T14:42:54.343Z" }, + { url = "https://files.pythonhosted.org/packages/e9/bd/e51a61b1054f09437acfbc2ff9106c30d1eb76bc1453d428399946781253/pillow-12.2.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e74473c875d78b8e9d5da2a70f7099549f9eb37ded4e2f6a463e60125bccd176", size = 7079655, upload-time = "2026-04-01T14:42:56.954Z" }, + { url = "https://files.pythonhosted.org/packages/6b/3d/45132c57d5fb4b5744567c3817026480ac7fc3ce5d4c47902bc0e7f6f853/pillow-12.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:56a3f9c60a13133a98ecff6197af34d7824de9b7b38c3654861a725c970c197b", size = 6503105, upload-time = "2026-04-01T14:42:59.847Z" }, + { url = "https://files.pythonhosted.org/packages/7d/2e/9df2fc1e82097b1df3dce58dc43286aa01068e918c07574711fcc53e6fb4/pillow-12.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:90e6f81de50ad6b534cab6e5aef77ff6e37722b2f5d908686f4a5c9eba17a909", size = 7203402, upload-time = "2026-04-01T14:43:02.664Z" }, + { url = "https://files.pythonhosted.org/packages/bd/2e/2941e42858ebb67e50ae741473de81c2984e6eff7b397017623c676e2e8d/pillow-12.2.0-cp311-cp311-win32.whl", hash = "sha256:8c984051042858021a54926eb597d6ee3012393ce9c181814115df4c60b9a808", size = 6378149, upload-time = "2026-04-01T14:43:05.274Z" }, + { url = "https://files.pythonhosted.org/packages/69/42/836b6f3cd7f3e5fa10a1f1a5420447c17966044c8fbf589cc0452d5502db/pillow-12.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:6e6b2a0c538fc200b38ff9eb6628228b77908c319a005815f2dde585a0664b60", size = 7082626, upload-time = "2026-04-01T14:43:08.557Z" }, + { url = "https://files.pythonhosted.org/packages/c2/88/549194b5d6f1f494b485e493edc6693c0a16f4ada488e5bd974ed1f42fad/pillow-12.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:9a8a34cc89c67a65ea7437ce257cea81a9dad65b29805f3ecee8c8fe8ff25ffe", size = 2463531, upload-time = "2026-04-01T14:43:10.743Z" }, + { url = "https://files.pythonhosted.org/packages/58/be/7482c8a5ebebbc6470b3eb791812fff7d5e0216c2be3827b30b8bb6603ed/pillow-12.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5", size = 5308279, upload-time = "2026-04-01T14:43:13.246Z" }, + { url = "https://files.pythonhosted.org/packages/d8/95/0a351b9289c2b5cbde0bacd4a83ebc44023e835490a727b2a3bd60ddc0f4/pillow-12.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421", size = 4695490, upload-time = "2026-04-01T14:43:15.584Z" }, + { url = "https://files.pythonhosted.org/packages/de/af/4e8e6869cbed569d43c416fad3dc4ecb944cb5d9492defaed89ddd6fe871/pillow-12.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987", size = 6284462, upload-time = "2026-04-01T14:43:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/9e/c05e19657fd57841e476be1ab46c4d501bffbadbafdc31a6d665f8b737b6/pillow-12.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76", size = 8094744, upload-time = "2026-04-01T14:43:20.716Z" }, + { url = "https://files.pythonhosted.org/packages/2b/54/1789c455ed10176066b6e7e6da1b01e50e36f94ba584dc68d9eebfe9156d/pillow-12.2.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005", size = 6398371, upload-time = "2026-04-01T14:43:23.443Z" }, + { url = "https://files.pythonhosted.org/packages/43/e3/fdc657359e919462369869f1c9f0e973f353f9a9ee295a39b1fea8ee1a77/pillow-12.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62f5409336adb0663b7caa0da5c7d9e7bdbaae9ce761d34669420c2a801b2780", size = 7087215, upload-time = "2026-04-01T14:43:26.758Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f8/2f6825e441d5b1959d2ca5adec984210f1ec086435b0ed5f52c19b3b8a6e/pillow-12.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:01afa7cf67f74f09523699b4e88c73fb55c13346d212a59a2db1f86b0a63e8c5", size = 6509783, upload-time = "2026-04-01T14:43:29.56Z" }, + { url = "https://files.pythonhosted.org/packages/67/f9/029a27095ad20f854f9dba026b3ea6428548316e057e6fc3545409e86651/pillow-12.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5", size = 7212112, upload-time = "2026-04-01T14:43:32.091Z" }, + { url = "https://files.pythonhosted.org/packages/be/42/025cfe05d1be22dbfdb4f264fe9de1ccda83f66e4fc3aac94748e784af04/pillow-12.2.0-cp312-cp312-win32.whl", hash = "sha256:58f62cc0f00fd29e64b29f4fd923ffdb3859c9f9e6105bfc37ba1d08994e8940", size = 6378489, upload-time = "2026-04-01T14:43:34.601Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7b/25a221d2c761c6a8ae21bfa3874988ff2583e19cf8a27bf2fee358df7942/pillow-12.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5", size = 7084129, upload-time = "2026-04-01T14:43:37.213Z" }, + { url = "https://files.pythonhosted.org/packages/10/e1/542a474affab20fd4a0f1836cb234e8493519da6b76899e30bcc5d990b8b/pillow-12.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414", size = 2463612, upload-time = "2026-04-01T14:43:39.421Z" }, + { url = "https://files.pythonhosted.org/packages/4a/01/53d10cf0dbad820a8db274d259a37ba50b88b24768ddccec07355382d5ad/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8297651f5b5679c19968abefd6bb84d95fe30ef712eb1b2d9b2d31ca61267f4c", size = 4100837, upload-time = "2026-04-01T14:43:41.506Z" }, + { url = "https://files.pythonhosted.org/packages/0f/98/f3a6657ecb698c937f6c76ee564882945f29b79bad496abcba0e84659ec5/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:50d8520da2a6ce0af445fa6d648c4273c3eeefbc32d7ce049f22e8b5c3daecc2", size = 4176528, upload-time = "2026-04-01T14:43:43.773Z" }, + { url = "https://files.pythonhosted.org/packages/69/bc/8986948f05e3ea490b8442ea1c1d4d990b24a7e43d8a51b2c7d8b1dced36/pillow-12.2.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:766cef22385fa1091258ad7e6216792b156dc16d8d3fa607e7545b2b72061f1c", size = 3640401, upload-time = "2026-04-01T14:43:45.87Z" }, + { url = "https://files.pythonhosted.org/packages/34/46/6c717baadcd62bc8ed51d238d521ab651eaa74838291bda1f86fe1f864c9/pillow-12.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5d2fd0fa6b5d9d1de415060363433f28da8b1526c1c129020435e186794b3795", size = 5308094, upload-time = "2026-04-01T14:43:48.438Z" }, + { url = "https://files.pythonhosted.org/packages/71/43/905a14a8b17fdb1ccb58d282454490662d2cb89a6bfec26af6d3520da5ec/pillow-12.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:56b25336f502b6ed02e889f4ece894a72612fe885889a6e8c4c80239ff6e5f5f", size = 4695402, upload-time = "2026-04-01T14:43:51.292Z" }, + { url = "https://files.pythonhosted.org/packages/73/dd/42107efcb777b16fa0393317eac58f5b5cf30e8392e266e76e51cff28c3d/pillow-12.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f1c943e96e85df3d3478f7b691f229887e143f81fedab9b20205349ab04d73ed", size = 6280005, upload-time = "2026-04-01T14:43:54.242Z" }, + { url = "https://files.pythonhosted.org/packages/a8/68/b93e09e5e8549019e61acf49f65b1a8530765a7f812c77a7461bca7e4494/pillow-12.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:03f6fab9219220f041c74aeaa2939ff0062bd5c364ba9ce037197f4c6d498cd9", size = 8090669, upload-time = "2026-04-01T14:43:57.335Z" }, + { url = "https://files.pythonhosted.org/packages/4b/6e/3ccb54ce8ec4ddd1accd2d89004308b7b0b21c4ac3d20fa70af4760a4330/pillow-12.2.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cdfebd752ec52bf5bb4e35d9c64b40826bc5b40a13df7c3cda20a2c03a0f5ed", size = 6395194, upload-time = "2026-04-01T14:43:59.864Z" }, + { url = "https://files.pythonhosted.org/packages/67/ee/21d4e8536afd1a328f01b359b4d3997b291ffd35a237c877b331c1c3b71c/pillow-12.2.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eedf4b74eda2b5a4b2b2fb4c006d6295df3bf29e459e198c90ea48e130dc75c3", size = 7082423, upload-time = "2026-04-01T14:44:02.74Z" }, + { url = "https://files.pythonhosted.org/packages/78/5f/e9f86ab0146464e8c133fe85df987ed9e77e08b29d8d35f9f9f4d6f917ba/pillow-12.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:00a2865911330191c0b818c59103b58a5e697cae67042366970a6b6f1b20b7f9", size = 6505667, upload-time = "2026-04-01T14:44:05.381Z" }, + { url = "https://files.pythonhosted.org/packages/ed/1e/409007f56a2fdce61584fd3acbc2bbc259857d555196cedcadc68c015c82/pillow-12.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1e1757442ed87f4912397c6d35a0db6a7b52592156014706f17658ff58bbf795", size = 7208580, upload-time = "2026-04-01T14:44:08.39Z" }, + { url = "https://files.pythonhosted.org/packages/23/c4/7349421080b12fb35414607b8871e9534546c128a11965fd4a7002ccfbee/pillow-12.2.0-cp313-cp313-win32.whl", hash = "sha256:144748b3af2d1b358d41286056d0003f47cb339b8c43a9ea42f5fea4d8c66b6e", size = 6375896, upload-time = "2026-04-01T14:44:11.197Z" }, + { url = "https://files.pythonhosted.org/packages/3f/82/8a3739a5e470b3c6cbb1d21d315800d8e16bff503d1f16b03a4ec3212786/pillow-12.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:390ede346628ccc626e5730107cde16c42d3836b89662a115a921f28440e6a3b", size = 7081266, upload-time = "2026-04-01T14:44:13.947Z" }, + { url = "https://files.pythonhosted.org/packages/c3/25/f968f618a062574294592f668218f8af564830ccebdd1fa6200f598e65c5/pillow-12.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:8023abc91fba39036dbce14a7d6535632f99c0b857807cbbbf21ecc9f4717f06", size = 2463508, upload-time = "2026-04-01T14:44:16.312Z" }, + { url = "https://files.pythonhosted.org/packages/4d/a4/b342930964e3cb4dce5038ae34b0eab4653334995336cd486c5a8c25a00c/pillow-12.2.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:042db20a421b9bafecc4b84a8b6e444686bd9d836c7fd24542db3e7df7baad9b", size = 5309927, upload-time = "2026-04-01T14:44:18.89Z" }, + { url = "https://files.pythonhosted.org/packages/9f/de/23198e0a65a9cf06123f5435a5d95cea62a635697f8f03d134d3f3a96151/pillow-12.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:dd025009355c926a84a612fecf58bb315a3f6814b17ead51a8e48d3823d9087f", size = 4698624, upload-time = "2026-04-01T14:44:21.115Z" }, + { url = "https://files.pythonhosted.org/packages/01/a6/1265e977f17d93ea37aa28aa81bad4fa597933879fac2520d24e021c8da3/pillow-12.2.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88ddbc66737e277852913bd1e07c150cc7bb124539f94c4e2df5344494e0a612", size = 6321252, upload-time = "2026-04-01T14:44:23.663Z" }, + { url = "https://files.pythonhosted.org/packages/3c/83/5982eb4a285967baa70340320be9f88e57665a387e3a53a7f0db8231a0cd/pillow-12.2.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d362d1878f00c142b7e1a16e6e5e780f02be8195123f164edf7eddd911eefe7c", size = 8126550, upload-time = "2026-04-01T14:44:26.772Z" }, + { url = "https://files.pythonhosted.org/packages/4e/48/6ffc514adce69f6050d0753b1a18fd920fce8cac87620d5a31231b04bfc5/pillow-12.2.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c727a6d53cb0018aadd8018c2b938376af27914a68a492f59dfcaca650d5eea", size = 6433114, upload-time = "2026-04-01T14:44:29.615Z" }, + { url = "https://files.pythonhosted.org/packages/36/a3/f9a77144231fb8d40ee27107b4463e205fa4677e2ca2548e14da5cf18dce/pillow-12.2.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd8c21c98c5cc60653bcb311bef2ce0401642b7ce9d09e03a7da87c878289d4", size = 7115667, upload-time = "2026-04-01T14:44:32.773Z" }, + { url = "https://files.pythonhosted.org/packages/c1/fc/ac4ee3041e7d5a565e1c4fd72a113f03b6394cc72ab7089d27608f8aaccb/pillow-12.2.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9f08483a632889536b8139663db60f6724bfcb443c96f1b18855860d7d5c0fd4", size = 6538966, upload-time = "2026-04-01T14:44:35.252Z" }, + { url = "https://files.pythonhosted.org/packages/c0/a8/27fb307055087f3668f6d0a8ccb636e7431d56ed0750e07a60547b1e083e/pillow-12.2.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dac8d77255a37e81a2efcbd1fc05f1c15ee82200e6c240d7e127e25e365c39ea", size = 7238241, upload-time = "2026-04-01T14:44:37.875Z" }, + { url = "https://files.pythonhosted.org/packages/ad/4b/926ab182c07fccae9fcb120043464e1ff1564775ec8864f21a0ebce6ac25/pillow-12.2.0-cp313-cp313t-win32.whl", hash = "sha256:ee3120ae9dff32f121610bb08e4313be87e03efeadfc6c0d18f89127e24d0c24", size = 6379592, upload-time = "2026-04-01T14:44:40.336Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c4/f9e476451a098181b30050cc4c9a3556b64c02cf6497ea421ac047e89e4b/pillow-12.2.0-cp313-cp313t-win_amd64.whl", hash = "sha256:325ca0528c6788d2a6c3d40e3568639398137346c3d6e66bb61db96b96511c98", size = 7085542, upload-time = "2026-04-01T14:44:43.251Z" }, + { url = "https://files.pythonhosted.org/packages/00/a4/285f12aeacbe2d6dc36c407dfbbe9e96d4a80b0fb710a337f6d2ad978c75/pillow-12.2.0-cp313-cp313t-win_arm64.whl", hash = "sha256:2e5a76d03a6c6dcef67edabda7a52494afa4035021a79c8558e14af25313d453", size = 2465765, upload-time = "2026-04-01T14:44:45.996Z" }, + { url = "https://files.pythonhosted.org/packages/bf/98/4595daa2365416a86cb0d495248a393dfc84e96d62ad080c8546256cb9c0/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3adc9215e8be0448ed6e814966ecf3d9952f0ea40eb14e89a102b87f450660d8", size = 4100848, upload-time = "2026-04-01T14:44:48.48Z" }, + { url = "https://files.pythonhosted.org/packages/0b/79/40184d464cf89f6663e18dfcf7ca21aae2491fff1a16127681bf1fa9b8cf/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:6a9adfc6d24b10f89588096364cc726174118c62130c817c2837c60cf08a392b", size = 4176515, upload-time = "2026-04-01T14:44:51.353Z" }, + { url = "https://files.pythonhosted.org/packages/b0/63/703f86fd4c422a9cf722833670f4f71418fb116b2853ff7da722ea43f184/pillow-12.2.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6a6e67ea2e6feda684ed370f9a1c52e7a243631c025ba42149a2cc5934dec295", size = 3640159, upload-time = "2026-04-01T14:44:53.588Z" }, + { url = "https://files.pythonhosted.org/packages/71/e0/fb22f797187d0be2270f83500aab851536101b254bfa1eae10795709d283/pillow-12.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2bb4a8d594eacdfc59d9e5ad972aa8afdd48d584ffd5f13a937a664c3e7db0ed", size = 5312185, upload-time = "2026-04-01T14:44:56.039Z" }, + { url = "https://files.pythonhosted.org/packages/ba/8c/1a9e46228571de18f8e28f16fabdfc20212a5d019f3e3303452b3f0a580d/pillow-12.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:80b2da48193b2f33ed0c32c38140f9d3186583ce7d516526d462645fd98660ae", size = 4695386, upload-time = "2026-04-01T14:44:58.663Z" }, + { url = "https://files.pythonhosted.org/packages/70/62/98f6b7f0c88b9addd0e87c217ded307b36be024d4ff8869a812b241d1345/pillow-12.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22db17c68434de69d8ecfc2fe821569195c0c373b25cccb9cbdacf2c6e53c601", size = 6280384, upload-time = "2026-04-01T14:45:01.5Z" }, + { url = "https://files.pythonhosted.org/packages/5e/03/688747d2e91cfbe0e64f316cd2e8005698f76ada3130d0194664174fa5de/pillow-12.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b14cc0106cd9aecda615dd6903840a058b4700fcb817687d0ee4fc8b6e389be", size = 8091599, upload-time = "2026-04-01T14:45:04.5Z" }, + { url = "https://files.pythonhosted.org/packages/f6/35/577e22b936fcdd66537329b33af0b4ccfefaeabd8aec04b266528cddb33c/pillow-12.2.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cbeb542b2ebc6fcdacabf8aca8c1a97c9b3ad3927d46b8723f9d4f033288a0f", size = 6396021, upload-time = "2026-04-01T14:45:07.117Z" }, + { url = "https://files.pythonhosted.org/packages/11/8d/d2532ad2a603ca2b93ad9f5135732124e57811d0168155852f37fbce2458/pillow-12.2.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bfd07bc812fbd20395212969e41931001fd59eb55a60658b0e5710872e95286", size = 7083360, upload-time = "2026-04-01T14:45:09.763Z" }, + { url = "https://files.pythonhosted.org/packages/5e/26/d325f9f56c7e039034897e7380e9cc202b1e368bfd04d4cbe6a441f02885/pillow-12.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9aba9a17b623ef750a4d11b742cbafffeb48a869821252b30ee21b5e91392c50", size = 6507628, upload-time = "2026-04-01T14:45:12.378Z" }, + { url = "https://files.pythonhosted.org/packages/5f/f7/769d5632ffb0988f1c5e7660b3e731e30f7f8ec4318e94d0a5d674eb65a4/pillow-12.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:deede7c263feb25dba4e82ea23058a235dcc2fe1f6021025dc71f2b618e26104", size = 7209321, upload-time = "2026-04-01T14:45:15.122Z" }, + { url = "https://files.pythonhosted.org/packages/6a/7a/c253e3c645cd47f1aceea6a8bacdba9991bf45bb7dfe927f7c893e89c93c/pillow-12.2.0-cp314-cp314-win32.whl", hash = "sha256:632ff19b2778e43162304d50da0181ce24ac5bb8180122cbe1bf4673428328c7", size = 6479723, upload-time = "2026-04-01T14:45:17.797Z" }, + { url = "https://files.pythonhosted.org/packages/cd/8b/601e6566b957ca50e28725cb6c355c59c2c8609751efbecd980db44e0349/pillow-12.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:4e6c62e9d237e9b65fac06857d511e90d8461a32adcc1b9065ea0c0fa3a28150", size = 7217400, upload-time = "2026-04-01T14:45:20.529Z" }, + { url = "https://files.pythonhosted.org/packages/d6/94/220e46c73065c3e2951bb91c11a1fb636c8c9ad427ac3ce7d7f3359b9b2f/pillow-12.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:b1c1fbd8a5a1af3412a0810d060a78b5136ec0836c8a4ef9aa11807f2a22f4e1", size = 2554835, upload-time = "2026-04-01T14:45:23.162Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ab/1b426a3974cb0e7da5c29ccff4807871d48110933a57207b5a676cccc155/pillow-12.2.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:57850958fe9c751670e49b2cecf6294acc99e562531f4bd317fa5ddee2068463", size = 5314225, upload-time = "2026-04-01T14:45:25.637Z" }, + { url = "https://files.pythonhosted.org/packages/19/1e/dce46f371be2438eecfee2a1960ee2a243bbe5e961890146d2dee1ff0f12/pillow-12.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d5d38f1411c0ed9f97bcb49b7bd59b6b7c314e0e27420e34d99d844b9ce3b6f3", size = 4698541, upload-time = "2026-04-01T14:45:28.355Z" }, + { url = "https://files.pythonhosted.org/packages/55/c3/7fbecf70adb3a0c33b77a300dc52e424dc22ad8cdc06557a2e49523b703d/pillow-12.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c0a9f29ca8e79f09de89293f82fc9b0270bb4af1d58bc98f540cc4aedf03166", size = 6322251, upload-time = "2026-04-01T14:45:30.924Z" }, + { url = "https://files.pythonhosted.org/packages/1c/3c/7fbc17cfb7e4fe0ef1642e0abc17fc6c94c9f7a16be41498e12e2ba60408/pillow-12.2.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1610dd6c61621ae1cf811bef44d77e149ce3f7b95afe66a4512f8c59f25d9ebe", size = 8127807, upload-time = "2026-04-01T14:45:33.908Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c3/a8ae14d6defd2e448493ff512fae903b1e9bd40b72efb6ec55ce0048c8ce/pillow-12.2.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a34329707af4f73cf1782a36cd2289c0368880654a2c11f027bcee9052d35dd", size = 6433935, upload-time = "2026-04-01T14:45:36.623Z" }, + { url = "https://files.pythonhosted.org/packages/6e/32/2880fb3a074847ac159d8f902cb43278a61e85f681661e7419e6596803ed/pillow-12.2.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e9c4f5b3c546fa3458a29ab22646c1c6c787ea8f5ef51300e5a60300736905e", size = 7116720, upload-time = "2026-04-01T14:45:39.258Z" }, + { url = "https://files.pythonhosted.org/packages/46/87/495cc9c30e0129501643f24d320076f4cc54f718341df18cc70ec94c44e1/pillow-12.2.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06", size = 6540498, upload-time = "2026-04-01T14:45:41.879Z" }, + { url = "https://files.pythonhosted.org/packages/18/53/773f5edca692009d883a72211b60fdaf8871cbef075eaa9d577f0a2f989e/pillow-12.2.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f278f034eb75b4e8a13a54a876cc4a5ab39173d2cdd93a638e1b467fc545ac43", size = 7239413, upload-time = "2026-04-01T14:45:44.705Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e4/4b64a97d71b2a83158134abbb2f5bd3f8a2ea691361282f010998f339ec7/pillow-12.2.0-cp314-cp314t-win32.whl", hash = "sha256:6bb77b2dcb06b20f9f4b4a8454caa581cd4dd0643a08bacf821216a16d9c8354", size = 6482084, upload-time = "2026-04-01T14:45:47.568Z" }, + { url = "https://files.pythonhosted.org/packages/ba/13/306d275efd3a3453f72114b7431c877d10b1154014c1ebbedd067770d629/pillow-12.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6562ace0d3fb5f20ed7290f1f929cae41b25ae29528f2af1722966a0a02e2aa1", size = 7225152, upload-time = "2026-04-01T14:45:50.032Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6e/cf826fae916b8658848d7b9f38d88da6396895c676e8086fc0988073aaf8/pillow-12.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb", size = 2556579, upload-time = "2026-04-01T14:45:52.529Z" }, + { url = "https://files.pythonhosted.org/packages/4e/b7/2437044fb910f499610356d1352e3423753c98e34f915252aafecc64889f/pillow-12.2.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0538bd5e05efec03ae613fd89c4ce0368ecd2ba239cc25b9f9be7ed426b0af1f", size = 5273969, upload-time = "2026-04-01T14:45:55.538Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f4/8316e31de11b780f4ac08ef3654a75555e624a98db1056ecb2122d008d5a/pillow-12.2.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:394167b21da716608eac917c60aa9b969421b5dcbbe02ae7f013e7b85811c69d", size = 4659674, upload-time = "2026-04-01T14:45:58.093Z" }, + { url = "https://files.pythonhosted.org/packages/d4/37/664fca7201f8bb2aa1d20e2c3d5564a62e6ae5111741966c8319ca802361/pillow-12.2.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5d04bfa02cc2d23b497d1e90a0f927070043f6cbf303e738300532379a4b4e0f", size = 5288479, upload-time = "2026-04-01T14:46:01.141Z" }, + { url = "https://files.pythonhosted.org/packages/49/62/5b0ed78fce87346be7a5cfcfaaad91f6a1f98c26f86bdbafa2066c647ef6/pillow-12.2.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0c838a5125cee37e68edec915651521191cef1e6aa336b855f495766e77a366e", size = 7032230, upload-time = "2026-04-01T14:46:03.874Z" }, + { url = "https://files.pythonhosted.org/packages/c3/28/ec0fc38107fc32536908034e990c47914c57cd7c5a3ece4d8d8f7ffd7e27/pillow-12.2.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a6c9fa44005fa37a91ebfc95d081e8079757d2e904b27103f4f5fa6f0bf78c0", size = 5355404, upload-time = "2026-04-01T14:46:06.33Z" }, + { url = "https://files.pythonhosted.org/packages/5e/8b/51b0eddcfa2180d60e41f06bd6d0a62202b20b59c68f5a132e615b75aecf/pillow-12.2.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:25373b66e0dd5905ed63fa3cae13c82fbddf3079f2c8bf15c6fb6a35586324c1", size = 6002215, upload-time = "2026-04-01T14:46:08.83Z" }, + { url = "https://files.pythonhosted.org/packages/bc/60/5382c03e1970de634027cee8e1b7d39776b778b81812aaf45b694dfe9e28/pillow-12.2.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:bfa9c230d2fe991bed5318a5f119bd6780cda2915cca595393649fc118ab895e", size = 7080946, upload-time = "2026-04-01T14:46:11.734Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cf/86/0248f086a84f01b37aaec0fa567b397df1a119f73c16f6c7a9aac73ea309/platformdirs-4.5.1.tar.gz", hash = "sha256:61d5cdcc6065745cdd94f0f878977f8de9437be93de97c1c12f853c9c0cdcbda", size = 21715, upload-time = "2025-12-05T13:52:58.638Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/28/3bfe2fa5a7b9c46fe7e13c97bda14c895fb10fa2ebf1d0abb90e0cea7ee1/platformdirs-4.5.1-py3-none-any.whl", hash = "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31", size = 18731, upload-time = "2025-12-05T13:52:56.823Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pre-commit" +version = "4.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/f1/6d86a29246dfd2e9b6237f0b5823717f60cad94d47ddc26afa916d21f525/pre_commit-4.5.1.tar.gz", hash = "sha256:eb545fcff725875197837263e977ea257a402056661f09dae08e4b149b030a61", size = 198232, upload-time = "2025-12-16T21:14:33.552Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/19/fd3ef348460c80af7bb4669ea7926651d1f95c23ff2df18b9d24bab4f3fa/pre_commit-4.5.1-py2.py3-none-any.whl", hash = "sha256:3b3afd891e97337708c1674210f8eba659b52a38ea5f822ff142d10786221f77", size = 226437, upload-time = "2025-12-16T21:14:32.409Z" }, +] + +[[package]] +name = "protobuf" +version = "6.33.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/25/7c72c307aafc96fa87062aa6291d9f7c94836e43214d43722e86037aac02/protobuf-6.33.5.tar.gz", hash = "sha256:6ddcac2a081f8b7b9642c09406bc6a4290128fce5f471cddd165960bb9119e5c", size = 444465, upload-time = "2026-01-29T21:51:33.494Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/79/af92d0a8369732b027e6d6084251dd8e782c685c72da161bd4a2e00fbabb/protobuf-6.33.5-cp310-abi3-win32.whl", hash = "sha256:d71b040839446bac0f4d162e758bea99c8251161dae9d0983a3b88dee345153b", size = 425769, upload-time = "2026-01-29T21:51:21.751Z" }, + { url = "https://files.pythonhosted.org/packages/55/75/bb9bc917d10e9ee13dee8607eb9ab963b7cf8be607c46e7862c748aa2af7/protobuf-6.33.5-cp310-abi3-win_amd64.whl", hash = "sha256:3093804752167bcab3998bec9f1048baae6e29505adaf1afd14a37bddede533c", size = 437118, upload-time = "2026-01-29T21:51:24.022Z" }, + { url = "https://files.pythonhosted.org/packages/a2/6b/e48dfc1191bc5b52950246275bf4089773e91cb5ba3592621723cdddca62/protobuf-6.33.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:a5cb85982d95d906df1e2210e58f8e4f1e3cdc088e52c921a041f9c9a0386de5", size = 427766, upload-time = "2026-01-29T21:51:25.413Z" }, + { url = "https://files.pythonhosted.org/packages/4e/b1/c79468184310de09d75095ed1314b839eb2f72df71097db9d1404a1b2717/protobuf-6.33.5-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:9b71e0281f36f179d00cbcb119cb19dec4d14a81393e5ea220f64b286173e190", size = 324638, upload-time = "2026-01-29T21:51:26.423Z" }, + { url = "https://files.pythonhosted.org/packages/c5/f5/65d838092fd01c44d16037953fd4c2cc851e783de9b8f02b27ec4ffd906f/protobuf-6.33.5-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:8afa18e1d6d20af15b417e728e9f60f3aa108ee76f23c3b2c07a2c3b546d3afd", size = 339411, upload-time = "2026-01-29T21:51:27.446Z" }, + { url = "https://files.pythonhosted.org/packages/9b/53/a9443aa3ca9ba8724fdfa02dd1887c1bcd8e89556b715cfbacca6b63dbec/protobuf-6.33.5-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:cbf16ba3350fb7b889fca858fb215967792dc125b35c7976ca4818bee3521cf0", size = 323465, upload-time = "2026-01-29T21:51:28.925Z" }, + { url = "https://files.pythonhosted.org/packages/57/bf/2086963c69bdac3d7cff1cc7ff79b8ce5ea0bec6797a017e1be338a46248/protobuf-6.33.5-py3-none-any.whl", hash = "sha256:69915a973dd0f60f31a08b8318b73eab2bd6a392c79184b3612226b0a3f8ec02", size = 170687, upload-time = "2026-01-29T21:51:32.557Z" }, +] + +[[package]] +name = "psutil" +version = "7.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/7c/31d1c3ceb1260301f87565f50689dc6da3db427ece1e1e012af22abca54e/psutil-7.2.0.tar.gz", hash = "sha256:2e4f8e1552f77d14dc96fb0f6240c5b34a37081c0889f0853b3b29a496e5ef64", size = 489863, upload-time = "2025-12-23T20:26:24.616Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/8e/b35aae6ed19bc4e2286cac4832e4d522fcf00571867b0a85a3f77ef96a80/psutil-7.2.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:c31e927555539132a00380c971816ea43d089bf4bd5f3e918ed8c16776d68474", size = 129593, upload-time = "2025-12-23T20:26:28.019Z" }, + { url = "https://files.pythonhosted.org/packages/61/a2/773d17d74e122bbffe08b97f73f2d4a01ef53fb03b98e61b8e4f64a9c6b9/psutil-7.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:db8e44e766cef86dea47d9a1fa535d38dc76449e5878a92f33683b7dba5bfcb2", size = 130104, upload-time = "2025-12-23T20:26:30.27Z" }, + { url = "https://files.pythonhosted.org/packages/0d/e3/d3a9b3f4bd231abbd70a988beb2e3edd15306051bccbfc4472bd34a56e01/psutil-7.2.0-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:85ef849ac92169dedc59a7ac2fb565f47b3468fbe1524bf748746bc21afb94c7", size = 180579, upload-time = "2025-12-23T20:26:32.628Z" }, + { url = "https://files.pythonhosted.org/packages/66/f8/6c73044424aabe1b7824d4d4504029d406648286d8fe7ba8c4682e0d3042/psutil-7.2.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26782bdbae2f5c14ce9ebe8ad2411dc2ca870495e0cd90f8910ede7fa5e27117", size = 183171, upload-time = "2025-12-23T20:26:34.972Z" }, + { url = "https://files.pythonhosted.org/packages/48/7d/76d7a863340885d41826562225a566683e653ee6c9ba03c9f3856afa7d80/psutil-7.2.0-cp313-cp313t-win_amd64.whl", hash = "sha256:b7665f612d3b38a583391b95969667a53aaf6c5706dc27a602c9a4874fbf09e4", size = 139055, upload-time = "2025-12-23T20:26:36.848Z" }, + { url = "https://files.pythonhosted.org/packages/a0/48/200054ada0ae4872c8a71db54f3eb6a9af4101680ee6830d373b7fda526b/psutil-7.2.0-cp313-cp313t-win_arm64.whl", hash = "sha256:4413373c174520ae28a24a8974ad8ce6b21f060d27dde94e25f8c73a7effe57a", size = 134737, upload-time = "2025-12-23T20:26:38.784Z" }, + { url = "https://files.pythonhosted.org/packages/44/86/98da45dff471b93ef5ce5bcaefa00e3038295a7880a77cf74018243d37fb/psutil-7.2.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2f2f53fd114e7946dfba3afb98c9b7c7f376009447360ca15bfb73f2066f84c7", size = 129692, upload-time = "2025-12-23T20:26:40.623Z" }, + { url = "https://files.pythonhosted.org/packages/50/ee/10eae91ba4ad071c92db3c178ba861f30406342de9f0ddbe6d51fd741236/psutil-7.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e65c41d7e60068f60ce43b31a3a7fc90deb0dfd34ffc824a2574c2e5279b377e", size = 130110, upload-time = "2025-12-23T20:26:42.569Z" }, + { url = "https://files.pythonhosted.org/packages/87/3a/2b2897443d56fedbbc34ac68a0dc7d55faa05d555372a2f989109052f86d/psutil-7.2.0-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cc66d21366850a4261412ce994ae9976bba9852dafb4f2fa60db68ed17ff5281", size = 181487, upload-time = "2025-12-23T20:26:44.633Z" }, + { url = "https://files.pythonhosted.org/packages/11/66/44308428f7333db42c5ea7390c52af1b38f59b80b80c437291f58b5dfdad/psutil-7.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e025d67b42b8f22b096d5d20f5171de0e0fefb2f0ce983a13c5a1b5ed9872706", size = 184320, upload-time = "2025-12-23T20:26:46.83Z" }, + { url = "https://files.pythonhosted.org/packages/18/28/d2feadc7f18e501c5ce687c377db7dca924585418fd694272b8e488ea99f/psutil-7.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:45f6b91f7ad63414d6454fd609e5e3556d0e1038d5d9c75a1368513bdf763f57", size = 140372, upload-time = "2025-12-23T20:26:49.334Z" }, + { url = "https://files.pythonhosted.org/packages/b2/1d/48381f5fd0425aa054c4ee3de24f50de3d6c347019f3aec75f357377d447/psutil-7.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:87b18a19574139d60a546e88b5f5b9cbad598e26cdc790d204ab95d7024f03ee", size = 135400, upload-time = "2025-12-23T20:26:51.585Z" }, + { url = "https://files.pythonhosted.org/packages/40/c5/a49160bf3e165b7b93a60579a353cf5d939d7f878fe5fd369110f1d18043/psutil-7.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:977a2fcd132d15cb05b32b2d85b98d087cad039b0ce435731670ba74da9e6133", size = 128116, upload-time = "2025-12-23T20:26:53.516Z" }, + { url = "https://files.pythonhosted.org/packages/10/a1/c75feb480f60cd768fb6ed00ac362a16a33e5076ec8475a22d8162fb2659/psutil-7.2.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:24151011c21fadd94214d7139d7c6c54569290d7e553989bdf0eab73b13beb8c", size = 128925, upload-time = "2025-12-23T20:26:55.573Z" }, + { url = "https://files.pythonhosted.org/packages/12/ff/e93136587c00a543f4bc768b157fac2c47cd77b180d4f4e5c6efb6ea53a2/psutil-7.2.0-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91f211ba9279e7c61d9d8f84b713cfc38fa161cb0597d5cb3f1ca742f6848254", size = 154666, upload-time = "2025-12-23T20:26:57.312Z" }, + { url = "https://files.pythonhosted.org/packages/b8/dd/4c2de9c3827c892599d277a69d2224136800870a8a88a80981de905de28d/psutil-7.2.0-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f37415188b7ea98faf90fed51131181646c59098b077550246e2e092e127418b", size = 156109, upload-time = "2025-12-23T20:26:58.851Z" }, + { url = "https://files.pythonhosted.org/packages/81/3f/090943c682d3629968dd0b04826ddcbc760ee1379021dbe316e2ddfcd01b/psutil-7.2.0-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0d12c7ce6ed1128cd81fd54606afa054ac7dbb9773469ebb58cf2f171c49f2ac", size = 148081, upload-time = "2025-12-23T20:27:01.318Z" }, + { url = "https://files.pythonhosted.org/packages/c4/88/c39648ebb8ec182d0364af53cdefe6eddb5f3872ba718b5855a8ff65d6d4/psutil-7.2.0-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ca0faef7976530940dcd39bc5382d0d0d5eb023b186a4901ca341bd8d8684151", size = 147376, upload-time = "2025-12-23T20:27:03.347Z" }, + { url = "https://files.pythonhosted.org/packages/01/a2/5b39e08bd9b27476bc7cce7e21c71a481ad60b81ffac49baf02687a50d7f/psutil-7.2.0-cp37-abi3-win_amd64.whl", hash = "sha256:abdb74137ca232d20250e9ad471f58d500e7743bc8253ba0bfbf26e570c0e437", size = 136910, upload-time = "2025-12-23T20:27:05.289Z" }, + { url = "https://files.pythonhosted.org/packages/59/54/53839db1258c1eaeb4ded57ff202144ebc75b23facc05a74fd98d338b0c6/psutil-7.2.0-cp37-abi3-win_arm64.whl", hash = "sha256:284e71038b3139e7ab3834b63b3eb5aa5565fcd61a681ec746ef9a0a8c457fd2", size = 133807, upload-time = "2025-12-23T20:27:06.825Z" }, +] + +[[package]] +name = "pybtex" +version = "0.25.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "latexcodec" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5f/bc/c2be05ca72f8c103670e983df8be26d1e288bc6556f487fa8cccaa27779f/pybtex-0.25.1.tar.gz", hash = "sha256:9eaf90267c7e83e225af89fea65c370afbf65f458220d3946a9e3049e1eca491", size = 406157, upload-time = "2025-06-26T13:27:41.903Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/68/ceb5d6679baa326261f5d3e5113d9cfed6efef2810afd9f18bffb8ed312b/pybtex-0.25.1-py2.py3-none-any.whl", hash = "sha256:9053b0d619409a0a83f38abad5d9921de5f7b3ede00742beafcd9f10ad0d8c5c", size = 127437, upload-time = "2025-06-26T13:27:43.585Z" }, +] + +[[package]] +name = "pybtex-docutils" +version = "1.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docutils" }, + { name = "pybtex" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/84/796ea94d26188a853660f81bded39f8de4cfe595130aef0dea1088705a11/pybtex-docutils-1.0.3.tar.gz", hash = "sha256:3a7ebdf92b593e00e8c1c538aa9a20bca5d92d84231124715acc964d51d93c6b", size = 18348, upload-time = "2023-08-22T18:47:54.833Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/b1/ce1f4596211efb5410e178a803f08e59b20bedb66837dcf41e21c54f9ec1/pybtex_docutils-1.0.3-py3-none-any.whl", hash = "sha256:8fd290d2ae48e32fcb54d86b0efb8d573198653c7e2447d5bec5847095f430b9", size = 6385, upload-time = "2023-08-22T06:43:20.513Z" }, +] + +[[package]] +name = "pycparser" +version = "2.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/cf/d2d3b9f5699fb1e4615c8e32ff220203e43b248e1dfcc6736ad9057731ca/pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2", size = 173734, upload-time = "2025-09-09T13:23:47.91Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934", size = 118140, upload-time = "2025-09-09T13:23:46.651Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pyparsing" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/c1/1d9de9aeaa1b89b0186e5fe23294ff6517fce1bc69149185577cd31016b2/pyparsing-3.3.1.tar.gz", hash = "sha256:47fad0f17ac1e2cad3de3b458570fbc9b03560aa029ed5e16ee5554da9a2251c", size = 1550512, upload-time = "2025-12-23T03:14:04.391Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/40/2614036cdd416452f5bf98ec037f38a1afb17f327cb8e6b652d4729e0af8/pyparsing-3.3.1-py3-none-any.whl", hash = "sha256:023b5e7e5520ad96642e2c6db4cb683d3970bd640cdf7115049a6e9c3682df82", size = 121793, upload-time = "2025-12-23T03:14:02.103Z" }, +] + +[[package]] +name = "pyreadline3" +version = "3.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/49/4cea918a08f02817aabae639e3d0ac046fef9f9180518a3ad394e22da148/pyreadline3-3.5.4.tar.gz", hash = "sha256:8d57d53039a1c75adba8e50dd3d992b28143480816187ea5efbd5c78e6c885b7", size = 99839, upload-time = "2024-09-19T02:40:10.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/dc/491b7661614ab97483abf2056be1deee4dc2490ecbf7bff9ab5cdbac86e1/pyreadline3-3.5.4-py3-none-any.whl", hash = "sha256:eaf8e6cc3c49bcccf145fc6067ba8643d1df34d604a1ec0eccbf7a18e6d3fae6", size = 83178, upload-time = "2024-09-19T02:40:08.598Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, +] + +[[package]] +name = "pytest-cov" +version = "7.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage", extra = ["toml"] }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/f7/c933acc76f5208b3b00089573cf6a2bc26dc80a8aece8f52bb7d6b1855ca/pytest_cov-7.0.0.tar.gz", hash = "sha256:33c97eda2e049a0c5298e91f519302a1334c26ac65c1a483d6206fd458361af1", size = 54328, upload-time = "2025-09-09T10:57:02.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl", hash = "sha256:3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861", size = 22424, upload-time = "2025-09-09T10:57:00.695Z" }, +] + +[[package]] +name = "pytest-timeout" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/82/4c9ecabab13363e72d880f2fb504c5f750433b2b6f16e99f4ec21ada284c/pytest_timeout-2.4.0.tar.gz", hash = "sha256:7e68e90b01f9eff71332b25001f85c75495fc4e3a836701876183c4bcfd0540a", size = 17973, upload-time = "2025-05-05T19:44:34.99Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "regex" +version = "2025.11.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/a9/546676f25e573a4cf00fe8e119b78a37b6a8fe2dc95cda877b30889c9c45/regex-2025.11.3.tar.gz", hash = "sha256:1fedc720f9bb2494ce31a58a1631f9c82df6a09b49c19517ea5cc280b4541e01", size = 414669, upload-time = "2025-11-03T21:34:22.089Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/90/4fb5056e5f03a7048abd2b11f598d464f0c167de4f2a51aa868c376b8c70/regex-2025.11.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:eadade04221641516fa25139273505a1c19f9bf97589a05bc4cfcd8b4a618031", size = 488081, upload-time = "2025-11-03T21:31:11.946Z" }, + { url = "https://files.pythonhosted.org/packages/85/23/63e481293fac8b069d84fba0299b6666df720d875110efd0338406b5d360/regex-2025.11.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:feff9e54ec0dd3833d659257f5c3f5322a12eee58ffa360984b716f8b92983f4", size = 290554, upload-time = "2025-11-03T21:31:13.387Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9d/b101d0262ea293a0066b4522dfb722eb6a8785a8c3e084396a5f2c431a46/regex-2025.11.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3b30bc921d50365775c09a7ed446359e5c0179e9e2512beec4a60cbcef6ddd50", size = 288407, upload-time = "2025-11-03T21:31:14.809Z" }, + { url = "https://files.pythonhosted.org/packages/0c/64/79241c8209d5b7e00577ec9dca35cd493cc6be35b7d147eda367d6179f6d/regex-2025.11.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f99be08cfead2020c7ca6e396c13543baea32343b7a9a5780c462e323bd8872f", size = 793418, upload-time = "2025-11-03T21:31:16.556Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e2/23cd5d3573901ce8f9757c92ca4db4d09600b865919b6d3e7f69f03b1afd/regex-2025.11.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6dd329a1b61c0ee95ba95385fb0c07ea0d3fe1a21e1349fa2bec272636217118", size = 860448, upload-time = "2025-11-03T21:31:18.12Z" }, + { url = "https://files.pythonhosted.org/packages/2a/4c/aecf31beeaa416d0ae4ecb852148d38db35391aac19c687b5d56aedf3a8b/regex-2025.11.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4c5238d32f3c5269d9e87be0cf096437b7622b6920f5eac4fd202468aaeb34d2", size = 907139, upload-time = "2025-11-03T21:31:20.753Z" }, + { url = "https://files.pythonhosted.org/packages/61/22/b8cb00df7d2b5e0875f60628594d44dba283e951b1ae17c12f99e332cc0a/regex-2025.11.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10483eefbfb0adb18ee9474498c9a32fcf4e594fbca0543bb94c48bac6183e2e", size = 800439, upload-time = "2025-11-03T21:31:22.069Z" }, + { url = "https://files.pythonhosted.org/packages/02/a8/c4b20330a5cdc7a8eb265f9ce593f389a6a88a0c5f280cf4d978f33966bc/regex-2025.11.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:78c2d02bb6e1da0720eedc0bad578049cad3f71050ef8cd065ecc87691bed2b0", size = 782965, upload-time = "2025-11-03T21:31:23.598Z" }, + { url = "https://files.pythonhosted.org/packages/b4/4c/ae3e52988ae74af4b04d2af32fee4e8077f26e51b62ec2d12d246876bea2/regex-2025.11.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e6b49cd2aad93a1790ce9cffb18964f6d3a4b0b3dbdbd5de094b65296fce6e58", size = 854398, upload-time = "2025-11-03T21:31:25.008Z" }, + { url = "https://files.pythonhosted.org/packages/06/d1/a8b9cf45874eda14b2e275157ce3b304c87e10fb38d9fc26a6e14eb18227/regex-2025.11.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:885b26aa3ee56433b630502dc3d36ba78d186a00cc535d3806e6bfd9ed3c70ab", size = 845897, upload-time = "2025-11-03T21:31:26.427Z" }, + { url = "https://files.pythonhosted.org/packages/ea/fe/1830eb0236be93d9b145e0bd8ab499f31602fe0999b1f19e99955aa8fe20/regex-2025.11.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ddd76a9f58e6a00f8772e72cff8ebcff78e022be95edf018766707c730593e1e", size = 788906, upload-time = "2025-11-03T21:31:28.078Z" }, + { url = "https://files.pythonhosted.org/packages/66/47/dc2577c1f95f188c1e13e2e69d8825a5ac582ac709942f8a03af42ed6e93/regex-2025.11.3-cp311-cp311-win32.whl", hash = "sha256:3e816cc9aac1cd3cc9a4ec4d860f06d40f994b5c7b4d03b93345f44e08cc68bf", size = 265812, upload-time = "2025-11-03T21:31:29.72Z" }, + { url = "https://files.pythonhosted.org/packages/50/1e/15f08b2f82a9bbb510621ec9042547b54d11e83cb620643ebb54e4eb7d71/regex-2025.11.3-cp311-cp311-win_amd64.whl", hash = "sha256:087511f5c8b7dfbe3a03f5d5ad0c2a33861b1fc387f21f6f60825a44865a385a", size = 277737, upload-time = "2025-11-03T21:31:31.422Z" }, + { url = "https://files.pythonhosted.org/packages/f4/fc/6500eb39f5f76c5e47a398df82e6b535a5e345f839581012a418b16f9cc3/regex-2025.11.3-cp311-cp311-win_arm64.whl", hash = "sha256:1ff0d190c7f68ae7769cd0313fe45820ba07ffebfddfaa89cc1eb70827ba0ddc", size = 270290, upload-time = "2025-11-03T21:31:33.041Z" }, + { url = "https://files.pythonhosted.org/packages/e8/74/18f04cb53e58e3fb107439699bd8375cf5a835eec81084e0bddbd122e4c2/regex-2025.11.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bc8ab71e2e31b16e40868a40a69007bc305e1109bd4658eb6cad007e0bf67c41", size = 489312, upload-time = "2025-11-03T21:31:34.343Z" }, + { url = "https://files.pythonhosted.org/packages/78/3f/37fcdd0d2b1e78909108a876580485ea37c91e1acf66d3bb8e736348f441/regex-2025.11.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:22b29dda7e1f7062a52359fca6e58e548e28c6686f205e780b02ad8ef710de36", size = 291256, upload-time = "2025-11-03T21:31:35.675Z" }, + { url = "https://files.pythonhosted.org/packages/bf/26/0a575f58eb23b7ebd67a45fccbc02ac030b737b896b7e7a909ffe43ffd6a/regex-2025.11.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3a91e4a29938bc1a082cc28fdea44be420bf2bebe2665343029723892eb073e1", size = 288921, upload-time = "2025-11-03T21:31:37.07Z" }, + { url = "https://files.pythonhosted.org/packages/ea/98/6a8dff667d1af907150432cf5abc05a17ccd32c72a3615410d5365ac167a/regex-2025.11.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b884f4226602ad40c5d55f52bf91a9df30f513864e0054bad40c0e9cf1afb7", size = 798568, upload-time = "2025-11-03T21:31:38.784Z" }, + { url = "https://files.pythonhosted.org/packages/64/15/92c1db4fa4e12733dd5a526c2dd2b6edcbfe13257e135fc0f6c57f34c173/regex-2025.11.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3e0b11b2b2433d1c39c7c7a30e3f3d0aeeea44c2a8d0bae28f6b95f639927a69", size = 864165, upload-time = "2025-11-03T21:31:40.559Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e7/3ad7da8cdee1ce66c7cd37ab5ab05c463a86ffeb52b1a25fe7bd9293b36c/regex-2025.11.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87eb52a81ef58c7ba4d45c3ca74e12aa4b4e77816f72ca25258a85b3ea96cb48", size = 912182, upload-time = "2025-11-03T21:31:42.002Z" }, + { url = "https://files.pythonhosted.org/packages/84/bd/9ce9f629fcb714ffc2c3faf62b6766ecb7a585e1e885eb699bcf130a5209/regex-2025.11.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a12ab1f5c29b4e93db518f5e3872116b7e9b1646c9f9f426f777b50d44a09e8c", size = 803501, upload-time = "2025-11-03T21:31:43.815Z" }, + { url = "https://files.pythonhosted.org/packages/7c/0f/8dc2e4349d8e877283e6edd6c12bdcebc20f03744e86f197ab6e4492bf08/regex-2025.11.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7521684c8c7c4f6e88e35ec89680ee1aa8358d3f09d27dfbdf62c446f5d4c695", size = 787842, upload-time = "2025-11-03T21:31:45.353Z" }, + { url = "https://files.pythonhosted.org/packages/f9/73/cff02702960bc185164d5619c0c62a2f598a6abff6695d391b096237d4ab/regex-2025.11.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7fe6e5440584e94cc4b3f5f4d98a25e29ca12dccf8873679a635638349831b98", size = 858519, upload-time = "2025-11-03T21:31:46.814Z" }, + { url = "https://files.pythonhosted.org/packages/61/83/0e8d1ae71e15bc1dc36231c90b46ee35f9d52fab2e226b0e039e7ea9c10a/regex-2025.11.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:8e026094aa12b43f4fd74576714e987803a315c76edb6b098b9809db5de58f74", size = 850611, upload-time = "2025-11-03T21:31:48.289Z" }, + { url = "https://files.pythonhosted.org/packages/c8/f5/70a5cdd781dcfaa12556f2955bf170cd603cb1c96a1827479f8faea2df97/regex-2025.11.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:435bbad13e57eb5606a68443af62bed3556de2f46deb9f7d4237bc2f1c9fb3a0", size = 789759, upload-time = "2025-11-03T21:31:49.759Z" }, + { url = "https://files.pythonhosted.org/packages/59/9b/7c29be7903c318488983e7d97abcf8ebd3830e4c956c4c540005fcfb0462/regex-2025.11.3-cp312-cp312-win32.whl", hash = "sha256:3839967cf4dc4b985e1570fd8d91078f0c519f30491c60f9ac42a8db039be204", size = 266194, upload-time = "2025-11-03T21:31:51.53Z" }, + { url = "https://files.pythonhosted.org/packages/1a/67/3b92df89f179d7c367be654ab5626ae311cb28f7d5c237b6bb976cd5fbbb/regex-2025.11.3-cp312-cp312-win_amd64.whl", hash = "sha256:e721d1b46e25c481dc5ded6f4b3f66c897c58d2e8cfdf77bbced84339108b0b9", size = 277069, upload-time = "2025-11-03T21:31:53.151Z" }, + { url = "https://files.pythonhosted.org/packages/d7/55/85ba4c066fe5094d35b249c3ce8df0ba623cfd35afb22d6764f23a52a1c5/regex-2025.11.3-cp312-cp312-win_arm64.whl", hash = "sha256:64350685ff08b1d3a6fff33f45a9ca183dc1d58bbfe4981604e70ec9801bbc26", size = 270330, upload-time = "2025-11-03T21:31:54.514Z" }, + { url = "https://files.pythonhosted.org/packages/e1/a7/dda24ebd49da46a197436ad96378f17df30ceb40e52e859fc42cac45b850/regex-2025.11.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c1e448051717a334891f2b9a620fe36776ebf3dd8ec46a0b877c8ae69575feb4", size = 489081, upload-time = "2025-11-03T21:31:55.9Z" }, + { url = "https://files.pythonhosted.org/packages/19/22/af2dc751aacf88089836aa088a1a11c4f21a04707eb1b0478e8e8fb32847/regex-2025.11.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9b5aca4d5dfd7fbfbfbdaf44850fcc7709a01146a797536a8f84952e940cca76", size = 291123, upload-time = "2025-11-03T21:31:57.758Z" }, + { url = "https://files.pythonhosted.org/packages/a3/88/1a3ea5672f4b0a84802ee9891b86743438e7c04eb0b8f8c4e16a42375327/regex-2025.11.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:04d2765516395cf7dda331a244a3282c0f5ae96075f728629287dfa6f76ba70a", size = 288814, upload-time = "2025-11-03T21:32:01.12Z" }, + { url = "https://files.pythonhosted.org/packages/fb/8c/f5987895bf42b8ddeea1b315c9fedcfe07cadee28b9c98cf50d00adcb14d/regex-2025.11.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d9903ca42bfeec4cebedba8022a7c97ad2aab22e09573ce9976ba01b65e4361", size = 798592, upload-time = "2025-11-03T21:32:03.006Z" }, + { url = "https://files.pythonhosted.org/packages/99/2a/6591ebeede78203fa77ee46a1c36649e02df9eaa77a033d1ccdf2fcd5d4e/regex-2025.11.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:639431bdc89d6429f6721625e8129413980ccd62e9d3f496be618a41d205f160", size = 864122, upload-time = "2025-11-03T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/94/d6/be32a87cf28cf8ed064ff281cfbd49aefd90242a83e4b08b5a86b38e8eb4/regex-2025.11.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f117efad42068f9715677c8523ed2be1518116d1c49b1dd17987716695181efe", size = 912272, upload-time = "2025-11-03T21:32:06.148Z" }, + { url = "https://files.pythonhosted.org/packages/62/11/9bcef2d1445665b180ac7f230406ad80671f0fc2a6ffb93493b5dd8cd64c/regex-2025.11.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4aecb6f461316adf9f1f0f6a4a1a3d79e045f9b71ec76055a791affa3b285850", size = 803497, upload-time = "2025-11-03T21:32:08.162Z" }, + { url = "https://files.pythonhosted.org/packages/e5/a7/da0dc273d57f560399aa16d8a68ae7f9b57679476fc7ace46501d455fe84/regex-2025.11.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3b3a5f320136873cc5561098dfab677eea139521cb9a9e8db98b7e64aef44cbc", size = 787892, upload-time = "2025-11-03T21:32:09.769Z" }, + { url = "https://files.pythonhosted.org/packages/da/4b/732a0c5a9736a0b8d6d720d4945a2f1e6f38f87f48f3173559f53e8d5d82/regex-2025.11.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:75fa6f0056e7efb1f42a1c34e58be24072cb9e61a601340cc1196ae92326a4f9", size = 858462, upload-time = "2025-11-03T21:32:11.769Z" }, + { url = "https://files.pythonhosted.org/packages/0c/f5/a2a03df27dc4c2d0c769220f5110ba8c4084b0bfa9ab0f9b4fcfa3d2b0fc/regex-2025.11.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:dbe6095001465294f13f1adcd3311e50dd84e5a71525f20a10bd16689c61ce0b", size = 850528, upload-time = "2025-11-03T21:32:13.906Z" }, + { url = "https://files.pythonhosted.org/packages/d6/09/e1cd5bee3841c7f6eb37d95ca91cdee7100b8f88b81e41c2ef426910891a/regex-2025.11.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:454d9b4ae7881afbc25015b8627c16d88a597479b9dea82b8c6e7e2e07240dc7", size = 789866, upload-time = "2025-11-03T21:32:15.748Z" }, + { url = "https://files.pythonhosted.org/packages/eb/51/702f5ea74e2a9c13d855a6a85b7f80c30f9e72a95493260193c07f3f8d74/regex-2025.11.3-cp313-cp313-win32.whl", hash = "sha256:28ba4d69171fc6e9896337d4fc63a43660002b7da53fc15ac992abcf3410917c", size = 266189, upload-time = "2025-11-03T21:32:17.493Z" }, + { url = "https://files.pythonhosted.org/packages/8b/00/6e29bb314e271a743170e53649db0fdb8e8ff0b64b4f425f5602f4eb9014/regex-2025.11.3-cp313-cp313-win_amd64.whl", hash = "sha256:bac4200befe50c670c405dc33af26dad5a3b6b255dd6c000d92fe4629f9ed6a5", size = 277054, upload-time = "2025-11-03T21:32:19.042Z" }, + { url = "https://files.pythonhosted.org/packages/25/f1/b156ff9f2ec9ac441710764dda95e4edaf5f36aca48246d1eea3f1fd96ec/regex-2025.11.3-cp313-cp313-win_arm64.whl", hash = "sha256:2292cd5a90dab247f9abe892ac584cb24f0f54680c73fcb4a7493c66c2bf2467", size = 270325, upload-time = "2025-11-03T21:32:21.338Z" }, + { url = "https://files.pythonhosted.org/packages/20/28/fd0c63357caefe5680b8ea052131acbd7f456893b69cc2a90cc3e0dc90d4/regex-2025.11.3-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:1eb1ebf6822b756c723e09f5186473d93236c06c579d2cc0671a722d2ab14281", size = 491984, upload-time = "2025-11-03T21:32:23.466Z" }, + { url = "https://files.pythonhosted.org/packages/df/ec/7014c15626ab46b902b3bcc4b28a7bae46d8f281fc7ea9c95e22fcaaa917/regex-2025.11.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:1e00ec2970aab10dc5db34af535f21fcf32b4a31d99e34963419636e2f85ae39", size = 292673, upload-time = "2025-11-03T21:32:25.034Z" }, + { url = "https://files.pythonhosted.org/packages/23/ab/3b952ff7239f20d05f1f99e9e20188513905f218c81d52fb5e78d2bf7634/regex-2025.11.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a4cb042b615245d5ff9b3794f56be4138b5adc35a4166014d31d1814744148c7", size = 291029, upload-time = "2025-11-03T21:32:26.528Z" }, + { url = "https://files.pythonhosted.org/packages/21/7e/3dc2749fc684f455f162dcafb8a187b559e2614f3826877d3844a131f37b/regex-2025.11.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:44f264d4bf02f3176467d90b294d59bf1db9fe53c141ff772f27a8b456b2a9ed", size = 807437, upload-time = "2025-11-03T21:32:28.363Z" }, + { url = "https://files.pythonhosted.org/packages/1b/0b/d529a85ab349c6a25d1ca783235b6e3eedf187247eab536797021f7126c6/regex-2025.11.3-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7be0277469bf3bd7a34a9c57c1b6a724532a0d235cd0dc4e7f4316f982c28b19", size = 873368, upload-time = "2025-11-03T21:32:30.4Z" }, + { url = "https://files.pythonhosted.org/packages/7d/18/2d868155f8c9e3e9d8f9e10c64e9a9f496bb8f7e037a88a8bed26b435af6/regex-2025.11.3-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0d31e08426ff4b5b650f68839f5af51a92a5b51abd8554a60c2fbc7c71f25d0b", size = 914921, upload-time = "2025-11-03T21:32:32.123Z" }, + { url = "https://files.pythonhosted.org/packages/2d/71/9d72ff0f354fa783fe2ba913c8734c3b433b86406117a8db4ea2bf1c7a2f/regex-2025.11.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e43586ce5bd28f9f285a6e729466841368c4a0353f6fd08d4ce4630843d3648a", size = 812708, upload-time = "2025-11-03T21:32:34.305Z" }, + { url = "https://files.pythonhosted.org/packages/e7/19/ce4bf7f5575c97f82b6e804ffb5c4e940c62609ab2a0d9538d47a7fdf7d4/regex-2025.11.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:0f9397d561a4c16829d4e6ff75202c1c08b68a3bdbfe29dbfcdb31c9830907c6", size = 795472, upload-time = "2025-11-03T21:32:36.364Z" }, + { url = "https://files.pythonhosted.org/packages/03/86/fd1063a176ffb7b2315f9a1b08d17b18118b28d9df163132615b835a26ee/regex-2025.11.3-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:dd16e78eb18ffdb25ee33a0682d17912e8cc8a770e885aeee95020046128f1ce", size = 868341, upload-time = "2025-11-03T21:32:38.042Z" }, + { url = "https://files.pythonhosted.org/packages/12/43/103fb2e9811205e7386366501bc866a164a0430c79dd59eac886a2822950/regex-2025.11.3-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:ffcca5b9efe948ba0661e9df0fa50d2bc4b097c70b9810212d6b62f05d83b2dd", size = 854666, upload-time = "2025-11-03T21:32:40.079Z" }, + { url = "https://files.pythonhosted.org/packages/7d/22/e392e53f3869b75804762c7c848bd2dd2abf2b70fb0e526f58724638bd35/regex-2025.11.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c56b4d162ca2b43318ac671c65bd4d563e841a694ac70e1a976ac38fcf4ca1d2", size = 799473, upload-time = "2025-11-03T21:32:42.148Z" }, + { url = "https://files.pythonhosted.org/packages/4f/f9/8bd6b656592f925b6845fcbb4d57603a3ac2fb2373344ffa1ed70aa6820a/regex-2025.11.3-cp313-cp313t-win32.whl", hash = "sha256:9ddc42e68114e161e51e272f667d640f97e84a2b9ef14b7477c53aac20c2d59a", size = 268792, upload-time = "2025-11-03T21:32:44.13Z" }, + { url = "https://files.pythonhosted.org/packages/e5/87/0e7d603467775ff65cd2aeabf1b5b50cc1c3708556a8b849a2fa4dd1542b/regex-2025.11.3-cp313-cp313t-win_amd64.whl", hash = "sha256:7a7c7fdf755032ffdd72c77e3d8096bdcb0eb92e89e17571a196f03d88b11b3c", size = 280214, upload-time = "2025-11-03T21:32:45.853Z" }, + { url = "https://files.pythonhosted.org/packages/8d/d0/2afc6f8e94e2b64bfb738a7c2b6387ac1699f09f032d363ed9447fd2bb57/regex-2025.11.3-cp313-cp313t-win_arm64.whl", hash = "sha256:df9eb838c44f570283712e7cff14c16329a9f0fb19ca492d21d4b7528ee6821e", size = 271469, upload-time = "2025-11-03T21:32:48.026Z" }, + { url = "https://files.pythonhosted.org/packages/31/e9/f6e13de7e0983837f7b6d238ad9458800a874bf37c264f7923e63409944c/regex-2025.11.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:9697a52e57576c83139d7c6f213d64485d3df5bf84807c35fa409e6c970801c6", size = 489089, upload-time = "2025-11-03T21:32:50.027Z" }, + { url = "https://files.pythonhosted.org/packages/a3/5c/261f4a262f1fa65141c1b74b255988bd2fa020cc599e53b080667d591cfc/regex-2025.11.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e18bc3f73bd41243c9b38a6d9f2366cd0e0137a9aebe2d8ff76c5b67d4c0a3f4", size = 291059, upload-time = "2025-11-03T21:32:51.682Z" }, + { url = "https://files.pythonhosted.org/packages/8e/57/f14eeb7f072b0e9a5a090d1712741fd8f214ec193dba773cf5410108bb7d/regex-2025.11.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:61a08bcb0ec14ff4e0ed2044aad948d0659604f824cbd50b55e30b0ec6f09c73", size = 288900, upload-time = "2025-11-03T21:32:53.569Z" }, + { url = "https://files.pythonhosted.org/packages/3c/6b/1d650c45e99a9b327586739d926a1cd4e94666b1bd4af90428b36af66dc7/regex-2025.11.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c9c30003b9347c24bcc210958c5d167b9e4f9be786cb380a7d32f14f9b84674f", size = 799010, upload-time = "2025-11-03T21:32:55.222Z" }, + { url = "https://files.pythonhosted.org/packages/99/ee/d66dcbc6b628ce4e3f7f0cbbb84603aa2fc0ffc878babc857726b8aab2e9/regex-2025.11.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4e1e592789704459900728d88d41a46fe3969b82ab62945560a31732ffc19a6d", size = 864893, upload-time = "2025-11-03T21:32:57.239Z" }, + { url = "https://files.pythonhosted.org/packages/bf/2d/f238229f1caba7ac87a6c4153d79947fb0261415827ae0f77c304260c7d3/regex-2025.11.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6538241f45eb5a25aa575dbba1069ad786f68a4f2773a29a2bd3dd1f9de787be", size = 911522, upload-time = "2025-11-03T21:32:59.274Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3d/22a4eaba214a917c80e04f6025d26143690f0419511e0116508e24b11c9b/regex-2025.11.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bce22519c989bb72a7e6b36a199384c53db7722fe669ba891da75907fe3587db", size = 803272, upload-time = "2025-11-03T21:33:01.393Z" }, + { url = "https://files.pythonhosted.org/packages/84/b1/03188f634a409353a84b5ef49754b97dbcc0c0f6fd6c8ede505a8960a0a4/regex-2025.11.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:66d559b21d3640203ab9075797a55165d79017520685fb407b9234d72ab63c62", size = 787958, upload-time = "2025-11-03T21:33:03.379Z" }, + { url = "https://files.pythonhosted.org/packages/99/6a/27d072f7fbf6fadd59c64d210305e1ff865cc3b78b526fd147db768c553b/regex-2025.11.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:669dcfb2e38f9e8c69507bace46f4889e3abbfd9b0c29719202883c0a603598f", size = 859289, upload-time = "2025-11-03T21:33:05.374Z" }, + { url = "https://files.pythonhosted.org/packages/9a/70/1b3878f648e0b6abe023172dacb02157e685564853cc363d9961bcccde4e/regex-2025.11.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:32f74f35ff0f25a5021373ac61442edcb150731fbaa28286bbc8bb1582c89d02", size = 850026, upload-time = "2025-11-03T21:33:07.131Z" }, + { url = "https://files.pythonhosted.org/packages/dd/d5/68e25559b526b8baab8e66839304ede68ff6727237a47727d240006bd0ff/regex-2025.11.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e6c7a21dffba883234baefe91bc3388e629779582038f75d2a5be918e250f0ed", size = 789499, upload-time = "2025-11-03T21:33:09.141Z" }, + { url = "https://files.pythonhosted.org/packages/fc/df/43971264857140a350910d4e33df725e8c94dd9dee8d2e4729fa0d63d49e/regex-2025.11.3-cp314-cp314-win32.whl", hash = "sha256:795ea137b1d809eb6836b43748b12634291c0ed55ad50a7d72d21edf1cd565c4", size = 271604, upload-time = "2025-11-03T21:33:10.9Z" }, + { url = "https://files.pythonhosted.org/packages/01/6f/9711b57dc6894a55faf80a4c1b5aa4f8649805cb9c7aef46f7d27e2b9206/regex-2025.11.3-cp314-cp314-win_amd64.whl", hash = "sha256:9f95fbaa0ee1610ec0fc6b26668e9917a582ba80c52cc6d9ada15e30aa9ab9ad", size = 280320, upload-time = "2025-11-03T21:33:12.572Z" }, + { url = "https://files.pythonhosted.org/packages/f1/7e/f6eaa207d4377481f5e1775cdeb5a443b5a59b392d0065f3417d31d80f87/regex-2025.11.3-cp314-cp314-win_arm64.whl", hash = "sha256:dfec44d532be4c07088c3de2876130ff0fbeeacaa89a137decbbb5f665855a0f", size = 273372, upload-time = "2025-11-03T21:33:14.219Z" }, + { url = "https://files.pythonhosted.org/packages/c3/06/49b198550ee0f5e4184271cee87ba4dfd9692c91ec55289e6282f0f86ccf/regex-2025.11.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:ba0d8a5d7f04f73ee7d01d974d47c5834f8a1b0224390e4fe7c12a3a92a78ecc", size = 491985, upload-time = "2025-11-03T21:33:16.555Z" }, + { url = "https://files.pythonhosted.org/packages/ce/bf/abdafade008f0b1c9da10d934034cb670432d6cf6cbe38bbb53a1cfd6cf8/regex-2025.11.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:442d86cf1cfe4faabf97db7d901ef58347efd004934da045c745e7b5bd57ac49", size = 292669, upload-time = "2025-11-03T21:33:18.32Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ef/0c357bb8edbd2ad8e273fcb9e1761bc37b8acbc6e1be050bebd6475f19c1/regex-2025.11.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fd0a5e563c756de210bb964789b5abe4f114dacae9104a47e1a649b910361536", size = 291030, upload-time = "2025-11-03T21:33:20.048Z" }, + { url = "https://files.pythonhosted.org/packages/79/06/edbb67257596649b8fb088d6aeacbcb248ac195714b18a65e018bf4c0b50/regex-2025.11.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf3490bcbb985a1ae97b2ce9ad1c0f06a852d5b19dde9b07bdf25bf224248c95", size = 807674, upload-time = "2025-11-03T21:33:21.797Z" }, + { url = "https://files.pythonhosted.org/packages/f4/d9/ad4deccfce0ea336296bd087f1a191543bb99ee1c53093dcd4c64d951d00/regex-2025.11.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3809988f0a8b8c9dcc0f92478d6501fac7200b9ec56aecf0ec21f4a2ec4b6009", size = 873451, upload-time = "2025-11-03T21:33:23.741Z" }, + { url = "https://files.pythonhosted.org/packages/13/75/a55a4724c56ef13e3e04acaab29df26582f6978c000ac9cd6810ad1f341f/regex-2025.11.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f4ff94e58e84aedb9c9fce66d4ef9f27a190285b451420f297c9a09f2b9abee9", size = 914980, upload-time = "2025-11-03T21:33:25.999Z" }, + { url = "https://files.pythonhosted.org/packages/67/1e/a1657ee15bd9116f70d4a530c736983eed997b361e20ecd8f5ca3759d5c5/regex-2025.11.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eb542fd347ce61e1321b0a6b945d5701528dca0cd9759c2e3bb8bd57e47964d", size = 812852, upload-time = "2025-11-03T21:33:27.852Z" }, + { url = "https://files.pythonhosted.org/packages/b8/6f/f7516dde5506a588a561d296b2d0044839de06035bb486b326065b4c101e/regex-2025.11.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d6c2d5919075a1f2e413c00b056ea0c2f065b3f5fe83c3d07d325ab92dce51d6", size = 795566, upload-time = "2025-11-03T21:33:32.364Z" }, + { url = "https://files.pythonhosted.org/packages/d9/dd/3d10b9e170cc16fb34cb2cef91513cf3df65f440b3366030631b2984a264/regex-2025.11.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:3f8bf11a4827cc7ce5a53d4ef6cddd5ad25595d3c1435ef08f76825851343154", size = 868463, upload-time = "2025-11-03T21:33:34.459Z" }, + { url = "https://files.pythonhosted.org/packages/f5/8e/935e6beff1695aa9085ff83195daccd72acc82c81793df480f34569330de/regex-2025.11.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:22c12d837298651e5550ac1d964e4ff57c3f56965fc1812c90c9fb2028eaf267", size = 854694, upload-time = "2025-11-03T21:33:36.793Z" }, + { url = "https://files.pythonhosted.org/packages/92/12/10650181a040978b2f5720a6a74d44f841371a3d984c2083fc1752e4acf6/regex-2025.11.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:62ba394a3dda9ad41c7c780f60f6e4a70988741415ae96f6d1bf6c239cf01379", size = 799691, upload-time = "2025-11-03T21:33:39.079Z" }, + { url = "https://files.pythonhosted.org/packages/67/90/8f37138181c9a7690e7e4cb388debbd389342db3c7381d636d2875940752/regex-2025.11.3-cp314-cp314t-win32.whl", hash = "sha256:4bf146dca15cdd53224a1bf46d628bd7590e4a07fbb69e720d561aea43a32b38", size = 274583, upload-time = "2025-11-03T21:33:41.302Z" }, + { url = "https://files.pythonhosted.org/packages/8f/cd/867f5ec442d56beb56f5f854f40abcfc75e11d10b11fdb1869dd39c63aaf/regex-2025.11.3-cp314-cp314t-win_amd64.whl", hash = "sha256:adad1a1bcf1c9e76346e091d22d23ac54ef28e1365117d99521631078dfec9de", size = 284286, upload-time = "2025-11-03T21:33:43.324Z" }, + { url = "https://files.pythonhosted.org/packages/20/31/32c0c4610cbc070362bf1d2e4ea86d1ea29014d400a6d6c2486fcfd57766/regex-2025.11.3-cp314-cp314t-win_arm64.whl", hash = "sha256:c54f768482cef41e219720013cd05933b6f971d9562544d691c68699bf2b6801", size = 274741, upload-time = "2025-11-03T21:33:45.557Z" }, +] + +[[package]] +name = "requests" +version = "2.32.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, +] + +[[package]] +name = "rich" +version = "14.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/c6/f3b320c27991c46f43ee9d856302c70dc2d0fb2dba4842ff739d5f46b393/rich-14.3.3.tar.gz", hash = "sha256:b8daa0b9e4eef54dd8cf7c86c03713f53241884e814f4e2f5fb342fe520f639b", size = 230582, upload-time = "2026-02-19T17:23:12.474Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/25/b208c5683343959b670dc001595f2f3737e051da617f66c31f7c4fa93abc/rich-14.3.3-py3-none-any.whl", hash = "sha256:793431c1f8619afa7d3b52b2cdec859562b950ea0d4b6b505397612db8d5362d", size = 310458, upload-time = "2026-02-19T17:23:13.732Z" }, +] + +[[package]] +name = "roman-numerals" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/f9/41dc953bbeb056c17d5f7a519f50fdf010bd0553be2d630bc69d1e022703/roman_numerals-4.1.0.tar.gz", hash = "sha256:1af8b147eb1405d5839e78aeb93131690495fe9da5c91856cb33ad55a7f1e5b2", size = 9077, upload-time = "2025-12-17T18:25:34.381Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl", hash = "sha256:647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7", size = 7676, upload-time = "2025-12-17T18:25:33.098Z" }, +] + +[[package]] +name = "roman-numerals-py" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "roman-numerals" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/b5/de96fca640f4f656eb79bbee0e79aeec52e3e0e359f8a3e6a0d366378b64/roman_numerals_py-4.1.0.tar.gz", hash = "sha256:f5d7b2b4ca52dd855ef7ab8eb3590f428c0b1ea480736ce32b01fef2a5f8daf9", size = 4274, upload-time = "2025-12-17T18:25:41.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/2c/daca29684cbe9fd4bc711f8246da3c10adca1ccc4d24436b17572eb2590e/roman_numerals_py-4.1.0-py3-none-any.whl", hash = "sha256:553114c1167141c1283a51743759723ecd05604a1b6b507225e91dc1a6df0780", size = 4547, upload-time = "2025-12-17T18:25:40.136Z" }, +] + +[[package]] +name = "ruff" +version = "0.14.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/08/52232a877978dd8f9cf2aeddce3e611b40a63287dfca29b6b8da791f5e8d/ruff-0.14.10.tar.gz", hash = "sha256:9a2e830f075d1a42cd28420d7809ace390832a490ed0966fe373ba288e77aaf4", size = 5859763, upload-time = "2025-12-18T19:28:57.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/01/933704d69f3f05ee16ef11406b78881733c186fe14b6a46b05cfcaf6d3b2/ruff-0.14.10-py3-none-linux_armv6l.whl", hash = "sha256:7a3ce585f2ade3e1f29ec1b92df13e3da262178df8c8bdf876f48fa0e8316c49", size = 13527080, upload-time = "2025-12-18T19:29:25.642Z" }, + { url = "https://files.pythonhosted.org/packages/df/58/a0349197a7dfa603ffb7f5b0470391efa79ddc327c1e29c4851e85b09cc5/ruff-0.14.10-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:674f9be9372907f7257c51f1d4fc902cb7cf014b9980152b802794317941f08f", size = 13797320, upload-time = "2025-12-18T19:29:02.571Z" }, + { url = "https://files.pythonhosted.org/packages/7b/82/36be59f00a6082e38c23536df4e71cdbc6af8d7c707eade97fcad5c98235/ruff-0.14.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d85713d522348837ef9df8efca33ccb8bd6fcfc86a2cde3ccb4bc9d28a18003d", size = 12918434, upload-time = "2025-12-18T19:28:51.202Z" }, + { url = "https://files.pythonhosted.org/packages/a6/00/45c62a7f7e34da92a25804f813ebe05c88aa9e0c25e5cb5a7d23dd7450e3/ruff-0.14.10-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6987ebe0501ae4f4308d7d24e2d0fe3d7a98430f5adfd0f1fead050a740a3a77", size = 13371961, upload-time = "2025-12-18T19:29:04.991Z" }, + { url = "https://files.pythonhosted.org/packages/40/31/a5906d60f0405f7e57045a70f2d57084a93ca7425f22e1d66904769d1628/ruff-0.14.10-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:16a01dfb7b9e4eee556fbfd5392806b1b8550c9b4a9f6acd3dbe6812b193c70a", size = 13275629, upload-time = "2025-12-18T19:29:21.381Z" }, + { url = "https://files.pythonhosted.org/packages/3e/60/61c0087df21894cf9d928dc04bcd4fb10e8b2e8dca7b1a276ba2155b2002/ruff-0.14.10-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7165d31a925b7a294465fa81be8c12a0e9b60fb02bf177e79067c867e71f8b1f", size = 14029234, upload-time = "2025-12-18T19:29:00.132Z" }, + { url = "https://files.pythonhosted.org/packages/44/84/77d911bee3b92348b6e5dab5a0c898d87084ea03ac5dc708f46d88407def/ruff-0.14.10-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:c561695675b972effb0c0a45db233f2c816ff3da8dcfbe7dfc7eed625f218935", size = 15449890, upload-time = "2025-12-18T19:28:53.573Z" }, + { url = "https://files.pythonhosted.org/packages/e9/36/480206eaefa24a7ec321582dda580443a8f0671fdbf6b1c80e9c3e93a16a/ruff-0.14.10-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4bb98fcbbc61725968893682fd4df8966a34611239c9fd07a1f6a07e7103d08e", size = 15123172, upload-time = "2025-12-18T19:29:23.453Z" }, + { url = "https://files.pythonhosted.org/packages/5c/38/68e414156015ba80cef5473d57919d27dfb62ec804b96180bafdeaf0e090/ruff-0.14.10-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f24b47993a9d8cb858429e97bdf8544c78029f09b520af615c1d261bf827001d", size = 14460260, upload-time = "2025-12-18T19:29:27.808Z" }, + { url = "https://files.pythonhosted.org/packages/b3/19/9e050c0dca8aba824d67cc0db69fb459c28d8cd3f6855b1405b3f29cc91d/ruff-0.14.10-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59aabd2e2c4fd614d2862e7939c34a532c04f1084476d6833dddef4afab87e9f", size = 14229978, upload-time = "2025-12-18T19:29:11.32Z" }, + { url = "https://files.pythonhosted.org/packages/51/eb/e8dd1dd6e05b9e695aa9dd420f4577debdd0f87a5ff2fedda33c09e9be8c/ruff-0.14.10-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:213db2b2e44be8625002dbea33bb9c60c66ea2c07c084a00d55732689d697a7f", size = 14338036, upload-time = "2025-12-18T19:29:09.184Z" }, + { url = "https://files.pythonhosted.org/packages/6a/12/f3e3a505db7c19303b70af370d137795fcfec136d670d5de5391e295c134/ruff-0.14.10-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:b914c40ab64865a17a9a5b67911d14df72346a634527240039eb3bd650e5979d", size = 13264051, upload-time = "2025-12-18T19:29:13.431Z" }, + { url = "https://files.pythonhosted.org/packages/08/64/8c3a47eaccfef8ac20e0484e68e0772013eb85802f8a9f7603ca751eb166/ruff-0.14.10-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1484983559f026788e3a5c07c81ef7d1e97c1c78ed03041a18f75df104c45405", size = 13283998, upload-time = "2025-12-18T19:29:06.994Z" }, + { url = "https://files.pythonhosted.org/packages/12/84/534a5506f4074e5cc0529e5cd96cfc01bb480e460c7edf5af70d2bcae55e/ruff-0.14.10-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c70427132db492d25f982fffc8d6c7535cc2fd2c83fc8888f05caaa248521e60", size = 13601891, upload-time = "2025-12-18T19:28:55.811Z" }, + { url = "https://files.pythonhosted.org/packages/0d/1e/14c916087d8598917dbad9b2921d340f7884824ad6e9c55de948a93b106d/ruff-0.14.10-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:5bcf45b681e9f1ee6445d317ce1fa9d6cba9a6049542d1c3d5b5958986be8830", size = 14336660, upload-time = "2025-12-18T19:29:16.531Z" }, + { url = "https://files.pythonhosted.org/packages/f2/1c/d7b67ab43f30013b47c12b42d1acd354c195351a3f7a1d67f59e54227ede/ruff-0.14.10-py3-none-win32.whl", hash = "sha256:104c49fc7ab73f3f3a758039adea978869a918f31b73280db175b43a2d9b51d6", size = 13196187, upload-time = "2025-12-18T19:29:19.006Z" }, + { url = "https://files.pythonhosted.org/packages/fb/9c/896c862e13886fae2af961bef3e6312db9ebc6adc2b156fe95e615dee8c1/ruff-0.14.10-py3-none-win_amd64.whl", hash = "sha256:466297bd73638c6bdf06485683e812db1c00c7ac96d4ddd0294a338c62fdc154", size = 14661283, upload-time = "2025-12-18T19:29:30.16Z" }, + { url = "https://files.pythonhosted.org/packages/74/31/b0e29d572670dca3674eeee78e418f20bdf97fa8aa9ea71380885e175ca0/ruff-0.14.10-py3-none-win_arm64.whl", hash = "sha256:e51d046cf6dda98a4633b8a8a771451107413b0f07183b2bef03f075599e44e6", size = 13729839, upload-time = "2025-12-18T19:28:48.636Z" }, +] + +[[package]] +name = "safetensors" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/29/9c/6e74567782559a63bd040a236edca26fd71bc7ba88de2ef35d75df3bca5e/safetensors-0.7.0.tar.gz", hash = "sha256:07663963b67e8bd9f0b8ad15bb9163606cd27cc5a1b96235a50d8369803b96b0", size = 200878, upload-time = "2025-11-19T15:18:43.199Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/47/aef6c06649039accf914afef490268e1067ed82be62bcfa5b7e886ad15e8/safetensors-0.7.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c82f4d474cf725255d9e6acf17252991c3c8aac038d6ef363a4bf8be2f6db517", size = 467781, upload-time = "2025-11-19T15:18:35.84Z" }, + { url = "https://files.pythonhosted.org/packages/e8/00/374c0c068e30cd31f1e1b46b4b5738168ec79e7689ca82ee93ddfea05109/safetensors-0.7.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:94fd4858284736bb67a897a41608b5b0c2496c9bdb3bf2af1fa3409127f20d57", size = 447058, upload-time = "2025-11-19T15:18:34.416Z" }, + { url = "https://files.pythonhosted.org/packages/f1/06/578ffed52c2296f93d7fd2d844cabfa92be51a587c38c8afbb8ae449ca89/safetensors-0.7.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e07d91d0c92a31200f25351f4acb2bc6aff7f48094e13ebb1d0fb995b54b6542", size = 491748, upload-time = "2025-11-19T15:18:09.79Z" }, + { url = "https://files.pythonhosted.org/packages/ae/33/1debbbb70e4791dde185edb9413d1fe01619255abb64b300157d7f15dddd/safetensors-0.7.0-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8469155f4cb518bafb4acf4865e8bb9d6804110d2d9bdcaa78564b9fd841e104", size = 503881, upload-time = "2025-11-19T15:18:16.145Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1c/40c2ca924d60792c3be509833df711b553c60effbd91da6f5284a83f7122/safetensors-0.7.0-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:54bef08bf00a2bff599982f6b08e8770e09cc012d7bba00783fc7ea38f1fb37d", size = 623463, upload-time = "2025-11-19T15:18:21.11Z" }, + { url = "https://files.pythonhosted.org/packages/9b/3a/13784a9364bd43b0d61eef4bea2845039bc2030458b16594a1bd787ae26e/safetensors-0.7.0-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:42cb091236206bb2016d245c377ed383aa7f78691748f3bb6ee1bfa51ae2ce6a", size = 532855, upload-time = "2025-11-19T15:18:25.719Z" }, + { url = "https://files.pythonhosted.org/packages/a0/60/429e9b1cb3fc651937727befe258ea24122d9663e4d5709a48c9cbfceecb/safetensors-0.7.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dac7252938f0696ddea46f5e855dd3138444e82236e3be475f54929f0c510d48", size = 507152, upload-time = "2025-11-19T15:18:33.023Z" }, + { url = "https://files.pythonhosted.org/packages/3c/a8/4b45e4e059270d17af60359713ffd83f97900d45a6afa73aaa0d737d48b6/safetensors-0.7.0-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1d060c70284127fa805085d8f10fbd0962792aed71879d00864acda69dbab981", size = 541856, upload-time = "2025-11-19T15:18:31.075Z" }, + { url = "https://files.pythonhosted.org/packages/06/87/d26d8407c44175d8ae164a95b5a62707fcc445f3c0c56108e37d98070a3d/safetensors-0.7.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:cdab83a366799fa730f90a4ebb563e494f28e9e92c4819e556152ad55e43591b", size = 674060, upload-time = "2025-11-19T15:18:37.211Z" }, + { url = "https://files.pythonhosted.org/packages/11/f5/57644a2ff08dc6325816ba7217e5095f17269dada2554b658442c66aed51/safetensors-0.7.0-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:672132907fcad9f2aedcb705b2d7b3b93354a2aec1b2f706c4db852abe338f85", size = 771715, upload-time = "2025-11-19T15:18:38.689Z" }, + { url = "https://files.pythonhosted.org/packages/86/31/17883e13a814bd278ae6e266b13282a01049b0c81341da7fd0e3e71a80a3/safetensors-0.7.0-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:5d72abdb8a4d56d4020713724ba81dac065fedb7f3667151c4a637f1d3fb26c0", size = 714377, upload-time = "2025-11-19T15:18:40.162Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d8/0c8a7dc9b41dcac53c4cbf9df2b9c83e0e0097203de8b37a712b345c0be5/safetensors-0.7.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b0f6d66c1c538d5a94a73aa9ddca8ccc4227e6c9ff555322ea40bdd142391dd4", size = 677368, upload-time = "2025-11-19T15:18:41.627Z" }, + { url = "https://files.pythonhosted.org/packages/05/e5/cb4b713c8a93469e3c5be7c3f8d77d307e65fe89673e731f5c2bfd0a9237/safetensors-0.7.0-cp38-abi3-win32.whl", hash = "sha256:c74af94bf3ac15ac4d0f2a7c7b4663a15f8c2ab15ed0fc7531ca61d0835eccba", size = 326423, upload-time = "2025-11-19T15:18:45.74Z" }, + { url = "https://files.pythonhosted.org/packages/5d/e6/ec8471c8072382cb91233ba7267fd931219753bb43814cbc71757bfd4dab/safetensors-0.7.0-cp38-abi3-win_amd64.whl", hash = "sha256:d1239932053f56f3456f32eb9625590cc7582e905021f94636202a864d470755", size = 341380, upload-time = "2025-11-19T15:18:44.427Z" }, +] + +[[package]] +name = "setuptools" +version = "80.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" }, +] + +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "snowballstemmer" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/75/a7/9810d872919697c9d01295633f5d574fb416d47e535f258272ca1f01f447/snowballstemmer-3.0.1.tar.gz", hash = "sha256:6d5eeeec8e9f84d4d56b847692bacf79bc2c8e90c7f80ca4444ff8b6f2e52895", size = 105575, upload-time = "2025-05-09T16:34:51.843Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl", hash = "sha256:6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064", size = 103274, upload-time = "2025-05-09T16:34:50.371Z" }, +] + +[[package]] +name = "soupsieve" +version = "2.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/89/23/adf3796d740536d63a6fbda113d07e60c734b6ed5d3058d1e47fc0495e47/soupsieve-2.8.1.tar.gz", hash = "sha256:4cf733bc50fa805f5df4b8ef4740fc0e0fa6218cf3006269afd3f9d6d80fd350", size = 117856, upload-time = "2025-12-18T13:50:34.655Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/f3/b67d6ea49ca9154453b6d70b34ea22f3996b9fa55da105a79d8732227adc/soupsieve-2.8.1-py3-none-any.whl", hash = "sha256:a11fe2a6f3d76ab3cf2de04eb339c1be5b506a8a47f2ceb6d139803177f85434", size = 36710, upload-time = "2025-12-18T13:50:33.267Z" }, +] + +[[package]] +name = "sphinx" +version = "8.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals-py" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/ad/4360e50ed56cb483667b8e6dadf2d3fda62359593faabbe749a27c4eaca6/sphinx-8.2.3.tar.gz", hash = "sha256:398ad29dee7f63a75888314e9424d40f52ce5a6a87ae88e7071e80af296ec348", size = 8321876, upload-time = "2025-03-02T22:31:59.658Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/53/136e9eca6e0b9dc0e1962e2c908fbea2e5ac000c2a2fbd9a35797958c48b/sphinx-8.2.3-py3-none-any.whl", hash = "sha256:4405915165f13521d875a8c29c8970800a0141c14cc5416a38feca4ea5d9b9c3", size = 3589741, upload-time = "2025-03-02T22:31:56.836Z" }, +] + +[[package]] +name = "sphinx-autodoc-defaultargs" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/7c/8edb5d817ebeb146073be629ed58f89b292a643ee111bee3c703be05a7a1/sphinx-autodoc-defaultargs-0.1.2.tar.gz", hash = "sha256:b11cd9ac43510c1f1e9a3f9469a370e6bbe2bc2cb9e67cb79987f4dcd3be5be3", size = 16664, upload-time = "2021-05-02T11:55:04.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/3f/a352e58671b1e6d932a6a4b28b45ff58791788ab7fb6a1f9a7162411198d/sphinx_autodoc_defaultargs-0.1.2-py3-none-any.whl", hash = "sha256:1513b1aea8c20348f1763dbfd7ff11f86cd77f2ce3246c73e1405a2a67fea3ff", size = 8976, upload-time = "2021-05-02T11:55:03.309Z" }, +] + +[[package]] +name = "sphinx-autodoc-typehints" +version = "3.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/4f/4fd5583678bb7dc8afa69e9b309e6a99ee8d79ad3a4728f4e52fd7cb37c7/sphinx_autodoc_typehints-3.5.2.tar.gz", hash = "sha256:5fcd4a3eb7aa89424c1e2e32bedca66edc38367569c9169a80f4b3e934171fdb", size = 37839, upload-time = "2025-10-16T00:50:15.743Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/f2/9657c98a66973b7c35bfd48ba65d1922860de9598fbb535cd96e3f58a908/sphinx_autodoc_typehints-3.5.2-py3-none-any.whl", hash = "sha256:0accd043619f53c86705958e323b419e41667917045ac9215d7be1b493648d8c", size = 21184, upload-time = "2025-10-16T00:50:13.973Z" }, +] + +[[package]] +name = "sphinx-basic-ng" +version = "1.0.0b2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/0b/a866924ded68efec7a1759587a4e478aec7559d8165fac8b2ad1c0e774d6/sphinx_basic_ng-1.0.0b2.tar.gz", hash = "sha256:9ec55a47c90c8c002b5960c57492ec3021f5193cb26cebc2dc4ea226848651c9", size = 20736, upload-time = "2023-07-08T18:40:54.166Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/dd/018ce05c532a22007ac58d4f45232514cd9d6dd0ee1dc374e309db830983/sphinx_basic_ng-1.0.0b2-py3-none-any.whl", hash = "sha256:eb09aedbabfb650607e9b4b68c9d240b90b1e1be221d6ad71d61c52e29f7932b", size = 22496, upload-time = "2023-07-08T18:40:52.659Z" }, +] + +[[package]] +name = "sphinx-copybutton" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/2b/a964715e7f5295f77509e59309959f4125122d648f86b4fe7d70ca1d882c/sphinx-copybutton-0.5.2.tar.gz", hash = "sha256:4cf17c82fb9646d1bc9ca92ac280813a3b605d8c421225fd9913154103ee1fbd", size = 23039, upload-time = "2023-04-14T08:10:22.998Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/48/1ea60e74949eecb12cdd6ac43987f9fd331156388dcc2319b45e2ebb81bf/sphinx_copybutton-0.5.2-py3-none-any.whl", hash = "sha256:fb543fd386d917746c9a2c50360c7905b605726b9355cd26e9974857afeae06e", size = 13343, upload-time = "2023-04-14T08:10:20.844Z" }, +] + +[[package]] +name = "sphinx-design" +version = "0.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2b/69/b34e0cb5336f09c6866d53b4a19d76c227cdec1bbc7ac4de63ca7d58c9c7/sphinx_design-0.6.1.tar.gz", hash = "sha256:b44eea3719386d04d765c1a8257caca2b3e6f8421d7b3a5e742c0fd45f84e632", size = 2193689, upload-time = "2024-08-02T13:48:44.277Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/43/65c0acbd8cc6f50195a3a1fc195c404988b15c67090e73c7a41a9f57d6bd/sphinx_design-0.6.1-py3-none-any.whl", hash = "sha256:b11f37db1a802a183d61b159d9a202314d4d2fe29c163437001324fe2f19549c", size = 2215338, upload-time = "2024-08-02T13:48:42.106Z" }, +] + +[[package]] +name = "sphinx-notfound-page" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/b2/67603444a8ee97b4a8ea71b0a9d6bab1727ed65e362c87e02f818ee57b8a/sphinx_notfound_page-1.1.0.tar.gz", hash = "sha256:913e1754370bb3db201d9300d458a8b8b5fb22e9246a816643a819a9ea2b8067", size = 7392, upload-time = "2025-01-28T18:45:02.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/d4/019fe439c840a7966012bbb95ccbdd81c5c10271749706793b43beb05145/sphinx_notfound_page-1.1.0-py3-none-any.whl", hash = "sha256:835dc76ff7914577a1f58d80a2c8418fb6138c0932c8da8adce4d9096fbcd389", size = 8167, upload-time = "2025-01-28T18:45:00.465Z" }, +] + +[[package]] +name = "sphinxcontrib-applehelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/6e/b837e84a1a704953c62ef8776d45c3e8d759876b4a84fe14eba2859106fe/sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1", size = 20053, upload-time = "2024-07-29T01:09:00.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5", size = 119300, upload-time = "2024-07-29T01:08:58.99Z" }, +] + +[[package]] +name = "sphinxcontrib-bibtex" +version = "2.6.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docutils" }, + { name = "pybtex" }, + { name = "pybtex-docutils" }, + { name = "sphinx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/83/1488c9879f2fa3c2cbd6f666c7a3a42a1fa9e08462bec73281fa6c092cba/sphinxcontrib_bibtex-2.6.5.tar.gz", hash = "sha256:9b3224dd6fece9268ebd8c905dc0a83ff2f6c54148a9235fe70e9d1e9ff149c0", size = 118462, upload-time = "2025-06-27T10:40:14.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/a0/3a612da94f828f26cabb247817393e79472c32b12c49222bf85fb6d7b6c8/sphinxcontrib_bibtex-2.6.5-py3-none-any.whl", hash = "sha256:455ea4509642ea0b28ede3721550273626f85af65af01f161bfd8e19dc1edd7d", size = 40410, upload-time = "2025-06-27T10:40:12.274Z" }, +] + +[[package]] +name = "sphinxcontrib-devhelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/d2/5beee64d3e4e747f316bae86b55943f51e82bb86ecd325883ef65741e7da/sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad", size = 12967, upload-time = "2024-07-29T01:09:23.417Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2", size = 82530, upload-time = "2024-07-29T01:09:21.945Z" }, +] + +[[package]] +name = "sphinxcontrib-gtagjs" +version = "0.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/81/63d8832f82656a1d4208eccf7caa441370e274b560d2a6d56a9cc80afd57/sphinxcontrib-gtagjs-0.2.1.tar.gz", hash = "sha256:46423c2e9313396bc8872bc1cc40bebd862cac976c21811390b081b33738886b", size = 3294, upload-time = "2022-09-02T07:32:34.316Z" } + +[[package]] +name = "sphinxcontrib-htmlhelp" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/93/983afd9aa001e5201eab16b5a444ed5b9b0a7a010541e0ddfbbfd0b2470c/sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9", size = 22617, upload-time = "2024-07-29T01:09:37.889Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8", size = 98705, upload-time = "2024-07-29T01:09:36.407Z" }, +] + +[[package]] +name = "sphinxcontrib-jsmath" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/e8/9ed3830aeed71f17c026a07a5097edcf44b692850ef215b161b8ad875729/sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8", size = 5787, upload-time = "2019-01-21T16:10:16.347Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", size = 5071, upload-time = "2019-01-21T16:10:14.333Z" }, +] + +[[package]] +name = "sphinxcontrib-qthelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/68/bc/9104308fc285eb3e0b31b67688235db556cd5b0ef31d96f30e45f2e51cae/sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab", size = 17165, upload-time = "2024-07-29T01:09:56.435Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb", size = 88743, upload-time = "2024-07-29T01:09:54.885Z" }, +] + +[[package]] +name = "sphinxcontrib-serializinghtml" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/44/6716b257b0aa6bfd51a1b31665d1c205fb12cb5ad56de752dfa15657de2f/sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d", size = 16080, upload-time = "2024-07-29T01:10:09.332Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", size = 92072, upload-time = "2024-07-29T01:10:08.203Z" }, +] + +[[package]] +name = "sphinxcontrib-youtube" +version = "1.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, + { name = "sphinx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0a/56/fcc4aa7ec202c90cdd74ba6cf13ed5c6f1210295af6e98598b47c7f3833f/sphinxcontrib_youtube-1.4.1.tar.gz", hash = "sha256:eb7871c8af47fd2b5c9727615354b7f95bce554be8be45b9fa8e5bc022f88059", size = 6023, upload-time = "2023-09-22T11:13:20.674Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/18/02842653778221bae87147d343a85dcd7c86ae5d86d32c5ab34003623db3/sphinxcontrib_youtube-1.4.1-py2.py3-none-any.whl", hash = "sha256:de9cb454f066d580a1e7ad64efae7dd9e12c1b1567a31faa330b1aeaeed40460", size = 6511, upload-time = "2023-09-22T11:13:19.094Z" }, +] + +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + +[[package]] +name = "tokenizers" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/46/fb6854cec3278fbfa4a75b50232c77622bc517ac886156e6afbfa4d8fc6e/tokenizers-0.22.1.tar.gz", hash = "sha256:61de6522785310a309b3407bac22d99c4db5dba349935e99e4d15ea2226af2d9", size = 363123, upload-time = "2025-09-19T09:49:23.424Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/33/f4b2d94ada7ab297328fc671fed209368ddb82f965ec2224eb1892674c3a/tokenizers-0.22.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:59fdb013df17455e5f950b4b834a7b3ee2e0271e6378ccb33aa74d178b513c73", size = 3069318, upload-time = "2025-09-19T09:49:11.848Z" }, + { url = "https://files.pythonhosted.org/packages/1c/58/2aa8c874d02b974990e89ff95826a4852a8b2a273c7d1b4411cdd45a4565/tokenizers-0.22.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:8d4e484f7b0827021ac5f9f71d4794aaef62b979ab7608593da22b1d2e3c4edc", size = 2926478, upload-time = "2025-09-19T09:49:09.759Z" }, + { url = "https://files.pythonhosted.org/packages/1e/3b/55e64befa1e7bfea963cf4b787b2cea1011362c4193f5477047532ce127e/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19d2962dd28bc67c1f205ab180578a78eef89ac60ca7ef7cbe9635a46a56422a", size = 3256994, upload-time = "2025-09-19T09:48:56.701Z" }, + { url = "https://files.pythonhosted.org/packages/71/0b/fbfecf42f67d9b7b80fde4aabb2b3110a97fac6585c9470b5bff103a80cb/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:38201f15cdb1f8a6843e6563e6e79f4abd053394992b9bbdf5213ea3469b4ae7", size = 3153141, upload-time = "2025-09-19T09:48:59.749Z" }, + { url = "https://files.pythonhosted.org/packages/17/a9/b38f4e74e0817af8f8ef925507c63c6ae8171e3c4cb2d5d4624bf58fca69/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1cbe5454c9a15df1b3443c726063d930c16f047a3cc724b9e6e1a91140e5a21", size = 3508049, upload-time = "2025-09-19T09:49:05.868Z" }, + { url = "https://files.pythonhosted.org/packages/d2/48/dd2b3dac46bb9134a88e35d72e1aa4869579eacc1a27238f1577270773ff/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e7d094ae6312d69cc2a872b54b91b309f4f6fbce871ef28eb27b52a98e4d0214", size = 3710730, upload-time = "2025-09-19T09:49:01.832Z" }, + { url = "https://files.pythonhosted.org/packages/93/0e/ccabc8d16ae4ba84a55d41345207c1e2ea88784651a5a487547d80851398/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:afd7594a56656ace95cdd6df4cca2e4059d294c5cfb1679c57824b605556cb2f", size = 3412560, upload-time = "2025-09-19T09:49:03.867Z" }, + { url = "https://files.pythonhosted.org/packages/d0/c6/dc3a0db5a6766416c32c034286d7c2d406da1f498e4de04ab1b8959edd00/tokenizers-0.22.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2ef6063d7a84994129732b47e7915e8710f27f99f3a3260b8a38fc7ccd083f4", size = 3250221, upload-time = "2025-09-19T09:49:07.664Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a6/2c8486eef79671601ff57b093889a345dd3d576713ef047776015dc66de7/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ba0a64f450b9ef412c98f6bcd2a50c6df6e2443b560024a09fa6a03189726879", size = 9345569, upload-time = "2025-09-19T09:49:14.214Z" }, + { url = "https://files.pythonhosted.org/packages/6b/16/32ce667f14c35537f5f605fe9bea3e415ea1b0a646389d2295ec348d5657/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:331d6d149fa9c7d632cde4490fb8bbb12337fa3a0232e77892be656464f4b446", size = 9271599, upload-time = "2025-09-19T09:49:16.639Z" }, + { url = "https://files.pythonhosted.org/packages/51/7c/a5f7898a3f6baa3fc2685c705e04c98c1094c523051c805cdd9306b8f87e/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:607989f2ea68a46cb1dfbaf3e3aabdf3f21d8748312dbeb6263d1b3b66c5010a", size = 9533862, upload-time = "2025-09-19T09:49:19.146Z" }, + { url = "https://files.pythonhosted.org/packages/36/65/7e75caea90bc73c1dd8d40438adf1a7bc26af3b8d0a6705ea190462506e1/tokenizers-0.22.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a0f307d490295717726598ef6fa4f24af9d484809223bbc253b201c740a06390", size = 9681250, upload-time = "2025-09-19T09:49:21.501Z" }, + { url = "https://files.pythonhosted.org/packages/30/2c/959dddef581b46e6209da82df3b78471e96260e2bc463f89d23b1bf0e52a/tokenizers-0.22.1-cp39-abi3-win32.whl", hash = "sha256:b5120eed1442765cd90b903bb6cfef781fd8fe64e34ccaecbae4c619b7b12a82", size = 2472003, upload-time = "2025-09-19T09:49:27.089Z" }, + { url = "https://files.pythonhosted.org/packages/b3/46/e33a8c93907b631a99377ef4c5f817ab453d0b34f93529421f42ff559671/tokenizers-0.22.1-cp39-abi3-win_amd64.whl", hash = "sha256:65fd6e3fb11ca1e78a6a93602490f134d1fdeb13bcef99389d5102ea318ed138", size = 2674684, upload-time = "2025-09-19T09:49:24.953Z" }, +] + +[[package]] +name = "tomli" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/ed/3f73f72945444548f33eba9a87fc7a6e969915e7b1acc8260b30e1f76a2f/tomli-2.3.0.tar.gz", hash = "sha256:64be704a875d2a59753d80ee8a533c3fe183e3f06807ff7dc2232938ccb01549", size = 17392, upload-time = "2025-10-08T22:01:47.119Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/2e/299f62b401438d5fe1624119c723f5d877acc86a4c2492da405626665f12/tomli-2.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:88bd15eb972f3664f5ed4b57c1634a97153b4bac4479dcb6a495f41921eb7f45", size = 153236, upload-time = "2025-10-08T22:01:00.137Z" }, + { url = "https://files.pythonhosted.org/packages/86/7f/d8fffe6a7aefdb61bced88fcb5e280cfd71e08939da5894161bd71bea022/tomli-2.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:883b1c0d6398a6a9d29b508c331fa56adbcdff647f6ace4dfca0f50e90dfd0ba", size = 148084, upload-time = "2025-10-08T22:01:01.63Z" }, + { url = "https://files.pythonhosted.org/packages/47/5c/24935fb6a2ee63e86d80e4d3b58b222dafaf438c416752c8b58537c8b89a/tomli-2.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d1381caf13ab9f300e30dd8feadb3de072aeb86f1d34a8569453ff32a7dea4bf", size = 234832, upload-time = "2025-10-08T22:01:02.543Z" }, + { url = "https://files.pythonhosted.org/packages/89/da/75dfd804fc11e6612846758a23f13271b76d577e299592b4371a4ca4cd09/tomli-2.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0e285d2649b78c0d9027570d4da3425bdb49830a6156121360b3f8511ea3441", size = 242052, upload-time = "2025-10-08T22:01:03.836Z" }, + { url = "https://files.pythonhosted.org/packages/70/8c/f48ac899f7b3ca7eb13af73bacbc93aec37f9c954df3c08ad96991c8c373/tomli-2.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0a154a9ae14bfcf5d8917a59b51ffd5a3ac1fd149b71b47a3a104ca4edcfa845", size = 239555, upload-time = "2025-10-08T22:01:04.834Z" }, + { url = "https://files.pythonhosted.org/packages/ba/28/72f8afd73f1d0e7829bfc093f4cb98ce0a40ffc0cc997009ee1ed94ba705/tomli-2.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:74bf8464ff93e413514fefd2be591c3b0b23231a77f901db1eb30d6f712fc42c", size = 245128, upload-time = "2025-10-08T22:01:05.84Z" }, + { url = "https://files.pythonhosted.org/packages/b6/eb/a7679c8ac85208706d27436e8d421dfa39d4c914dcf5fa8083a9305f58d9/tomli-2.3.0-cp311-cp311-win32.whl", hash = "sha256:00b5f5d95bbfc7d12f91ad8c593a1659b6387b43f054104cda404be6bda62456", size = 96445, upload-time = "2025-10-08T22:01:06.896Z" }, + { url = "https://files.pythonhosted.org/packages/0a/fe/3d3420c4cb1ad9cb462fb52967080575f15898da97e21cb6f1361d505383/tomli-2.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:4dc4ce8483a5d429ab602f111a93a6ab1ed425eae3122032db7e9acf449451be", size = 107165, upload-time = "2025-10-08T22:01:08.107Z" }, + { url = "https://files.pythonhosted.org/packages/ff/b7/40f36368fcabc518bb11c8f06379a0fd631985046c038aca08c6d6a43c6e/tomli-2.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d7d86942e56ded512a594786a5ba0a5e521d02529b3826e7761a05138341a2ac", size = 154891, upload-time = "2025-10-08T22:01:09.082Z" }, + { url = "https://files.pythonhosted.org/packages/f9/3f/d9dd692199e3b3aab2e4e4dd948abd0f790d9ded8cd10cbaae276a898434/tomli-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:73ee0b47d4dad1c5e996e3cd33b8a76a50167ae5f96a2607cbe8cc773506ab22", size = 148796, upload-time = "2025-10-08T22:01:10.266Z" }, + { url = "https://files.pythonhosted.org/packages/60/83/59bff4996c2cf9f9387a0f5a3394629c7efa5ef16142076a23a90f1955fa/tomli-2.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:792262b94d5d0a466afb5bc63c7daa9d75520110971ee269152083270998316f", size = 242121, upload-time = "2025-10-08T22:01:11.332Z" }, + { url = "https://files.pythonhosted.org/packages/45/e5/7c5119ff39de8693d6baab6c0b6dcb556d192c165596e9fc231ea1052041/tomli-2.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f195fe57ecceac95a66a75ac24d9d5fbc98ef0962e09b2eddec5d39375aae52", size = 250070, upload-time = "2025-10-08T22:01:12.498Z" }, + { url = "https://files.pythonhosted.org/packages/45/12/ad5126d3a278f27e6701abde51d342aa78d06e27ce2bb596a01f7709a5a2/tomli-2.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e31d432427dcbf4d86958c184b9bfd1e96b5b71f8eb17e6d02531f434fd335b8", size = 245859, upload-time = "2025-10-08T22:01:13.551Z" }, + { url = "https://files.pythonhosted.org/packages/fb/a1/4d6865da6a71c603cfe6ad0e6556c73c76548557a8d658f9e3b142df245f/tomli-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7b0882799624980785240ab732537fcfc372601015c00f7fc367c55308c186f6", size = 250296, upload-time = "2025-10-08T22:01:14.614Z" }, + { url = "https://files.pythonhosted.org/packages/a0/b7/a7a7042715d55c9ba6e8b196d65d2cb662578b4d8cd17d882d45322b0d78/tomli-2.3.0-cp312-cp312-win32.whl", hash = "sha256:ff72b71b5d10d22ecb084d345fc26f42b5143c5533db5e2eaba7d2d335358876", size = 97124, upload-time = "2025-10-08T22:01:15.629Z" }, + { url = "https://files.pythonhosted.org/packages/06/1e/f22f100db15a68b520664eb3328fb0ae4e90530887928558112c8d1f4515/tomli-2.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:1cb4ed918939151a03f33d4242ccd0aa5f11b3547d0cf30f7c74a408a5b99878", size = 107698, upload-time = "2025-10-08T22:01:16.51Z" }, + { url = "https://files.pythonhosted.org/packages/89/48/06ee6eabe4fdd9ecd48bf488f4ac783844fd777f547b8d1b61c11939974e/tomli-2.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5192f562738228945d7b13d4930baffda67b69425a7f0da96d360b0a3888136b", size = 154819, upload-time = "2025-10-08T22:01:17.964Z" }, + { url = "https://files.pythonhosted.org/packages/f1/01/88793757d54d8937015c75dcdfb673c65471945f6be98e6a0410fba167ed/tomli-2.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:be71c93a63d738597996be9528f4abe628d1adf5e6eb11607bc8fe1a510b5dae", size = 148766, upload-time = "2025-10-08T22:01:18.959Z" }, + { url = "https://files.pythonhosted.org/packages/42/17/5e2c956f0144b812e7e107f94f1cc54af734eb17b5191c0bbfb72de5e93e/tomli-2.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4665508bcbac83a31ff8ab08f424b665200c0e1e645d2bd9ab3d3e557b6185b", size = 240771, upload-time = "2025-10-08T22:01:20.106Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f4/0fbd014909748706c01d16824eadb0307115f9562a15cbb012cd9b3512c5/tomli-2.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4021923f97266babc6ccab9f5068642a0095faa0a51a246a6a02fccbb3514eaf", size = 248586, upload-time = "2025-10-08T22:01:21.164Z" }, + { url = "https://files.pythonhosted.org/packages/30/77/fed85e114bde5e81ecf9bc5da0cc69f2914b38f4708c80ae67d0c10180c5/tomli-2.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4ea38c40145a357d513bffad0ed869f13c1773716cf71ccaa83b0fa0cc4e42f", size = 244792, upload-time = "2025-10-08T22:01:22.417Z" }, + { url = "https://files.pythonhosted.org/packages/55/92/afed3d497f7c186dc71e6ee6d4fcb0acfa5f7d0a1a2878f8beae379ae0cc/tomli-2.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ad805ea85eda330dbad64c7ea7a4556259665bdf9d2672f5dccc740eb9d3ca05", size = 248909, upload-time = "2025-10-08T22:01:23.859Z" }, + { url = "https://files.pythonhosted.org/packages/f8/84/ef50c51b5a9472e7265ce1ffc7f24cd4023d289e109f669bdb1553f6a7c2/tomli-2.3.0-cp313-cp313-win32.whl", hash = "sha256:97d5eec30149fd3294270e889b4234023f2c69747e555a27bd708828353ab606", size = 96946, upload-time = "2025-10-08T22:01:24.893Z" }, + { url = "https://files.pythonhosted.org/packages/b2/b7/718cd1da0884f281f95ccfa3a6cc572d30053cba64603f79d431d3c9b61b/tomli-2.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:0c95ca56fbe89e065c6ead5b593ee64b84a26fca063b5d71a1122bf26e533999", size = 107705, upload-time = "2025-10-08T22:01:26.153Z" }, + { url = "https://files.pythonhosted.org/packages/19/94/aeafa14a52e16163008060506fcb6aa1949d13548d13752171a755c65611/tomli-2.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:cebc6fe843e0733ee827a282aca4999b596241195f43b4cc371d64fc6639da9e", size = 154244, upload-time = "2025-10-08T22:01:27.06Z" }, + { url = "https://files.pythonhosted.org/packages/db/e4/1e58409aa78eefa47ccd19779fc6f36787edbe7d4cd330eeeedb33a4515b/tomli-2.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4c2ef0244c75aba9355561272009d934953817c49f47d768070c3c94355c2aa3", size = 148637, upload-time = "2025-10-08T22:01:28.059Z" }, + { url = "https://files.pythonhosted.org/packages/26/b6/d1eccb62f665e44359226811064596dd6a366ea1f985839c566cd61525ae/tomli-2.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c22a8bf253bacc0cf11f35ad9808b6cb75ada2631c2d97c971122583b129afbc", size = 241925, upload-time = "2025-10-08T22:01:29.066Z" }, + { url = "https://files.pythonhosted.org/packages/70/91/7cdab9a03e6d3d2bb11beae108da5bdc1c34bdeb06e21163482544ddcc90/tomli-2.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0eea8cc5c5e9f89c9b90c4896a8deefc74f518db5927d0e0e8d4a80953d774d0", size = 249045, upload-time = "2025-10-08T22:01:31.98Z" }, + { url = "https://files.pythonhosted.org/packages/15/1b/8c26874ed1f6e4f1fcfeb868db8a794cbe9f227299402db58cfcc858766c/tomli-2.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b74a0e59ec5d15127acdabd75ea17726ac4c5178ae51b85bfe39c4f8a278e879", size = 245835, upload-time = "2025-10-08T22:01:32.989Z" }, + { url = "https://files.pythonhosted.org/packages/fd/42/8e3c6a9a4b1a1360c1a2a39f0b972cef2cc9ebd56025168c4137192a9321/tomli-2.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b5870b50c9db823c595983571d1296a6ff3e1b88f734a4c8f6fc6188397de005", size = 253109, upload-time = "2025-10-08T22:01:34.052Z" }, + { url = "https://files.pythonhosted.org/packages/22/0c/b4da635000a71b5f80130937eeac12e686eefb376b8dee113b4a582bba42/tomli-2.3.0-cp314-cp314-win32.whl", hash = "sha256:feb0dacc61170ed7ab602d3d972a58f14ee3ee60494292d384649a3dc38ef463", size = 97930, upload-time = "2025-10-08T22:01:35.082Z" }, + { url = "https://files.pythonhosted.org/packages/b9/74/cb1abc870a418ae99cd5c9547d6bce30701a954e0e721821df483ef7223c/tomli-2.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:b273fcbd7fc64dc3600c098e39136522650c49bca95df2d11cf3b626422392c8", size = 107964, upload-time = "2025-10-08T22:01:36.057Z" }, + { url = "https://files.pythonhosted.org/packages/54/78/5c46fff6432a712af9f792944f4fcd7067d8823157949f4e40c56b8b3c83/tomli-2.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:940d56ee0410fa17ee1f12b817b37a4d4e4dc4d27340863cc67236c74f582e77", size = 163065, upload-time = "2025-10-08T22:01:37.27Z" }, + { url = "https://files.pythonhosted.org/packages/39/67/f85d9bd23182f45eca8939cd2bc7050e1f90c41f4a2ecbbd5963a1d1c486/tomli-2.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f85209946d1fe94416debbb88d00eb92ce9cd5266775424ff81bc959e001acaf", size = 159088, upload-time = "2025-10-08T22:01:38.235Z" }, + { url = "https://files.pythonhosted.org/packages/26/5a/4b546a0405b9cc0659b399f12b6adb750757baf04250b148d3c5059fc4eb/tomli-2.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a56212bdcce682e56b0aaf79e869ba5d15a6163f88d5451cbde388d48b13f530", size = 268193, upload-time = "2025-10-08T22:01:39.712Z" }, + { url = "https://files.pythonhosted.org/packages/42/4f/2c12a72ae22cf7b59a7fe75b3465b7aba40ea9145d026ba41cb382075b0e/tomli-2.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5f3ffd1e098dfc032d4d3af5c0ac64f6d286d98bc148698356847b80fa4de1b", size = 275488, upload-time = "2025-10-08T22:01:40.773Z" }, + { url = "https://files.pythonhosted.org/packages/92/04/a038d65dbe160c3aa5a624e93ad98111090f6804027d474ba9c37c8ae186/tomli-2.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e01decd096b1530d97d5d85cb4dff4af2d8347bd35686654a004f8dea20fc67", size = 272669, upload-time = "2025-10-08T22:01:41.824Z" }, + { url = "https://files.pythonhosted.org/packages/be/2f/8b7c60a9d1612a7cbc39ffcca4f21a73bf368a80fc25bccf8253e2563267/tomli-2.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8a35dd0e643bb2610f156cca8db95d213a90015c11fee76c946aa62b7ae7e02f", size = 279709, upload-time = "2025-10-08T22:01:43.177Z" }, + { url = "https://files.pythonhosted.org/packages/7e/46/cc36c679f09f27ded940281c38607716c86cf8ba4a518d524e349c8b4874/tomli-2.3.0-cp314-cp314t-win32.whl", hash = "sha256:a1f7f282fe248311650081faafa5f4732bdbfef5d45fe3f2e702fbc6f2d496e0", size = 107563, upload-time = "2025-10-08T22:01:44.233Z" }, + { url = "https://files.pythonhosted.org/packages/84/ff/426ca8683cf7b753614480484f6437f568fd2fda2edbdf57a2d3d8b27a0b/tomli-2.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:70a251f8d4ba2d9ac2542eecf008b3c8a9fc5c3f9f02c56a9d7952612be2fdba", size = 119756, upload-time = "2025-10-08T22:01:45.234Z" }, + { url = "https://files.pythonhosted.org/packages/77/b8/0135fadc89e73be292b473cb820b4f5a08197779206b33191e801feeae40/tomli-2.3.0-py3-none-any.whl", hash = "sha256:e95b1af3c5b07d9e643909b5abbec77cd9f1217e6d0bca72b0234736b9fb1f1b", size = 14408, upload-time = "2025-10-08T22:01:46.04Z" }, +] + +[[package]] +name = "torch" +version = "2.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cufile-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvshmem-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "setuptools", marker = "python_full_version >= '3.12'" }, + { name = "sympy" }, + { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/db/c064112ac0089af3d2f7a2b5bfbabf4aa407a78b74f87889e524b91c5402/torch-2.9.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:62b3fd888277946918cba4478cf849303da5359f0fb4e3bfb86b0533ba2eaf8d", size = 104220430, upload-time = "2025-11-12T15:20:31.705Z" }, + { url = "https://files.pythonhosted.org/packages/56/be/76eaa36c9cd032d3b01b001e2c5a05943df75f26211f68fae79e62f87734/torch-2.9.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d033ff0ac3f5400df862a51bdde9bad83561f3739ea0046e68f5401ebfa67c1b", size = 899821446, upload-time = "2025-11-12T15:20:15.544Z" }, + { url = "https://files.pythonhosted.org/packages/47/cc/7a2949e38dfe3244c4df21f0e1c27bce8aedd6c604a587dd44fc21017cb4/torch-2.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:0d06b30a9207b7c3516a9e0102114024755a07045f0c1d2f2a56b1819ac06bcb", size = 110973074, upload-time = "2025-11-12T15:21:39.958Z" }, + { url = "https://files.pythonhosted.org/packages/1e/ce/7d251155a783fb2c1bb6837b2b7023c622a2070a0a72726ca1df47e7ea34/torch-2.9.1-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:52347912d868653e1528b47cafaf79b285b98be3f4f35d5955389b1b95224475", size = 74463887, upload-time = "2025-11-12T15:20:36.611Z" }, + { url = "https://files.pythonhosted.org/packages/0f/27/07c645c7673e73e53ded71705045d6cb5bae94c4b021b03aa8d03eee90ab/torch-2.9.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:da5f6f4d7f4940a173e5572791af238cb0b9e21b1aab592bd8b26da4c99f1cd6", size = 104126592, upload-time = "2025-11-12T15:20:41.62Z" }, + { url = "https://files.pythonhosted.org/packages/19/17/e377a460603132b00760511299fceba4102bd95db1a0ee788da21298ccff/torch-2.9.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:27331cd902fb4322252657f3902adf1c4f6acad9dcad81d8df3ae14c7c4f07c4", size = 899742281, upload-time = "2025-11-12T15:22:17.602Z" }, + { url = "https://files.pythonhosted.org/packages/b1/1a/64f5769025db846a82567fa5b7d21dba4558a7234ee631712ee4771c436c/torch-2.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:81a285002d7b8cfd3fdf1b98aa8df138d41f1a8334fd9ea37511517cedf43083", size = 110940568, upload-time = "2025-11-12T15:21:18.689Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ab/07739fd776618e5882661d04c43f5b5586323e2f6a2d7d84aac20d8f20bd/torch-2.9.1-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:c0d25d1d8e531b8343bea0ed811d5d528958f1dcbd37e7245bc686273177ad7e", size = 74479191, upload-time = "2025-11-12T15:21:25.816Z" }, + { url = "https://files.pythonhosted.org/packages/20/60/8fc5e828d050bddfab469b3fe78e5ab9a7e53dda9c3bdc6a43d17ce99e63/torch-2.9.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:c29455d2b910b98738131990394da3e50eea8291dfeb4b12de71ecf1fdeb21cb", size = 104135743, upload-time = "2025-11-12T15:21:34.936Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b7/6d3f80e6918213babddb2a37b46dbb14c15b14c5f473e347869a51f40e1f/torch-2.9.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:524de44cd13931208ba2c4bde9ec7741fd4ae6bfd06409a604fc32f6520c2bc9", size = 899749493, upload-time = "2025-11-12T15:24:36.356Z" }, + { url = "https://files.pythonhosted.org/packages/a6/47/c7843d69d6de8938c1cbb1eba426b1d48ddf375f101473d3e31a5fc52b74/torch-2.9.1-cp313-cp313-win_amd64.whl", hash = "sha256:545844cc16b3f91e08ce3b40e9c2d77012dd33a48d505aed34b7740ed627a1b2", size = 110944162, upload-time = "2025-11-12T15:21:53.151Z" }, + { url = "https://files.pythonhosted.org/packages/28/0e/2a37247957e72c12151b33a01e4df651d9d155dd74d8cfcbfad15a79b44a/torch-2.9.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5be4bf7496f1e3ffb1dd44b672adb1ac3f081f204c5ca81eba6442f5f634df8e", size = 74830751, upload-time = "2025-11-12T15:21:43.792Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f7/7a18745edcd7b9ca2381aa03353647bca8aace91683c4975f19ac233809d/torch-2.9.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:30a3e170a84894f3652434b56d59a64a2c11366b0ed5776fab33c2439396bf9a", size = 104142929, upload-time = "2025-11-12T15:21:48.319Z" }, + { url = "https://files.pythonhosted.org/packages/f4/dd/f1c0d879f2863ef209e18823a988dc7a1bf40470750e3ebe927efdb9407f/torch-2.9.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:8301a7b431e51764629208d0edaa4f9e4c33e6df0f2f90b90e261d623df6a4e2", size = 899748978, upload-time = "2025-11-12T15:23:04.568Z" }, + { url = "https://files.pythonhosted.org/packages/1f/9f/6986b83a53b4d043e36f3f898b798ab51f7f20fdf1a9b01a2720f445043d/torch-2.9.1-cp313-cp313t-win_amd64.whl", hash = "sha256:2e1c42c0ae92bf803a4b2409fdfed85e30f9027a66887f5e7dcdbc014c7531db", size = 111176995, upload-time = "2025-11-12T15:22:01.618Z" }, + { url = "https://files.pythonhosted.org/packages/40/60/71c698b466dd01e65d0e9514b5405faae200c52a76901baf6906856f17e4/torch-2.9.1-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:2c14b3da5df416cf9cb5efab83aa3056f5b8cd8620b8fde81b4987ecab730587", size = 74480347, upload-time = "2025-11-12T15:21:57.648Z" }, + { url = "https://files.pythonhosted.org/packages/48/50/c4b5112546d0d13cc9eaa1c732b823d676a9f49ae8b6f97772f795874a03/torch-2.9.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1edee27a7c9897f4e0b7c14cfc2f3008c571921134522d5b9b5ec4ebbc69041a", size = 74433245, upload-time = "2025-11-12T15:22:39.027Z" }, + { url = "https://files.pythonhosted.org/packages/81/c9/2628f408f0518b3bae49c95f5af3728b6ab498c8624ab1e03a43dd53d650/torch-2.9.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:19d144d6b3e29921f1fc70503e9f2fc572cde6a5115c0c0de2f7ca8b1483e8b6", size = 104134804, upload-time = "2025-11-12T15:22:35.222Z" }, + { url = "https://files.pythonhosted.org/packages/28/fc/5bc91d6d831ae41bf6e9e6da6468f25330522e92347c9156eb3f1cb95956/torch-2.9.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:c432d04376f6d9767a9852ea0def7b47a7bbc8e7af3b16ac9cf9ce02b12851c9", size = 899747132, upload-time = "2025-11-12T15:23:36.068Z" }, + { url = "https://files.pythonhosted.org/packages/63/5d/e8d4e009e52b6b2cf1684bde2a6be157b96fb873732542fb2a9a99e85a83/torch-2.9.1-cp314-cp314-win_amd64.whl", hash = "sha256:d187566a2cdc726fc80138c3cdb260970fab1c27e99f85452721f7759bbd554d", size = 110934845, upload-time = "2025-11-12T15:22:48.367Z" }, + { url = "https://files.pythonhosted.org/packages/bd/b2/2d15a52516b2ea3f414643b8de68fa4cb220d3877ac8b1028c83dc8ca1c4/torch-2.9.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cb10896a1f7fedaddbccc2017ce6ca9ecaaf990f0973bdfcf405439750118d2c", size = 74823558, upload-time = "2025-11-12T15:22:43.392Z" }, + { url = "https://files.pythonhosted.org/packages/86/5c/5b2e5d84f5b9850cd1e71af07524d8cbb74cba19379800f1f9f7c997fc70/torch-2.9.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:0a2bd769944991c74acf0c4ef23603b9c777fdf7637f115605a4b2d8023110c7", size = 104145788, upload-time = "2025-11-12T15:23:52.109Z" }, + { url = "https://files.pythonhosted.org/packages/a9/8c/3da60787bcf70add986c4ad485993026ac0ca74f2fc21410bc4eb1bb7695/torch-2.9.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:07c8a9660bc9414c39cac530ac83b1fb1b679d7155824144a40a54f4a47bfa73", size = 899735500, upload-time = "2025-11-12T15:24:08.788Z" }, + { url = "https://files.pythonhosted.org/packages/db/2b/f7818f6ec88758dfd21da46b6cd46af9d1b3433e53ddbb19ad1e0da17f9b/torch-2.9.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c88d3299ddeb2b35dcc31753305612db485ab6f1823e37fb29451c8b2732b87e", size = 111163659, upload-time = "2025-11-12T15:23:20.009Z" }, +] + +[[package]] +name = "tqdm" +version = "4.67.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737, upload-time = "2024-11-24T20:12:22.481Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" }, +] + +[[package]] +name = "transformers" +version = "5.0.0rc3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "requests" }, + { name = "safetensors" }, + { name = "tokenizers" }, + { name = "tqdm" }, + { name = "typer-slim" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3f/a3/7c116a8d85f69ea7749cf4c2df79e64c35d028e5fc7ea0168f299d03b8c7/transformers-5.0.0rc3.tar.gz", hash = "sha256:a0315b92b7e087617ade42ec9e6e92ee7620541cc5d6a3331886c52cbe306f5c", size = 8388520, upload-time = "2026-01-14T16:49:02.952Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/f2/ae2b8968764253bdf38a48dee3c299b8d0bedf7c8ffbe3449fca9bd95338/transformers-5.0.0rc3-py3-none-any.whl", hash = "sha256:383fad27f4f73092d330e45fae384681e5c8521e1dc1cf6cb1a297780e68bf2d", size = 10107087, upload-time = "2026-01-14T16:48:59.393Z" }, +] + +[[package]] +name = "triton" +version = "3.5.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/72/ec90c3519eaf168f22cb1757ad412f3a2add4782ad3a92861c9ad135d886/triton-3.5.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:61413522a48add32302353fdbaaf92daaaab06f6b5e3229940d21b5207f47579", size = 170425802, upload-time = "2025-11-11T17:40:53.209Z" }, + { url = "https://files.pythonhosted.org/packages/f2/50/9a8358d3ef58162c0a415d173cfb45b67de60176e1024f71fbc4d24c0b6d/triton-3.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d2c6b915a03888ab931a9fd3e55ba36785e1fe70cbea0b40c6ef93b20fc85232", size = 170470207, upload-time = "2025-11-11T17:41:00.253Z" }, + { url = "https://files.pythonhosted.org/packages/27/46/8c3bbb5b0a19313f50edcaa363b599e5a1a5ac9683ead82b9b80fe497c8d/triton-3.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3f4346b6ebbd4fad18773f5ba839114f4826037c9f2f34e0148894cd5dd3dba", size = 170470410, upload-time = "2025-11-11T17:41:06.319Z" }, + { url = "https://files.pythonhosted.org/packages/37/92/e97fcc6b2c27cdb87ce5ee063d77f8f26f19f06916aa680464c8104ef0f6/triton-3.5.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0b4d2c70127fca6a23e247f9348b8adde979d2e7a20391bfbabaac6aebc7e6a8", size = 170579924, upload-time = "2025-11-11T17:41:12.455Z" }, + { url = "https://files.pythonhosted.org/packages/a4/e6/c595c35e5c50c4bc56a7bac96493dad321e9e29b953b526bbbe20f9911d0/triton-3.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0637b1efb1db599a8e9dc960d53ab6e4637db7d4ab6630a0974705d77b14b60", size = 170480488, upload-time = "2025-11-11T17:41:18.222Z" }, + { url = "https://files.pythonhosted.org/packages/16/b5/b0d3d8b901b6a04ca38df5e24c27e53afb15b93624d7fd7d658c7cd9352a/triton-3.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bac7f7d959ad0f48c0e97d6643a1cc0fd5786fe61cb1f83b537c6b2d54776478", size = 170582192, upload-time = "2025-11-11T17:41:23.963Z" }, +] + +[[package]] +name = "typer" +version = "0.24.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "click" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/24/cb09efec5cc954f7f9b930bf8279447d24618bb6758d4f6adf2574c41780/typer-0.24.1.tar.gz", hash = "sha256:e39b4732d65fbdcde189ae76cf7cd48aeae72919dea1fdfc16593be016256b45", size = 118613, upload-time = "2026-02-21T16:54:40.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/91/48db081e7a63bb37284f9fbcefda7c44c277b18b0e13fbc36ea2335b71e6/typer-0.24.1-py3-none-any.whl", hash = "sha256:112c1f0ce578bfb4cab9ffdabc68f031416ebcc216536611ba21f04e9aa84c9e", size = 56085, upload-time = "2026-02-21T16:54:41.616Z" }, +] + +[[package]] +name = "typer-slim" +version = "0.24.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/a7/e6aecc4b4eb59598829a3b5076a93aff291b4fdaa2ded25efc4e1f4d219c/typer_slim-0.24.0.tar.gz", hash = "sha256:f0ed36127183f52ae6ced2ecb2521789995992c521a46083bfcdbb652d22ad34", size = 4776, upload-time = "2026-02-16T22:08:51.2Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/24/5480c20380dfd18cf33d14784096dca45a24eae6102e91d49a718d3b6855/typer_slim-0.24.0-py3-none-any.whl", hash = "sha256:d5d7ee1ee2834d5020c7c616ed5e0d0f29b9a4b1dd283bdebae198ec09778d0e", size = 3394, upload-time = "2026-02-16T22:08:49.92Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + +[[package]] +name = "virtualenv" +version = "20.36.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distlib" }, + { name = "filelock" }, + { name = "platformdirs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/aa/a3/4d310fa5f00863544e1d0f4de93bddec248499ccf97d4791bc3122c9d4f3/virtualenv-20.36.1.tar.gz", hash = "sha256:8befb5c81842c641f8ee658481e42641c68b5eab3521d8e092d18320902466ba", size = 6032239, upload-time = "2026-01-09T18:21:01.296Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/2a/dc2228b2888f51192c7dc766106cd475f1b768c10caaf9727659726f7391/virtualenv-20.36.1-py3-none-any.whl", hash = "sha256:575a8d6b124ef88f6f51d56d656132389f961062a9177016a50e4f507bbcc19f", size = 6008258, upload-time = "2026-01-09T18:20:59.425Z" }, +] + +[[package]] +name = "zipp" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, +]